11590Srgrimes/*- 21590Srgrimes * Copyright (c) 2003 Jake Burkholder. 31590Srgrimes * Copyright (c) 2005 - 2011 Marius Strobl <marius@FreeBSD.org> 41590Srgrimes * All rights reserved. 51590Srgrimes * 61590Srgrimes * Redistribution and use in source and binary forms, with or without 71590Srgrimes * modification, are permitted provided that the following conditions 81590Srgrimes * are met: 91590Srgrimes * 1. Redistributions of source code must retain the above copyright 101590Srgrimes * notice, this list of conditions and the following disclaimer. 111590Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 121590Srgrimes * notice, this list of conditions and the following disclaimer in the 131590Srgrimes * documentation and/or other materials provided with the distribution. 141590Srgrimes * 151590Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 161590Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 171590Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 181590Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 191590Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 201590Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 211590Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 221590Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 231590Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 241590Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 251590Srgrimes * SUCH DAMAGE. 261590Srgrimes */ 271590Srgrimes 281590Srgrimes#include <sys/cdefs.h> 291590Srgrimes__FBSDID("$FreeBSD$"); 301590Srgrimes 311590Srgrimes#include <sys/param.h> 321590Srgrimes#include <sys/systm.h> 331590Srgrimes#include <sys/lock.h> 341590Srgrimes#include <sys/mutex.h> 351590Srgrimes#include <sys/smp.h> 361590Srgrimes 371590Srgrimes#include <vm/vm.h> 3841568Sarchie#include <vm/pmap.h> 391590Srgrimes 401590Srgrimes#include <machine/asi.h> 411590Srgrimes#include <machine/cache.h> 421590Srgrimes#include <machine/cpu.h> 431590Srgrimes#include <machine/cpufunc.h> 4456597Smharo#include <machine/dcr.h> 4541568Sarchie#include <machine/lsu.h> 4656597Smharo#include <machine/smp.h> 471590Srgrimes#include <machine/tlb.h> 4899112Sobrien#include <machine/ver.h> 4999112Sobrien#include <machine/vmparam.h> 501590Srgrimes 511590Srgrimes#define CHEETAH_ICACHE_TAG_LOWER 0x30 5278158Sroam#define CHEETAH_T16_ENTRIES 16 531590Srgrimes#define CHEETAH_DT512_ENTRIES 512 541590Srgrimes#define CHEETAH_IT128_ENTRIES 128 551590Srgrimes#define CHEETAH_IT512_ENTRIES 512 561590Srgrimes 5778158Sroam/* 581590Srgrimes * CPU-specific initialization for Sun Cheetah and later CPUs 59129678Spjd */ 60132201Stjrvoid 61139813Spjdcheetah_init(u_int cpu_impl) 621590Srgrimes{ 631590Srgrimes u_long val; 641590Srgrimes 6556597Smharo /* Ensure the TSB Extension Registers hold 0 as TSB_Base. */ 6623693Speter 671590Srgrimes stxa(AA_DMMU_TSB_PEXT_REG, ASI_DMMU, 0); 6878158Sroam stxa(AA_IMMU_TSB_PEXT_REG, ASI_IMMU, 0); 6978158Sroam membar(Sync); 7078158Sroam 7178158Sroam stxa(AA_DMMU_TSB_SEXT_REG, ASI_DMMU, 0); 7278158Sroam /* 7378158Sroam * NB: the secondary context was removed from the iMMU. 74128772Skientzle */ 7592920Simp membar(Sync); 76184656Smlaier 77184656Smlaier stxa(AA_DMMU_TSB_NEXT_REG, ASI_DMMU, 0); 78184656Smlaier stxa(AA_IMMU_TSB_NEXT_REG, ASI_IMMU, 0); 79184656Smlaier membar(Sync); 801590Srgrimes 81184656Smlaier /* 82184733Smlaier * Configure the first large dTLB to hold 4MB pages (e.g. for direct 83184733Smlaier * mappings) for all three contexts and ensure the second one is set 84158339Smaxim * up to hold 8k pages for them. Note that this is constraint by 851590Srgrimes * US-IV+, whose large dTLBs can only hold entries of certain page 86100822Sdwmalone * sizes each. 871590Srgrimes * For US-IV+, additionally ensure that the large iTLB is set up to 8832097Sjkh * hold 8k pages for nucleus and primary context (still no secondary 8932097Sjkh * iMMU context. 90184733Smlaier * NB: according to documentation, changing the page size of the same 9132097Sjkh * context requires a context demap before changing the corresponding 9232097Sjkh * page size, but we hardly can flush our locked pages here, so we use 9332097Sjkh * a demap all instead. 94176561Skeramida */ 95176561Skeramida stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); 9632097Sjkh membar(Sync); 9787216Smarkm val = (TS_4M << TLB_PCXR_N_PGSZ0_SHIFT) | 981590Srgrimes (TS_8K << TLB_PCXR_N_PGSZ1_SHIFT) | 99132201Stjr (TS_4M << TLB_PCXR_P_PGSZ0_SHIFT) | 100132201Stjr (TS_8K << TLB_PCXR_P_PGSZ1_SHIFT); 101176561Skeramida if (cpu_impl == CPU_IMPL_ULTRASPARCIVp) 102184733Smlaier val |= (TS_8K << TLB_PCXR_N_PGSZ_I_SHIFT) | 103128772Skientzle (TS_8K << TLB_PCXR_P_PGSZ_I_SHIFT); 1041590Srgrimes stxa(AA_DMMU_PCXR, ASI_DMMU, val); 10532097Sjkh val = (TS_4M << TLB_SCXR_S_PGSZ0_SHIFT) | 106184654Smlaier (TS_8K << TLB_SCXR_S_PGSZ1_SHIFT); 107184733Smlaier stxa(AA_DMMU_SCXR, ASI_DMMU, val); 108184733Smlaier flush(KERNBASE); 10919120Sscrappy 11078158Sroam /* 111128772Skientzle * Ensure DCR_IFPOE is disabled as long as we haven't implemented 112184733Smlaier * support for it (if ever) as most if not all firmware versions 1131590Srgrimes * apparently turn it on. Not making use of DCR_IFPOE should also 114184733Smlaier * avoid Cheetah erratum #109. 115184733Smlaier */ 116184733Smlaier val = rd(asr18) & ~DCR_IFPOE; 117184733Smlaier if (cpu_impl == CPU_IMPL_ULTRASPARCIVp) { 118184733Smlaier /* 119184733Smlaier * Ensure the branch prediction mode is set to PC indexing 120184733Smlaier * in order to work around US-IV+ erratum #2. 121184733Smlaier */ 122184733Smlaier val = (val & ~DCR_BPM_MASK) | DCR_BPM_PC; 123184733Smlaier /* 124184733Smlaier * XXX disable dTLB parity error reporting as otherwise we 125184733Smlaier * get seemingly false positives when copying in the user 126184654Smlaier * window by simulating a fill trap on return to usermode in 127184654Smlaier * case single issue is disabled, which thus appears to be 128184654Smlaier * a CPU bug. 129184654Smlaier */ 130184654Smlaier val &= ~DCR_DTPE; 131184654Smlaier } 132184654Smlaier wr(asr18, val, 0); 133184654Smlaier} 13419120Sscrappy 135184654Smlaier/* 136184654Smlaier * Enable level 1 caches. 137184654Smlaier */ 138184654Smlaiervoid 139184654Smlaiercheetah_cache_enable(u_int cpu_impl) 140184654Smlaier{ 141184654Smlaier u_long lsu; 142184654Smlaier 143184654Smlaier lsu = ldxa(0, ASI_LSU_CTL_REG); 144184654Smlaier if (cpu_impl == CPU_IMPL_ULTRASPARCIII) { 145184654Smlaier /* Disable P$ due to US-III erratum #18. */ 146184654Smlaier lsu &= ~LSU_PE; 147184654Smlaier } 148184654Smlaier stxa(0, ASI_LSU_CTL_REG, lsu | LSU_IC | LSU_DC); 149184654Smlaier flush(KERNBASE); 150184654Smlaier} 151184654Smlaier 152184654Smlaier/* 153184654Smlaier * Flush all lines from the level 1 caches. 154184654Smlaier */ 155184654Smlaiervoid 156184654Smlaiercheetah_cache_flush(void) 157184654Smlaier{ 158184654Smlaier u_long addr, lsu; 159184654Smlaier register_t s; 160184654Smlaier 161184654Smlaier s = intr_disable(); 162184654Smlaier for (addr = 0; addr < PCPU_GET(cache.dc_size); 163184654Smlaier addr += PCPU_GET(cache.dc_linesize)) 164184654Smlaier /* 165184654Smlaier * Note that US-IV+ additionally require a membar #Sync before 166184733Smlaier * a load or store to ASI_DCACHE_TAG. 167184654Smlaier */ 168184654Smlaier __asm __volatile( 169184654Smlaier "membar #Sync;" 170184654Smlaier "stxa %%g0, [%0] %1;" 171184654Smlaier "membar #Sync" 172184654Smlaier : : "r" (addr), "n" (ASI_DCACHE_TAG)); 173184733Smlaier 174184654Smlaier /* The I$ must be disabled when flushing it so ensure it's off. */ 175184654Smlaier lsu = ldxa(0, ASI_LSU_CTL_REG); 176184654Smlaier stxa(0, ASI_LSU_CTL_REG, lsu & ~(LSU_IC)); 177184654Smlaier flush(KERNBASE); 178184654Smlaier for (addr = CHEETAH_ICACHE_TAG_LOWER; 179184654Smlaier addr < PCPU_GET(cache.ic_size) * 2; 180184654Smlaier addr += PCPU_GET(cache.ic_linesize) * 2) 181184654Smlaier __asm __volatile( 182184654Smlaier "stxa %%g0, [%0] %1;" 183184654Smlaier "membar #Sync" 184184654Smlaier : : "r" (addr), "n" (ASI_ICACHE_TAG)); 185184654Smlaier stxa(0, ASI_LSU_CTL_REG, lsu); 186184654Smlaier flush(KERNBASE); 1871590Srgrimes intr_restore(s); 18832097Sjkh} 1891590Srgrimes 1901590Srgrimes/* 1911590Srgrimes * Flush a physical page from the data cache. 1921590Srgrimes */ 1931590Srgrimesvoid 1941590Srgrimescheetah_dcache_page_inval(vm_paddr_t spa) 1951590Srgrimes{ 1961590Srgrimes vm_paddr_t pa; 1971590Srgrimes void *cookie; 1981590Srgrimes 1991590Srgrimes KASSERT((spa & PAGE_MASK) == 0, 2001590Srgrimes ("%s: pa not page aligned", __func__)); 2011590Srgrimes cookie = ipi_dcache_page_inval(tl_ipi_cheetah_dcache_page_inval, spa); 2021590Srgrimes for (pa = spa; pa < spa + PAGE_SIZE; 2031590Srgrimes pa += PCPU_GET(cache.dc_linesize)) 20432097Sjkh stxa_sync(pa, ASI_DCACHE_INVALIDATE, 0); 20532097Sjkh ipi_wait(cookie); 20632097Sjkh} 20732097Sjkh 20832097Sjkh/* 20932097Sjkh * Flush a physical page from the intsruction cache. Instruction cache 21032097Sjkh * consistency is maintained by hardware. 2111590Srgrimes */ 2121590Srgrimesvoid 21332097Sjkhcheetah_icache_page_inval(vm_paddr_t pa __unused) 21432097Sjkh{ 2151590Srgrimes 2161590Srgrimes} 21732097Sjkh 21832097Sjkh/* 21932097Sjkh * Flush all non-locked mappings from the TLBs. 220184733Smlaier */ 221184733Smlaiervoid 222184733Smlaiercheetah_tlb_flush_nonlocked(void) 22332097Sjkh{ 22432097Sjkh 2251590Srgrimes stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0); 22619120Sscrappy stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0); 2271590Srgrimes flush(KERNBASE); 22832097Sjkh} 22919120Sscrappy 23019120Sscrappy/* 23119120Sscrappy * Flush all user mappings from the TLBs. 23232097Sjkh */ 2331590Srgrimesvoid 2341590Srgrimescheetah_tlb_flush_user(void) 2351590Srgrimes{ 2361590Srgrimes u_long data, tag; 23787216Smarkm register_t s; 2381590Srgrimes u_int i, slot; 2391590Srgrimes 2401590Srgrimes /* 241184733Smlaier * We read ASI_{D,I}TLB_DATA_ACCESS_REG twice back-to-back in order 242184733Smlaier * to work around errata of USIII and beyond. 2431590Srgrimes */ 244184733Smlaier for (i = 0; i < CHEETAH_T16_ENTRIES; i++) { 245184733Smlaier slot = TLB_DAR_SLOT(TLB_DAR_T16, i); 246184733Smlaier s = intr_disable(); 247184733Smlaier (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 248184733Smlaier data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 24932097Sjkh intr_restore(s); 250128772Skientzle tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); 2511590Srgrimes if ((data & TD_V) != 0 && (data & TD_L) == 0 && 25232097Sjkh TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 2531590Srgrimes stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); 25432097Sjkh s = intr_disable(); 2551590Srgrimes (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 256184654Smlaier data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 257184654Smlaier intr_restore(s); 258184654Smlaier tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); 259184654Smlaier if ((data & TD_V) != 0 && (data & TD_L) == 0 && 260184654Smlaier TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 261184654Smlaier stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); 2621590Srgrimes } 26378158Sroam for (i = 0; i < CHEETAH_DT512_ENTRIES; i++) { 264184733Smlaier slot = TLB_DAR_SLOT(TLB_DAR_DT512_0, i); 265184733Smlaier s = intr_disable(); 266184733Smlaier (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 267184654Smlaier data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 268184733Smlaier intr_restore(s); 269128772Skientzle tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); 270184654Smlaier if ((data & TD_V) != 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 271184654Smlaier stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); 272184733Smlaier slot = TLB_DAR_SLOT(TLB_DAR_DT512_1, i); 273184654Smlaier s = intr_disable(); 274184654Smlaier (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 275184654Smlaier data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 276184733Smlaier intr_restore(s); 277184654Smlaier tag = ldxa(slot, ASI_DTLB_TAG_READ_REG); 27858601Scharnier if ((data & TD_V) != 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 279184654Smlaier stxa_sync(slot, ASI_DTLB_DATA_ACCESS_REG, 0); 280184654Smlaier } 281184654Smlaier if (PCPU_GET(impl) == CPU_IMPL_ULTRASPARCIVp) { 282184654Smlaier for (i = 0; i < CHEETAH_IT512_ENTRIES; i++) { 283184654Smlaier slot = TLB_DAR_SLOT(TLB_DAR_IT512, i); 284184654Smlaier s = intr_disable(); 285184654Smlaier (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 286184654Smlaier data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 287184654Smlaier intr_restore(s); 288184654Smlaier tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); 289184654Smlaier if ((data & TD_V) != 0 && 290184654Smlaier TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 29132097Sjkh stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); 292184654Smlaier } 293184654Smlaier } else { 294184654Smlaier for (i = 0; i < CHEETAH_IT128_ENTRIES; i++) { 29532097Sjkh slot = TLB_DAR_SLOT(TLB_DAR_IT128, i); 29678158Sroam s = intr_disable(); 297184733Smlaier (void)ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 298184733Smlaier data = ldxa(slot, ASI_ITLB_DATA_ACCESS_REG); 299184733Smlaier tag = ldxa(slot, ASI_ITLB_TAG_READ_REG); 300184733Smlaier intr_restore(s); 301184654Smlaier if ((data & TD_V) != 0 && 302184654Smlaier TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 303184733Smlaier stxa_sync(slot, ASI_ITLB_DATA_ACCESS_REG, 0); 304184654Smlaier } 305184654Smlaier } 306184654Smlaier} 307184733Smlaier