1209243Sjchandra/*- 2209243Sjchandra * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org> 3209243Sjchandra * All rights reserved. 4209243Sjchandra * 5209243Sjchandra * Redistribution and use in source and binary forms, with or without 6209243Sjchandra * modification, are permitted provided that the following conditions 7209243Sjchandra * are met: 8209243Sjchandra * 1. Redistributions of source code must retain the above copyright 9209243Sjchandra * notice, this list of conditions and the following disclaimer. 10209243Sjchandra * 2. Redistributions in binary form must reproduce the above copyright 11209243Sjchandra * notice, this list of conditions and the following disclaimer in the 12209243Sjchandra * documentation and/or other materials provided with the distribution. 13209243Sjchandra * 14209243Sjchandra * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15209243Sjchandra * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16209243Sjchandra * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17209243Sjchandra * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18209243Sjchandra * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19209243Sjchandra * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20209243Sjchandra * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21209243Sjchandra * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22209243Sjchandra * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23209243Sjchandra * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24209243Sjchandra * SUCH DAMAGE. 25209243Sjchandra * 26209243Sjchandra * $FreeBSD$ 27209243Sjchandra */ 28209243Sjchandra 29209243Sjchandra#include "opt_ddb.h" 30209243Sjchandra 31209243Sjchandra#include <sys/param.h> 32209243Sjchandra#include <sys/kernel.h> 33209243Sjchandra#include <sys/systm.h> 34209243Sjchandra#include <sys/pcpu.h> 35209243Sjchandra#include <sys/smp.h> 36209243Sjchandra 37209243Sjchandra#include <vm/vm.h> 38241123Salc#include <vm/pmap.h> 39209243Sjchandra 40209243Sjchandra#include <machine/pte.h> 41209243Sjchandra#include <machine/tlb.h> 42209243Sjchandra 43249415Sjchandra#if defined(CPU_CNMIPS) 44249415Sjchandra#define MIPS_MAX_TLB_ENTRIES 128 45249415Sjchandra#elif defined(CPU_NLM) 46249415Sjchandra#define MIPS_MAX_TLB_ENTRIES (2048 + 128) 47249415Sjchandra#else 48249415Sjchandra#define MIPS_MAX_TLB_ENTRIES 64 49249415Sjchandra#endif 50249415Sjchandra 51209243Sjchandrastruct tlb_state { 52209243Sjchandra unsigned wired; 53209243Sjchandra struct tlb_entry { 54209243Sjchandra register_t entryhi; 55209243Sjchandra register_t entrylo0; 56209243Sjchandra register_t entrylo1; 57255935Sadrian register_t pagemask; 58209243Sjchandra } entry[MIPS_MAX_TLB_ENTRIES]; 59209243Sjchandra}; 60209243Sjchandra 61209243Sjchandrastatic struct tlb_state tlb_state[MAXCPU]; 62209243Sjchandra 63209243Sjchandra#if 0 64209243Sjchandra/* 65209243Sjchandra * PageMask must increment in steps of 2 bits. 66209243Sjchandra */ 67209243SjchandraCOMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0); 68209243Sjchandra#endif 69209243Sjchandra 70209243Sjchandrastatic inline void 71209243Sjchandratlb_probe(void) 72209243Sjchandra{ 73209243Sjchandra __asm __volatile ("tlbp" : : : "memory"); 74209243Sjchandra mips_cp0_sync(); 75209243Sjchandra} 76209243Sjchandra 77209243Sjchandrastatic inline void 78209243Sjchandratlb_read(void) 79209243Sjchandra{ 80209243Sjchandra __asm __volatile ("tlbr" : : : "memory"); 81209243Sjchandra mips_cp0_sync(); 82209243Sjchandra} 83209243Sjchandra 84209243Sjchandrastatic inline void 85209243Sjchandratlb_write_indexed(void) 86209243Sjchandra{ 87209243Sjchandra __asm __volatile ("tlbwi" : : : "memory"); 88209243Sjchandra mips_cp0_sync(); 89209243Sjchandra} 90209243Sjchandra 91209243Sjchandrastatic inline void 92209243Sjchandratlb_write_random(void) 93209243Sjchandra{ 94209243Sjchandra __asm __volatile ("tlbwr" : : : "memory"); 95209243Sjchandra mips_cp0_sync(); 96209243Sjchandra} 97209243Sjchandra 98209243Sjchandrastatic void tlb_invalidate_one(unsigned); 99209243Sjchandra 100209243Sjchandravoid 101209243Sjchandratlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1) 102209243Sjchandra{ 103209645Sjchandra register_t asid; 104209243Sjchandra register_t s; 105209243Sjchandra 106209243Sjchandra va &= ~PAGE_MASK; 107209243Sjchandra 108209243Sjchandra s = intr_disable(); 109209243Sjchandra asid = mips_rd_entryhi() & TLBHI_ASID_MASK; 110209243Sjchandra 111209243Sjchandra mips_wr_index(i); 112209243Sjchandra mips_wr_pagemask(0); 113209243Sjchandra mips_wr_entryhi(TLBHI_ENTRY(va, 0)); 114209243Sjchandra mips_wr_entrylo0(pte0); 115209243Sjchandra mips_wr_entrylo1(pte1); 116209243Sjchandra tlb_write_indexed(); 117209243Sjchandra 118209243Sjchandra mips_wr_entryhi(asid); 119209243Sjchandra intr_restore(s); 120209243Sjchandra} 121209243Sjchandra 122209243Sjchandravoid 123209243Sjchandratlb_invalidate_address(struct pmap *pmap, vm_offset_t va) 124209243Sjchandra{ 125209645Sjchandra register_t asid; 126209243Sjchandra register_t s; 127209243Sjchandra int i; 128209243Sjchandra 129209243Sjchandra va &= ~PAGE_MASK; 130209243Sjchandra 131209243Sjchandra s = intr_disable(); 132209243Sjchandra asid = mips_rd_entryhi() & TLBHI_ASID_MASK; 133209243Sjchandra 134209243Sjchandra mips_wr_pagemask(0); 135209243Sjchandra mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap))); 136209243Sjchandra tlb_probe(); 137209243Sjchandra i = mips_rd_index(); 138209243Sjchandra if (i >= 0) 139209243Sjchandra tlb_invalidate_one(i); 140209243Sjchandra 141209243Sjchandra mips_wr_entryhi(asid); 142209243Sjchandra intr_restore(s); 143209243Sjchandra} 144209243Sjchandra 145209243Sjchandravoid 146209243Sjchandratlb_invalidate_all(void) 147209243Sjchandra{ 148209645Sjchandra register_t asid; 149209243Sjchandra register_t s; 150209243Sjchandra unsigned i; 151209243Sjchandra 152209243Sjchandra s = intr_disable(); 153209243Sjchandra asid = mips_rd_entryhi() & TLBHI_ASID_MASK; 154209243Sjchandra 155209243Sjchandra for (i = mips_rd_wired(); i < num_tlbentries; i++) 156209243Sjchandra tlb_invalidate_one(i); 157209243Sjchandra 158209243Sjchandra mips_wr_entryhi(asid); 159209243Sjchandra intr_restore(s); 160209243Sjchandra} 161209243Sjchandra 162209243Sjchandravoid 163209243Sjchandratlb_invalidate_all_user(struct pmap *pmap) 164209243Sjchandra{ 165209645Sjchandra register_t asid; 166209243Sjchandra register_t s; 167209243Sjchandra unsigned i; 168209243Sjchandra 169209243Sjchandra s = intr_disable(); 170209243Sjchandra asid = mips_rd_entryhi() & TLBHI_ASID_MASK; 171209243Sjchandra 172209243Sjchandra for (i = mips_rd_wired(); i < num_tlbentries; i++) { 173209243Sjchandra register_t uasid; 174209243Sjchandra 175209243Sjchandra mips_wr_index(i); 176209243Sjchandra tlb_read(); 177209243Sjchandra 178209243Sjchandra uasid = mips_rd_entryhi() & TLBHI_ASID_MASK; 179209243Sjchandra if (pmap == NULL) { 180209243Sjchandra /* 181209243Sjchandra * Invalidate all non-kernel entries. 182209243Sjchandra */ 183209243Sjchandra if (uasid == 0) 184209243Sjchandra continue; 185209243Sjchandra } else { 186209243Sjchandra /* 187209243Sjchandra * Invalidate this pmap's entries. 188209243Sjchandra */ 189209243Sjchandra if (uasid != pmap_asid(pmap)) 190209243Sjchandra continue; 191209243Sjchandra } 192209243Sjchandra tlb_invalidate_one(i); 193209243Sjchandra } 194209243Sjchandra 195209243Sjchandra mips_wr_entryhi(asid); 196209243Sjchandra intr_restore(s); 197209243Sjchandra} 198209243Sjchandra 199241123Salc/* 200241123Salc * Invalidates any TLB entries that map a virtual page from the specified 201241123Salc * address range. If "end" is zero, then every virtual page is considered to 202241123Salc * be within the address range's upper bound. 203241123Salc */ 204241123Salcvoid 205241123Salctlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 206241123Salc{ 207241123Salc register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi; 208241123Salc int i; 209241123Salc 210241123Salc KASSERT(start < end || (end == 0 && start > 0), 211241123Salc ("tlb_invalidate_range: invalid range")); 212241123Salc 213241123Salc /* 214241123Salc * Truncate the virtual address "start" to an even page frame number, 215241123Salc * and round the virtual address "end" to an even page frame number. 216241123Salc */ 217241123Salc start &= ~((1 << TLBMASK_SHIFT) - 1); 218241123Salc end = (end + (1 << TLBMASK_SHIFT) - 1) & ~((1 << TLBMASK_SHIFT) - 1); 219241123Salc 220241123Salc s = intr_disable(); 221241123Salc save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK; 222241123Salc 223241123Salc asid = pmap_asid(pmap); 224241123Salc start_hi = TLBHI_ENTRY(start, asid); 225241123Salc end_hi = TLBHI_ENTRY(end, asid); 226241123Salc 227241123Salc /* 228241123Salc * Select the fastest method for invalidating the TLB entries. 229241123Salc */ 230241123Salc if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 && 231241123Salc start >= -(num_tlbentries << TLBMASK_SHIFT))) { 232241123Salc /* 233241123Salc * The virtual address range is small compared to the size of 234241123Salc * the TLB. Probe the TLB for each even numbered page frame 235241123Salc * within the virtual address range. 236241123Salc */ 237241123Salc for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) { 238241123Salc mips_wr_pagemask(0); 239241123Salc mips_wr_entryhi(hi); 240241123Salc tlb_probe(); 241241123Salc i = mips_rd_index(); 242241123Salc if (i >= 0) 243241123Salc tlb_invalidate_one(i); 244241123Salc } 245241123Salc } else { 246241123Salc /* 247241123Salc * The virtual address range is large compared to the size of 248241123Salc * the TLB. Test every non-wired TLB entry. 249241123Salc */ 250241123Salc for (i = mips_rd_wired(); i < num_tlbentries; i++) { 251241123Salc mips_wr_index(i); 252241123Salc tlb_read(); 253241123Salc hi = mips_rd_entryhi(); 254241123Salc if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi || 255241123Salc end == 0)) { 256241123Salc /* 257241123Salc * If "hi" is a large page that spans 258241123Salc * "start_hi", then it must be invalidated. 259241123Salc */ 260241123Salc hi_pagemask = mips_rd_pagemask(); 261241123Salc if (hi >= (start_hi & ~(hi_pagemask << 262241123Salc TLBMASK_SHIFT))) 263241123Salc tlb_invalidate_one(i); 264241123Salc } 265241123Salc } 266241123Salc } 267241123Salc 268241123Salc mips_wr_entryhi(save_asid); 269241123Salc intr_restore(s); 270241123Salc} 271241123Salc 272209243Sjchandra/* XXX Only if DDB? */ 273209243Sjchandravoid 274209243Sjchandratlb_save(void) 275209243Sjchandra{ 276249415Sjchandra unsigned ntlb, i, cpu; 277209243Sjchandra 278209243Sjchandra cpu = PCPU_GET(cpuid); 279249415Sjchandra if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) 280249415Sjchandra ntlb = MIPS_MAX_TLB_ENTRIES; 281249415Sjchandra else 282249415Sjchandra ntlb = num_tlbentries; 283209243Sjchandra tlb_state[cpu].wired = mips_rd_wired(); 284249415Sjchandra for (i = 0; i < ntlb; i++) { 285209243Sjchandra mips_wr_index(i); 286209243Sjchandra tlb_read(); 287209243Sjchandra 288209243Sjchandra tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi(); 289255935Sadrian tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask(); 290209243Sjchandra tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0(); 291209243Sjchandra tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1(); 292209243Sjchandra } 293209243Sjchandra} 294209243Sjchandra 295209243Sjchandravoid 296209243Sjchandratlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte) 297209243Sjchandra{ 298209645Sjchandra register_t asid; 299209243Sjchandra register_t s; 300209243Sjchandra int i; 301209243Sjchandra 302209243Sjchandra va &= ~PAGE_MASK; 303209243Sjchandra pte &= ~TLBLO_SWBITS_MASK; 304209243Sjchandra 305209243Sjchandra s = intr_disable(); 306209243Sjchandra asid = mips_rd_entryhi() & TLBHI_ASID_MASK; 307209243Sjchandra 308209243Sjchandra mips_wr_pagemask(0); 309209243Sjchandra mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap))); 310209243Sjchandra tlb_probe(); 311209243Sjchandra i = mips_rd_index(); 312209243Sjchandra if (i >= 0) { 313209243Sjchandra tlb_read(); 314209243Sjchandra 315209243Sjchandra if ((va & PAGE_SIZE) == 0) { 316209243Sjchandra mips_wr_entrylo0(pte); 317209243Sjchandra } else { 318209243Sjchandra mips_wr_entrylo1(pte); 319209243Sjchandra } 320209243Sjchandra tlb_write_indexed(); 321209243Sjchandra } 322209243Sjchandra 323209243Sjchandra mips_wr_entryhi(asid); 324209243Sjchandra intr_restore(s); 325209243Sjchandra} 326209243Sjchandra 327209243Sjchandrastatic void 328209243Sjchandratlb_invalidate_one(unsigned i) 329209243Sjchandra{ 330209243Sjchandra /* XXX an invalid ASID? */ 331209243Sjchandra mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0)); 332209243Sjchandra mips_wr_entrylo0(0); 333209243Sjchandra mips_wr_entrylo1(0); 334209243Sjchandra mips_wr_pagemask(0); 335209243Sjchandra mips_wr_index(i); 336209243Sjchandra tlb_write_indexed(); 337209243Sjchandra} 338209243Sjchandra 339209243Sjchandra#ifdef DDB 340209243Sjchandra#include <ddb/ddb.h> 341209243Sjchandra 342209243SjchandraDB_SHOW_COMMAND(tlb, ddb_dump_tlb) 343209243Sjchandra{ 344255935Sadrian register_t ehi, elo0, elo1, epagemask; 345249415Sjchandra unsigned i, cpu, ntlb; 346209243Sjchandra 347209243Sjchandra /* 348209243Sjchandra * XXX 349209243Sjchandra * The worst conversion from hex to decimal ever. 350209243Sjchandra */ 351209243Sjchandra if (have_addr) 352209243Sjchandra cpu = ((addr >> 4) % 16) * 10 + (addr % 16); 353209243Sjchandra else 354209243Sjchandra cpu = PCPU_GET(cpuid); 355209243Sjchandra 356209243Sjchandra if (cpu < 0 || cpu >= mp_ncpus) { 357209243Sjchandra db_printf("Invalid CPU %u\n", cpu); 358209243Sjchandra return; 359209243Sjchandra } 360249415Sjchandra if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) { 361249415Sjchandra ntlb = MIPS_MAX_TLB_ENTRIES; 362249415Sjchandra db_printf("Warning: Only %d of %d TLB entries saved!\n", 363249415Sjchandra ntlb, num_tlbentries); 364249415Sjchandra } else 365249415Sjchandra ntlb = num_tlbentries; 366209243Sjchandra 367209243Sjchandra if (cpu == PCPU_GET(cpuid)) 368209243Sjchandra tlb_save(); 369209243Sjchandra 370209243Sjchandra db_printf("Beginning TLB dump for CPU %u...\n", cpu); 371249415Sjchandra for (i = 0; i < ntlb; i++) { 372209243Sjchandra if (i == tlb_state[cpu].wired) { 373209243Sjchandra if (i != 0) 374209243Sjchandra db_printf("^^^ WIRED ENTRIES ^^^\n"); 375209243Sjchandra else 376209243Sjchandra db_printf("(No wired entries.)\n"); 377209243Sjchandra } 378209243Sjchandra 379209243Sjchandra /* XXX PageMask. */ 380209243Sjchandra ehi = tlb_state[cpu].entry[i].entryhi; 381209243Sjchandra elo0 = tlb_state[cpu].entry[i].entrylo0; 382209243Sjchandra elo1 = tlb_state[cpu].entry[i].entrylo1; 383255935Sadrian epagemask = tlb_state[cpu].entry[i].pagemask; 384209243Sjchandra 385209243Sjchandra if (elo0 == 0 && elo1 == 0) 386209243Sjchandra continue; 387209243Sjchandra 388255935Sadrian db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask); 389209243Sjchandra db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0)); 390209243Sjchandra db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1)); 391209243Sjchandra } 392209243Sjchandra db_printf("Finished.\n"); 393209243Sjchandra} 394209243Sjchandra#endif 395