1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completely out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24#include <linux/bug.h> 25#include <linux/export.h> 26#include <linux/kernel.h> 27#include <linux/types.h> 28#include <linux/smp.h> 29#include <linux/string.h> 30#include <linux/cache.h> 31#include <linux/pgtable.h> 32 33#include <asm/cacheflush.h> 34#include <asm/cpu-type.h> 35#include <asm/mipsregs.h> 36#include <asm/mmu_context.h> 37#include <asm/regdef.h> 38#include <asm/uasm.h> 39#include <asm/setup.h> 40#include <asm/tlbex.h> 41 42static int mips_xpa_disabled; 43 44static int __init xpa_disable(char *s) 45{ 46 mips_xpa_disabled = 1; 47 48 return 1; 49} 50 51__setup("noxpa", xpa_disable); 52 53/* 54 * TLB load/store/modify handlers. 55 * 56 * Only the fastpath gets synthesized at runtime, the slowpath for 57 * do_page_fault remains normal asm. 58 */ 59extern void tlb_do_page_fault_0(void); 60extern void tlb_do_page_fault_1(void); 61 62struct work_registers { 63 int r1; 64 int r2; 65 int r3; 66}; 67 68struct tlb_reg_save { 69 unsigned long a; 70 unsigned long b; 71} ____cacheline_aligned_in_smp; 72 73static struct tlb_reg_save handler_reg_save[NR_CPUS]; 74 75static inline int r45k_bvahwbug(void) 76{ 77 /* XXX: We should probe for the presence of this bug, but we don't. */ 78 return 0; 79} 80 81static inline int r4k_250MHZhwbug(void) 82{ 83 /* XXX: We should probe for the presence of this bug, but we don't. */ 84 return 0; 85} 86 87extern int sb1250_m3_workaround_needed(void); 88 89static inline int __maybe_unused bcm1250_m3_war(void) 90{ 91 if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS)) 92 return sb1250_m3_workaround_needed(); 93 return 0; 94} 95 96static inline int __maybe_unused r10000_llsc_war(void) 97{ 98 return IS_ENABLED(CONFIG_WAR_R10000_LLSC); 99} 100 101static int use_bbit_insns(void) 102{ 103 switch (current_cpu_type()) { 104 case CPU_CAVIUM_OCTEON: 105 case CPU_CAVIUM_OCTEON_PLUS: 106 case CPU_CAVIUM_OCTEON2: 107 case CPU_CAVIUM_OCTEON3: 108 return 1; 109 default: 110 return 0; 111 } 112} 113 114static int use_lwx_insns(void) 115{ 116 switch (current_cpu_type()) { 117 case CPU_CAVIUM_OCTEON2: 118 case CPU_CAVIUM_OCTEON3: 119 return 1; 120 default: 121 return 0; 122 } 123} 124#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 125 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 126static bool scratchpad_available(void) 127{ 128 return true; 129} 130static int scratchpad_offset(int i) 131{ 132 /* 133 * CVMSEG starts at address -32768 and extends for 134 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 135 */ 136 i += 1; /* Kernel use starts at the top and works down. */ 137 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 138} 139#else 140static bool scratchpad_available(void) 141{ 142 return false; 143} 144static int scratchpad_offset(int i) 145{ 146 BUG(); 147 /* Really unreachable, but evidently some GCC want this. */ 148 return 0; 149} 150#endif 151/* 152 * Found by experiment: At least some revisions of the 4kc throw under 153 * some circumstances a machine check exception, triggered by invalid 154 * values in the index register. Delaying the tlbp instruction until 155 * after the next branch, plus adding an additional nop in front of 156 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 157 * why; it's not an issue caused by the core RTL. 158 * 159 */ 160static int m4kc_tlbp_war(void) 161{ 162 return current_cpu_type() == CPU_4KC; 163} 164 165/* Handle labels (which must be positive integers). */ 166enum label_id { 167 label_second_part = 1, 168 label_leave, 169 label_vmalloc, 170 label_vmalloc_done, 171 label_tlbw_hazard_0, 172 label_split = label_tlbw_hazard_0 + 8, 173 label_tlbl_goaround1, 174 label_tlbl_goaround2, 175 label_nopage_tlbl, 176 label_nopage_tlbs, 177 label_nopage_tlbm, 178 label_smp_pgtable_change, 179 label_r3000_write_probe_fail, 180 label_large_segbits_fault, 181#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 182 label_tlb_huge_update, 183#endif 184}; 185 186UASM_L_LA(_second_part) 187UASM_L_LA(_leave) 188UASM_L_LA(_vmalloc) 189UASM_L_LA(_vmalloc_done) 190/* _tlbw_hazard_x is handled differently. */ 191UASM_L_LA(_split) 192UASM_L_LA(_tlbl_goaround1) 193UASM_L_LA(_tlbl_goaround2) 194UASM_L_LA(_nopage_tlbl) 195UASM_L_LA(_nopage_tlbs) 196UASM_L_LA(_nopage_tlbm) 197UASM_L_LA(_smp_pgtable_change) 198UASM_L_LA(_r3000_write_probe_fail) 199UASM_L_LA(_large_segbits_fault) 200#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 201UASM_L_LA(_tlb_huge_update) 202#endif 203 204static int hazard_instance; 205 206static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) 207{ 208 switch (instance) { 209 case 0 ... 7: 210 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); 211 return; 212 default: 213 BUG(); 214 } 215} 216 217static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) 218{ 219 switch (instance) { 220 case 0 ... 7: 221 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); 222 break; 223 default: 224 BUG(); 225 } 226} 227 228/* 229 * pgtable bits are assigned dynamically depending on processor feature 230 * and statically based on kernel configuration. This spits out the actual 231 * values the kernel is using. Required to make sense from disassembled 232 * TLB exception handlers. 233 */ 234static void output_pgtable_bits_defines(void) 235{ 236#define pr_define(fmt, ...) \ 237 pr_debug("#define " fmt, ##__VA_ARGS__) 238 239 pr_debug("#include <asm/asm.h>\n"); 240 pr_debug("#include <asm/regdef.h>\n"); 241 pr_debug("\n"); 242 243 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); 244 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); 245 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); 246 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); 247 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); 248#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 249 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); 250#endif 251#ifdef _PAGE_NO_EXEC_SHIFT 252 if (cpu_has_rixi) 253 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); 254#endif 255 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); 256 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); 257 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); 258 pr_define("PFN_PTE_SHIFT %d\n", PFN_PTE_SHIFT); 259 pr_debug("\n"); 260} 261 262static inline void dump_handler(const char *symbol, const void *start, const void *end) 263{ 264 unsigned int count = (end - start) / sizeof(u32); 265 const u32 *handler = start; 266 int i; 267 268 pr_debug("LEAF(%s)\n", symbol); 269 270 pr_debug("\t.set push\n"); 271 pr_debug("\t.set noreorder\n"); 272 273 for (i = 0; i < count; i++) 274 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]); 275 276 pr_debug("\t.set\tpop\n"); 277 278 pr_debug("\tEND(%s)\n", symbol); 279} 280 281#ifdef CONFIG_64BIT 282# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 283#else 284# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 285#endif 286 287/* The worst case length of the handler is around 18 instructions for 288 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 289 * Maximum space available is 32 instructions for R3000 and 64 290 * instructions for R4000. 291 * 292 * We deliberately chose a buffer size of 128, so we won't scribble 293 * over anything important on overflow before we panic. 294 */ 295static u32 tlb_handler[128]; 296 297/* simply assume worst case size for labels and relocs */ 298static struct uasm_label labels[128]; 299static struct uasm_reloc relocs[128]; 300 301static int check_for_high_segbits; 302static bool fill_includes_sw_bits; 303 304static unsigned int kscratch_used_mask; 305 306static inline int __maybe_unused c0_kscratch(void) 307{ 308 return 31; 309} 310 311static int allocate_kscratch(void) 312{ 313 int r; 314 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 315 316 r = ffs(a); 317 318 if (r == 0) 319 return -1; 320 321 r--; /* make it zero based */ 322 323 kscratch_used_mask |= (1 << r); 324 325 return r; 326} 327 328static int scratch_reg; 329int pgd_reg; 330EXPORT_SYMBOL_GPL(pgd_reg); 331enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 332 333static struct work_registers build_get_work_registers(u32 **p) 334{ 335 struct work_registers r; 336 337 if (scratch_reg >= 0) { 338 /* Save in CPU local C0_KScratch? */ 339 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); 340 r.r1 = GPR_K0; 341 r.r2 = GPR_K1; 342 r.r3 = GPR_AT; 343 return r; 344 } 345 346 if (num_possible_cpus() > 1) { 347 /* Get smp_processor_id */ 348 UASM_i_CPUID_MFC0(p, GPR_K0, SMP_CPUID_REG); 349 UASM_i_SRL_SAFE(p, GPR_K0, GPR_K0, SMP_CPUID_REGSHIFT); 350 351 /* handler_reg_save index in GPR_K0 */ 352 UASM_i_SLL(p, GPR_K0, GPR_K0, ilog2(sizeof(struct tlb_reg_save))); 353 354 UASM_i_LA(p, GPR_K1, (long)&handler_reg_save); 355 UASM_i_ADDU(p, GPR_K0, GPR_K0, GPR_K1); 356 } else { 357 UASM_i_LA(p, GPR_K0, (long)&handler_reg_save); 358 } 359 /* GPR_K0 now points to save area, save $1 and $2 */ 360 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), GPR_K0); 361 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), GPR_K0); 362 363 r.r1 = GPR_K1; 364 r.r2 = 1; 365 r.r3 = 2; 366 return r; 367} 368 369static void build_restore_work_registers(u32 **p) 370{ 371 if (scratch_reg >= 0) { 372 uasm_i_ehb(p); 373 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 374 return; 375 } 376 /* GPR_K0 already points to save area, restore $1 and $2 */ 377 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), GPR_K0); 378 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), GPR_K0); 379} 380 381#ifndef CONFIG_MIPS_PGD_C0_CONTEXT 382 383/* 384 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 385 * we cannot do r3000 under these circumstances. 386 * 387 * The R3000 TLB handler is simple. 388 */ 389static void build_r3000_tlb_refill_handler(void) 390{ 391 long pgdc = (long)pgd_current; 392 u32 *p; 393 394 memset(tlb_handler, 0, sizeof(tlb_handler)); 395 p = tlb_handler; 396 397 uasm_i_mfc0(&p, GPR_K0, C0_BADVADDR); 398 uasm_i_lui(&p, GPR_K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 399 uasm_i_lw(&p, GPR_K1, uasm_rel_lo(pgdc), GPR_K1); 400 uasm_i_srl(&p, GPR_K0, GPR_K0, 22); /* load delay */ 401 uasm_i_sll(&p, GPR_K0, GPR_K0, 2); 402 uasm_i_addu(&p, GPR_K1, GPR_K1, GPR_K0); 403 uasm_i_mfc0(&p, GPR_K0, C0_CONTEXT); 404 uasm_i_lw(&p, GPR_K1, 0, GPR_K1); /* cp0 delay */ 405 uasm_i_andi(&p, GPR_K0, GPR_K0, 0xffc); /* load delay */ 406 uasm_i_addu(&p, GPR_K1, GPR_K1, GPR_K0); 407 uasm_i_lw(&p, GPR_K0, 0, GPR_K1); 408 uasm_i_nop(&p); /* load delay */ 409 uasm_i_mtc0(&p, GPR_K0, C0_ENTRYLO0); 410 uasm_i_mfc0(&p, GPR_K1, C0_EPC); /* cp0 delay */ 411 uasm_i_tlbwr(&p); /* cp0 delay */ 412 uasm_i_jr(&p, GPR_K1); 413 uasm_i_rfe(&p); /* branch delay */ 414 415 if (p > tlb_handler + 32) 416 panic("TLB refill handler space exceeded"); 417 418 pr_debug("Wrote TLB refill handler (%u instructions).\n", 419 (unsigned int)(p - tlb_handler)); 420 421 memcpy((void *)ebase, tlb_handler, 0x80); 422 local_flush_icache_range(ebase, ebase + 0x80); 423 dump_handler("r3000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x80)); 424} 425#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 426 427/* 428 * The R4000 TLB handler is much more complicated. We have two 429 * consecutive handler areas with 32 instructions space each. 430 * Since they aren't used at the same time, we can overflow in the 431 * other one.To keep things simple, we first assume linear space, 432 * then we relocate it to the final handler layout as needed. 433 */ 434static u32 final_handler[64]; 435 436/* 437 * Hazards 438 * 439 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 440 * 2. A timing hazard exists for the TLBP instruction. 441 * 442 * stalling_instruction 443 * TLBP 444 * 445 * The JTLB is being read for the TLBP throughout the stall generated by the 446 * previous instruction. This is not really correct as the stalling instruction 447 * can modify the address used to access the JTLB. The failure symptom is that 448 * the TLBP instruction will use an address created for the stalling instruction 449 * and not the address held in C0_ENHI and thus report the wrong results. 450 * 451 * The software work-around is to not allow the instruction preceding the TLBP 452 * to stall - make it an NOP or some other instruction guaranteed not to stall. 453 * 454 * Errata 2 will not be fixed. This errata is also on the R5000. 455 * 456 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 457 */ 458static void __maybe_unused build_tlb_probe_entry(u32 **p) 459{ 460 switch (current_cpu_type()) { 461 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 462 case CPU_R4600: 463 case CPU_R4700: 464 case CPU_R5000: 465 case CPU_NEVADA: 466 uasm_i_nop(p); 467 uasm_i_tlbp(p); 468 break; 469 470 default: 471 uasm_i_tlbp(p); 472 break; 473 } 474} 475 476void build_tlb_write_entry(u32 **p, struct uasm_label **l, 477 struct uasm_reloc **r, 478 enum tlb_write_entry wmode) 479{ 480 void(*tlbw)(u32 **) = NULL; 481 482 switch (wmode) { 483 case tlb_random: tlbw = uasm_i_tlbwr; break; 484 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 485 } 486 487 if (cpu_has_mips_r2_r6) { 488 if (cpu_has_mips_r2_exec_hazard) 489 uasm_i_ehb(p); 490 tlbw(p); 491 return; 492 } 493 494 switch (current_cpu_type()) { 495 case CPU_R4000PC: 496 case CPU_R4000SC: 497 case CPU_R4000MC: 498 case CPU_R4400PC: 499 case CPU_R4400SC: 500 case CPU_R4400MC: 501 /* 502 * This branch uses up a mtc0 hazard nop slot and saves 503 * two nops after the tlbw instruction. 504 */ 505 uasm_bgezl_hazard(p, r, hazard_instance); 506 tlbw(p); 507 uasm_bgezl_label(l, p, hazard_instance); 508 hazard_instance++; 509 uasm_i_nop(p); 510 break; 511 512 case CPU_R4600: 513 case CPU_R4700: 514 uasm_i_nop(p); 515 tlbw(p); 516 uasm_i_nop(p); 517 break; 518 519 case CPU_R5000: 520 case CPU_NEVADA: 521 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 522 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 523 tlbw(p); 524 break; 525 526 case CPU_R4300: 527 case CPU_5KC: 528 case CPU_TX49XX: 529 case CPU_PR4450: 530 uasm_i_nop(p); 531 tlbw(p); 532 break; 533 534 case CPU_R10000: 535 case CPU_R12000: 536 case CPU_R14000: 537 case CPU_R16000: 538 case CPU_4KC: 539 case CPU_4KEC: 540 case CPU_M14KC: 541 case CPU_M14KEC: 542 case CPU_SB1: 543 case CPU_SB1A: 544 case CPU_4KSC: 545 case CPU_20KC: 546 case CPU_25KF: 547 case CPU_BMIPS32: 548 case CPU_BMIPS3300: 549 case CPU_BMIPS4350: 550 case CPU_BMIPS4380: 551 case CPU_BMIPS5000: 552 case CPU_LOONGSON2EF: 553 case CPU_LOONGSON64: 554 case CPU_R5500: 555 if (m4kc_tlbp_war()) 556 uasm_i_nop(p); 557 fallthrough; 558 case CPU_ALCHEMY: 559 tlbw(p); 560 break; 561 562 case CPU_RM7000: 563 uasm_i_nop(p); 564 uasm_i_nop(p); 565 uasm_i_nop(p); 566 uasm_i_nop(p); 567 tlbw(p); 568 break; 569 570 case CPU_XBURST: 571 tlbw(p); 572 uasm_i_nop(p); 573 break; 574 575 default: 576 panic("No TLB refill handler yet (CPU type: %d)", 577 current_cpu_type()); 578 break; 579 } 580} 581EXPORT_SYMBOL_GPL(build_tlb_write_entry); 582 583static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 584 unsigned int reg) 585{ 586 if (_PAGE_GLOBAL_SHIFT == 0) { 587 /* pte_t is already in EntryLo format */ 588 return; 589 } 590 591 if (cpu_has_rixi && _PAGE_NO_EXEC != 0) { 592 if (fill_includes_sw_bits) { 593 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 594 } else { 595 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); 596 UASM_i_ROTR(p, reg, reg, 597 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); 598 } 599 } else { 600#ifdef CONFIG_PHYS_ADDR_T_64BIT 601 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 602#else 603 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 604#endif 605 } 606} 607 608#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 609 610static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, 611 unsigned int tmp, enum label_id lid, 612 int restore_scratch) 613{ 614 if (restore_scratch) { 615 /* 616 * Ensure the MFC0 below observes the value written to the 617 * KScratch register by the prior MTC0. 618 */ 619 if (scratch_reg >= 0) 620 uasm_i_ehb(p); 621 622 /* Reset default page size */ 623 if (PM_DEFAULT_MASK >> 16) { 624 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 625 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 626 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 627 uasm_il_b(p, r, lid); 628 } else if (PM_DEFAULT_MASK) { 629 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 630 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 631 uasm_il_b(p, r, lid); 632 } else { 633 uasm_i_mtc0(p, 0, C0_PAGEMASK); 634 uasm_il_b(p, r, lid); 635 } 636 if (scratch_reg >= 0) 637 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 638 else 639 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 640 } else { 641 /* Reset default page size */ 642 if (PM_DEFAULT_MASK >> 16) { 643 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 644 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 645 uasm_il_b(p, r, lid); 646 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 647 } else if (PM_DEFAULT_MASK) { 648 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 649 uasm_il_b(p, r, lid); 650 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 651 } else { 652 uasm_il_b(p, r, lid); 653 uasm_i_mtc0(p, 0, C0_PAGEMASK); 654 } 655 } 656} 657 658static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, 659 struct uasm_reloc **r, 660 unsigned int tmp, 661 enum tlb_write_entry wmode, 662 int restore_scratch) 663{ 664 /* Set huge page tlb entry size */ 665 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 666 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 667 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 668 669 build_tlb_write_entry(p, l, r, wmode); 670 671 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 672} 673 674/* 675 * Check if Huge PTE is present, if so then jump to LABEL. 676 */ 677static void 678build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 679 unsigned int pmd, int lid) 680{ 681 UASM_i_LW(p, tmp, 0, pmd); 682 if (use_bbit_insns()) { 683 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 684 } else { 685 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 686 uasm_il_bnez(p, r, tmp, lid); 687 } 688} 689 690static void build_huge_update_entries(u32 **p, unsigned int pte, 691 unsigned int tmp) 692{ 693 int small_sequence; 694 695 /* 696 * A huge PTE describes an area the size of the 697 * configured huge page size. This is twice the 698 * of the large TLB entry size we intend to use. 699 * A TLB entry half the size of the configured 700 * huge page size is configured into entrylo0 701 * and entrylo1 to cover the contiguous huge PTE 702 * address space. 703 */ 704 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 705 706 /* We can clobber tmp. It isn't used after this.*/ 707 if (!small_sequence) 708 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 709 710 build_convert_pte_to_entrylo(p, pte); 711 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 712 /* convert to entrylo1 */ 713 if (small_sequence) 714 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 715 else 716 UASM_i_ADDU(p, pte, pte, tmp); 717 718 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 719} 720 721static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, 722 struct uasm_label **l, 723 unsigned int pte, 724 unsigned int ptr, 725 unsigned int flush) 726{ 727#ifdef CONFIG_SMP 728 UASM_i_SC(p, pte, 0, ptr); 729 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 730 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 731#else 732 UASM_i_SW(p, pte, 0, ptr); 733#endif 734 if (cpu_has_ftlb && flush) { 735 BUG_ON(!cpu_has_tlbinv); 736 737 UASM_i_MFC0(p, ptr, C0_ENTRYHI); 738 uasm_i_ori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); 739 UASM_i_MTC0(p, ptr, C0_ENTRYHI); 740 build_tlb_write_entry(p, l, r, tlb_indexed); 741 742 uasm_i_xori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); 743 UASM_i_MTC0(p, ptr, C0_ENTRYHI); 744 build_huge_update_entries(p, pte, ptr); 745 build_huge_tlb_write_entry(p, l, r, pte, tlb_random, 0); 746 747 return; 748 } 749 750 build_huge_update_entries(p, pte, ptr); 751 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 752} 753#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 754 755#ifdef CONFIG_64BIT 756/* 757 * TMP and PTR are scratch. 758 * TMP will be clobbered, PTR will hold the pmd entry. 759 */ 760void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 761 unsigned int tmp, unsigned int ptr) 762{ 763#ifndef CONFIG_MIPS_PGD_C0_CONTEXT 764 long pgdc = (long)pgd_current; 765#endif 766 /* 767 * The vmalloc handling is not in the hotpath. 768 */ 769 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 770 771 if (check_for_high_segbits) { 772 /* 773 * The kernel currently implicitly assumes that the 774 * MIPS SEGBITS parameter for the processor is 775 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 776 * allocate virtual addresses outside the maximum 777 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 778 * that doesn't prevent user code from accessing the 779 * higher xuseg addresses. Here, we make sure that 780 * everything but the lower xuseg addresses goes down 781 * the module_alloc/vmalloc path. 782 */ 783 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); 784 uasm_il_bnez(p, r, ptr, label_vmalloc); 785 } else { 786 uasm_il_bltz(p, r, tmp, label_vmalloc); 787 } 788 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 789 790 if (pgd_reg != -1) { 791 /* pgd is in pgd_reg */ 792 if (cpu_has_ldpte) 793 UASM_i_MFC0(p, ptr, C0_PWBASE); 794 else 795 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 796 } else { 797#if defined(CONFIG_MIPS_PGD_C0_CONTEXT) 798 /* 799 * &pgd << 11 stored in CONTEXT [23..63]. 800 */ 801 UASM_i_MFC0(p, ptr, C0_CONTEXT); 802 803 /* Clear lower 23 bits of context. */ 804 uasm_i_dins(p, ptr, 0, 0, 23); 805 806 /* insert bit[63:59] of CAC_BASE into bit[11:6] of ptr */ 807 uasm_i_ori(p, ptr, ptr, ((u64)(CAC_BASE) >> 53)); 808 uasm_i_drotr(p, ptr, ptr, 11); 809#elif defined(CONFIG_SMP) 810 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); 811 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); 812 UASM_i_LA_mostly(p, tmp, pgdc); 813 uasm_i_daddu(p, ptr, ptr, tmp); 814 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 815 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 816#else 817 UASM_i_LA_mostly(p, ptr, pgdc); 818 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 819#endif 820 } 821 822 uasm_l_vmalloc_done(l, *p); 823 824 /* get pgd offset in bytes */ 825 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 826 827 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 828 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 829#ifndef __PAGETABLE_PUD_FOLDED 830 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 831 uasm_i_ld(p, ptr, 0, ptr); /* get pud pointer */ 832 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ 833 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); 834 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pud offset */ 835#endif 836#ifndef __PAGETABLE_PMD_FOLDED 837 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 838 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 839 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 840 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 841 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 842#endif 843} 844EXPORT_SYMBOL_GPL(build_get_pmde64); 845 846/* 847 * BVADDR is the faulting address, PTR is scratch. 848 * PTR will hold the pgd for vmalloc. 849 */ 850static void 851build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 852 unsigned int bvaddr, unsigned int ptr, 853 enum vmalloc64_mode mode) 854{ 855 long swpd = (long)swapper_pg_dir; 856 int single_insn_swpd; 857 int did_vmalloc_branch = 0; 858 859 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 860 861 uasm_l_vmalloc(l, *p); 862 863 if (mode != not_refill && check_for_high_segbits) { 864 if (single_insn_swpd) { 865 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 866 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 867 did_vmalloc_branch = 1; 868 /* fall through */ 869 } else { 870 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 871 } 872 } 873 if (!did_vmalloc_branch) { 874 if (single_insn_swpd) { 875 uasm_il_b(p, r, label_vmalloc_done); 876 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 877 } else { 878 UASM_i_LA_mostly(p, ptr, swpd); 879 uasm_il_b(p, r, label_vmalloc_done); 880 if (uasm_in_compat_space_p(swpd)) 881 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 882 else 883 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 884 } 885 } 886 if (mode != not_refill && check_for_high_segbits) { 887 uasm_l_large_segbits_fault(l, *p); 888 889 if (mode == refill_scratch && scratch_reg >= 0) 890 uasm_i_ehb(p); 891 892 /* 893 * We get here if we are an xsseg address, or if we are 894 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 895 * 896 * Ignoring xsseg (assume disabled so would generate 897 * (address errors?), the only remaining possibility 898 * is the upper xuseg addresses. On processors with 899 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 900 * addresses would have taken an address error. We try 901 * to mimic that here by taking a load/istream page 902 * fault. 903 */ 904 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 905 uasm_i_sync(p, 0); 906 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 907 uasm_i_jr(p, ptr); 908 909 if (mode == refill_scratch) { 910 if (scratch_reg >= 0) 911 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 912 else 913 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 914 } else { 915 uasm_i_nop(p); 916 } 917 } 918} 919 920#else /* !CONFIG_64BIT */ 921 922/* 923 * TMP and PTR are scratch. 924 * TMP will be clobbered, PTR will hold the pgd entry. 925 */ 926void build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 927{ 928 if (pgd_reg != -1) { 929 /* pgd is in pgd_reg */ 930 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); 931 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 932 } else { 933 long pgdc = (long)pgd_current; 934 935 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 936#ifdef CONFIG_SMP 937 uasm_i_mfc0(p, ptr, SMP_CPUID_REG); 938 UASM_i_LA_mostly(p, tmp, pgdc); 939 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); 940 uasm_i_addu(p, ptr, tmp, ptr); 941#else 942 UASM_i_LA_mostly(p, ptr, pgdc); 943#endif 944 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 945 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 946 } 947 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 948 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 949 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 950} 951EXPORT_SYMBOL_GPL(build_get_pgde32); 952 953#endif /* !CONFIG_64BIT */ 954 955static void build_adjust_context(u32 **p, unsigned int ctx) 956{ 957 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 958 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 959 960 if (shift) 961 UASM_i_SRL(p, ctx, ctx, shift); 962 uasm_i_andi(p, ctx, ctx, mask); 963} 964 965void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 966{ 967 /* 968 * Bug workaround for the Nevada. It seems as if under certain 969 * circumstances the move from cp0_context might produce a 970 * bogus result when the mfc0 instruction and its consumer are 971 * in a different cacheline or a load instruction, probably any 972 * memory reference, is between them. 973 */ 974 switch (current_cpu_type()) { 975 case CPU_NEVADA: 976 UASM_i_LW(p, ptr, 0, ptr); 977 GET_CONTEXT(p, tmp); /* get context reg */ 978 break; 979 980 default: 981 GET_CONTEXT(p, tmp); /* get context reg */ 982 UASM_i_LW(p, ptr, 0, ptr); 983 break; 984 } 985 986 build_adjust_context(p, tmp); 987 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 988} 989EXPORT_SYMBOL_GPL(build_get_ptep); 990 991void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) 992{ 993 int pte_off_even = 0; 994 int pte_off_odd = sizeof(pte_t); 995 996#if defined(CONFIG_CPU_MIPS32) && defined(CONFIG_PHYS_ADDR_T_64BIT) 997 /* The low 32 bits of EntryLo is stored in pte_high */ 998 pte_off_even += offsetof(pte_t, pte_high); 999 pte_off_odd += offsetof(pte_t, pte_high); 1000#endif 1001 1002 if (IS_ENABLED(CONFIG_XPA)) { 1003 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */ 1004 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1005 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); 1006 1007 if (cpu_has_xpa && !mips_xpa_disabled) { 1008 uasm_i_lw(p, tmp, 0, ptep); 1009 uasm_i_ext(p, tmp, tmp, 0, 24); 1010 uasm_i_mthc0(p, tmp, C0_ENTRYLO0); 1011 } 1012 1013 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */ 1014 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1015 UASM_i_MTC0(p, tmp, C0_ENTRYLO1); 1016 1017 if (cpu_has_xpa && !mips_xpa_disabled) { 1018 uasm_i_lw(p, tmp, sizeof(pte_t), ptep); 1019 uasm_i_ext(p, tmp, tmp, 0, 24); 1020 uasm_i_mthc0(p, tmp, C0_ENTRYLO1); 1021 } 1022 return; 1023 } 1024 1025 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */ 1026 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1027 if (r45k_bvahwbug()) 1028 build_tlb_probe_entry(p); 1029 build_convert_pte_to_entrylo(p, tmp); 1030 if (r4k_250MHZhwbug()) 1031 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1032 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1033 build_convert_pte_to_entrylo(p, ptep); 1034 if (r45k_bvahwbug()) 1035 uasm_i_mfc0(p, tmp, C0_INDEX); 1036 if (r4k_250MHZhwbug()) 1037 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1038 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1039} 1040EXPORT_SYMBOL_GPL(build_update_entries); 1041 1042struct mips_huge_tlb_info { 1043 int huge_pte; 1044 int restore_scratch; 1045 bool need_reload_pte; 1046}; 1047 1048static struct mips_huge_tlb_info 1049build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1050 struct uasm_reloc **r, unsigned int tmp, 1051 unsigned int ptr, int c0_scratch_reg) 1052{ 1053 struct mips_huge_tlb_info rv; 1054 unsigned int even, odd; 1055 int vmalloc_branch_delay_filled = 0; 1056 const int scratch = 1; /* Our extra working register */ 1057 1058 rv.huge_pte = scratch; 1059 rv.restore_scratch = 0; 1060 rv.need_reload_pte = false; 1061 1062 if (check_for_high_segbits) { 1063 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1064 1065 if (pgd_reg != -1) 1066 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1067 else 1068 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1069 1070 if (c0_scratch_reg >= 0) 1071 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1072 else 1073 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1074 1075 uasm_i_dsrl_safe(p, scratch, tmp, 1076 PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); 1077 uasm_il_bnez(p, r, scratch, label_vmalloc); 1078 1079 if (pgd_reg == -1) { 1080 vmalloc_branch_delay_filled = 1; 1081 /* Clear lower 23 bits of context. */ 1082 uasm_i_dins(p, ptr, 0, 0, 23); 1083 } 1084 } else { 1085 if (pgd_reg != -1) 1086 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1087 else 1088 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1089 1090 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1091 1092 if (c0_scratch_reg >= 0) 1093 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1094 else 1095 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1096 1097 if (pgd_reg == -1) 1098 /* Clear lower 23 bits of context. */ 1099 uasm_i_dins(p, ptr, 0, 0, 23); 1100 1101 uasm_il_bltz(p, r, tmp, label_vmalloc); 1102 } 1103 1104 if (pgd_reg == -1) { 1105 vmalloc_branch_delay_filled = 1; 1106 /* insert bit[63:59] of CAC_BASE into bit[11:6] of ptr */ 1107 uasm_i_ori(p, ptr, ptr, ((u64)(CAC_BASE) >> 53)); 1108 1109 uasm_i_drotr(p, ptr, ptr, 11); 1110 } 1111 1112#ifdef __PAGETABLE_PMD_FOLDED 1113#define LOC_PTEP scratch 1114#else 1115#define LOC_PTEP ptr 1116#endif 1117 1118 if (!vmalloc_branch_delay_filled) 1119 /* get pgd offset in bytes */ 1120 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1121 1122 uasm_l_vmalloc_done(l, *p); 1123 1124 /* 1125 * tmp ptr 1126 * fall-through case = badvaddr *pgd_current 1127 * vmalloc case = badvaddr swapper_pg_dir 1128 */ 1129 1130 if (vmalloc_branch_delay_filled) 1131 /* get pgd offset in bytes */ 1132 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1133 1134#ifdef __PAGETABLE_PMD_FOLDED 1135 GET_CONTEXT(p, tmp); /* get context reg */ 1136#endif 1137 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1138 1139 if (use_lwx_insns()) { 1140 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1141 } else { 1142 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1143 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1144 } 1145 1146#ifndef __PAGETABLE_PUD_FOLDED 1147 /* get pud offset in bytes */ 1148 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); 1149 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); 1150 1151 if (use_lwx_insns()) { 1152 UASM_i_LWX(p, ptr, scratch, ptr); 1153 } else { 1154 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1155 UASM_i_LW(p, ptr, 0, ptr); 1156 } 1157 /* ptr contains a pointer to PMD entry */ 1158 /* tmp contains the address */ 1159#endif 1160 1161#ifndef __PAGETABLE_PMD_FOLDED 1162 /* get pmd offset in bytes */ 1163 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1164 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1165 GET_CONTEXT(p, tmp); /* get context reg */ 1166 1167 if (use_lwx_insns()) { 1168 UASM_i_LWX(p, scratch, scratch, ptr); 1169 } else { 1170 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1171 UASM_i_LW(p, scratch, 0, ptr); 1172 } 1173#endif 1174 /* Adjust the context during the load latency. */ 1175 build_adjust_context(p, tmp); 1176 1177#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1178 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1179 /* 1180 * The in the LWX case we don't want to do the load in the 1181 * delay slot. It cannot issue in the same cycle and may be 1182 * speculative and unneeded. 1183 */ 1184 if (use_lwx_insns()) 1185 uasm_i_nop(p); 1186#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 1187 1188 1189 /* build_update_entries */ 1190 if (use_lwx_insns()) { 1191 even = ptr; 1192 odd = tmp; 1193 UASM_i_LWX(p, even, scratch, tmp); 1194 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1195 UASM_i_LWX(p, odd, scratch, tmp); 1196 } else { 1197 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1198 even = tmp; 1199 odd = ptr; 1200 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1201 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1202 } 1203 if (cpu_has_rixi) { 1204 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); 1205 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1206 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1207 } else { 1208 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1209 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1210 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1211 } 1212 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1213 1214 if (c0_scratch_reg >= 0) { 1215 uasm_i_ehb(p); 1216 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1217 build_tlb_write_entry(p, l, r, tlb_random); 1218 uasm_l_leave(l, *p); 1219 rv.restore_scratch = 1; 1220 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1221 build_tlb_write_entry(p, l, r, tlb_random); 1222 uasm_l_leave(l, *p); 1223 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1224 } else { 1225 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1226 build_tlb_write_entry(p, l, r, tlb_random); 1227 uasm_l_leave(l, *p); 1228 rv.restore_scratch = 1; 1229 } 1230 1231 uasm_i_eret(p); /* return from trap */ 1232 1233 return rv; 1234} 1235 1236/* 1237 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1238 * because EXL == 0. If we wrap, we can also use the 32 instruction 1239 * slots before the XTLB refill exception handler which belong to the 1240 * unused TLB refill exception. 1241 */ 1242#define MIPS64_REFILL_INSNS 32 1243 1244static void build_r4000_tlb_refill_handler(void) 1245{ 1246 u32 *p = tlb_handler; 1247 struct uasm_label *l = labels; 1248 struct uasm_reloc *r = relocs; 1249 u32 *f; 1250 unsigned int final_len; 1251 struct mips_huge_tlb_info htlb_info __maybe_unused; 1252 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1253 1254 memset(tlb_handler, 0, sizeof(tlb_handler)); 1255 memset(labels, 0, sizeof(labels)); 1256 memset(relocs, 0, sizeof(relocs)); 1257 memset(final_handler, 0, sizeof(final_handler)); 1258 1259 if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) { 1260 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, GPR_K0, GPR_K1, 1261 scratch_reg); 1262 vmalloc_mode = refill_scratch; 1263 } else { 1264 htlb_info.huge_pte = GPR_K0; 1265 htlb_info.restore_scratch = 0; 1266 htlb_info.need_reload_pte = true; 1267 vmalloc_mode = refill_noscratch; 1268 /* 1269 * create the plain linear handler 1270 */ 1271 if (bcm1250_m3_war()) { 1272 unsigned int segbits = 44; 1273 1274 uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR); 1275 uasm_i_dmfc0(&p, GPR_K1, C0_ENTRYHI); 1276 uasm_i_xor(&p, GPR_K0, GPR_K0, GPR_K1); 1277 uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 62); 1278 uasm_i_dsrl_safe(&p, GPR_K0, GPR_K0, 12 + 1); 1279 uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits); 1280 uasm_i_or(&p, GPR_K0, GPR_K0, GPR_K1); 1281 uasm_il_bnez(&p, &r, GPR_K0, label_leave); 1282 /* No need for uasm_i_nop */ 1283 } 1284 1285#ifdef CONFIG_64BIT 1286 build_get_pmde64(&p, &l, &r, GPR_K0, GPR_K1); /* get pmd in GPR_K1 */ 1287#else 1288 build_get_pgde32(&p, GPR_K0, GPR_K1); /* get pgd in GPR_K1 */ 1289#endif 1290 1291#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1292 build_is_huge_pte(&p, &r, GPR_K0, GPR_K1, label_tlb_huge_update); 1293#endif 1294 1295 build_get_ptep(&p, GPR_K0, GPR_K1); 1296 build_update_entries(&p, GPR_K0, GPR_K1); 1297 build_tlb_write_entry(&p, &l, &r, tlb_random); 1298 uasm_l_leave(&l, p); 1299 uasm_i_eret(&p); /* return from trap */ 1300 } 1301#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1302 uasm_l_tlb_huge_update(&l, p); 1303 if (htlb_info.need_reload_pte) 1304 UASM_i_LW(&p, htlb_info.huge_pte, 0, GPR_K1); 1305 build_huge_update_entries(&p, htlb_info.huge_pte, GPR_K1); 1306 build_huge_tlb_write_entry(&p, &l, &r, GPR_K0, tlb_random, 1307 htlb_info.restore_scratch); 1308#endif 1309 1310#ifdef CONFIG_64BIT 1311 build_get_pgd_vmalloc64(&p, &l, &r, GPR_K0, GPR_K1, vmalloc_mode); 1312#endif 1313 1314 /* 1315 * Overflow check: For the 64bit handler, we need at least one 1316 * free instruction slot for the wrap-around branch. In worst 1317 * case, if the intended insertion point is a delay slot, we 1318 * need three, with the second nop'ed and the third being 1319 * unused. 1320 */ 1321 switch (boot_cpu_type()) { 1322 default: 1323 if (sizeof(long) == 4) { 1324 fallthrough; 1325 case CPU_LOONGSON2EF: 1326 /* Loongson2 ebase is different than r4k, we have more space */ 1327 if ((p - tlb_handler) > 64) 1328 panic("TLB refill handler space exceeded"); 1329 /* 1330 * Now fold the handler in the TLB refill handler space. 1331 */ 1332 f = final_handler; 1333 /* Simplest case, just copy the handler. */ 1334 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1335 final_len = p - tlb_handler; 1336 break; 1337 } else { 1338 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1339 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1340 && uasm_insn_has_bdelay(relocs, 1341 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1342 panic("TLB refill handler space exceeded"); 1343 /* 1344 * Now fold the handler in the TLB refill handler space. 1345 */ 1346 f = final_handler + MIPS64_REFILL_INSNS; 1347 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1348 /* Just copy the handler. */ 1349 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1350 final_len = p - tlb_handler; 1351 } else { 1352#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1353 const enum label_id ls = label_tlb_huge_update; 1354#else 1355 const enum label_id ls = label_vmalloc; 1356#endif 1357 u32 *split; 1358 int ov = 0; 1359 int i; 1360 1361 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1362 ; 1363 BUG_ON(i == ARRAY_SIZE(labels)); 1364 split = labels[i].addr; 1365 1366 /* 1367 * See if we have overflown one way or the other. 1368 */ 1369 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1370 split < p - MIPS64_REFILL_INSNS) 1371 ov = 1; 1372 1373 if (ov) { 1374 /* 1375 * Split two instructions before the end. One 1376 * for the branch and one for the instruction 1377 * in the delay slot. 1378 */ 1379 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1380 1381 /* 1382 * If the branch would fall in a delay slot, 1383 * we must back up an additional instruction 1384 * so that it is no longer in a delay slot. 1385 */ 1386 if (uasm_insn_has_bdelay(relocs, split - 1)) 1387 split--; 1388 } 1389 /* Copy first part of the handler. */ 1390 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1391 f += split - tlb_handler; 1392 1393 if (ov) { 1394 /* Insert branch. */ 1395 uasm_l_split(&l, final_handler); 1396 uasm_il_b(&f, &r, label_split); 1397 if (uasm_insn_has_bdelay(relocs, split)) 1398 uasm_i_nop(&f); 1399 else { 1400 uasm_copy_handler(relocs, labels, 1401 split, split + 1, f); 1402 uasm_move_labels(labels, f, f + 1, -1); 1403 f++; 1404 split++; 1405 } 1406 } 1407 1408 /* Copy the rest of the handler. */ 1409 uasm_copy_handler(relocs, labels, split, p, final_handler); 1410 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1411 (p - split); 1412 } 1413 } 1414 break; 1415 } 1416 1417 uasm_resolve_relocs(relocs, labels); 1418 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1419 final_len); 1420 1421 memcpy((void *)ebase, final_handler, 0x100); 1422 local_flush_icache_range(ebase, ebase + 0x100); 1423 dump_handler("r4000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x100)); 1424} 1425 1426static void setup_pw(void) 1427{ 1428 unsigned int pwctl; 1429 unsigned long pgd_i, pgd_w; 1430#ifndef __PAGETABLE_PMD_FOLDED 1431 unsigned long pmd_i, pmd_w; 1432#endif 1433 unsigned long pt_i, pt_w; 1434 unsigned long pte_i, pte_w; 1435#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1436 unsigned long psn; 1437 1438 psn = ilog2(_PAGE_HUGE); /* bit used to indicate huge page */ 1439#endif 1440 pgd_i = PGDIR_SHIFT; /* 1st level PGD */ 1441#ifndef __PAGETABLE_PMD_FOLDED 1442 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_TABLE_ORDER; 1443 1444 pmd_i = PMD_SHIFT; /* 2nd level PMD */ 1445 pmd_w = PMD_SHIFT - PAGE_SHIFT; 1446#else 1447 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_TABLE_ORDER; 1448#endif 1449 1450 pt_i = PAGE_SHIFT; /* 3rd level PTE */ 1451 pt_w = PAGE_SHIFT - 3; 1452 1453 pte_i = ilog2(_PAGE_GLOBAL); 1454 pte_w = 0; 1455 pwctl = 1 << 30; /* Set PWDirExt */ 1456 1457#ifndef __PAGETABLE_PMD_FOLDED 1458 write_c0_pwfield(pgd_i << 24 | pmd_i << 12 | pt_i << 6 | pte_i); 1459 write_c0_pwsize(1 << 30 | pgd_w << 24 | pmd_w << 12 | pt_w << 6 | pte_w); 1460#else 1461 write_c0_pwfield(pgd_i << 24 | pt_i << 6 | pte_i); 1462 write_c0_pwsize(1 << 30 | pgd_w << 24 | pt_w << 6 | pte_w); 1463#endif 1464 1465#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1466 pwctl |= (1 << 6 | psn); 1467#endif 1468 write_c0_pwctl(pwctl); 1469 write_c0_kpgd((long)swapper_pg_dir); 1470 kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */ 1471} 1472 1473static void build_loongson3_tlb_refill_handler(void) 1474{ 1475 u32 *p = tlb_handler; 1476 struct uasm_label *l = labels; 1477 struct uasm_reloc *r = relocs; 1478 1479 memset(labels, 0, sizeof(labels)); 1480 memset(relocs, 0, sizeof(relocs)); 1481 memset(tlb_handler, 0, sizeof(tlb_handler)); 1482 1483 if (check_for_high_segbits) { 1484 uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR); 1485 uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 1486 PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); 1487 uasm_il_beqz(&p, &r, GPR_K1, label_vmalloc); 1488 uasm_i_nop(&p); 1489 1490 uasm_il_bgez(&p, &r, GPR_K0, label_large_segbits_fault); 1491 uasm_i_nop(&p); 1492 uasm_l_vmalloc(&l, p); 1493 } 1494 1495 uasm_i_dmfc0(&p, GPR_K1, C0_PGD); 1496 1497 uasm_i_lddir(&p, GPR_K0, GPR_K1, 3); /* global page dir */ 1498#ifndef __PAGETABLE_PMD_FOLDED 1499 uasm_i_lddir(&p, GPR_K1, GPR_K0, 1); /* middle page dir */ 1500#endif 1501 uasm_i_ldpte(&p, GPR_K1, 0); /* even */ 1502 uasm_i_ldpte(&p, GPR_K1, 1); /* odd */ 1503 uasm_i_tlbwr(&p); 1504 1505 /* restore page mask */ 1506 if (PM_DEFAULT_MASK >> 16) { 1507 uasm_i_lui(&p, GPR_K0, PM_DEFAULT_MASK >> 16); 1508 uasm_i_ori(&p, GPR_K0, GPR_K0, PM_DEFAULT_MASK & 0xffff); 1509 uasm_i_mtc0(&p, GPR_K0, C0_PAGEMASK); 1510 } else if (PM_DEFAULT_MASK) { 1511 uasm_i_ori(&p, GPR_K0, 0, PM_DEFAULT_MASK); 1512 uasm_i_mtc0(&p, GPR_K0, C0_PAGEMASK); 1513 } else { 1514 uasm_i_mtc0(&p, 0, C0_PAGEMASK); 1515 } 1516 1517 uasm_i_eret(&p); 1518 1519 if (check_for_high_segbits) { 1520 uasm_l_large_segbits_fault(&l, p); 1521 UASM_i_LA(&p, GPR_K1, (unsigned long)tlb_do_page_fault_0); 1522 uasm_i_jr(&p, GPR_K1); 1523 uasm_i_nop(&p); 1524 } 1525 1526 uasm_resolve_relocs(relocs, labels); 1527 memcpy((void *)(ebase + 0x80), tlb_handler, 0x80); 1528 local_flush_icache_range(ebase + 0x80, ebase + 0x100); 1529 dump_handler("loongson3_tlb_refill", 1530 (u32 *)(ebase + 0x80), (u32 *)(ebase + 0x100)); 1531} 1532 1533static void build_setup_pgd(void) 1534{ 1535 const int a0 = 4; 1536 const int __maybe_unused a1 = 5; 1537 const int __maybe_unused a2 = 6; 1538 u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd); 1539#ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1540 long pgdc = (long)pgd_current; 1541#endif 1542 1543 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); 1544 memset(labels, 0, sizeof(labels)); 1545 memset(relocs, 0, sizeof(relocs)); 1546 pgd_reg = allocate_kscratch(); 1547#ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1548 if (pgd_reg == -1) { 1549 struct uasm_label *l = labels; 1550 struct uasm_reloc *r = relocs; 1551 1552 /* PGD << 11 in c0_Context */ 1553 /* 1554 * If it is a ckseg0 address, convert to a physical 1555 * address. Shifting right by 29 and adding 4 will 1556 * result in zero for these addresses. 1557 * 1558 */ 1559 UASM_i_SRA(&p, a1, a0, 29); 1560 UASM_i_ADDIU(&p, a1, a1, 4); 1561 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1562 uasm_i_nop(&p); 1563 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1564 uasm_l_tlbl_goaround1(&l, p); 1565 UASM_i_SLL(&p, a0, a0, 11); 1566 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1567 uasm_i_jr(&p, 31); 1568 uasm_i_ehb(&p); 1569 } else { 1570 /* PGD in c0_KScratch */ 1571 if (cpu_has_ldpte) 1572 UASM_i_MTC0(&p, a0, C0_PWBASE); 1573 else 1574 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); 1575 uasm_i_jr(&p, 31); 1576 uasm_i_ehb(&p); 1577 } 1578#else 1579#ifdef CONFIG_SMP 1580 /* Save PGD to pgd_current[smp_processor_id()] */ 1581 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); 1582 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); 1583 UASM_i_LA_mostly(&p, a2, pgdc); 1584 UASM_i_ADDU(&p, a2, a2, a1); 1585 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); 1586#else 1587 UASM_i_LA_mostly(&p, a2, pgdc); 1588 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); 1589#endif /* SMP */ 1590 1591 /* if pgd_reg is allocated, save PGD also to scratch register */ 1592 if (pgd_reg != -1) { 1593 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); 1594 uasm_i_jr(&p, 31); 1595 uasm_i_ehb(&p); 1596 } else { 1597 uasm_i_jr(&p, 31); 1598 uasm_i_nop(&p); 1599 } 1600#endif 1601 if (p >= (u32 *)tlbmiss_handler_setup_pgd_end) 1602 panic("tlbmiss_handler_setup_pgd space exceeded"); 1603 1604 uasm_resolve_relocs(relocs, labels); 1605 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1606 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); 1607 1608 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd, 1609 tlbmiss_handler_setup_pgd_end); 1610} 1611 1612static void 1613iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1614{ 1615#ifdef CONFIG_SMP 1616 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 1617 uasm_i_sync(p, 0); 1618# ifdef CONFIG_PHYS_ADDR_T_64BIT 1619 if (cpu_has_64bits) 1620 uasm_i_lld(p, pte, 0, ptr); 1621 else 1622# endif 1623 UASM_i_LL(p, pte, 0, ptr); 1624#else 1625# ifdef CONFIG_PHYS_ADDR_T_64BIT 1626 if (cpu_has_64bits) 1627 uasm_i_ld(p, pte, 0, ptr); 1628 else 1629# endif 1630 UASM_i_LW(p, pte, 0, ptr); 1631#endif 1632} 1633 1634static void 1635iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1636 unsigned int mode, unsigned int scratch) 1637{ 1638 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1639 unsigned int swmode = mode & ~hwmode; 1640 1641 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_64bits) { 1642 uasm_i_lui(p, scratch, swmode >> 16); 1643 uasm_i_or(p, pte, pte, scratch); 1644 BUG_ON(swmode & 0xffff); 1645 } else { 1646 uasm_i_ori(p, pte, pte, mode); 1647 } 1648 1649#ifdef CONFIG_SMP 1650# ifdef CONFIG_PHYS_ADDR_T_64BIT 1651 if (cpu_has_64bits) 1652 uasm_i_scd(p, pte, 0, ptr); 1653 else 1654# endif 1655 UASM_i_SC(p, pte, 0, ptr); 1656 1657 if (r10000_llsc_war()) 1658 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1659 else 1660 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1661 1662# ifdef CONFIG_PHYS_ADDR_T_64BIT 1663 if (!cpu_has_64bits) { 1664 /* no uasm_i_nop needed */ 1665 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1666 uasm_i_ori(p, pte, pte, hwmode); 1667 BUG_ON(hwmode & ~0xffff); 1668 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1669 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1670 /* no uasm_i_nop needed */ 1671 uasm_i_lw(p, pte, 0, ptr); 1672 } else 1673 uasm_i_nop(p); 1674# else 1675 uasm_i_nop(p); 1676# endif 1677#else 1678# ifdef CONFIG_PHYS_ADDR_T_64BIT 1679 if (cpu_has_64bits) 1680 uasm_i_sd(p, pte, 0, ptr); 1681 else 1682# endif 1683 UASM_i_SW(p, pte, 0, ptr); 1684 1685# ifdef CONFIG_PHYS_ADDR_T_64BIT 1686 if (!cpu_has_64bits) { 1687 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1688 uasm_i_ori(p, pte, pte, hwmode); 1689 BUG_ON(hwmode & ~0xffff); 1690 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1691 uasm_i_lw(p, pte, 0, ptr); 1692 } 1693# endif 1694#endif 1695} 1696 1697/* 1698 * Check if PTE is present, if not then jump to LABEL. PTR points to 1699 * the page table where this PTE is located, PTE will be re-loaded 1700 * with its original value. 1701 */ 1702static void 1703build_pte_present(u32 **p, struct uasm_reloc **r, 1704 int pte, int ptr, int scratch, enum label_id lid) 1705{ 1706 int t = scratch >= 0 ? scratch : pte; 1707 int cur = pte; 1708 1709 if (cpu_has_rixi) { 1710 if (use_bbit_insns()) { 1711 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1712 uasm_i_nop(p); 1713 } else { 1714 if (_PAGE_PRESENT_SHIFT) { 1715 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); 1716 cur = t; 1717 } 1718 uasm_i_andi(p, t, cur, 1); 1719 uasm_il_beqz(p, r, t, lid); 1720 if (pte == t) 1721 /* You lose the SMP race :-(*/ 1722 iPTE_LW(p, pte, ptr); 1723 } 1724 } else { 1725 if (_PAGE_PRESENT_SHIFT) { 1726 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); 1727 cur = t; 1728 } 1729 uasm_i_andi(p, t, cur, 1730 (_PAGE_PRESENT | _PAGE_NO_READ) >> _PAGE_PRESENT_SHIFT); 1731 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT); 1732 uasm_il_bnez(p, r, t, lid); 1733 if (pte == t) 1734 /* You lose the SMP race :-(*/ 1735 iPTE_LW(p, pte, ptr); 1736 } 1737} 1738 1739/* Make PTE valid, store result in PTR. */ 1740static void 1741build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1742 unsigned int ptr, unsigned int scratch) 1743{ 1744 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1745 1746 iPTE_SW(p, r, pte, ptr, mode, scratch); 1747} 1748 1749/* 1750 * Check if PTE can be written to, if not branch to LABEL. Regardless 1751 * restore PTE with value from PTR when done. 1752 */ 1753static void 1754build_pte_writable(u32 **p, struct uasm_reloc **r, 1755 unsigned int pte, unsigned int ptr, int scratch, 1756 enum label_id lid) 1757{ 1758 int t = scratch >= 0 ? scratch : pte; 1759 int cur = pte; 1760 1761 if (_PAGE_PRESENT_SHIFT) { 1762 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); 1763 cur = t; 1764 } 1765 uasm_i_andi(p, t, cur, 1766 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT); 1767 uasm_i_xori(p, t, t, 1768 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT); 1769 uasm_il_bnez(p, r, t, lid); 1770 if (pte == t) 1771 /* You lose the SMP race :-(*/ 1772 iPTE_LW(p, pte, ptr); 1773 else 1774 uasm_i_nop(p); 1775} 1776 1777/* Make PTE writable, update software status bits as well, then store 1778 * at PTR. 1779 */ 1780static void 1781build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1782 unsigned int ptr, unsigned int scratch) 1783{ 1784 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1785 | _PAGE_DIRTY); 1786 1787 iPTE_SW(p, r, pte, ptr, mode, scratch); 1788} 1789 1790/* 1791 * Check if PTE can be modified, if not branch to LABEL. Regardless 1792 * restore PTE with value from PTR when done. 1793 */ 1794static void 1795build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1796 unsigned int pte, unsigned int ptr, int scratch, 1797 enum label_id lid) 1798{ 1799 if (use_bbit_insns()) { 1800 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1801 uasm_i_nop(p); 1802 } else { 1803 int t = scratch >= 0 ? scratch : pte; 1804 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT); 1805 uasm_i_andi(p, t, t, 1); 1806 uasm_il_beqz(p, r, t, lid); 1807 if (pte == t) 1808 /* You lose the SMP race :-(*/ 1809 iPTE_LW(p, pte, ptr); 1810 } 1811} 1812 1813#ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1814 1815 1816/* 1817 * R3000 style TLB load/store/modify handlers. 1818 */ 1819 1820/* 1821 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1822 * Then it returns. 1823 */ 1824static void 1825build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1826{ 1827 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1828 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1829 uasm_i_tlbwi(p); 1830 uasm_i_jr(p, tmp); 1831 uasm_i_rfe(p); /* branch delay */ 1832} 1833 1834/* 1835 * This places the pte into ENTRYLO0 and writes it with tlbwi 1836 * or tlbwr as appropriate. This is because the index register 1837 * may have the probe fail bit set as a result of a trap on a 1838 * kseg2 access, i.e. without refill. Then it returns. 1839 */ 1840static void 1841build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1842 struct uasm_reloc **r, unsigned int pte, 1843 unsigned int tmp) 1844{ 1845 uasm_i_mfc0(p, tmp, C0_INDEX); 1846 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1847 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1848 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1849 uasm_i_tlbwi(p); /* cp0 delay */ 1850 uasm_i_jr(p, tmp); 1851 uasm_i_rfe(p); /* branch delay */ 1852 uasm_l_r3000_write_probe_fail(l, *p); 1853 uasm_i_tlbwr(p); /* cp0 delay */ 1854 uasm_i_jr(p, tmp); 1855 uasm_i_rfe(p); /* branch delay */ 1856} 1857 1858static void 1859build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1860 unsigned int ptr) 1861{ 1862 long pgdc = (long)pgd_current; 1863 1864 uasm_i_mfc0(p, pte, C0_BADVADDR); 1865 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1866 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1867 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1868 uasm_i_sll(p, pte, pte, 2); 1869 uasm_i_addu(p, ptr, ptr, pte); 1870 uasm_i_mfc0(p, pte, C0_CONTEXT); 1871 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1872 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1873 uasm_i_addu(p, ptr, ptr, pte); 1874 uasm_i_lw(p, pte, 0, ptr); 1875 uasm_i_tlbp(p); /* load delay */ 1876} 1877 1878static void build_r3000_tlb_load_handler(void) 1879{ 1880 u32 *p = (u32 *)handle_tlbl; 1881 struct uasm_label *l = labels; 1882 struct uasm_reloc *r = relocs; 1883 1884 memset(p, 0, handle_tlbl_end - (char *)p); 1885 memset(labels, 0, sizeof(labels)); 1886 memset(relocs, 0, sizeof(relocs)); 1887 1888 build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1); 1889 build_pte_present(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbl); 1890 uasm_i_nop(&p); /* load delay */ 1891 build_make_valid(&p, &r, GPR_K0, GPR_K1, -1); 1892 build_r3000_tlb_reload_write(&p, &l, &r, GPR_K0, GPR_K1); 1893 1894 uasm_l_nopage_tlbl(&l, p); 1895 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1896 uasm_i_nop(&p); 1897 1898 if (p >= (u32 *)handle_tlbl_end) 1899 panic("TLB load handler fastpath space exceeded"); 1900 1901 uasm_resolve_relocs(relocs, labels); 1902 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1903 (unsigned int)(p - (u32 *)handle_tlbl)); 1904 1905 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_end); 1906} 1907 1908static void build_r3000_tlb_store_handler(void) 1909{ 1910 u32 *p = (u32 *)handle_tlbs; 1911 struct uasm_label *l = labels; 1912 struct uasm_reloc *r = relocs; 1913 1914 memset(p, 0, handle_tlbs_end - (char *)p); 1915 memset(labels, 0, sizeof(labels)); 1916 memset(relocs, 0, sizeof(relocs)); 1917 1918 build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1); 1919 build_pte_writable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbs); 1920 uasm_i_nop(&p); /* load delay */ 1921 build_make_write(&p, &r, GPR_K0, GPR_K1, -1); 1922 build_r3000_tlb_reload_write(&p, &l, &r, GPR_K0, GPR_K1); 1923 1924 uasm_l_nopage_tlbs(&l, p); 1925 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1926 uasm_i_nop(&p); 1927 1928 if (p >= (u32 *)handle_tlbs_end) 1929 panic("TLB store handler fastpath space exceeded"); 1930 1931 uasm_resolve_relocs(relocs, labels); 1932 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1933 (unsigned int)(p - (u32 *)handle_tlbs)); 1934 1935 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_end); 1936} 1937 1938static void build_r3000_tlb_modify_handler(void) 1939{ 1940 u32 *p = (u32 *)handle_tlbm; 1941 struct uasm_label *l = labels; 1942 struct uasm_reloc *r = relocs; 1943 1944 memset(p, 0, handle_tlbm_end - (char *)p); 1945 memset(labels, 0, sizeof(labels)); 1946 memset(relocs, 0, sizeof(relocs)); 1947 1948 build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1); 1949 build_pte_modifiable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbm); 1950 uasm_i_nop(&p); /* load delay */ 1951 build_make_write(&p, &r, GPR_K0, GPR_K1, -1); 1952 build_r3000_pte_reload_tlbwi(&p, GPR_K0, GPR_K1); 1953 1954 uasm_l_nopage_tlbm(&l, p); 1955 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1956 uasm_i_nop(&p); 1957 1958 if (p >= (u32 *)handle_tlbm_end) 1959 panic("TLB modify handler fastpath space exceeded"); 1960 1961 uasm_resolve_relocs(relocs, labels); 1962 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1963 (unsigned int)(p - (u32 *)handle_tlbm)); 1964 1965 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_end); 1966} 1967#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1968 1969static bool cpu_has_tlbex_tlbp_race(void) 1970{ 1971 /* 1972 * When a Hardware Table Walker is running it can replace TLB entries 1973 * at any time, leading to a race between it & the CPU. 1974 */ 1975 if (cpu_has_htw) 1976 return true; 1977 1978 /* 1979 * If the CPU shares FTLB RAM with its siblings then our entry may be 1980 * replaced at any time by a sibling performing a write to the FTLB. 1981 */ 1982 if (cpu_has_shared_ftlb_ram) 1983 return true; 1984 1985 /* In all other cases there ought to be no race condition to handle */ 1986 return false; 1987} 1988 1989/* 1990 * R4000 style TLB load/store/modify handlers. 1991 */ 1992static struct work_registers 1993build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1994 struct uasm_reloc **r) 1995{ 1996 struct work_registers wr = build_get_work_registers(p); 1997 1998#ifdef CONFIG_64BIT 1999 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 2000#else 2001 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 2002#endif 2003 2004#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2005 /* 2006 * For huge tlb entries, pmd doesn't contain an address but 2007 * instead contains the tlb pte. Check the PAGE_HUGE bit and 2008 * see if we need to jump to huge tlb processing. 2009 */ 2010 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 2011#endif 2012 2013 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 2014 UASM_i_LW(p, wr.r2, 0, wr.r2); 2015 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT - PTE_T_LOG2); 2016 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 2017 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 2018 2019#ifdef CONFIG_SMP 2020 uasm_l_smp_pgtable_change(l, *p); 2021#endif 2022 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 2023 if (!m4kc_tlbp_war()) { 2024 build_tlb_probe_entry(p); 2025 if (cpu_has_tlbex_tlbp_race()) { 2026 /* race condition happens, leaving */ 2027 uasm_i_ehb(p); 2028 uasm_i_mfc0(p, wr.r3, C0_INDEX); 2029 uasm_il_bltz(p, r, wr.r3, label_leave); 2030 uasm_i_nop(p); 2031 } 2032 } 2033 return wr; 2034} 2035 2036static void 2037build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 2038 struct uasm_reloc **r, unsigned int tmp, 2039 unsigned int ptr) 2040{ 2041 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 2042 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 2043 build_update_entries(p, tmp, ptr); 2044 build_tlb_write_entry(p, l, r, tlb_indexed); 2045 uasm_l_leave(l, *p); 2046 build_restore_work_registers(p); 2047 uasm_i_eret(p); /* return from trap */ 2048 2049#ifdef CONFIG_64BIT 2050 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 2051#endif 2052} 2053 2054static void build_r4000_tlb_load_handler(void) 2055{ 2056 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl); 2057 struct uasm_label *l = labels; 2058 struct uasm_reloc *r = relocs; 2059 struct work_registers wr; 2060 2061 memset(p, 0, handle_tlbl_end - (char *)p); 2062 memset(labels, 0, sizeof(labels)); 2063 memset(relocs, 0, sizeof(relocs)); 2064 2065 if (bcm1250_m3_war()) { 2066 unsigned int segbits = 44; 2067 2068 uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR); 2069 uasm_i_dmfc0(&p, GPR_K1, C0_ENTRYHI); 2070 uasm_i_xor(&p, GPR_K0, GPR_K0, GPR_K1); 2071 uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 62); 2072 uasm_i_dsrl_safe(&p, GPR_K0, GPR_K0, 12 + 1); 2073 uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits); 2074 uasm_i_or(&p, GPR_K0, GPR_K0, GPR_K1); 2075 uasm_il_bnez(&p, &r, GPR_K0, label_leave); 2076 /* No need for uasm_i_nop */ 2077 } 2078 2079 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2080 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 2081 if (m4kc_tlbp_war()) 2082 build_tlb_probe_entry(&p); 2083 2084 if (cpu_has_rixi && !cpu_has_rixiex) { 2085 /* 2086 * If the page is not _PAGE_VALID, RI or XI could not 2087 * have triggered it. Skip the expensive test.. 2088 */ 2089 if (use_bbit_insns()) { 2090 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 2091 label_tlbl_goaround1); 2092 } else { 2093 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 2094 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 2095 } 2096 uasm_i_nop(&p); 2097 2098 /* 2099 * Warn if something may race with us & replace the TLB entry 2100 * before we read it here. Everything with such races should 2101 * also have dedicated RiXi exception handlers, so this 2102 * shouldn't be hit. 2103 */ 2104 WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path"); 2105 2106 uasm_i_tlbr(&p); 2107 2108 if (cpu_has_mips_r2_exec_hazard) 2109 uasm_i_ehb(&p); 2110 2111 /* Examine entrylo 0 or 1 based on ptr. */ 2112 if (use_bbit_insns()) { 2113 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2114 } else { 2115 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 2116 uasm_i_beqz(&p, wr.r3, 8); 2117 } 2118 /* load it in the delay slot*/ 2119 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 2120 /* load it if ptr is odd */ 2121 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 2122 /* 2123 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 2124 * XI must have triggered it. 2125 */ 2126 if (use_bbit_insns()) { 2127 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 2128 uasm_i_nop(&p); 2129 uasm_l_tlbl_goaround1(&l, p); 2130 } else { 2131 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2132 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 2133 uasm_i_nop(&p); 2134 } 2135 uasm_l_tlbl_goaround1(&l, p); 2136 } 2137 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3); 2138 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2139 2140#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2141 /* 2142 * This is the entry point when build_r4000_tlbchange_handler_head 2143 * spots a huge page. 2144 */ 2145 uasm_l_tlb_huge_update(&l, p); 2146 iPTE_LW(&p, wr.r1, wr.r2); 2147 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 2148 build_tlb_probe_entry(&p); 2149 2150 if (cpu_has_rixi && !cpu_has_rixiex) { 2151 /* 2152 * If the page is not _PAGE_VALID, RI or XI could not 2153 * have triggered it. Skip the expensive test.. 2154 */ 2155 if (use_bbit_insns()) { 2156 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 2157 label_tlbl_goaround2); 2158 } else { 2159 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 2160 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2161 } 2162 uasm_i_nop(&p); 2163 2164 /* 2165 * Warn if something may race with us & replace the TLB entry 2166 * before we read it here. Everything with such races should 2167 * also have dedicated RiXi exception handlers, so this 2168 * shouldn't be hit. 2169 */ 2170 WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path"); 2171 2172 uasm_i_tlbr(&p); 2173 2174 if (cpu_has_mips_r2_exec_hazard) 2175 uasm_i_ehb(&p); 2176 2177 /* Examine entrylo 0 or 1 based on ptr. */ 2178 if (use_bbit_insns()) { 2179 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2180 } else { 2181 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 2182 uasm_i_beqz(&p, wr.r3, 8); 2183 } 2184 /* load it in the delay slot*/ 2185 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 2186 /* load it if ptr is odd */ 2187 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 2188 /* 2189 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 2190 * XI must have triggered it. 2191 */ 2192 if (use_bbit_insns()) { 2193 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 2194 } else { 2195 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2196 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2197 } 2198 if (PM_DEFAULT_MASK == 0) 2199 uasm_i_nop(&p); 2200 /* 2201 * We clobbered C0_PAGEMASK, restore it. On the other branch 2202 * it is restored in build_huge_tlb_write_entry. 2203 */ 2204 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 2205 2206 uasm_l_tlbl_goaround2(&l, p); 2207 } 2208 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 2209 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); 2210#endif 2211 2212 uasm_l_nopage_tlbl(&l, p); 2213 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 2214 uasm_i_sync(&p, 0); 2215 build_restore_work_registers(&p); 2216#ifdef CONFIG_CPU_MICROMIPS 2217 if ((unsigned long)tlb_do_page_fault_0 & 1) { 2218 uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_0)); 2219 uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_0)); 2220 uasm_i_jr(&p, GPR_K0); 2221 } else 2222#endif 2223 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2224 uasm_i_nop(&p); 2225 2226 if (p >= (u32 *)handle_tlbl_end) 2227 panic("TLB load handler fastpath space exceeded"); 2228 2229 uasm_resolve_relocs(relocs, labels); 2230 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2231 (unsigned int)(p - (u32 *)handle_tlbl)); 2232 2233 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_end); 2234} 2235 2236static void build_r4000_tlb_store_handler(void) 2237{ 2238 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs); 2239 struct uasm_label *l = labels; 2240 struct uasm_reloc *r = relocs; 2241 struct work_registers wr; 2242 2243 memset(p, 0, handle_tlbs_end - (char *)p); 2244 memset(labels, 0, sizeof(labels)); 2245 memset(relocs, 0, sizeof(relocs)); 2246 2247 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2248 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2249 if (m4kc_tlbp_war()) 2250 build_tlb_probe_entry(&p); 2251 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); 2252 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2253 2254#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2255 /* 2256 * This is the entry point when 2257 * build_r4000_tlbchange_handler_head spots a huge page. 2258 */ 2259 uasm_l_tlb_huge_update(&l, p); 2260 iPTE_LW(&p, wr.r1, wr.r2); 2261 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2262 build_tlb_probe_entry(&p); 2263 uasm_i_ori(&p, wr.r1, wr.r1, 2264 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2265 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); 2266#endif 2267 2268 uasm_l_nopage_tlbs(&l, p); 2269 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 2270 uasm_i_sync(&p, 0); 2271 build_restore_work_registers(&p); 2272#ifdef CONFIG_CPU_MICROMIPS 2273 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2274 uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2275 uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2276 uasm_i_jr(&p, GPR_K0); 2277 } else 2278#endif 2279 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2280 uasm_i_nop(&p); 2281 2282 if (p >= (u32 *)handle_tlbs_end) 2283 panic("TLB store handler fastpath space exceeded"); 2284 2285 uasm_resolve_relocs(relocs, labels); 2286 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2287 (unsigned int)(p - (u32 *)handle_tlbs)); 2288 2289 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_end); 2290} 2291 2292static void build_r4000_tlb_modify_handler(void) 2293{ 2294 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm); 2295 struct uasm_label *l = labels; 2296 struct uasm_reloc *r = relocs; 2297 struct work_registers wr; 2298 2299 memset(p, 0, handle_tlbm_end - (char *)p); 2300 memset(labels, 0, sizeof(labels)); 2301 memset(relocs, 0, sizeof(relocs)); 2302 2303 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2304 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2305 if (m4kc_tlbp_war()) 2306 build_tlb_probe_entry(&p); 2307 /* Present and writable bits set, set accessed and dirty bits. */ 2308 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); 2309 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2310 2311#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2312 /* 2313 * This is the entry point when 2314 * build_r4000_tlbchange_handler_head spots a huge page. 2315 */ 2316 uasm_l_tlb_huge_update(&l, p); 2317 iPTE_LW(&p, wr.r1, wr.r2); 2318 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2319 build_tlb_probe_entry(&p); 2320 uasm_i_ori(&p, wr.r1, wr.r1, 2321 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2322 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0); 2323#endif 2324 2325 uasm_l_nopage_tlbm(&l, p); 2326 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS)) 2327 uasm_i_sync(&p, 0); 2328 build_restore_work_registers(&p); 2329#ifdef CONFIG_CPU_MICROMIPS 2330 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2331 uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2332 uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2333 uasm_i_jr(&p, GPR_K0); 2334 } else 2335#endif 2336 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2337 uasm_i_nop(&p); 2338 2339 if (p >= (u32 *)handle_tlbm_end) 2340 panic("TLB modify handler fastpath space exceeded"); 2341 2342 uasm_resolve_relocs(relocs, labels); 2343 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2344 (unsigned int)(p - (u32 *)handle_tlbm)); 2345 2346 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_end); 2347} 2348 2349static void flush_tlb_handlers(void) 2350{ 2351 local_flush_icache_range((unsigned long)handle_tlbl, 2352 (unsigned long)handle_tlbl_end); 2353 local_flush_icache_range((unsigned long)handle_tlbs, 2354 (unsigned long)handle_tlbs_end); 2355 local_flush_icache_range((unsigned long)handle_tlbm, 2356 (unsigned long)handle_tlbm_end); 2357 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2358 (unsigned long)tlbmiss_handler_setup_pgd_end); 2359} 2360 2361static void print_htw_config(void) 2362{ 2363 unsigned long config; 2364 unsigned int pwctl; 2365 const int field = 2 * sizeof(unsigned long); 2366 2367 config = read_c0_pwfield(); 2368 pr_debug("PWField (0x%0*lx): GDI: 0x%02lx UDI: 0x%02lx MDI: 0x%02lx PTI: 0x%02lx PTEI: 0x%02lx\n", 2369 field, config, 2370 (config & MIPS_PWFIELD_GDI_MASK) >> MIPS_PWFIELD_GDI_SHIFT, 2371 (config & MIPS_PWFIELD_UDI_MASK) >> MIPS_PWFIELD_UDI_SHIFT, 2372 (config & MIPS_PWFIELD_MDI_MASK) >> MIPS_PWFIELD_MDI_SHIFT, 2373 (config & MIPS_PWFIELD_PTI_MASK) >> MIPS_PWFIELD_PTI_SHIFT, 2374 (config & MIPS_PWFIELD_PTEI_MASK) >> MIPS_PWFIELD_PTEI_SHIFT); 2375 2376 config = read_c0_pwsize(); 2377 pr_debug("PWSize (0x%0*lx): PS: 0x%lx GDW: 0x%02lx UDW: 0x%02lx MDW: 0x%02lx PTW: 0x%02lx PTEW: 0x%02lx\n", 2378 field, config, 2379 (config & MIPS_PWSIZE_PS_MASK) >> MIPS_PWSIZE_PS_SHIFT, 2380 (config & MIPS_PWSIZE_GDW_MASK) >> MIPS_PWSIZE_GDW_SHIFT, 2381 (config & MIPS_PWSIZE_UDW_MASK) >> MIPS_PWSIZE_UDW_SHIFT, 2382 (config & MIPS_PWSIZE_MDW_MASK) >> MIPS_PWSIZE_MDW_SHIFT, 2383 (config & MIPS_PWSIZE_PTW_MASK) >> MIPS_PWSIZE_PTW_SHIFT, 2384 (config & MIPS_PWSIZE_PTEW_MASK) >> MIPS_PWSIZE_PTEW_SHIFT); 2385 2386 pwctl = read_c0_pwctl(); 2387 pr_debug("PWCtl (0x%x): PWEn: 0x%x XK: 0x%x XS: 0x%x XU: 0x%x DPH: 0x%x HugePg: 0x%x Psn: 0x%x\n", 2388 pwctl, 2389 (pwctl & MIPS_PWCTL_PWEN_MASK) >> MIPS_PWCTL_PWEN_SHIFT, 2390 (pwctl & MIPS_PWCTL_XK_MASK) >> MIPS_PWCTL_XK_SHIFT, 2391 (pwctl & MIPS_PWCTL_XS_MASK) >> MIPS_PWCTL_XS_SHIFT, 2392 (pwctl & MIPS_PWCTL_XU_MASK) >> MIPS_PWCTL_XU_SHIFT, 2393 (pwctl & MIPS_PWCTL_DPH_MASK) >> MIPS_PWCTL_DPH_SHIFT, 2394 (pwctl & MIPS_PWCTL_HUGEPG_MASK) >> MIPS_PWCTL_HUGEPG_SHIFT, 2395 (pwctl & MIPS_PWCTL_PSN_MASK) >> MIPS_PWCTL_PSN_SHIFT); 2396} 2397 2398static void config_htw_params(void) 2399{ 2400 unsigned long pwfield, pwsize, ptei; 2401 unsigned int config; 2402 2403 /* 2404 * We are using 2-level page tables, so we only need to 2405 * setup GDW and PTW appropriately. UDW and MDW will remain 0. 2406 * The default value of GDI/UDI/MDI/PTI is 0xc. It is illegal to 2407 * write values less than 0xc in these fields because the entire 2408 * write will be dropped. As a result of which, we must preserve 2409 * the original reset values and overwrite only what we really want. 2410 */ 2411 2412 pwfield = read_c0_pwfield(); 2413 /* re-initialize the GDI field */ 2414 pwfield &= ~MIPS_PWFIELD_GDI_MASK; 2415 pwfield |= PGDIR_SHIFT << MIPS_PWFIELD_GDI_SHIFT; 2416 /* re-initialize the PTI field including the even/odd bit */ 2417 pwfield &= ~MIPS_PWFIELD_PTI_MASK; 2418 pwfield |= PAGE_SHIFT << MIPS_PWFIELD_PTI_SHIFT; 2419 if (CONFIG_PGTABLE_LEVELS >= 3) { 2420 pwfield &= ~MIPS_PWFIELD_MDI_MASK; 2421 pwfield |= PMD_SHIFT << MIPS_PWFIELD_MDI_SHIFT; 2422 } 2423 /* Set the PTEI right shift */ 2424 ptei = _PAGE_GLOBAL_SHIFT << MIPS_PWFIELD_PTEI_SHIFT; 2425 pwfield |= ptei; 2426 write_c0_pwfield(pwfield); 2427 /* Check whether the PTEI value is supported */ 2428 back_to_back_c0_hazard(); 2429 pwfield = read_c0_pwfield(); 2430 if (((pwfield & MIPS_PWFIELD_PTEI_MASK) << MIPS_PWFIELD_PTEI_SHIFT) 2431 != ptei) { 2432 pr_warn("Unsupported PTEI field value: 0x%lx. HTW will not be enabled", 2433 ptei); 2434 /* 2435 * Drop option to avoid HTW being enabled via another path 2436 * (eg htw_reset()) 2437 */ 2438 current_cpu_data.options &= ~MIPS_CPU_HTW; 2439 return; 2440 } 2441 2442 pwsize = ilog2(PTRS_PER_PGD) << MIPS_PWSIZE_GDW_SHIFT; 2443 pwsize |= ilog2(PTRS_PER_PTE) << MIPS_PWSIZE_PTW_SHIFT; 2444 if (CONFIG_PGTABLE_LEVELS >= 3) 2445 pwsize |= ilog2(PTRS_PER_PMD) << MIPS_PWSIZE_MDW_SHIFT; 2446 2447 /* Set pointer size to size of directory pointers */ 2448 if (IS_ENABLED(CONFIG_64BIT)) 2449 pwsize |= MIPS_PWSIZE_PS_MASK; 2450 /* PTEs may be multiple pointers long (e.g. with XPA) */ 2451 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT) 2452 & MIPS_PWSIZE_PTEW_MASK; 2453 2454 write_c0_pwsize(pwsize); 2455 2456 /* Make sure everything is set before we enable the HTW */ 2457 back_to_back_c0_hazard(); 2458 2459 /* 2460 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of 2461 * the pwctl fields. 2462 */ 2463 config = 1 << MIPS_PWCTL_PWEN_SHIFT; 2464 if (IS_ENABLED(CONFIG_64BIT)) 2465 config |= MIPS_PWCTL_XU_MASK; 2466 write_c0_pwctl(config); 2467 pr_info("Hardware Page Table Walker enabled\n"); 2468 2469 print_htw_config(); 2470} 2471 2472static void config_xpa_params(void) 2473{ 2474#ifdef CONFIG_XPA 2475 unsigned int pagegrain; 2476 2477 if (mips_xpa_disabled) { 2478 pr_info("Extended Physical Addressing (XPA) disabled\n"); 2479 return; 2480 } 2481 2482 pagegrain = read_c0_pagegrain(); 2483 write_c0_pagegrain(pagegrain | PG_ELPA); 2484 back_to_back_c0_hazard(); 2485 pagegrain = read_c0_pagegrain(); 2486 2487 if (pagegrain & PG_ELPA) 2488 pr_info("Extended Physical Addressing (XPA) enabled\n"); 2489 else 2490 panic("Extended Physical Addressing (XPA) disabled"); 2491#endif 2492} 2493 2494static void check_pabits(void) 2495{ 2496 unsigned long entry; 2497 unsigned pabits, fillbits; 2498 2499 if (!cpu_has_rixi || _PAGE_NO_EXEC == 0) { 2500 /* 2501 * We'll only be making use of the fact that we can rotate bits 2502 * into the fill if the CPU supports RIXI, so don't bother 2503 * probing this for CPUs which don't. 2504 */ 2505 return; 2506 } 2507 2508 write_c0_entrylo0(~0ul); 2509 back_to_back_c0_hazard(); 2510 entry = read_c0_entrylo0(); 2511 2512 /* clear all non-PFN bits */ 2513 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1); 2514 entry &= ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); 2515 2516 /* find a lower bound on PABITS, and upper bound on fill bits */ 2517 pabits = fls_long(entry) + 6; 2518 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0); 2519 2520 /* minus the RI & XI bits */ 2521 fillbits -= min_t(unsigned, fillbits, 2); 2522 2523 if (fillbits >= ilog2(_PAGE_NO_EXEC)) 2524 fill_includes_sw_bits = true; 2525 2526 pr_debug("Entry* registers contain %u fill bits\n", fillbits); 2527} 2528 2529void build_tlb_refill_handler(void) 2530{ 2531 /* 2532 * The refill handler is generated per-CPU, multi-node systems 2533 * may have local storage for it. The other handlers are only 2534 * needed once. 2535 */ 2536 static int run_once = 0; 2537 2538 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_rixi) 2539 panic("Kernels supporting XPA currently require CPUs with RIXI"); 2540 2541 output_pgtable_bits_defines(); 2542 check_pabits(); 2543 2544#ifdef CONFIG_64BIT 2545 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); 2546#endif 2547 2548 if (cpu_has_3kex) { 2549#ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2550 if (!run_once) { 2551 build_setup_pgd(); 2552 build_r3000_tlb_refill_handler(); 2553 build_r3000_tlb_load_handler(); 2554 build_r3000_tlb_store_handler(); 2555 build_r3000_tlb_modify_handler(); 2556 flush_tlb_handlers(); 2557 run_once++; 2558 } 2559#else 2560 panic("No R3000 TLB refill handler"); 2561#endif 2562 return; 2563 } 2564 2565 if (cpu_has_ldpte) 2566 setup_pw(); 2567 2568 if (!run_once) { 2569 scratch_reg = allocate_kscratch(); 2570 build_setup_pgd(); 2571 build_r4000_tlb_load_handler(); 2572 build_r4000_tlb_store_handler(); 2573 build_r4000_tlb_modify_handler(); 2574 if (cpu_has_ldpte) 2575 build_loongson3_tlb_refill_handler(); 2576 else 2577 build_r4000_tlb_refill_handler(); 2578 flush_tlb_handlers(); 2579 run_once++; 2580 } 2581 if (cpu_has_xpa) 2582 config_xpa_params(); 2583 if (cpu_has_htw) 2584 config_htw_params(); 2585} 2586