1/* sun4v_tlb_miss.S: Sun4v TLB miss handlers. 2 * 3 * Copyright (C) 2006 <davem@davemloft.net> 4 */ 5 6 .text 7 .align 32 8 9 /* Load ITLB fault information into VADDR and CTX, using BASE. */ 10#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \ 11 ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \ 12 ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX; 13 14 /* Load DTLB fault information into VADDR and CTX, using BASE. */ 15#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \ 16 ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \ 17 ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX; 18 19 /* DEST = (VADDR >> 22) 20 * 21 * Branch to ZERO_CTX_LABEL if context is zero. 22 */ 23#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \ 24 srlx VADDR, 22, DEST; \ 25 brz,pn CTX, ZERO_CTX_LABEL; \ 26 nop; 27 28 /* Create TSB pointer. This is something like: 29 * 30 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; 31 * tsb_base = tsb_reg & ~0x7UL; 32 * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); 33 * tsb_ptr = tsb_base + (tsb_index * 16); 34 */ 35#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ 36 and TSB_PTR, 0x7, TMP1; \ 37 mov 512, TMP2; \ 38 andn TSB_PTR, 0x7, TSB_PTR; \ 39 sllx TMP2, TMP1, TMP2; \ 40 srlx VADDR, HASH_SHIFT, TMP1; \ 41 sub TMP2, 1, TMP2; \ 42 and TMP1, TMP2, TMP1; \ 43 sllx TMP1, 4, TMP1; \ 44 add TSB_PTR, TMP1, TSB_PTR; 45 46sun4v_itlb_miss: 47 /* Load MMU Miss base into %g2. */ 48 ldxa [%g0] ASI_SCRATCHPAD, %g2 49 50 /* Load UTSB reg into %g1. */ 51 mov SCRATCHPAD_UTSBREG1, %g1 52 ldxa [%g1] ASI_SCRATCHPAD, %g1 53 54 LOAD_ITLB_INFO(%g2, %g4, %g5) 55 COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) 56 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) 57 58 /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ 59 ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 60 cmp %g2, %g6 61 bne,a,pn %xcc, tsb_miss_page_table_walk 62 mov FAULT_CODE_ITLB, %g3 63 andcc %g3, _PAGE_EXEC_4V, %g0 64 be,a,pn %xcc, tsb_do_fault 65 mov FAULT_CODE_ITLB, %g3 66 67 /* We have a valid entry, make hypervisor call to load 68 * I-TLB and return from trap. 69 * 70 * %g3: PTE 71 * %g4: vaddr 72 */ 73sun4v_itlb_load: 74 ldxa [%g0] ASI_SCRATCHPAD, %g6 75 mov %o0, %g1 ! save %o0 76 mov %o1, %g2 ! save %o1 77 mov %o2, %g5 ! save %o2 78 mov %o3, %g7 ! save %o3 79 mov %g4, %o0 ! vaddr 80 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx 81 mov %g3, %o2 ! PTE 82 mov HV_MMU_IMMU, %o3 ! flags 83 ta HV_MMU_MAP_ADDR_TRAP 84 brnz,pn %o0, sun4v_itlb_error 85 mov %g2, %o1 ! restore %o1 86 mov %g1, %o0 ! restore %o0 87 mov %g5, %o2 ! restore %o2 88 mov %g7, %o3 ! restore %o3 89 90 retry 91 92sun4v_dtlb_miss: 93 /* Load MMU Miss base into %g2. */ 94 ldxa [%g0] ASI_SCRATCHPAD, %g2 95 96 /* Load UTSB reg into %g1. */ 97 mov SCRATCHPAD_UTSBREG1, %g1 98 ldxa [%g1] ASI_SCRATCHPAD, %g1 99 100 LOAD_DTLB_INFO(%g2, %g4, %g5) 101 COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) 102 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) 103 104 /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ 105 ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 106 cmp %g2, %g6 107 bne,a,pn %xcc, tsb_miss_page_table_walk 108 mov FAULT_CODE_DTLB, %g3 109 110 /* We have a valid entry, make hypervisor call to load 111 * D-TLB and return from trap. 112 * 113 * %g3: PTE 114 * %g4: vaddr 115 */ 116sun4v_dtlb_load: 117 ldxa [%g0] ASI_SCRATCHPAD, %g6 118 mov %o0, %g1 ! save %o0 119 mov %o1, %g2 ! save %o1 120 mov %o2, %g5 ! save %o2 121 mov %o3, %g7 ! save %o3 122 mov %g4, %o0 ! vaddr 123 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx 124 mov %g3, %o2 ! PTE 125 mov HV_MMU_DMMU, %o3 ! flags 126 ta HV_MMU_MAP_ADDR_TRAP 127 brnz,pn %o0, sun4v_dtlb_error 128 mov %g2, %o1 ! restore %o1 129 mov %g1, %o0 ! restore %o0 130 mov %g5, %o2 ! restore %o2 131 mov %g7, %o3 ! restore %o3 132 133 retry 134 135sun4v_dtlb_prot: 136 SET_GL(1) 137 138 /* Load MMU Miss base into %g5. */ 139 ldxa [%g0] ASI_SCRATCHPAD, %g5 140 141 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 142 rdpr %tl, %g1 143 cmp %g1, 1 144 bgu,pn %xcc, winfix_trampoline 145 mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 146 ba,pt %xcc, sparc64_realfault_common 147 nop 148 149 /* Called from trap table: 150 * %g4: vaddr 151 * %g5: context 152 * %g6: TAG TARGET 153 */ 154sun4v_itsb_miss: 155 mov SCRATCHPAD_UTSBREG1, %g1 156 ldxa [%g1] ASI_SCRATCHPAD, %g1 157 brz,pn %g5, kvmap_itlb_4v 158 mov FAULT_CODE_ITLB, %g3 159 ba,a,pt %xcc, sun4v_tsb_miss_common 160 161 /* Called from trap table: 162 * %g4: vaddr 163 * %g5: context 164 * %g6: TAG TARGET 165 */ 166sun4v_dtsb_miss: 167 mov SCRATCHPAD_UTSBREG1, %g1 168 ldxa [%g1] ASI_SCRATCHPAD, %g1 169 brz,pn %g5, kvmap_dtlb_4v 170 mov FAULT_CODE_DTLB, %g3 171 172 /* fallthrough */ 173 174sun4v_tsb_miss_common: 175 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) 176 177 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 178 179#ifdef CONFIG_HUGETLB_PAGE 180 mov SCRATCHPAD_UTSBREG2, %g5 181 ldxa [%g5] ASI_SCRATCHPAD, %g5 182 cmp %g5, -1 183 be,pt %xcc, 80f 184 nop 185 COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) 186 187 /* That clobbered %g2, reload it. */ 188 ldxa [%g0] ASI_SCRATCHPAD, %g2 189 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 190 19180: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP] 192#endif 193 194 ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath 195 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 196 197sun4v_itlb_error: 198 sethi %hi(sun4v_err_itlb_vaddr), %g1 199 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] 200 sethi %hi(sun4v_err_itlb_ctx), %g1 201 ldxa [%g0] ASI_SCRATCHPAD, %g6 202 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 203 stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)] 204 sethi %hi(sun4v_err_itlb_pte), %g1 205 stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)] 206 sethi %hi(sun4v_err_itlb_error), %g1 207 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] 208 209 rdpr %tl, %g4 210 cmp %g4, 1 211 ble,pt %icc, 1f 212 sethi %hi(2f), %g7 213 ba,pt %xcc, etraptl1 214 or %g7, %lo(2f), %g7 215 2161: ba,pt %xcc, etrap 2172: or %g7, %lo(2b), %g7 218 mov %l4, %o1 219 call sun4v_itlb_error_report 220 add %sp, PTREGS_OFF, %o0 221 222 /* NOTREACHED */ 223 224sun4v_dtlb_error: 225 sethi %hi(sun4v_err_dtlb_vaddr), %g1 226 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] 227 sethi %hi(sun4v_err_dtlb_ctx), %g1 228 ldxa [%g0] ASI_SCRATCHPAD, %g6 229 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 230 stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)] 231 sethi %hi(sun4v_err_dtlb_pte), %g1 232 stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)] 233 sethi %hi(sun4v_err_dtlb_error), %g1 234 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] 235 236 rdpr %tl, %g4 237 cmp %g4, 1 238 ble,pt %icc, 1f 239 sethi %hi(2f), %g7 240 ba,pt %xcc, etraptl1 241 or %g7, %lo(2f), %g7 242 2431: ba,pt %xcc, etrap 2442: or %g7, %lo(2b), %g7 245 mov %l4, %o1 246 call sun4v_dtlb_error_report 247 add %sp, PTREGS_OFF, %o0 248 249 /* NOTREACHED */ 250 251 /* Instruction Access Exception, tl0. */ 252sun4v_iacc: 253 ldxa [%g0] ASI_SCRATCHPAD, %g2 254 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 255 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 256 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 257 sllx %g3, 16, %g3 258 or %g5, %g3, %g5 259 ba,pt %xcc, etrap 260 rd %pc, %g7 261 mov %l4, %o1 262 mov %l5, %o2 263 call sun4v_insn_access_exception 264 add %sp, PTREGS_OFF, %o0 265 ba,a,pt %xcc, rtrap 266 267 /* Instruction Access Exception, tl1. */ 268sun4v_iacc_tl1: 269 ldxa [%g0] ASI_SCRATCHPAD, %g2 270 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 271 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 272 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 273 sllx %g3, 16, %g3 274 or %g5, %g3, %g5 275 ba,pt %xcc, etraptl1 276 rd %pc, %g7 277 mov %l4, %o1 278 mov %l5, %o2 279 call sun4v_insn_access_exception_tl1 280 add %sp, PTREGS_OFF, %o0 281 ba,a,pt %xcc, rtrap 282 283 /* Data Access Exception, tl0. */ 284sun4v_dacc: 285 ldxa [%g0] ASI_SCRATCHPAD, %g2 286 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 287 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 288 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 289 sllx %g3, 16, %g3 290 or %g5, %g3, %g5 291 ba,pt %xcc, etrap 292 rd %pc, %g7 293 mov %l4, %o1 294 mov %l5, %o2 295 call sun4v_data_access_exception 296 add %sp, PTREGS_OFF, %o0 297 ba,a,pt %xcc, rtrap 298 299 /* Data Access Exception, tl1. */ 300sun4v_dacc_tl1: 301 ldxa [%g0] ASI_SCRATCHPAD, %g2 302 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 303 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 304 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 305 sllx %g3, 16, %g3 306 or %g5, %g3, %g5 307 ba,pt %xcc, etraptl1 308 rd %pc, %g7 309 mov %l4, %o1 310 mov %l5, %o2 311 call sun4v_data_access_exception_tl1 312 add %sp, PTREGS_OFF, %o0 313 ba,a,pt %xcc, rtrap 314 315 /* Memory Address Unaligned. */ 316sun4v_mna: 317 /* Window fixup? */ 318 rdpr %tl, %g2 319 cmp %g2, 1 320 ble,pt %icc, 1f 321 nop 322 323 SET_GL(1) 324 ldxa [%g0] ASI_SCRATCHPAD, %g2 325 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 326 mov HV_FAULT_TYPE_UNALIGNED, %g3 327 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4 328 sllx %g3, 16, %g3 329 or %g4, %g3, %g4 330 ba,pt %xcc, winfix_mna 331 rdpr %tpc, %g3 332 /* not reached */ 333 3341: ldxa [%g0] ASI_SCRATCHPAD, %g2 335 mov HV_FAULT_TYPE_UNALIGNED, %g3 336 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 337 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 338 sllx %g3, 16, %g3 339 or %g5, %g3, %g5 340 341 ba,pt %xcc, etrap 342 rd %pc, %g7 343 mov %l4, %o1 344 mov %l5, %o2 345 call sun4v_do_mna 346 add %sp, PTREGS_OFF, %o0 347 ba,a,pt %xcc, rtrap 348 349 /* Privileged Action. */ 350sun4v_privact: 351 ba,pt %xcc, etrap 352 rd %pc, %g7 353 call do_privact 354 add %sp, PTREGS_OFF, %o0 355 ba,a,pt %xcc, rtrap 356 357 /* Unaligned ldd float, tl0. */ 358sun4v_lddfmna: 359 ldxa [%g0] ASI_SCRATCHPAD, %g2 360 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 361 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 362 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 363 sllx %g3, 16, %g3 364 or %g5, %g3, %g5 365 ba,pt %xcc, etrap 366 rd %pc, %g7 367 mov %l4, %o1 368 mov %l5, %o2 369 call handle_lddfmna 370 add %sp, PTREGS_OFF, %o0 371 ba,a,pt %xcc, rtrap 372 373 /* Unaligned std float, tl0. */ 374sun4v_stdfmna: 375 ldxa [%g0] ASI_SCRATCHPAD, %g2 376 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 377 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 378 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 379 sllx %g3, 16, %g3 380 or %g5, %g3, %g5 381 ba,pt %xcc, etrap 382 rd %pc, %g7 383 mov %l4, %o1 384 mov %l5, %o2 385 call handle_stdfmna 386 add %sp, PTREGS_OFF, %o0 387 ba,a,pt %xcc, rtrap 388 389#define BRANCH_ALWAYS 0x10680000 390#define NOP 0x01000000 391#define SUN4V_DO_PATCH(OLD, NEW) \ 392 sethi %hi(NEW), %g1; \ 393 or %g1, %lo(NEW), %g1; \ 394 sethi %hi(OLD), %g2; \ 395 or %g2, %lo(OLD), %g2; \ 396 sub %g1, %g2, %g1; \ 397 sethi %hi(BRANCH_ALWAYS), %g3; \ 398 sll %g1, 11, %g1; \ 399 srl %g1, 11 + 2, %g1; \ 400 or %g3, %lo(BRANCH_ALWAYS), %g3; \ 401 or %g3, %g1, %g3; \ 402 stw %g3, [%g2]; \ 403 sethi %hi(NOP), %g3; \ 404 or %g3, %lo(NOP), %g3; \ 405 stw %g3, [%g2 + 0x4]; \ 406 flush %g2; 407 408 .globl sun4v_patch_tlb_handlers 409 .type sun4v_patch_tlb_handlers,#function 410sun4v_patch_tlb_handlers: 411 SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) 412 SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) 413 SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) 414 SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) 415 SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) 416 SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) 417 SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) 418 SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) 419 SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) 420 SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) 421 SUN4V_DO_PATCH(tl0_mna, sun4v_mna) 422 SUN4V_DO_PATCH(tl1_mna, sun4v_mna) 423 SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) 424 SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) 425 SUN4V_DO_PATCH(tl0_privact, sun4v_privact) 426 retl 427 nop 428 .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers 429