1/* sun4v_tlb_miss.S: Sun4v TLB miss handlers. 2 * 3 * Copyright (C) 2006 <davem@davemloft.net> 4 */ 5 6 .text 7 .align 32 8 9 /* Load ITLB fault information into VADDR and CTX, using BASE. */ 10#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \ 11 ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \ 12 ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX; 13 14 /* Load DTLB fault information into VADDR and CTX, using BASE. */ 15#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \ 16 ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \ 17 ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX; 18 19 /* DEST = (VADDR >> 22) 20 * 21 * Branch to ZERO_CTX_LABEL if context is zero. 22 */ 23#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \ 24 srlx VADDR, 22, DEST; \ 25 brz,pn CTX, ZERO_CTX_LABEL; \ 26 nop; 27 28 /* Create TSB pointer. This is something like: 29 * 30 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; 31 * tsb_base = tsb_reg & ~0x7UL; 32 * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); 33 * tsb_ptr = tsb_base + (tsb_index * 16); 34 */ 35#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ 36 and TSB_PTR, 0x7, TMP1; \ 37 mov 512, TMP2; \ 38 andn TSB_PTR, 0x7, TSB_PTR; \ 39 sllx TMP2, TMP1, TMP2; \ 40 srlx VADDR, HASH_SHIFT, TMP1; \ 41 sub TMP2, 1, TMP2; \ 42 and TMP1, TMP2, TMP1; \ 43 sllx TMP1, 4, TMP1; \ 44 add TSB_PTR, TMP1, TSB_PTR; 45 46sun4v_itlb_miss: 47 /* Load MMU Miss base into %g2. */ 48 ldxa [%g0] ASI_SCRATCHPAD, %g2 49 50 /* Load UTSB reg into %g1. */ 51 mov SCRATCHPAD_UTSBREG1, %g1 52 ldxa [%g1] ASI_SCRATCHPAD, %g1 53 54 LOAD_ITLB_INFO(%g2, %g4, %g5) 55 COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) 56 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) 57 58 /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ 59 ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 60 cmp %g2, %g6 61 bne,a,pn %xcc, tsb_miss_page_table_walk 62 mov FAULT_CODE_ITLB, %g3 63 andcc %g3, _PAGE_EXEC_4V, %g0 64 be,a,pn %xcc, tsb_do_fault 65 mov FAULT_CODE_ITLB, %g3 66 67 /* We have a valid entry, make hypervisor call to load 68 * I-TLB and return from trap. 69 * 70 * %g3: PTE 71 * %g4: vaddr 72 */ 73sun4v_itlb_load: 74 ldxa [%g0] ASI_SCRATCHPAD, %g6 75 mov %o0, %g1 ! save %o0 76 mov %o1, %g2 ! save %o1 77 mov %o2, %g5 ! save %o2 78 mov %o3, %g7 ! save %o3 79 mov %g4, %o0 ! vaddr 80 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx 81 mov %g3, %o2 ! PTE 82 mov HV_MMU_IMMU, %o3 ! flags 83 ta HV_MMU_MAP_ADDR_TRAP 84 brnz,pn %o0, sun4v_itlb_error 85 mov %g2, %o1 ! restore %o1 86 mov %g1, %o0 ! restore %o0 87 mov %g5, %o2 ! restore %o2 88 mov %g7, %o3 ! restore %o3 89 90 retry 91 92sun4v_dtlb_miss: 93 /* Load MMU Miss base into %g2. */ 94 ldxa [%g0] ASI_SCRATCHPAD, %g2 95 96 /* Load UTSB reg into %g1. */ 97 mov SCRATCHPAD_UTSBREG1, %g1 98 ldxa [%g1] ASI_SCRATCHPAD, %g1 99 100 LOAD_DTLB_INFO(%g2, %g4, %g5) 101 COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) 102 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) 103 104 /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ 105 ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 106 cmp %g2, %g6 107 bne,a,pn %xcc, tsb_miss_page_table_walk 108 mov FAULT_CODE_DTLB, %g3 109 110 /* We have a valid entry, make hypervisor call to load 111 * D-TLB and return from trap. 112 * 113 * %g3: PTE 114 * %g4: vaddr 115 */ 116sun4v_dtlb_load: 117 ldxa [%g0] ASI_SCRATCHPAD, %g6 118 mov %o0, %g1 ! save %o0 119 mov %o1, %g2 ! save %o1 120 mov %o2, %g5 ! save %o2 121 mov %o3, %g7 ! save %o3 122 mov %g4, %o0 ! vaddr 123 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx 124 mov %g3, %o2 ! PTE 125 mov HV_MMU_DMMU, %o3 ! flags 126 ta HV_MMU_MAP_ADDR_TRAP 127 brnz,pn %o0, sun4v_dtlb_error 128 mov %g2, %o1 ! restore %o1 129 mov %g1, %o0 ! restore %o0 130 mov %g5, %o2 ! restore %o2 131 mov %g7, %o3 ! restore %o3 132 133 retry 134 135sun4v_dtlb_prot: 136 SET_GL(1) 137 138 /* Load MMU Miss base into %g5. */ 139 ldxa [%g0] ASI_SCRATCHPAD, %g5 140 141 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 142 rdpr %tl, %g1 143 cmp %g1, 1 144 bgu,pn %xcc, winfix_trampoline 145 mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 146 ba,pt %xcc, sparc64_realfault_common 147 nop 148 149 /* Called from trap table: 150 * %g4: vaddr 151 * %g5: context 152 * %g6: TAG TARGET 153 */ 154sun4v_itsb_miss: 155 mov SCRATCHPAD_UTSBREG1, %g1 156 ldxa [%g1] ASI_SCRATCHPAD, %g1 157 brz,pn %g5, kvmap_itlb_4v 158 mov FAULT_CODE_ITLB, %g3 159 ba,a,pt %xcc, sun4v_tsb_miss_common 160 161 /* Called from trap table: 162 * %g4: vaddr 163 * %g5: context 164 * %g6: TAG TARGET 165 */ 166sun4v_dtsb_miss: 167 mov SCRATCHPAD_UTSBREG1, %g1 168 ldxa [%g1] ASI_SCRATCHPAD, %g1 169 brz,pn %g5, kvmap_dtlb_4v 170 mov FAULT_CODE_DTLB, %g3 171 172 /* fallthrough */ 173 174sun4v_tsb_miss_common: 175 COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) 176 177 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 178 179#ifdef CONFIG_HUGETLB_PAGE 180 mov SCRATCHPAD_UTSBREG2, %g5 181 ldxa [%g5] ASI_SCRATCHPAD, %g5 182 cmp %g5, -1 183 be,pt %xcc, 80f 184 nop 185 COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) 186 187 /* That clobbered %g2, reload it. */ 188 ldxa [%g0] ASI_SCRATCHPAD, %g2 189 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 190 19180: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP] 192#endif 193 194 ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath 195 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 196 197sun4v_itlb_error: 198 sethi %hi(sun4v_err_itlb_vaddr), %g1 199 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] 200 sethi %hi(sun4v_err_itlb_ctx), %g1 201 ldxa [%g0] ASI_SCRATCHPAD, %g6 202 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 203 stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)] 204 sethi %hi(sun4v_err_itlb_pte), %g1 205 stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)] 206 sethi %hi(sun4v_err_itlb_error), %g1 207 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] 208 209 rdpr %tl, %g4 210 cmp %g4, 1 211 ble,pt %icc, 1f 212 sethi %hi(2f), %g7 213 ba,pt %xcc, etraptl1 214 or %g7, %lo(2f), %g7 215 2161: ba,pt %xcc, etrap 2172: or %g7, %lo(2b), %g7 218 call sun4v_itlb_error_report 219 add %sp, PTREGS_OFF, %o0 220 221 /* NOTREACHED */ 222 223sun4v_dtlb_error: 224 sethi %hi(sun4v_err_dtlb_vaddr), %g1 225 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] 226 sethi %hi(sun4v_err_dtlb_ctx), %g1 227 ldxa [%g0] ASI_SCRATCHPAD, %g6 228 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 229 stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)] 230 sethi %hi(sun4v_err_dtlb_pte), %g1 231 stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)] 232 sethi %hi(sun4v_err_dtlb_error), %g1 233 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] 234 235 rdpr %tl, %g4 236 cmp %g4, 1 237 ble,pt %icc, 1f 238 sethi %hi(2f), %g7 239 ba,pt %xcc, etraptl1 240 or %g7, %lo(2f), %g7 241 2421: ba,pt %xcc, etrap 2432: or %g7, %lo(2b), %g7 244 call sun4v_dtlb_error_report 245 add %sp, PTREGS_OFF, %o0 246 247 /* NOTREACHED */ 248 249 /* Instruction Access Exception, tl0. */ 250sun4v_iacc: 251 ldxa [%g0] ASI_SCRATCHPAD, %g2 252 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 253 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 254 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 255 sllx %g3, 16, %g3 256 or %g5, %g3, %g5 257 ba,pt %xcc, etrap 258 rd %pc, %g7 259 mov %l4, %o1 260 mov %l5, %o2 261 call sun4v_insn_access_exception 262 add %sp, PTREGS_OFF, %o0 263 ba,a,pt %xcc, rtrap_clr_l6 264 265 /* Instruction Access Exception, tl1. */ 266sun4v_iacc_tl1: 267 ldxa [%g0] ASI_SCRATCHPAD, %g2 268 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 269 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 270 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 271 sllx %g3, 16, %g3 272 or %g5, %g3, %g5 273 ba,pt %xcc, etraptl1 274 rd %pc, %g7 275 mov %l4, %o1 276 mov %l5, %o2 277 call sun4v_insn_access_exception_tl1 278 add %sp, PTREGS_OFF, %o0 279 ba,a,pt %xcc, rtrap_clr_l6 280 281 /* Data Access Exception, tl0. */ 282sun4v_dacc: 283 ldxa [%g0] ASI_SCRATCHPAD, %g2 284 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 285 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 286 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 287 sllx %g3, 16, %g3 288 or %g5, %g3, %g5 289 ba,pt %xcc, etrap 290 rd %pc, %g7 291 mov %l4, %o1 292 mov %l5, %o2 293 call sun4v_data_access_exception 294 add %sp, PTREGS_OFF, %o0 295 ba,a,pt %xcc, rtrap_clr_l6 296 297 /* Data Access Exception, tl1. */ 298sun4v_dacc_tl1: 299 ldxa [%g0] ASI_SCRATCHPAD, %g2 300 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 301 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 302 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 303 sllx %g3, 16, %g3 304 or %g5, %g3, %g5 305 ba,pt %xcc, etraptl1 306 rd %pc, %g7 307 mov %l4, %o1 308 mov %l5, %o2 309 call sun4v_data_access_exception_tl1 310 add %sp, PTREGS_OFF, %o0 311 ba,a,pt %xcc, rtrap_clr_l6 312 313 /* Memory Address Unaligned. */ 314sun4v_mna: 315 /* Window fixup? */ 316 rdpr %tl, %g2 317 cmp %g2, 1 318 ble,pt %icc, 1f 319 nop 320 321 SET_GL(1) 322 ldxa [%g0] ASI_SCRATCHPAD, %g2 323 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 324 mov HV_FAULT_TYPE_UNALIGNED, %g3 325 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4 326 sllx %g3, 16, %g3 327 or %g4, %g3, %g4 328 ba,pt %xcc, winfix_mna 329 rdpr %tpc, %g3 330 /* not reached */ 331 3321: ldxa [%g0] ASI_SCRATCHPAD, %g2 333 mov HV_FAULT_TYPE_UNALIGNED, %g3 334 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 335 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 336 sllx %g3, 16, %g3 337 or %g5, %g3, %g5 338 339 ba,pt %xcc, etrap 340 rd %pc, %g7 341 mov %l4, %o1 342 mov %l5, %o2 343 call sun4v_do_mna 344 add %sp, PTREGS_OFF, %o0 345 ba,a,pt %xcc, rtrap_clr_l6 346 347 /* Privileged Action. */ 348sun4v_privact: 349 ba,pt %xcc, etrap 350 rd %pc, %g7 351 call do_privact 352 add %sp, PTREGS_OFF, %o0 353 ba,a,pt %xcc, rtrap_clr_l6 354 355 /* Unaligned ldd float, tl0. */ 356sun4v_lddfmna: 357 ldxa [%g0] ASI_SCRATCHPAD, %g2 358 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 359 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 360 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 361 sllx %g3, 16, %g3 362 or %g5, %g3, %g5 363 ba,pt %xcc, etrap 364 rd %pc, %g7 365 mov %l4, %o1 366 mov %l5, %o2 367 call handle_lddfmna 368 add %sp, PTREGS_OFF, %o0 369 ba,a,pt %xcc, rtrap_clr_l6 370 371 /* Unaligned std float, tl0. */ 372sun4v_stdfmna: 373 ldxa [%g0] ASI_SCRATCHPAD, %g2 374 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 375 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 376 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 377 sllx %g3, 16, %g3 378 or %g5, %g3, %g5 379 ba,pt %xcc, etrap 380 rd %pc, %g7 381 mov %l4, %o1 382 mov %l5, %o2 383 call handle_stdfmna 384 add %sp, PTREGS_OFF, %o0 385 ba,a,pt %xcc, rtrap_clr_l6 386 387#define BRANCH_ALWAYS 0x10680000 388#define NOP 0x01000000 389#define SUN4V_DO_PATCH(OLD, NEW) \ 390 sethi %hi(NEW), %g1; \ 391 or %g1, %lo(NEW), %g1; \ 392 sethi %hi(OLD), %g2; \ 393 or %g2, %lo(OLD), %g2; \ 394 sub %g1, %g2, %g1; \ 395 sethi %hi(BRANCH_ALWAYS), %g3; \ 396 sll %g1, 11, %g1; \ 397 srl %g1, 11 + 2, %g1; \ 398 or %g3, %lo(BRANCH_ALWAYS), %g3; \ 399 or %g3, %g1, %g3; \ 400 stw %g3, [%g2]; \ 401 sethi %hi(NOP), %g3; \ 402 or %g3, %lo(NOP), %g3; \ 403 stw %g3, [%g2 + 0x4]; \ 404 flush %g2; 405 406 .globl sun4v_patch_tlb_handlers 407 .type sun4v_patch_tlb_handlers,#function 408sun4v_patch_tlb_handlers: 409 SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) 410 SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) 411 SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) 412 SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) 413 SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) 414 SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) 415 SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) 416 SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) 417 SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) 418 SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) 419 SUN4V_DO_PATCH(tl0_mna, sun4v_mna) 420 SUN4V_DO_PATCH(tl1_mna, sun4v_mna) 421 SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) 422 SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) 423 SUN4V_DO_PATCH(tl0_privact, sun4v_privact) 424 retl 425 nop 426 .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers 427