exception.S revision 109810
1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2001 Jake Burkholder. 32 * All rights reserved. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 46 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 53 * SUCH DAMAGE. 54 * 55 * $FreeBSD: head/sys/sparc64/sparc64/exception.S 109810 2003-01-24 23:47:46Z jake $ 56 */ 57 58#include "opt_compat.h" 59#include "opt_ddb.h" 60 61#include <machine/asi.h> 62#include <machine/asmacros.h> 63#include <machine/ktr.h> 64#include <machine/pstate.h> 65#include <machine/trap.h> 66#include <machine/tstate.h> 67#include <machine/wstate.h> 68 69#include "assym.s" 70 71#define TSB_KERNEL_MASK 0x0 72#define TSB_KERNEL 0x0 73 74 .register %g2,#ignore 75 .register %g3,#ignore 76 .register %g6,#ignore 77 .register %g7,#ignore 78 79/* 80 * Atomically set the reference bit in a tte. 81 */ 82#define TTE_SET_BIT(r1, r2, r3, bit) \ 83 add r1, TTE_DATA, r1 ; \ 84 ldx [r1], r2 ; \ 859: or r2, bit, r3 ; \ 86 casxa [r1] ASI_N, r2, r3 ; \ 87 cmp r2, r3 ; \ 88 bne,pn %xcc, 9b ; \ 89 mov r3, r2 90 91#define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF) 92#define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W) 93 94/* 95 * Macros for spilling and filling live windows. 96 * 97 * NOTE: These macros use exactly 16 instructions, and it is assumed that the 98 * handler will not use more than 24 instructions total, to leave room for 99 * resume vectors which occupy the last 8 instructions. 100 */ 101 102#define SPILL(storer, base, size, asi) \ 103 storer %l0, [base + (0 * size)] asi ; \ 104 storer %l1, [base + (1 * size)] asi ; \ 105 storer %l2, [base + (2 * size)] asi ; \ 106 storer %l3, [base + (3 * size)] asi ; \ 107 storer %l4, [base + (4 * size)] asi ; \ 108 storer %l5, [base + (5 * size)] asi ; \ 109 storer %l6, [base + (6 * size)] asi ; \ 110 storer %l7, [base + (7 * size)] asi ; \ 111 storer %i0, [base + (8 * size)] asi ; \ 112 storer %i1, [base + (9 * size)] asi ; \ 113 storer %i2, [base + (10 * size)] asi ; \ 114 storer %i3, [base + (11 * size)] asi ; \ 115 storer %i4, [base + (12 * size)] asi ; \ 116 storer %i5, [base + (13 * size)] asi ; \ 117 storer %i6, [base + (14 * size)] asi ; \ 118 storer %i7, [base + (15 * size)] asi 119 120#define FILL(loader, base, size, asi) \ 121 loader [base + (0 * size)] asi, %l0 ; \ 122 loader [base + (1 * size)] asi, %l1 ; \ 123 loader [base + (2 * size)] asi, %l2 ; \ 124 loader [base + (3 * size)] asi, %l3 ; \ 125 loader [base + (4 * size)] asi, %l4 ; \ 126 loader [base + (5 * size)] asi, %l5 ; \ 127 loader [base + (6 * size)] asi, %l6 ; \ 128 loader [base + (7 * size)] asi, %l7 ; \ 129 loader [base + (8 * size)] asi, %i0 ; \ 130 loader [base + (9 * size)] asi, %i1 ; \ 131 loader [base + (10 * size)] asi, %i2 ; \ 132 loader [base + (11 * size)] asi, %i3 ; \ 133 loader [base + (12 * size)] asi, %i4 ; \ 134 loader [base + (13 * size)] asi, %i5 ; \ 135 loader [base + (14 * size)] asi, %i6 ; \ 136 loader [base + (15 * size)] asi, %i7 137 138#define ERRATUM50(reg) mov reg, reg 139 140#define KSTACK_SLOP 1024 141 142/* 143 * Sanity check the kernel stack and bail out if its wrong. 144 * XXX: doesn't handle being on the panic stack. 145 */ 146#define KSTACK_CHECK \ 147 dec 16, ASP_REG ; \ 148 stx %g1, [ASP_REG + 0] ; \ 149 stx %g2, [ASP_REG + 8] ; \ 150 add %sp, SPOFF, %g1 ; \ 151 andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ 152 bnz,a %xcc, tl1_kstack_fault ; \ 153 inc 16, ASP_REG ; \ 154 ldx [PCPU(CURTHREAD)], %g2 ; \ 155 ldx [%g2 + TD_KSTACK], %g2 ; \ 156 add %g2, KSTACK_SLOP, %g2 ; \ 157 subcc %g1, %g2, %g1 ; \ 158 ble,a %xcc, tl1_kstack_fault ; \ 159 inc 16, ASP_REG ; \ 160 set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ 161 cmp %g1, %g2 ; \ 162 bgt,a %xcc, tl1_kstack_fault ; \ 163 inc 16, ASP_REG ; \ 164 ldx [ASP_REG + 8], %g2 ; \ 165 ldx [ASP_REG + 0], %g1 ; \ 166 inc 16, ASP_REG 167 168ENTRY(tl1_kstack_fault) 169 rdpr %tl, %g1 1701: cmp %g1, 2 171 be,a 2f 172 nop 173 174#if KTR_COMPILE & KTR_TRAP 175 CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" 176 , %g2, %g3, %g4, 7, 8, 9) 177 rdpr %tl, %g3 178 stx %g3, [%g2 + KTR_PARM1] 179 rdpr %tpc, %g3 180 stx %g3, [%g2 + KTR_PARM1] 181 rdpr %tnpc, %g3 182 stx %g3, [%g2 + KTR_PARM1] 1839: 184#endif 185 186 sub %g1, 1, %g1 187 wrpr %g1, 0, %tl 188 ba,a %xcc, 1b 189 nop 190 1912: 192#if KTR_COMPILE & KTR_TRAP 193 CATR(KTR_TRAP, 194 "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" 195 , %g1, %g2, %g3, 7, 8, 9) 196 add %sp, SPOFF, %g2 197 stx %g2, [%g1 + KTR_PARM1] 198 ldx [PCPU(CURTHREAD)], %g2 199 ldx [%g2 + TD_KSTACK], %g2 200 stx %g2, [%g1 + KTR_PARM2] 201 rdpr %canrestore, %g2 202 stx %g2, [%g1 + KTR_PARM3] 203 rdpr %cansave, %g2 204 stx %g2, [%g1 + KTR_PARM4] 205 rdpr %otherwin, %g2 206 stx %g2, [%g1 + KTR_PARM5] 207 rdpr %wstate, %g2 208 stx %g2, [%g1 + KTR_PARM6] 2099: 210#endif 211 212 wrpr %g0, 0, %canrestore 213 wrpr %g0, 6, %cansave 214 wrpr %g0, 0, %otherwin 215 wrpr %g0, WSTATE_KERNEL, %wstate 216 217 sub ASP_REG, SPOFF + CCFSZ, %sp 218 clr %fp 219 220 set trap, %o2 221 b %xcc, tl1_trap 222 mov T_KSTACK_FAULT | T_KERNEL, %o0 223END(tl1_kstack_fault) 224 225/* 226 * Magic to resume from a spill or fill trap. If we get an alignment or an 227 * mmu fault during a spill or a fill, this macro will detect the fault and 228 * resume at a set instruction offset in the trap handler. 229 * 230 * To check if the previous trap was a spill/fill we convert the trapped pc 231 * to a trap type and verify that it is in the range of spill/fill vectors. 232 * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the 233 * tl bit allows us to detect both ranges with one test. 234 * 235 * This is: 236 * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 237 * 238 * To calculate the new pc we take advantage of the xor feature of wrpr. 239 * Forcing all the low bits of the trapped pc on we can produce any offset 240 * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. 241 * 242 * 0x7f ^ 0x1f == 0x60 243 * 0x1f == (0x80 - 0x60) - 1 244 * 245 * Which are the offset and xor value used to resume from alignment faults. 246 */ 247 248/* 249 * Determine if we have trapped inside of a spill/fill vector, and if so resume 250 * at a fixed instruction offset in the trap vector. Must be called on 251 * alternate globals. 252 */ 253#define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ 254 dec 16, ASP_REG ; \ 255 stx %g1, [ASP_REG + 0] ; \ 256 stx %g2, [ASP_REG + 8] ; \ 257 rdpr %tpc, %g1 ; \ 258 ERRATUM50(%g1) ; \ 259 rdpr %tba, %g2 ; \ 260 sub %g1, %g2, %g2 ; \ 261 srlx %g2, 5, %g2 ; \ 262 andn %g2, 0x200, %g2 ; \ 263 cmp %g2, 0x80 ; \ 264 blu,pt %xcc, 9f ; \ 265 cmp %g2, 0x100 ; \ 266 bgeu,pt %xcc, 9f ; \ 267 or %g1, 0x7f, %g1 ; \ 268 wrpr %g1, xor, %tnpc ; \ 269 stxa_g0_sfsr ; \ 270 ldx [ASP_REG + 8], %g2 ; \ 271 ldx [ASP_REG + 0], %g1 ; \ 272 inc 16, ASP_REG ; \ 273 done ; \ 2749: ldx [ASP_REG + 8], %g2 ; \ 275 ldx [ASP_REG + 0], %g1 ; \ 276 inc 16, ASP_REG 277 278/* 279 * For certain faults we need to clear the sfsr mmu register before returning. 280 */ 281#define RSF_CLR_SFSR \ 282 wr %g0, ASI_DMMU, %asi ; \ 283 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 284 285#define RSF_XOR(off) ((0x80 - off) - 1) 286 287/* 288 * Instruction offsets in spill and fill trap handlers for handling certain 289 * nested traps, and corresponding xor constants for wrpr. 290 */ 291#define RSF_OFF_ALIGN 0x60 292#define RSF_OFF_MMU 0x70 293 294#define RESUME_SPILLFILL_ALIGN \ 295 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) 296#define RESUME_SPILLFILL_MMU \ 297 RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) 298#define RESUME_SPILLFILL_MMU_CLR_SFSR \ 299 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) 300 301/* 302 * Constant to add to %tnpc when taking a fill trap just before returning to 303 * user mode. 304 */ 305#define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill 306 307/* 308 * Retry a spill or fill with a different wstate due to an alignment fault. 309 * We may just be using the wrong stack offset. 310 */ 311#define RSF_ALIGN_RETRY(ws) \ 312 wrpr %g0, (ws), %wstate ; \ 313 retry ; \ 314 .align 16 315 316/* 317 * Generate a T_SPILL or T_FILL trap if the window operation fails. 318 */ 319#define RSF_TRAP(type) \ 320 b %xcc, tl0_sftrap ; \ 321 mov type, %g2 ; \ 322 .align 16 323 324/* 325 * Game over if the window operation fails. 326 */ 327#define RSF_FATAL(type) \ 328 b %xcc, rsf_fatal ; \ 329 mov type, %g2 ; \ 330 .align 16 331 332/* 333 * Magic to resume from a failed fill a few instructions after the corrsponding 334 * restore. This is used on return from the kernel to usermode. 335 */ 336#define RSF_FILL_MAGIC \ 337 rdpr %tnpc, %g1 ; \ 338 add %g1, RSF_FILL_INC, %g1 ; \ 339 wrpr %g1, 0, %tnpc ; \ 340 done ; \ 341 .align 16 342 343/* 344 * Spill to the pcb if a spill to the user stack in kernel mode fails. 345 */ 346#define RSF_SPILL_TOPCB \ 347 b,a %xcc, tl1_spill_topcb ; \ 348 nop ; \ 349 .align 16 350 351ENTRY(rsf_fatal) 352#if KTR_COMPILE & KTR_TRAP 353 CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" 354 , %g1, %g3, %g4, 7, 8, 9) 355 rdpr %tt, %g3 356 stx %g3, [%g1 + KTR_PARM1] 357 stx %g2, [%g1 + KTR_PARM2] 3589: 359#endif 360 361 KSTACK_CHECK 362 363 sir 364END(rsf_fatal) 365 366 .comm intrnames, IV_MAX * 8 367 .comm eintrnames, 0 368 369 .comm intrcnt, IV_MAX * 8 370 .comm eintrcnt, 0 371 372/* 373 * Trap table and associated macros 374 * 375 * Due to its size a trap table is an inherently hard thing to represent in 376 * code in a clean way. There are approximately 1024 vectors, of 8 or 32 377 * instructions each, many of which are identical. The way that this is 378 * layed out is the instructions (8 or 32) for the actual trap vector appear 379 * as an AS macro. In general this code branches to tl0_trap or tl1_trap, 380 * but if not supporting code can be placed just after the definition of the 381 * macro. The macros are then instantiated in a different section (.trap), 382 * which is setup to be placed by the linker at the beginning of .text, and the 383 * code around the macros is moved to the end of trap table. In this way the 384 * code that must be sequential in memory can be split up, and located near 385 * its supporting code so that it is easier to follow. 386 */ 387 388 /* 389 * Clean window traps occur when %cleanwin is zero to ensure that data 390 * is not leaked between address spaces in registers. 391 */ 392 .macro clean_window 393 clr %o0 394 clr %o1 395 clr %o2 396 clr %o3 397 clr %o4 398 clr %o5 399 clr %o6 400 clr %o7 401 clr %l0 402 clr %l1 403 clr %l2 404 clr %l3 405 clr %l4 406 clr %l5 407 clr %l6 408 rdpr %cleanwin, %l7 409 inc %l7 410 wrpr %l7, 0, %cleanwin 411 clr %l7 412 retry 413 .align 128 414 .endm 415 416 /* 417 * Stack fixups for entry from user mode. We are still running on the 418 * user stack, and with its live registers, so we must save soon. We 419 * are on alternate globals so we do have some registers. Set the 420 * transitional window state, and do the save. If this traps we 421 * we attempt to spill a window to the user stack. If this fails, 422 * we spill the window to the pcb and continue. Spilling to the pcb 423 * must not fail. 424 * 425 * NOTE: Must be called with alternate globals and clobbers %g1. 426 */ 427 428 .macro tl0_split 429 rdpr %wstate, %g1 430 wrpr %g1, WSTATE_TRANSITION, %wstate 431 save 432 .endm 433 434 .macro tl0_setup type 435 tl0_split 436 clr %o1 437 set trap, %o2 438 ba %xcc, tl0_utrap 439 mov \type, %o0 440 .endm 441 442 /* 443 * Generic trap type. Call trap() with the specified type. 444 */ 445 .macro tl0_gen type 446 tl0_setup \type 447 .align 32 448 .endm 449 450 /* 451 * This is used to suck up the massive swaths of reserved trap types. 452 * Generates count "reserved" trap vectors. 453 */ 454 .macro tl0_reserved count 455 .rept \count 456 tl0_gen T_RESERVED 457 .endr 458 .endm 459 460 .macro tl1_split 461 rdpr %wstate, %g1 462 wrpr %g1, WSTATE_NESTED, %wstate 463 save %sp, -(CCFSZ + TF_SIZEOF), %sp 464 .endm 465 466 .macro tl1_setup type 467 tl1_split 468 clr %o1 469 set trap, %o2 470 b %xcc, tl1_trap 471 mov \type | T_KERNEL, %o0 472 .endm 473 474 .macro tl1_gen type 475 tl1_setup \type 476 .align 32 477 .endm 478 479 .macro tl1_reserved count 480 .rept \count 481 tl1_gen T_RESERVED 482 .endr 483 .endm 484 485 .macro tl0_fp_restore 486 wr %g0, FPRS_FEF, %fprs 487 wr %g0, ASI_BLK_S, %asi 488 ldda [PCB_REG + PCB_FPSTATE + FP_FB0] %asi, %f0 489 ldda [PCB_REG + PCB_FPSTATE + FP_FB1] %asi, %f16 490 ldda [PCB_REG + PCB_FPSTATE + FP_FB2] %asi, %f32 491 ldda [PCB_REG + PCB_FPSTATE + FP_FB3] %asi, %f48 492 membar #Sync 493 done 494 .align 32 495 .endm 496 497 .macro tl0_insn_excptn 498 wrpr %g0, PSTATE_ALT, %pstate 499 wr %g0, ASI_IMMU, %asi 500 rdpr %tpc, %g3 501 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 502 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 503 membar #Sync 504 b %xcc, tl0_sfsr_trap 505 mov T_INSTRUCTION_EXCEPTION, %g2 506 .align 32 507 .endm 508 509 .macro tl0_data_excptn 510 wrpr %g0, PSTATE_ALT, %pstate 511 wr %g0, ASI_DMMU, %asi 512 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 513 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 514 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 515 membar #Sync 516 b %xcc, tl0_sfsr_trap 517 mov T_DATA_EXCEPTION, %g2 518 .align 32 519 .endm 520 521 .macro tl0_align 522 wr %g0, ASI_DMMU, %asi 523 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 524 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 525 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 526 membar #Sync 527 b %xcc, tl0_sfsr_trap 528 mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 529 .align 32 530 .endm 531 532ENTRY(tl0_sfsr_trap) 533 tl0_split 534 clr %o1 535 set trap, %o2 536 mov %g3, %o4 537 mov %g4, %o5 538 ba %xcc, tl0_utrap 539 mov %g2, %o0 540END(tl0_sfsr_trap) 541 542 .macro tl0_intr level, mask 543 tl0_split 544 set \mask, %o1 545 b %xcc, tl0_intr 546 mov \level, %o0 547 .align 32 548 .endm 549 550#define INTR(level, traplvl) \ 551 tl ## traplvl ## _intr level, 1 << level 552 553#define TICK(traplvl) \ 554 tl ## traplvl ## _intr PIL_TICK, 1 555 556#define INTR_LEVEL(tl) \ 557 INTR(1, tl) ; \ 558 INTR(2, tl) ; \ 559 INTR(3, tl) ; \ 560 INTR(4, tl) ; \ 561 INTR(5, tl) ; \ 562 INTR(6, tl) ; \ 563 INTR(7, tl) ; \ 564 INTR(8, tl) ; \ 565 INTR(9, tl) ; \ 566 INTR(10, tl) ; \ 567 INTR(11, tl) ; \ 568 INTR(12, tl) ; \ 569 INTR(13, tl) ; \ 570 TICK(tl) ; \ 571 INTR(15, tl) ; 572 573 .macro tl0_intr_level 574 INTR_LEVEL(0) 575 .endm 576 577 .macro intr_vector 578 ldxa [%g0] ASI_INTR_RECEIVE, %g1 579 andcc %g1, IRSR_BUSY, %g0 580 bnz,a,pt %xcc, intr_vector 581 nop 582 sir 583 .align 32 584 .endm 585 586 .macro immu_miss_user 587 /* 588 * Initialize the page size walker. 589 */ 590 mov TS_MIN, %g2 591 592 /* 593 * Loop over all supported page sizes. 594 */ 595 596 /* 597 * Compute the page shift for the page size we are currently looking 598 * for. 599 */ 6001: add %g2, %g2, %g3 601 add %g3, %g2, %g3 602 add %g3, PAGE_SHIFT, %g3 603 604 /* 605 * Extract the virtual page number from the contents of the tag 606 * access register. 607 */ 608 srlx %g1, %g3, %g3 609 610 /* 611 * Compute the tte bucket address. 612 */ 613 ldxa [%g0 + AA_IMMU_TSB] %asi, %g5 614 and %g3, TSB_BUCKET_MASK, %g4 615 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 616 add %g4, %g5, %g4 617 618 /* 619 * Compute the tte tag target. 620 */ 621 sllx %g3, TV_SIZE_BITS, %g3 622 or %g3, %g2, %g3 623 624 /* 625 * Loop over the ttes in this bucket 626 */ 627 628 /* 629 * Load the tte. Note that this instruction may fault, clobbering 630 * the contents of the tag access register, %g5, %g6, and %g7. We 631 * do not use %g5, and %g6 and %g7 are not used until this instruction 632 * completes successfully. 633 */ 6342: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 635 636 /* 637 * Check that its valid and executable and that the tte tags match. 638 */ 639 brgez,pn %g7, 3f 640 andcc %g7, TD_EXEC, %g0 641 bz,pn %xcc, 3f 642 cmp %g3, %g6 643 bne,pn %xcc, 3f 644 EMPTY 645 646 /* 647 * We matched a tte, load the tlb. 648 */ 649 650 /* 651 * Set the reference bit, if it's currently clear. 652 */ 653 andcc %g7, TD_REF, %g0 654 bz,a,pn %xcc, tl0_immu_miss_set_ref 655 nop 656 657 /* 658 * Load the tte tag and data into the tlb and retry the instruction. 659 */ 660 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 661 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 662 retry 663 664 /* 665 * Advance to the next tte in this bucket, and check the low bits 666 * of the bucket pointer to see if we've finished the bucket. 667 */ 6683: add %g4, 1 << TTE_SHIFT, %g4 669 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 670 bnz,pt %xcc, 2b 671 EMPTY 672 673 /* 674 * See if we just checked the largest page size, and advance to the 675 * next one if not. 676 */ 677 cmp %g2, TS_MAX 678 bne,pt %xcc, 1b 679 add %g2, 1, %g2 680 .endm 681 682 .macro tl0_immu_miss 683 /* 684 * Load the virtual page number and context from the tag access 685 * register. We ignore the context. 686 */ 687 wr %g0, ASI_IMMU, %asi 688 ldxa [%g0 + AA_IMMU_TAR] %asi, %g1 689 690 /* 691 * Try a fast inline lookup of the user tsb. 692 */ 693 immu_miss_user 694 695 /* 696 * Not in user tsb, call c code. 697 */ 698 ba,a %xcc, tl0_immu_miss_trap 699 .align 128 700 .endm 701 702ENTRY(tl0_immu_miss_set_ref) 703 /* 704 * Set the reference bit. 705 */ 706 TTE_SET_REF(%g4, %g2, %g3) 707 708 /* 709 * May have become invalid during casxa, in which case start over. 710 */ 711 brgez,pn %g2, 1f 712 nop 713 714 /* 715 * Load the tte tag and data into the tlb and retry the instruction. 716 */ 717 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 718 stxa %g2, [%g0] ASI_ITLB_DATA_IN_REG 7191: retry 720END(tl0_immu_miss_set_ref) 721 722ENTRY(tl0_immu_miss_trap) 723 /* 724 * Put back the contents of the tag access register, in case we 725 * faulted. 726 */ 727 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 728 membar #Sync 729 730 /* 731 * Switch to alternate globals. 732 */ 733 wrpr %g0, PSTATE_ALT, %pstate 734 735 /* 736 * Reload the tag access register. 737 */ 738 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 739 740 /* 741 * Save the tag access register, and call common trap code. 742 */ 743 tl0_split 744 clr %o1 745 set trap, %o2 746 mov %g2, %o3 747 b %xcc, tl0_trap 748 mov T_INSTRUCTION_MISS, %o0 749END(tl0_immu_miss_trap) 750 751 .macro dmmu_miss_user 752 /* 753 * Initialize the page size walker. 754 */ 755 mov TS_MIN, %g2 756 757 /* 758 * Loop over all supported page sizes. 759 */ 760 761 /* 762 * Compute the page shift for the page size we are currently looking 763 * for. 764 */ 7651: add %g2, %g2, %g3 766 add %g3, %g2, %g3 767 add %g3, PAGE_SHIFT, %g3 768 769 /* 770 * Extract the virtual page number from the contents of the tag 771 * access register. 772 */ 773 srlx %g1, %g3, %g3 774 775 /* 776 * Compute the tte bucket address. 777 */ 778 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 779 and %g3, TSB_BUCKET_MASK, %g4 780 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 781 add %g4, %g5, %g4 782 783 /* 784 * Compute the tte tag target. 785 */ 786 sllx %g3, TV_SIZE_BITS, %g3 787 or %g3, %g2, %g3 788 789 /* 790 * Loop over the ttes in this bucket 791 */ 792 793 /* 794 * Load the tte. Note that this instruction may fault, clobbering 795 * the contents of the tag access register, %g5, %g6, and %g7. We 796 * do not use %g5, and %g6 and %g7 are not used until this instruction 797 * completes successfully. 798 */ 7992: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 800 801 /* 802 * Check that its valid and that the virtual page numbers match. 803 */ 804 brgez,pn %g7, 3f 805 cmp %g3, %g6 806 bne,pn %xcc, 3f 807 EMPTY 808 809 /* 810 * We matched a tte, load the tlb. 811 */ 812 813 /* 814 * Set the reference bit, if it's currently clear. 815 */ 816 andcc %g7, TD_REF, %g0 817 bz,a,pn %xcc, dmmu_miss_user_set_ref 818 nop 819 820 /* 821 * Load the tte tag and data into the tlb and retry the instruction. 822 */ 823 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 824 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 825 retry 826 827 /* 828 * Advance to the next tte in this bucket, and check the low bits 829 * of the bucket pointer to see if we've finished the bucket. 830 */ 8313: add %g4, 1 << TTE_SHIFT, %g4 832 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 833 bnz,pt %xcc, 2b 834 EMPTY 835 836 /* 837 * See if we just checked the largest page size, and advance to the 838 * next one if not. 839 */ 840 cmp %g2, TS_MAX 841 bne,pt %xcc, 1b 842 add %g2, 1, %g2 843 .endm 844 845ENTRY(dmmu_miss_user_set_ref) 846 /* 847 * Set the reference bit. 848 */ 849 TTE_SET_REF(%g4, %g2, %g3) 850 851 /* 852 * May have become invalid during casxa, in which case start over. 853 */ 854 brgez,pn %g2, 1f 855 nop 856 857 /* 858 * Load the tte tag and data into the tlb and retry the instruction. 859 */ 860 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 861 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 8621: retry 863END(dmmu_miss_user_set_ref) 864 865 .macro tl0_dmmu_miss 866 /* 867 * Load the virtual page number and context from the tag access 868 * register. We ignore the context. 869 */ 870 wr %g0, ASI_DMMU, %asi 871 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 872 873 /* 874 * Try a fast inline lookup of the primary tsb. 875 */ 876 dmmu_miss_user 877 878 /* 879 * Not in user tsb, call c code. 880 */ 881 ba,a %xcc, tl0_dmmu_miss_trap 882 .align 128 883 .endm 884 885ENTRY(tl0_dmmu_miss_trap) 886 /* 887 * Put back the contents of the tag access register, in case we 888 * faulted. 889 */ 890 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 891 membar #Sync 892 893 /* 894 * Switch to alternate globals. 895 */ 896 wrpr %g0, PSTATE_ALT, %pstate 897 898 /* 899 * Reload the tag access register. 900 */ 901 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 902 903 /* 904 * Save the tag access register and call common trap code. 905 */ 906 tl0_split 907 clr %o1 908 set trap, %o2 909 mov %g2, %o3 910 b %xcc, tl0_trap 911 mov T_DATA_MISS, %o0 912END(tl0_dmmu_miss_trap) 913 914 .macro dmmu_prot_user 915 /* 916 * Initialize the page size walker. 917 */ 918 mov TS_MIN, %g2 919 920 /* 921 * Loop over all supported page sizes. 922 */ 923 924 /* 925 * Compute the page shift for the page size we are currently looking 926 * for. 927 */ 9281: add %g2, %g2, %g3 929 add %g3, %g2, %g3 930 add %g3, PAGE_SHIFT, %g3 931 932 /* 933 * Extract the virtual page number from the contents of the tag 934 * access register. 935 */ 936 srlx %g1, %g3, %g3 937 938 /* 939 * Compute the tte bucket address. 940 */ 941 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 942 and %g3, TSB_BUCKET_MASK, %g4 943 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 944 add %g4, %g5, %g4 945 946 /* 947 * Compute the tte tag target. 948 */ 949 sllx %g3, TV_SIZE_BITS, %g3 950 or %g3, %g2, %g3 951 952 /* 953 * Loop over the ttes in this bucket 954 */ 955 956 /* 957 * Load the tte. Note that this instruction may fault, clobbering 958 * the contents of the tag access register, %g5, %g6, and %g7. We 959 * do not use %g5, and %g6 and %g7 are not used until this instruction 960 * completes successfully. 961 */ 9622: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 963 964 /* 965 * Check that its valid and writable and that the virtual page 966 * numbers match. 967 */ 968 brgez,pn %g7, 4f 969 andcc %g7, TD_SW, %g0 970 bz,pn %xcc, 4f 971 cmp %g3, %g6 972 bne,pn %xcc, 4f 973 nop 974 975 /* 976 * Set the hardware write bit. 977 */ 978 TTE_SET_W(%g4, %g2, %g3) 979 980 /* 981 * Delete the old TLB entry and clear the sfsr. 982 */ 983 srlx %g1, PAGE_SHIFT, %g3 984 sllx %g3, PAGE_SHIFT, %g3 985 stxa %g0, [%g3] ASI_DMMU_DEMAP 986 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 987 membar #Sync 988 989 /* 990 * May have become invalid during casxa, in which case start over. 991 */ 992 brgez,pn %g2, 3f 993 or %g2, TD_W, %g2 994 995 /* 996 * Load the tte data into the tlb and retry the instruction. 997 */ 998 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 999 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 10003: retry 1001 1002 /* 1003 * Check the low bits to see if we've finished the bucket. 1004 */ 10054: add %g4, 1 << TTE_SHIFT, %g4 1006 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 1007 bnz,pt %xcc, 2b 1008 EMPTY 1009 1010 /* 1011 * See if we just checked the largest page size, and advance to the 1012 * next one if not. 1013 */ 1014 cmp %g2, TS_MAX 1015 bne,pt %xcc, 1b 1016 add %g2, 1, %g2 1017 .endm 1018 1019 .macro tl0_dmmu_prot 1020 ba,a %xcc, tl0_dmmu_prot_1 1021 nop 1022 .align 128 1023 .endm 1024 1025ENTRY(tl0_dmmu_prot_1) 1026 /* 1027 * Load the virtual page number and context from the tag access 1028 * register. We ignore the context. 1029 */ 1030 wr %g0, ASI_DMMU, %asi 1031 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 1032 1033 /* 1034 * Try a fast inline lookup of the tsb. 1035 */ 1036 dmmu_prot_user 1037 1038 /* 1039 * Not in user tsb, call c code. 1040 */ 1041 b,a %xcc, tl0_dmmu_prot_trap 1042 nop 1043END(tl0_dmmu_prot_1) 1044 1045ENTRY(tl0_dmmu_prot_trap) 1046 /* 1047 * Put back the contents of the tag access register, in case we 1048 * faulted. 1049 */ 1050 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1051 membar #Sync 1052 1053 /* 1054 * Switch to alternate globals. 1055 */ 1056 wrpr %g0, PSTATE_ALT, %pstate 1057 1058 /* 1059 * Load the tar, sfar and sfsr. 1060 */ 1061 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1062 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1063 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1064 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1065 membar #Sync 1066 1067 /* 1068 * Save the mmu registers and call common trap code. 1069 */ 1070 tl0_split 1071 clr %o1 1072 set trap, %o2 1073 mov %g2, %o3 1074 mov %g3, %o4 1075 mov %g4, %o5 1076 ba %xcc, tl0_utrap 1077 mov T_DATA_PROTECTION, %o0 1078END(tl0_dmmu_prot_trap) 1079 1080 .macro tl0_spill_0_n 1081 wr %g0, ASI_AIUP, %asi 1082 SPILL(stxa, %sp + SPOFF, 8, %asi) 1083 saved 1084 retry 1085 .align 32 1086 RSF_TRAP(T_SPILL) 1087 RSF_TRAP(T_SPILL) 1088 .endm 1089 1090 .macro tl0_spill_1_n 1091 wr %g0, ASI_AIUP, %asi 1092 SPILL(stwa, %sp, 4, %asi) 1093 saved 1094 retry 1095 .align 32 1096 RSF_TRAP(T_SPILL) 1097 RSF_TRAP(T_SPILL) 1098 .endm 1099 1100 .macro tl0_fill_0_n 1101 wr %g0, ASI_AIUP, %asi 1102 FILL(ldxa, %sp + SPOFF, 8, %asi) 1103 restored 1104 retry 1105 .align 32 1106 RSF_TRAP(T_FILL) 1107 RSF_TRAP(T_FILL) 1108 .endm 1109 1110 .macro tl0_fill_1_n 1111 wr %g0, ASI_AIUP, %asi 1112 FILL(lduwa, %sp, 4, %asi) 1113 restored 1114 retry 1115 .align 32 1116 RSF_TRAP(T_FILL) 1117 RSF_TRAP(T_FILL) 1118 .endm 1119 1120ENTRY(tl0_sftrap) 1121 rdpr %tstate, %g1 1122 and %g1, TSTATE_CWP_MASK, %g1 1123 wrpr %g1, 0, %cwp 1124 tl0_split 1125 clr %o1 1126 set trap, %o2 1127 b %xcc, tl0_trap 1128 mov %g2, %o0 1129END(tl0_sftrap) 1130 1131 .macro tl0_spill_bad count 1132 .rept \count 1133 sir 1134 .align 128 1135 .endr 1136 .endm 1137 1138 .macro tl0_fill_bad count 1139 .rept \count 1140 sir 1141 .align 128 1142 .endr 1143 .endm 1144 1145 .macro tl0_syscall 1146 tl0_split 1147 clr %o1 1148 set syscall, %o2 1149 ba %xcc, tl0_trap 1150 mov T_SYSCALL, %o0 1151 .align 32 1152 .endm 1153 1154 .macro tl1_insn_excptn 1155 wrpr %g0, PSTATE_ALT, %pstate 1156 wr %g0, ASI_IMMU, %asi 1157 rdpr %tpc, %g3 1158 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 1159 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 1160 membar #Sync 1161 b %xcc, tl1_insn_exceptn_trap 1162 mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 1163 .align 32 1164 .endm 1165 1166ENTRY(tl1_insn_exceptn_trap) 1167 tl1_split 1168 clr %o1 1169 set trap, %o2 1170 mov %g3, %o4 1171 mov %g4, %o5 1172 b %xcc, tl1_trap 1173 mov %g2, %o0 1174END(tl1_insn_exceptn_trap) 1175 1176 .macro tl1_data_excptn 1177 wrpr %g0, PSTATE_ALT, %pstate 1178 b,a %xcc, tl1_data_excptn_trap 1179 nop 1180 .align 32 1181 .endm 1182 1183ENTRY(tl1_data_excptn_trap) 1184 RESUME_SPILLFILL_MMU_CLR_SFSR 1185 b %xcc, tl1_sfsr_trap 1186 mov T_DATA_EXCEPTION | T_KERNEL, %g2 1187END(tl1_data_excptn_trap) 1188 1189 .macro tl1_align 1190 b,a %xcc, tl1_align_trap 1191 nop 1192 .align 32 1193 .endm 1194 1195ENTRY(tl1_align_trap) 1196 RESUME_SPILLFILL_ALIGN 1197 b %xcc, tl1_sfsr_trap 1198 mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 1199END(tl1_data_excptn_trap) 1200 1201ENTRY(tl1_sfsr_trap) 1202 wr %g0, ASI_DMMU, %asi 1203 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1204 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1205 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1206 membar #Sync 1207 1208 tl1_split 1209 clr %o1 1210 set trap, %o2 1211 mov %g3, %o4 1212 mov %g4, %o5 1213 b %xcc, tl1_trap 1214 mov %g2, %o0 1215END(tl1_sfsr_trap) 1216 1217 .macro tl1_intr level, mask 1218 tl1_split 1219 set \mask, %o1 1220 b %xcc, tl1_intr 1221 mov \level, %o0 1222 .align 32 1223 .endm 1224 1225 .macro tl1_intr_level 1226 INTR_LEVEL(1) 1227 .endm 1228 1229 .macro tl1_immu_miss 1230 /* 1231 * Load the context and the virtual page number from the tag access 1232 * register. We ignore the context. 1233 */ 1234 wr %g0, ASI_IMMU, %asi 1235 ldxa [%g0 + AA_IMMU_TAR] %asi, %g5 1236 1237 /* 1238 * Compute the address of the tte. The tsb mask and address of the 1239 * tsb are patched at startup. 1240 */ 1241 .globl tl1_immu_miss_patch_1 1242tl1_immu_miss_patch_1: 1243 sethi %hi(TSB_KERNEL_MASK), %g6 1244 or %g6, %lo(TSB_KERNEL_MASK), %g6 1245 sethi %hi(TSB_KERNEL), %g7 1246 1247 srlx %g5, TAR_VPN_SHIFT, %g5 1248 and %g5, %g6, %g6 1249 sllx %g6, TTE_SHIFT, %g6 1250 add %g6, %g7, %g6 1251 1252 /* 1253 * Load the tte. 1254 */ 1255 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1256 1257 /* 1258 * Check that its valid and executable and that the virtual page 1259 * numbers match. 1260 */ 1261 brgez,pn %g7, tl1_immu_miss_trap 1262 andcc %g7, TD_EXEC, %g0 1263 bz,pn %xcc, tl1_immu_miss_trap 1264 srlx %g6, TV_SIZE_BITS, %g6 1265 cmp %g5, %g6 1266 bne,pn %xcc, tl1_immu_miss_trap 1267 EMPTY 1268 1269 /* 1270 * Set the reference bit if its currently clear. 1271 */ 1272 andcc %g7, TD_REF, %g0 1273 bz,a,pn %xcc, tl1_immu_miss_set_ref 1274 nop 1275 1276 /* 1277 * Load the tte data into the TLB and retry the instruction. 1278 */ 1279 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 1280 retry 1281 .align 128 1282 .endm 1283 1284ENTRY(tl1_immu_miss_set_ref) 1285 /* 1286 * Recompute the tte address, which we clobbered loading the tte. The 1287 * tsb mask and address of the tsb are patched at startup. 1288 */ 1289 .globl tl1_immu_miss_patch_2 1290tl1_immu_miss_patch_2: 1291 sethi %hi(TSB_KERNEL_MASK), %g6 1292 or %g6, %lo(TSB_KERNEL_MASK), %g6 1293 sethi %hi(TSB_KERNEL), %g7 1294 1295 and %g5, %g6, %g5 1296 sllx %g5, TTE_SHIFT, %g5 1297 add %g5, %g7, %g5 1298 1299 /* 1300 * Set the reference bit. 1301 */ 1302 TTE_SET_REF(%g5, %g6, %g7) 1303 1304 /* 1305 * May have become invalid during casxa, in which case start over. 1306 */ 1307 brgez,pn %g6, 1f 1308 nop 1309 1310 /* 1311 * Load the tte data into the TLB and retry the instruction. 1312 */ 1313 stxa %g6, [%g0] ASI_ITLB_DATA_IN_REG 13141: retry 1315END(tl1_immu_miss_set_ref) 1316 1317ENTRY(tl1_immu_miss_trap) 1318 /* 1319 * Switch to alternate globals. 1320 */ 1321 wrpr %g0, PSTATE_ALT, %pstate 1322 1323 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 1324 1325 tl1_split 1326 clr %o1 1327 set trap, %o2 1328 mov %g2, %o3 1329 b %xcc, tl1_trap 1330 mov T_INSTRUCTION_MISS | T_KERNEL, %o0 1331END(tl1_immu_miss_trap) 1332 1333 .macro tl1_dmmu_miss 1334 /* 1335 * Load the context and the virtual page number from the tag access 1336 * register. 1337 */ 1338 wr %g0, ASI_DMMU, %asi 1339 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1340 1341 /* 1342 * Extract the context from the contents of the tag access register. 1343 * If its non-zero this is a fault on a user address. Note that the 1344 * faulting address is passed in %g1. 1345 */ 1346 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1347 brnz,a,pn %g6, tl1_dmmu_miss_user 1348 mov %g5, %g1 1349 1350 /* 1351 * Check for the direct mapped physical region. These addresses have 1352 * the high bit set so they are negative. 1353 */ 1354 brlz,pn %g5, tl1_dmmu_miss_direct 1355 EMPTY 1356 1357 /* 1358 * Compute the address of the tte. The tsb mask and address of the 1359 * tsb are patched at startup. 1360 */ 1361 .globl tl1_dmmu_miss_patch_1 1362tl1_dmmu_miss_patch_1: 1363 sethi %hi(TSB_KERNEL_MASK), %g6 1364 or %g6, %lo(TSB_KERNEL_MASK), %g6 1365 sethi %hi(TSB_KERNEL), %g7 1366 1367 srlx %g5, TAR_VPN_SHIFT, %g5 1368 and %g5, %g6, %g6 1369 sllx %g6, TTE_SHIFT, %g6 1370 add %g6, %g7, %g6 1371 1372 /* 1373 * Load the tte. 1374 */ 1375 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1376 1377 /* 1378 * Check that its valid and that the virtual page numbers match. 1379 */ 1380 brgez,pn %g7, tl1_dmmu_miss_trap 1381 srlx %g6, TV_SIZE_BITS, %g6 1382 cmp %g5, %g6 1383 bne,pn %xcc, tl1_dmmu_miss_trap 1384 EMPTY 1385 1386 /* 1387 * Set the reference bit if its currently clear. 1388 */ 1389 andcc %g7, TD_REF, %g0 1390 bz,a,pt %xcc, tl1_dmmu_miss_set_ref 1391 nop 1392 1393 /* 1394 * Load the tte data into the TLB and retry the instruction. 1395 */ 1396 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 1397 retry 1398 .align 128 1399 .endm 1400 1401ENTRY(tl1_dmmu_miss_set_ref) 1402 /* 1403 * Recompute the tte address, which we clobbered loading the tte. The 1404 * tsb mask and address of the tsb are patched at startup. 1405 */ 1406 .globl tl1_dmmu_miss_patch_2 1407tl1_dmmu_miss_patch_2: 1408 sethi %hi(TSB_KERNEL_MASK), %g6 1409 or %g6, %lo(TSB_KERNEL_MASK), %g6 1410 sethi %hi(TSB_KERNEL), %g7 1411 1412 and %g5, %g6, %g5 1413 sllx %g5, TTE_SHIFT, %g5 1414 add %g5, %g7, %g5 1415 1416 /* 1417 * Set the reference bit. 1418 */ 1419 TTE_SET_REF(%g5, %g6, %g7) 1420 1421 /* 1422 * May have become invalid during casxa, in which case start over. 1423 */ 1424 brgez,pn %g6, 1f 1425 nop 1426 1427 /* 1428 * Load the tte data into the TLB and retry the instruction. 1429 */ 1430 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 14311: retry 1432END(tl1_dmmu_miss_set_ref) 1433 1434ENTRY(tl1_dmmu_miss_trap) 1435 /* 1436 * Switch to alternate globals. 1437 */ 1438 wrpr %g0, PSTATE_ALT, %pstate 1439 1440 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1441 1442 KSTACK_CHECK 1443 1444 tl1_split 1445 clr %o1 1446 set trap, %o2 1447 mov %g2, %o3 1448 b %xcc, tl1_trap 1449 mov T_DATA_MISS | T_KERNEL, %o0 1450END(tl1_dmmu_miss_trap) 1451 1452ENTRY(tl1_dmmu_miss_direct) 1453 /* 1454 * Mask off the high bits of the virtual address to get the physical 1455 * address, and or in the tte bits. The virtual address bits that 1456 * correspond to the tte valid and page size bits are left set, so 1457 * they don't have to be included in the tte bits below. We know they 1458 * are set because the virtual address is in the upper va hole. 1459 */ 1460 setx TLB_DIRECT_TO_TTE_MASK, %g7, %g6 1461 and %g5, %g6, %g5 1462 or %g5, TD_CP | TD_CV | TD_W, %g5 1463 1464 /* 1465 * Load the tte data into the TLB and retry the instruction. 1466 */ 1467 stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG 1468 retry 1469END(tl1_dmmu_miss_direct) 1470 1471ENTRY(tl1_dmmu_miss_user) 1472 /* 1473 * Try a fast inline lookup of the user tsb. 1474 */ 1475 dmmu_miss_user 1476 1477 /* 1478 * Put back the contents of the tag access register, in case we 1479 * faulted. 1480 */ 1481 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1482 membar #Sync 1483 1484 /* 1485 * Switch to alternate globals. 1486 */ 1487 wrpr %g0, PSTATE_ALT, %pstate 1488 1489 /* 1490 * Handle faults during window spill/fill. 1491 */ 1492 RESUME_SPILLFILL_MMU 1493 1494 /* 1495 * Reload the tag access register. 1496 */ 1497 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1498 1499 tl1_split 1500 clr %o1 1501 set trap, %o2 1502 mov %g2, %o3 1503 b %xcc, tl1_trap 1504 mov T_DATA_MISS | T_KERNEL, %o0 1505END(tl1_dmmu_miss_user) 1506 1507 .macro tl1_dmmu_prot 1508 ba,a %xcc, tl1_dmmu_prot_1 1509 nop 1510 .align 128 1511 .endm 1512 1513ENTRY(tl1_dmmu_prot_1) 1514 /* 1515 * Load the context and the virtual page number from the tag access 1516 * register. 1517 */ 1518 wr %g0, ASI_DMMU, %asi 1519 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1520 1521 /* 1522 * Extract the context from the contents of the tag access register. 1523 * If its non-zero this is a fault on a user address. Note that the 1524 * faulting address is passed in %g1. 1525 */ 1526 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1527 brnz,a,pn %g6, tl1_dmmu_prot_user 1528 mov %g5, %g1 1529 1530 /* 1531 * Compute the address of the tte. The tsb mask and address of the 1532 * tsb are patched at startup. 1533 */ 1534 .globl tl1_dmmu_prot_patch_1 1535tl1_dmmu_prot_patch_1: 1536 sethi %hi(TSB_KERNEL_MASK), %g6 1537 or %g6, %lo(TSB_KERNEL_MASK), %g6 1538 sethi %hi(TSB_KERNEL), %g7 1539 1540 srlx %g5, TAR_VPN_SHIFT, %g5 1541 and %g5, %g6, %g6 1542 sllx %g6, TTE_SHIFT, %g6 1543 add %g6, %g7, %g6 1544 1545 /* 1546 * Load the tte. 1547 */ 1548 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1549 1550 /* 1551 * Check that its valid and writeable and that the virtual page 1552 * numbers match. 1553 */ 1554 brgez,pn %g7, tl1_dmmu_prot_trap 1555 andcc %g7, TD_SW, %g0 1556 bz,pn %xcc, tl1_dmmu_prot_trap 1557 srlx %g6, TV_SIZE_BITS, %g6 1558 cmp %g5, %g6 1559 bne,pn %xcc, tl1_dmmu_prot_trap 1560 EMPTY 1561 1562 /* 1563 * Delete the old TLB entry and clear the sfsr. 1564 */ 1565 sllx %g5, TAR_VPN_SHIFT, %g6 1566 or %g6, TLB_DEMAP_NUCLEUS, %g6 1567 stxa %g0, [%g6] ASI_DMMU_DEMAP 1568 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1569 membar #Sync 1570 1571 /* 1572 * Recompute the tte address, which we clobbered loading the tte. The 1573 * tsb mask and address of the tsb are patched at startup. 1574 */ 1575 .globl tl1_dmmu_prot_patch_2 1576tl1_dmmu_prot_patch_2: 1577 sethi %hi(TSB_KERNEL_MASK), %g6 1578 or %g6, %lo(TSB_KERNEL_MASK), %g6 1579 sethi %hi(TSB_KERNEL), %g7 1580 1581 and %g5, %g6, %g5 1582 sllx %g5, TTE_SHIFT, %g5 1583 add %g5, %g7, %g5 1584 1585 /* 1586 * Set the hardware write bit. 1587 */ 1588 TTE_SET_W(%g5, %g6, %g7) 1589 1590 /* 1591 * May have become invalid during casxa, in which case start over. 1592 */ 1593 brgez,pn %g6, 1f 1594 or %g6, TD_W, %g6 1595 1596 /* 1597 * Load the tte data into the TLB and retry the instruction. 1598 */ 1599 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 16001: retry 1601END(tl1_dmmu_prot_1) 1602 1603ENTRY(tl1_dmmu_prot_user) 1604 /* 1605 * Try a fast inline lookup of the user tsb. 1606 */ 1607 dmmu_prot_user 1608 1609 /* 1610 * Put back the contents of the tag access register, in case we 1611 * faulted. 1612 */ 1613 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1614 membar #Sync 1615 1616 /* 1617 * Switch to alternate globals. 1618 */ 1619 wrpr %g0, PSTATE_ALT, %pstate 1620 1621 /* Handle faults during window spill/fill. */ 1622 RESUME_SPILLFILL_MMU_CLR_SFSR 1623 1624 b,a %xcc, tl1_dmmu_prot_trap 1625 nop 1626END(tl1_dmmu_prot_user) 1627 1628ENTRY(tl1_dmmu_prot_trap) 1629 /* 1630 * Switch to alternate globals. 1631 */ 1632 wrpr %g0, PSTATE_ALT, %pstate 1633 1634 /* 1635 * Load the sfar, sfsr and tar. Clear the sfsr. 1636 */ 1637 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1638 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1639 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1640 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1641 membar #Sync 1642 1643 tl1_split 1644 clr %o1 1645 set trap, %o2 1646 mov %g2, %o3 1647 mov %g3, %o4 1648 mov %g4, %o5 1649 b %xcc, tl1_trap 1650 mov T_DATA_PROTECTION | T_KERNEL, %o0 1651END(tl1_dmmu_prot_trap) 1652 1653 .macro tl1_spill_0_n 1654 SPILL(stx, %sp + SPOFF, 8, EMPTY) 1655 saved 1656 retry 1657 .align 32 1658 RSF_FATAL(T_SPILL) 1659 RSF_FATAL(T_SPILL) 1660 .endm 1661 1662 .macro tl1_spill_2_n 1663 wr %g0, ASI_AIUP, %asi 1664 SPILL(stxa, %sp + SPOFF, 8, %asi) 1665 saved 1666 retry 1667 .align 32 1668 RSF_SPILL_TOPCB 1669 RSF_SPILL_TOPCB 1670 .endm 1671 1672 .macro tl1_spill_3_n 1673 wr %g0, ASI_AIUP, %asi 1674 SPILL(stwa, %sp, 4, %asi) 1675 saved 1676 retry 1677 .align 32 1678 RSF_SPILL_TOPCB 1679 RSF_SPILL_TOPCB 1680 .endm 1681 1682 .macro tl1_spill_0_o 1683 wr %g0, ASI_AIUP, %asi 1684 SPILL(stxa, %sp + SPOFF, 8, %asi) 1685 saved 1686 retry 1687 .align 32 1688 RSF_SPILL_TOPCB 1689 RSF_SPILL_TOPCB 1690 .endm 1691 1692 .macro tl1_spill_1_o 1693 wr %g0, ASI_AIUP, %asi 1694 SPILL(stwa, %sp, 4, %asi) 1695 saved 1696 retry 1697 .align 32 1698 RSF_SPILL_TOPCB 1699 RSF_SPILL_TOPCB 1700 .endm 1701 1702 .macro tl1_spill_2_o 1703 RSF_SPILL_TOPCB 1704 .align 128 1705 .endm 1706 1707 .macro tl1_fill_0_n 1708 FILL(ldx, %sp + SPOFF, 8, EMPTY) 1709 restored 1710 retry 1711 .align 32 1712 RSF_FATAL(T_FILL) 1713 RSF_FATAL(T_FILL) 1714 .endm 1715 1716 .macro tl1_fill_2_n 1717 wr %g0, ASI_AIUP, %asi 1718 FILL(ldxa, %sp + SPOFF, 8, %asi) 1719 restored 1720 retry 1721 .align 32 1722 RSF_FILL_MAGIC 1723 RSF_FILL_MAGIC 1724 .endm 1725 1726 .macro tl1_fill_3_n 1727 wr %g0, ASI_AIUP, %asi 1728 FILL(lduwa, %sp, 4, %asi) 1729 restored 1730 retry 1731 .align 32 1732 RSF_FILL_MAGIC 1733 RSF_FILL_MAGIC 1734 .endm 1735 1736/* 1737 * This is used to spill windows that are still occupied with user 1738 * data on kernel entry to the pcb. 1739 */ 1740ENTRY(tl1_spill_topcb) 1741 wrpr %g0, PSTATE_ALT, %pstate 1742 1743 /* Free some globals for our use. */ 1744 dec 24, ASP_REG 1745 stx %g1, [ASP_REG + 0] 1746 stx %g2, [ASP_REG + 8] 1747 stx %g3, [ASP_REG + 16] 1748 1749 ldx [PCB_REG + PCB_NSAVED], %g1 1750 1751 sllx %g1, PTR_SHIFT, %g2 1752 add %g2, PCB_REG, %g2 1753 stx %sp, [%g2 + PCB_RWSP] 1754 1755 sllx %g1, RW_SHIFT, %g2 1756 add %g2, PCB_REG, %g2 1757 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) 1758 1759 inc %g1 1760 stx %g1, [PCB_REG + PCB_NSAVED] 1761 1762#if KTR_COMPILE & KTR_TRAP 1763 CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" 1764 , %g1, %g2, %g3, 7, 8, 9) 1765 rdpr %tpc, %g2 1766 stx %g2, [%g1 + KTR_PARM1] 1767 rdpr %tnpc, %g2 1768 stx %g2, [%g1 + KTR_PARM2] 1769 stx %sp, [%g1 + KTR_PARM3] 1770 ldx [PCB_REG + PCB_NSAVED], %g2 1771 stx %g2, [%g1 + KTR_PARM4] 17729: 1773#endif 1774 1775 saved 1776 1777 ldx [ASP_REG + 16], %g3 1778 ldx [ASP_REG + 8], %g2 1779 ldx [ASP_REG + 0], %g1 1780 inc 24, ASP_REG 1781 retry 1782END(tl1_spill_topcb) 1783 1784 .macro tl1_spill_bad count 1785 .rept \count 1786 sir 1787 .align 128 1788 .endr 1789 .endm 1790 1791 .macro tl1_fill_bad count 1792 .rept \count 1793 sir 1794 .align 128 1795 .endr 1796 .endm 1797 1798 .macro tl1_soft count 1799 .rept \count 1800 tl1_gen T_SOFT | T_KERNEL 1801 .endr 1802 .endm 1803 1804 .sect .trap 1805 .align 0x8000 1806 .globl tl0_base 1807 1808tl0_base: 1809 tl0_reserved 8 ! 0x0-0x7 1810tl0_insn_excptn: 1811 tl0_insn_excptn ! 0x8 1812 tl0_reserved 1 ! 0x9 1813tl0_insn_error: 1814 tl0_gen T_INSTRUCTION_ERROR ! 0xa 1815 tl0_reserved 5 ! 0xb-0xf 1816tl0_insn_illegal: 1817 tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 1818tl0_priv_opcode: 1819 tl0_gen T_PRIVILEGED_OPCODE ! 0x11 1820 tl0_reserved 14 ! 0x12-0x1f 1821tl0_fp_disabled: 1822 tl0_gen T_FP_DISABLED ! 0x20 1823tl0_fp_ieee: 1824 tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 1825tl0_fp_other: 1826 tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 1827tl0_tag_ovflw: 1828 tl0_gen T_TAG_OFERFLOW ! 0x23 1829tl0_clean_window: 1830 clean_window ! 0x24 1831tl0_divide: 1832 tl0_gen T_DIVISION_BY_ZERO ! 0x28 1833 tl0_reserved 7 ! 0x29-0x2f 1834tl0_data_excptn: 1835 tl0_data_excptn ! 0x30 1836 tl0_reserved 1 ! 0x31 1837tl0_data_error: 1838 tl0_gen T_DATA_ERROR ! 0x32 1839 tl0_reserved 1 ! 0x33 1840tl0_align: 1841 tl0_align ! 0x34 1842tl0_align_lddf: 1843 tl0_gen T_RESERVED ! 0x35 1844tl0_align_stdf: 1845 tl0_gen T_RESERVED ! 0x36 1846tl0_priv_action: 1847 tl0_gen T_PRIVILEGED_ACTION ! 0x37 1848 tl0_reserved 9 ! 0x38-0x40 1849tl0_intr_level: 1850 tl0_intr_level ! 0x41-0x4f 1851 tl0_reserved 16 ! 0x50-0x5f 1852tl0_intr_vector: 1853 intr_vector ! 0x60 1854tl0_watch_phys: 1855 tl0_gen T_PA_WATCHPOINT ! 0x61 1856tl0_watch_virt: 1857 tl0_gen T_VA_WATCHPOINT ! 0x62 1858tl0_ecc: 1859 tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 1860tl0_immu_miss: 1861 tl0_immu_miss ! 0x64 1862tl0_dmmu_miss: 1863 tl0_dmmu_miss ! 0x68 1864tl0_dmmu_prot: 1865 tl0_dmmu_prot ! 0x6c 1866 tl0_reserved 16 ! 0x70-0x7f 1867tl0_spill_0_n: 1868 tl0_spill_0_n ! 0x80 1869tl0_spill_1_n: 1870 tl0_spill_1_n ! 0x84 1871 tl0_spill_bad 14 ! 0x88-0xbf 1872tl0_fill_0_n: 1873 tl0_fill_0_n ! 0xc0 1874tl0_fill_1_n: 1875 tl0_fill_1_n ! 0xc4 1876 tl0_fill_bad 14 ! 0xc8-0xff 1877tl0_soft: 1878 tl0_gen T_SYSCALL ! 0x100 1879 tl0_gen T_BREAKPOINT ! 0x101 1880 tl0_gen T_DIVISION_BY_ZERO ! 0x102 1881 tl0_reserved 1 ! 0x103 1882 tl0_gen T_CLEAN_WINDOW ! 0x104 1883 tl0_gen T_RANGE_CHECK ! 0x105 1884 tl0_gen T_FIX_ALIGNMENT ! 0x106 1885 tl0_gen T_INTEGER_OVERFLOW ! 0x107 1886 tl0_gen T_SYSCALL ! 0x108 1887#ifdef COMPAT_FREEBSD4 1888 tl0_syscall ! 0x109 1889#else 1890 tl0_gen T_SYSCALL ! 0x109 1891#endif 1892 tl0_fp_restore ! 0x10a 1893 tl0_reserved 5 ! 0x10b-0x10f 1894 tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 1895 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 1896 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 1897 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 1898 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 1899 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 1900 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 1901 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 1902 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 1903 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 1904 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a 1905 tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b 1906 tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c 1907 tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d 1908 tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e 1909 tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f 1910 tl0_reserved 32 ! 0x120-0x13f 1911 tl0_gen T_SYSCALL ! 0x140 1912 tl0_syscall ! 0x141 1913 tl0_gen T_SYSCALL ! 0x142 1914 tl0_gen T_SYSCALL ! 0x143 1915 tl0_reserved 188 ! 0x144-0x1ff 1916 1917tl1_base: 1918 tl1_reserved 8 ! 0x200-0x207 1919tl1_insn_excptn: 1920 tl1_insn_excptn ! 0x208 1921 tl1_reserved 1 ! 0x209 1922tl1_insn_error: 1923 tl1_gen T_INSTRUCTION_ERROR ! 0x20a 1924 tl1_reserved 5 ! 0x20b-0x20f 1925tl1_insn_illegal: 1926 tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 1927tl1_priv_opcode: 1928 tl1_gen T_PRIVILEGED_OPCODE ! 0x211 1929 tl1_reserved 14 ! 0x212-0x21f 1930tl1_fp_disabled: 1931 tl1_gen T_FP_DISABLED ! 0x220 1932tl1_fp_ieee: 1933 tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 1934tl1_fp_other: 1935 tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 1936tl1_tag_ovflw: 1937 tl1_gen T_TAG_OFERFLOW ! 0x223 1938tl1_clean_window: 1939 clean_window ! 0x224 1940tl1_divide: 1941 tl1_gen T_DIVISION_BY_ZERO ! 0x228 1942 tl1_reserved 7 ! 0x229-0x22f 1943tl1_data_excptn: 1944 tl1_data_excptn ! 0x230 1945 tl1_reserved 1 ! 0x231 1946tl1_data_error: 1947 tl1_gen T_DATA_ERROR ! 0x232 1948 tl1_reserved 1 ! 0x233 1949tl1_align: 1950 tl1_align ! 0x234 1951tl1_align_lddf: 1952 tl1_gen T_RESERVED ! 0x235 1953tl1_align_stdf: 1954 tl1_gen T_RESERVED ! 0x236 1955tl1_priv_action: 1956 tl1_gen T_PRIVILEGED_ACTION ! 0x237 1957 tl1_reserved 9 ! 0x238-0x240 1958tl1_intr_level: 1959 tl1_intr_level ! 0x241-0x24f 1960 tl1_reserved 16 ! 0x250-0x25f 1961tl1_intr_vector: 1962 intr_vector ! 0x260 1963tl1_watch_phys: 1964 tl1_gen T_PA_WATCHPOINT ! 0x261 1965tl1_watch_virt: 1966 tl1_gen T_VA_WATCHPOINT ! 0x262 1967tl1_ecc: 1968 tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 1969tl1_immu_miss: 1970 tl1_immu_miss ! 0x264 1971tl1_dmmu_miss: 1972 tl1_dmmu_miss ! 0x268 1973tl1_dmmu_prot: 1974 tl1_dmmu_prot ! 0x26c 1975 tl1_reserved 16 ! 0x270-0x27f 1976tl1_spill_0_n: 1977 tl1_spill_0_n ! 0x280 1978 tl1_spill_bad 1 ! 0x284 1979tl1_spill_2_n: 1980 tl1_spill_2_n ! 0x288 1981tl1_spill_3_n: 1982 tl1_spill_3_n ! 0x29c 1983 tl1_spill_bad 4 ! 0x290-0x29f 1984tl1_spill_0_o: 1985 tl1_spill_0_o ! 0x2a0 1986tl1_spill_1_o: 1987 tl1_spill_1_o ! 0x2a4 1988tl1_spill_2_o: 1989 tl1_spill_2_o ! 0x2a8 1990 tl1_spill_bad 5 ! 0x2ac-0x2bf 1991tl1_fill_0_n: 1992 tl1_fill_0_n ! 0x2c0 1993 tl1_fill_bad 1 ! 0x2c4 1994tl1_fill_2_n: 1995 tl1_fill_2_n ! 0x2d0 1996tl1_fill_3_n: 1997 tl1_fill_3_n ! 0x2d4 1998 tl1_fill_bad 12 ! 0x2d8-0x2ff 1999 tl1_reserved 1 ! 0x300 2000tl1_breakpoint: 2001 tl1_gen T_BREAKPOINT ! 0x301 2002 tl1_gen T_RSTRWP_PHYS ! 0x302 2003 tl1_gen T_RSTRWP_VIRT ! 0x303 2004 tl1_reserved 252 ! 0x304-0x3ff 2005 2006/* 2007 * User trap entry point. 2008 * 2009 * void tl0_utrap(u_long type, u_long o1, u_long o2, u_long tar, u_long sfar, 2010 * u_long sfsr) 2011 * 2012 * This handles redirecting a trap back to usermode as a user trap. The user 2013 * program must have first registered a trap handler with the kernel using 2014 * sysarch(SPARC_UTRAP_INSTALL). The trap handler is passed enough state 2015 * for it to return to the trapping code directly, it will not return through 2016 * the kernel. The trap type is passed in %o0, all out registers must be 2017 * passed through to tl0_trap or to usermode untouched. Note that the 2018 * parameters passed in out registers may be used by the user trap handler. 2019 * Do not change the registers they are passed in or you will break the ABI. 2020 * 2021 * If the trap type allows user traps, setup state to execute the user trap 2022 * handler and bounce back to usermode, otherwise branch to tl0_trap. 2023 */ 2024ENTRY(tl0_utrap) 2025 /* 2026 * Check if the trap type allows user traps. 2027 */ 2028 cmp %o0, UT_MAX 2029 bge,a,pt %xcc, tl0_trap 2030 nop 2031 2032 /* 2033 * Load the user trap handler from the utrap table. 2034 */ 2035 ldx [PCPU(CURTHREAD)], %l0 2036 ldx [%l0 + TD_PROC], %l0 2037 ldx [%l0 + P_MD + MD_UTRAP], %l0 2038 brz,pt %l0, tl0_trap 2039 sllx %o0, PTR_SHIFT, %l1 2040 ldx [%l0 + %l1], %l0 2041 brz,a,pt %l0, tl0_trap 2042 nop 2043 2044 /* 2045 * If the save we did on entry to the kernel had to spill a window 2046 * to the pcb, pretend we took a spill trap instead. Any windows 2047 * that are in the pcb must be copied out or the fill handler will 2048 * not be able to find them, since the user trap handler returns 2049 * directly to the trapping code. Note that we only support precise 2050 * user traps, which implies that the condition that caused the trap 2051 * in the first place is still valid, so it will occur again when we 2052 * re-execute the trapping instruction. 2053 */ 2054 ldx [PCB_REG + PCB_NSAVED], %l1 2055 brnz,a,pn %l1, tl0_trap 2056 mov T_SPILL, %o0 2057 2058 /* 2059 * Pass %fsr in %l4, %tstate in %l5, %tpc in %l6 and %tnpc in %l7. 2060 * The ABI specifies only %l6 and %l7, but we need to pass %fsr or 2061 * it may be clobbered by an interrupt before the user trap code 2062 * can read it, and we must pass %tstate in order to restore %ccr 2063 * and %asi. The %fsr must be stored to memory, so we use the 2064 * temporary stack for that. 2065 */ 2066 rd %fprs, %l1 2067 or %l1, FPRS_FEF, %l2 2068 wr %l2, 0, %fprs 2069 dec 8, ASP_REG 2070 stx %fsr, [ASP_REG] 2071 ldx [ASP_REG], %l4 2072 inc 8, ASP_REG 2073 wr %l1, 0, %fprs 2074 2075 rdpr %tstate, %l5 2076 rdpr %tpc, %l6 2077 rdpr %tnpc, %l7 2078 2079 /* 2080 * Setup %tnpc to return to. 2081 */ 2082 wrpr %l0, 0, %tnpc 2083 2084 /* 2085 * Setup %wstate for return, clear WSTATE_TRANSITION. 2086 */ 2087 rdpr %wstate, %l1 2088 and %l1, WSTATE_NORMAL_MASK, %l1 2089 wrpr %l1, 0, %wstate 2090 2091 /* 2092 * Setup %tstate for return, change the saved cwp to point to the 2093 * current window instead of the window at the time of the trap. 2094 */ 2095 andn %l5, TSTATE_CWP_MASK, %l1 2096 rdpr %cwp, %l2 2097 wrpr %l1, %l2, %tstate 2098 2099 /* 2100 * Setup %sp. Userland processes will crash if this is not setup. 2101 */ 2102 sub %fp, CCFSZ, %sp 2103 2104 /* 2105 * Execute the user trap handler. 2106 */ 2107 done 2108END(tl0_utrap) 2109 2110/* 2111 * (Real) User trap entry point. 2112 * 2113 * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2114 * u_int sfsr) 2115 * 2116 * The following setup has been performed: 2117 * - the windows have been split and the active user window has been saved 2118 * (maybe just to the pcb) 2119 * - we are on alternate globals and interrupts are disabled 2120 * 2121 * We switch to the kernel stack, build a trapframe, switch to normal 2122 * globals, enable interrupts and call trap. 2123 * 2124 * NOTE: We must be very careful setting up the per-cpu pointer. We know that 2125 * it has been pre-set in alternate globals, so we read it from there and setup 2126 * the normal %g7 *before* enabling interrupts. This avoids any possibility 2127 * of cpu migration and using the wrong pcpup. 2128 */ 2129ENTRY(tl0_trap) 2130 /* 2131 * Force kernel store order. 2132 */ 2133 wrpr %g0, PSTATE_ALT, %pstate 2134 2135 rdpr %tstate, %l0 2136 rdpr %tpc, %l1 2137 rdpr %tnpc, %l2 2138 rd %y, %l3 2139 rd %fprs, %l4 2140 rdpr %wstate, %l5 2141 2142#if KTR_COMPILE & KTR_TRAP 2143 CATR(KTR_TRAP, 2144 "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2145 , %g1, %g2, %g3, 7, 8, 9) 2146 ldx [PCPU(CURTHREAD)], %g2 2147 stx %g2, [%g1 + KTR_PARM1] 2148 stx %o0, [%g1 + KTR_PARM2] 2149 rdpr %pil, %g2 2150 stx %g2, [%g1 + KTR_PARM3] 2151 stx %l1, [%g1 + KTR_PARM4] 2152 stx %l2, [%g1 + KTR_PARM5] 2153 stx %i6, [%g1 + KTR_PARM6] 21549: 2155#endif 2156 21571: and %l5, WSTATE_NORMAL_MASK, %l5 2158 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2159 wrpr %l5, WSTATE_KERNEL, %wstate 2160 rdpr %canrestore, %l6 2161 wrpr %l6, 0, %otherwin 2162 wrpr %g0, 0, %canrestore 2163 2164 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2165 2166 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2167 stx %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2168 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2169 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2170 stx %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2171 2172 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2173 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2174 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2175 stx %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2176 stx %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2177 stx %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2178 2179 wr %g0, FPRS_FEF, %fprs 2180 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2181 rd %gsr, %l6 2182 stx %l6, [%sp + SPOFF + CCFSZ + TF_GSR] 2183 wr %g0, 0, %fprs 2184 2185 mov PCB_REG, %l0 2186 mov PCPU_REG, %l1 2187 wrpr %g0, PSTATE_NORMAL, %pstate 2188 2189 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2190 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2191 2192 mov %l0, PCB_REG 2193 mov %l1, PCPU_REG 2194 wrpr %g0, PSTATE_KERNEL, %pstate 2195 2196 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2197 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2198 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2199 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2200 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2201 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2202 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2203 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2204 2205 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2206 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2207 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2208 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2209 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2210 2211 set tl0_ret - 8, %o7 2212 jmpl %o2, %g0 2213 add %sp, CCFSZ + SPOFF, %o0 2214END(tl0_trap) 2215 2216/* 2217 * void tl0_intr(u_int level, u_int mask) 2218 */ 2219ENTRY(tl0_intr) 2220 /* 2221 * Force kernel store order. 2222 */ 2223 wrpr %g0, PSTATE_ALT, %pstate 2224 2225 rdpr %tstate, %l0 2226 rdpr %tpc, %l1 2227 rdpr %tnpc, %l2 2228 rd %y, %l3 2229 rd %fprs, %l4 2230 rdpr %wstate, %l5 2231 2232#if KTR_COMPILE & KTR_INTR 2233 CATR(KTR_INTR, 2234 "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2235 , %g1, %g2, %g3, 7, 8, 9) 2236 ldx [PCPU(CURTHREAD)], %g2 2237 stx %g2, [%g1 + KTR_PARM1] 2238 stx %o0, [%g1 + KTR_PARM2] 2239 rdpr %pil, %g2 2240 stx %g2, [%g1 + KTR_PARM3] 2241 stx %l1, [%g1 + KTR_PARM4] 2242 stx %l2, [%g1 + KTR_PARM5] 2243 stx %i6, [%g1 + KTR_PARM6] 22449: 2245#endif 2246 2247 wrpr %o0, 0, %pil 2248 wr %o1, 0, %clear_softint 2249 2250 and %l5, WSTATE_NORMAL_MASK, %l5 2251 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2252 wrpr %l5, WSTATE_KERNEL, %wstate 2253 rdpr %canrestore, %l6 2254 wrpr %l6, 0, %otherwin 2255 wrpr %g0, 0, %canrestore 2256 2257 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2258 2259 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2260 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2261 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2262 stx %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2263 stx %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2264 stx %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2265 2266 wr %g0, FPRS_FEF, %fprs 2267 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2268 rd %gsr, %l6 2269 stx %l6, [%sp + SPOFF + CCFSZ + TF_GSR] 2270 wr %g0, 0, %fprs 2271 2272 mov %o0, %l3 2273 mov T_INTERRUPT, %o1 2274 2275 stx %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2276 stx %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2277 2278 mov PCB_REG, %l0 2279 mov PCPU_REG, %l1 2280 wrpr %g0, PSTATE_NORMAL, %pstate 2281 2282 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2283 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2284 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2285 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2286 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2287 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2288 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2289 2290 mov %l0, PCB_REG 2291 mov %l1, PCPU_REG 2292 wrpr %g0, PSTATE_KERNEL, %pstate 2293 2294 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2295 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2296 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2297 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2298 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2299 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2300 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2301 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2302 2303 call critical_enter 2304 nop 2305 2306 SET(cnt+V_INTR, %l1, %l0) 2307 ATOMIC_INC_INT(%l0, %l1, %l2) 2308 2309 SET(intr_handlers, %l1, %l0) 2310 sllx %l3, IH_SHIFT, %l1 2311 ldx [%l0 + %l1], %l1 2312 KASSERT(%l1, "tl0_intr: ih null") 2313 call %l1 2314 add %sp, CCFSZ + SPOFF, %o0 2315 2316 call critical_exit 2317 nop 2318 2319 b,a %xcc, tl0_ret 2320 nop 2321END(tl0_intr) 2322 2323/* 2324 * Initiate return to usermode. 2325 * 2326 * Called with a trapframe on the stack. The window that was setup in 2327 * tl0_trap may have been used by "fast" trap handlers that pretend to be 2328 * leaf functions, so all ins and locals may have been clobbered since 2329 * then. 2330 * 2331 * This code is rather long and complicated. 2332 */ 2333ENTRY(tl0_ret) 2334 /* 2335 * Check for pending asts atomically with returning. We must raise 2336 * the pil before checking, and if no asts are found the pil must 2337 * remain raised until the retry is executed, or we risk missing asts 2338 * caused by interrupts occuring after the test. If the pil is lowered, 2339 * as it is when we call ast, the check must be re-executed. 2340 */ 2341 wrpr %g0, PIL_TICK, %pil 2342 ldx [PCPU(CURTHREAD)], %l0 2343 ldx [%l0 + TD_KSE], %l1 2344 lduw [%l1 + KE_FLAGS], %l2 2345 and %l2, KEF_ASTPENDING | KEF_NEEDRESCHED, %l2 2346 brz,a,pt %l2, 1f 2347 nop 2348 2349 /* 2350 * We have an ast. Re-enable interrupts and handle it, then restart 2351 * the return sequence. 2352 */ 2353 wrpr %g0, 0, %pil 2354 call ast 2355 add %sp, CCFSZ + SPOFF, %o0 2356 ba,a %xcc, tl0_ret 2357 nop 2358 2359 /* 2360 * Check for windows that were spilled to the pcb and need to be 2361 * copied out. This must be the last thing that is done before the 2362 * return to usermode. If there are still user windows in the cpu 2363 * and we call a nested function after this, which causes them to be 2364 * spilled to the pcb, they will not be copied out and the stack will 2365 * be inconsistent. 2366 */ 23671: ldx [PCB_REG + PCB_NSAVED], %l1 2368 brz,a,pt %l1, 2f 2369 nop 2370 wrpr %g0, 0, %pil 2371 mov T_SPILL, %o0 2372 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2373 call trap 2374 add %sp, SPOFF + CCFSZ, %o0 2375 ba,a %xcc, tl0_ret 2376 nop 2377 2378 /* 2379 * Restore the out and most global registers from the trapframe. 2380 * The ins will become the outs when we restore below. 2381 */ 23822: ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 2383 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 2384 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 2385 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 2386 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 2387 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 2388 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 2389 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 2390 2391 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2392 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2393 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2394 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2395 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2396 2397 /* 2398 * Load everything we need to restore below before disabling 2399 * interrupts. 2400 */ 2401 ldx [%sp + SPOFF + CCFSZ + TF_FPRS], %l0 2402 ldx [%sp + SPOFF + CCFSZ + TF_GSR], %l1 2403 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2404 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l3 2405 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l4 2406 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l5 2407 ldx [%sp + SPOFF + CCFSZ + TF_WSTATE], %l6 2408 2409 /* 2410 * Disable interrupts to restore the special globals. They are not 2411 * saved and restored for all kernel traps, so an interrupt at the 2412 * wrong time would clobber them. 2413 */ 2414 wrpr %g0, PSTATE_NORMAL, %pstate 2415 2416 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 2417 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 2418 2419 /* 2420 * Switch to alternate globals. This frees up some registers we 2421 * can use after the restore changes our window. 2422 */ 2423 wrpr %g0, PSTATE_ALT, %pstate 2424 2425 /* 2426 * Drop %pil to zero. It must have been zero at the time of the 2427 * trap, since we were in usermode, but it was raised above in 2428 * order to check for asts atomically. We have interrupts disabled 2429 * so any interrupts will not be serviced until we complete the 2430 * return to usermode. 2431 */ 2432 wrpr %g0, 0, %pil 2433 2434 /* 2435 * Save %fprs in an alternate global so it can be restored after the 2436 * restore instruction below. If we restore it before the restore, 2437 * and the restore traps we may run for a while with floating point 2438 * enabled in the kernel, which we want to avoid. 2439 */ 2440 mov %l0, %g1 2441 2442 /* 2443 * Restore %fsr and %gsr. These need floating point enabled in %fprs, 2444 * so we set it temporarily and then clear it. 2445 */ 2446 wr %g0, FPRS_FEF, %fprs 2447 ldx [%sp + SPOFF + CCFSZ + TF_FSR], %fsr 2448 wr %l1, 0, %gsr 2449 wr %g0, 0, %fprs 2450 2451 /* 2452 * Restore program counters. This could be done after the restore 2453 * but we're out of alternate globals to store them in... 2454 */ 2455 wrpr %l2, 0, %tnpc 2456 wrpr %l3, 0, %tpc 2457 2458 /* 2459 * Save %tstate in an alternate global and clear the %cwp field. %cwp 2460 * will be affected by the restore below and we need to make sure it 2461 * points to the current window at that time, not the window that was 2462 * active at the time of the trap. 2463 */ 2464 andn %l4, TSTATE_CWP_MASK, %g2 2465 2466 /* 2467 * Restore %y. Could also be below if we had more alternate globals. 2468 */ 2469 wr %l5, 0, %y 2470 2471 /* 2472 * Setup %wstate for return. We need to restore the user window state 2473 * which we saved in wstate.other when we trapped. We also need to 2474 * set the transition bit so the restore will be handled specially 2475 * if it traps, use the xor feature of wrpr to do that. 2476 */ 2477 srlx %l6, WSTATE_OTHER_SHIFT, %g3 2478 wrpr %g3, WSTATE_TRANSITION, %wstate 2479 2480 /* 2481 * Setup window management registers for return. If not all user 2482 * windows were spilled in the kernel %otherwin will be non-zero, 2483 * so we need to transfer it to %canrestore to correctly restore 2484 * those windows. Otherwise everything gets set to zero and the 2485 * restore below will fill a window directly from the user stack. 2486 */ 2487 rdpr %otherwin, %o0 2488 wrpr %o0, 0, %canrestore 2489 wrpr %g0, 0, %otherwin 2490 wrpr %o0, 0, %cleanwin 2491 2492 /* 2493 * Now do the restore. If this instruction causes a fill trap which 2494 * fails to fill a window from the user stack, we will resume at 2495 * tl0_ret_fill_end and call back into the kernel. 2496 */ 2497 restore 2498tl0_ret_fill: 2499 2500 /* 2501 * We made it. We're back in the window that was active at the time 2502 * of the trap, and ready to return to usermode. 2503 */ 2504 2505 /* 2506 * Restore %frps. This was saved in an alternate global above. 2507 */ 2508 wr %g1, 0, %fprs 2509 2510 /* 2511 * Fixup %tstate so the saved %cwp points to the current window and 2512 * restore it. 2513 */ 2514 rdpr %cwp, %g4 2515 wrpr %g2, %g4, %tstate 2516 2517 /* 2518 * Restore the user window state. The transition bit was set above 2519 * for special handling of the restore, this clears it. 2520 */ 2521 wrpr %g3, 0, %wstate 2522 2523#if KTR_COMPILE & KTR_TRAP 2524 CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2525 , %g2, %g3, %g4, 7, 8, 9) 2526 ldx [PCPU(CURTHREAD)], %g3 2527 stx %g3, [%g2 + KTR_PARM1] 2528 rdpr %pil, %g3 2529 stx %g3, [%g2 + KTR_PARM2] 2530 rdpr %tpc, %g3 2531 stx %g3, [%g2 + KTR_PARM3] 2532 rdpr %tnpc, %g3 2533 stx %g3, [%g2 + KTR_PARM4] 2534 stx %sp, [%g2 + KTR_PARM5] 25359: 2536#endif 2537 2538 /* 2539 * Return to usermode. 2540 */ 2541 retry 2542tl0_ret_fill_end: 2543 2544#if KTR_COMPILE & KTR_TRAP 2545 CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" 2546 , %l0, %l1, %l2, 7, 8, 9) 2547 rdpr %pstate, %l1 2548 stx %l1, [%l0 + KTR_PARM1] 2549 stx %l5, [%l0 + KTR_PARM2] 2550 stx %sp, [%l0 + KTR_PARM3] 25519: 2552#endif 2553 2554 /* 2555 * The restore above caused a fill trap and the fill handler was 2556 * unable to fill a window from the user stack. The special fill 2557 * handler recognized this and punted, sending us here. We need 2558 * to carefully undo any state that was restored before the restore 2559 * was executed and call trap again. Trap will copyin a window 2560 * from the user stack which will fault in the page we need so the 2561 * restore above will succeed when we try again. If this fails 2562 * the process has trashed its stack, so we kill it. 2563 */ 2564 2565 /* 2566 * Restore the kernel window state. This was saved in %l6 above, and 2567 * since the restore failed we're back in the same window. 2568 */ 2569 wrpr %l6, 0, %wstate 2570 2571 /* 2572 * Restore the normal globals which have predefined values in the 2573 * kernel. We clobbered them above restoring the user's globals 2574 * so this is very important. 2575 * XXX PSTATE_ALT must already be set. 2576 */ 2577 wrpr %g0, PSTATE_ALT, %pstate 2578 mov PCB_REG, %o0 2579 mov PCPU_REG, %o1 2580 wrpr %g0, PSTATE_NORMAL, %pstate 2581 mov %o0, PCB_REG 2582 mov %o1, PCPU_REG 2583 wrpr %g0, PSTATE_KERNEL, %pstate 2584 2585 /* 2586 * Simulate a fill trap and then start the whole return sequence over 2587 * again. This is special because it only copies in 1 window, not 2 2588 * as we would for a normal failed fill. This may be the first time 2589 * the process has been run, so there may not be 2 windows worth of 2590 * stack to copyin. 2591 */ 2592 mov T_FILL_RET, %o0 2593 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2594 call trap 2595 add %sp, SPOFF + CCFSZ, %o0 2596 ba,a %xcc, tl0_ret 2597 nop 2598END(tl0_ret) 2599 2600/* 2601 * Kernel trap entry point 2602 * 2603 * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2604 * u_int sfsr) 2605 * 2606 * This is easy because the stack is already setup and the windows don't need 2607 * to be split. We build a trapframe and call trap(), the same as above, but 2608 * the outs don't need to be saved. 2609 */ 2610ENTRY(tl1_trap) 2611 rdpr %tstate, %l0 2612 rdpr %tpc, %l1 2613 rdpr %tnpc, %l2 2614 rdpr %pil, %l3 2615 rd %y, %l4 2616 rdpr %wstate, %l5 2617 2618#if KTR_COMPILE & KTR_TRAP 2619 CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" 2620 , %g1, %g2, %g3, 7, 8, 9) 2621 ldx [PCPU(CURTHREAD)], %g2 2622 stx %g2, [%g1 + KTR_PARM1] 2623 stx %o0, [%g1 + KTR_PARM2] 2624 stx %l3, [%g1 + KTR_PARM3] 2625 stx %l1, [%g1 + KTR_PARM4] 2626 stx %i6, [%g1 + KTR_PARM5] 26279: 2628#endif 2629 2630 wrpr %g0, 1, %tl 2631 2632 and %l5, WSTATE_OTHER_MASK, %l5 2633 wrpr %l5, WSTATE_KERNEL, %wstate 2634 2635 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2636 stx %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2637 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2638 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2639 stx %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2640 2641 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2642 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2643 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2644 stx %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2645 stx %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2646 2647 mov PCB_REG, %l0 2648 mov PCPU_REG, %l1 2649 wrpr %g0, PSTATE_NORMAL, %pstate 2650 2651 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2652 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2653 2654 mov %l0, PCB_REG 2655 mov %l1, PCPU_REG 2656 wrpr %g0, PSTATE_KERNEL, %pstate 2657 2658 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2659 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2660 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2661 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2662 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2663 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2664 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2665 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2666 2667 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2668 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2669 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2670 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2671 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2672 2673 set tl1_ret - 8, %o7 2674 jmpl %o2, %g0 2675 add %sp, CCFSZ + SPOFF, %o0 2676END(tl1_trap) 2677 2678ENTRY(tl1_ret) 2679 ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 2680 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 2681 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 2682 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 2683 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 2684 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 2685 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 2686 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 2687 2688 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2689 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2690 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2691 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2692 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2693 2694 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2695 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2696 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2697 ldx [%sp + SPOFF + CCFSZ + TF_PIL], %l3 2698 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 2699 2700 set VM_MIN_PROM_ADDRESS, %l5 2701 cmp %l1, %l5 2702 bl,a,pt %xcc, 1f 2703 nop 2704 2705 wrpr %g0, PSTATE_NORMAL, %pstate 2706 2707 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 2708 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 2709 27101: wrpr %g0, PSTATE_ALT, %pstate 2711 2712 andn %l0, TSTATE_CWP_MASK, %g1 2713 mov %l1, %g2 2714 mov %l2, %g3 2715 2716 wrpr %l3, 0, %pil 2717 wr %l4, 0, %y 2718 2719 restore 2720 2721 wrpr %g0, 2, %tl 2722 2723 rdpr %cwp, %g4 2724 wrpr %g1, %g4, %tstate 2725 wrpr %g2, 0, %tpc 2726 wrpr %g3, 0, %tnpc 2727 2728#if KTR_COMPILE & KTR_TRAP 2729 CATR(KTR_TRAP, "tl1_ret: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2730 , %g2, %g3, %g4, 7, 8, 9) 2731 ldx [PCPU(CURTHREAD)], %g3 2732 stx %g3, [%g2 + KTR_PARM1] 2733 rdpr %pil, %g3 2734 stx %g3, [%g2 + KTR_PARM2] 2735 rdpr %tstate, %g3 2736 stx %g3, [%g2 + KTR_PARM3] 2737 rdpr %tpc, %g3 2738 stx %g3, [%g2 + KTR_PARM4] 2739 stx %sp, [%g2 + KTR_PARM5] 27409: 2741#endif 2742 2743 retry 2744END(tl1_ret) 2745 2746/* 2747 * void tl1_intr(u_int level, u_int mask) 2748 */ 2749ENTRY(tl1_intr) 2750 rdpr %tstate, %l0 2751 rdpr %tpc, %l1 2752 rdpr %tnpc, %l2 2753 rdpr %pil, %l3 2754 rd %y, %l4 2755 rdpr %wstate, %l5 2756 2757#if KTR_COMPILE & KTR_INTR 2758 CATR(KTR_INTR, 2759 "tl1_intr: td=%p level=%#lx pil=%#lx pc=%#lx sp=%#lx" 2760 , %g1, %g2, %g3, 7, 8, 9) 2761 ldx [PCPU(CURTHREAD)], %g2 2762 stx %g2, [%g1 + KTR_PARM1] 2763 stx %o0, [%g1 + KTR_PARM2] 2764 stx %l3, [%g1 + KTR_PARM3] 2765 stx %l1, [%g1 + KTR_PARM4] 2766 stx %i6, [%g1 + KTR_PARM5] 27679: 2768#endif 2769 2770 wrpr %o0, 0, %pil 2771 wr %o1, 0, %clear_softint 2772 2773 wrpr %g0, 1, %tl 2774 2775 and %l5, WSTATE_OTHER_MASK, %l5 2776 wrpr %l5, WSTATE_KERNEL, %wstate 2777 2778 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2779 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2780 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2781 stx %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2782 stx %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2783 2784 mov %o0, %l7 2785 mov T_INTERRUPT | T_KERNEL, %o1 2786 2787 stx %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2788 stx %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2789 2790 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2791 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2792 2793 mov PCB_REG, %l4 2794 mov PCPU_REG, %l5 2795 wrpr %g0, PSTATE_NORMAL, %pstate 2796 2797 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2798 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2799 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2800 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2801 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2802 2803 mov %l4, PCB_REG 2804 mov %l5, PCPU_REG 2805 wrpr %g0, PSTATE_KERNEL, %pstate 2806 2807 call critical_enter 2808 nop 2809 2810 SET(cnt+V_INTR, %l5, %l4) 2811 ATOMIC_INC_INT(%l4, %l5, %l6) 2812 2813 SET(intr_handlers, %l5, %l4) 2814 sllx %l7, IH_SHIFT, %l5 2815 ldx [%l4 + %l5], %l5 2816 KASSERT(%l5, "tl1_intr: ih null") 2817 call %l5 2818 add %sp, CCFSZ + SPOFF, %o0 2819 2820 call critical_exit 2821 nop 2822 2823 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 2824 2825 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2826 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2827 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2828 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2829 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2830 2831 wrpr %g0, PSTATE_ALT, %pstate 2832 2833 andn %l0, TSTATE_CWP_MASK, %g1 2834 mov %l1, %g2 2835 mov %l2, %g3 2836 wrpr %l3, 0, %pil 2837 wr %l4, 0, %y 2838 2839 restore 2840 2841 wrpr %g0, 2, %tl 2842 2843 rdpr %cwp, %g4 2844 wrpr %g1, %g4, %tstate 2845 wrpr %g2, 0, %tpc 2846 wrpr %g3, 0, %tnpc 2847 2848#if KTR_COMPILE & KTR_INTR 2849 CATR(KTR_INTR, "tl1_intr: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2850 , %g2, %g3, %g4, 7, 8, 9) 2851 ldx [PCPU(CURTHREAD)], %g3 2852 stx %g3, [%g2 + KTR_PARM1] 2853 rdpr %pil, %g3 2854 stx %g3, [%g2 + KTR_PARM2] 2855 rdpr %tstate, %g3 2856 stx %g3, [%g2 + KTR_PARM3] 2857 rdpr %tpc, %g3 2858 stx %g3, [%g2 + KTR_PARM4] 2859 stx %sp, [%g2 + KTR_PARM5] 28609: 2861#endif 2862 2863 retry 2864END(tl1_intr) 2865 2866/* 2867 * Freshly forked processes come here when switched to for the first time. 2868 * The arguments to fork_exit() have been setup in the locals, we must move 2869 * them to the outs. 2870 */ 2871ENTRY(fork_trampoline) 2872#if KTR_COMPILE & KTR_PROC 2873 CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" 2874 , %g1, %g2, %g3, 7, 8, 9) 2875 ldx [PCPU(CURTHREAD)], %g2 2876 stx %g2, [%g1 + KTR_PARM1] 2877 ldx [%g2 + TD_PROC], %g2 2878 add %g2, P_COMM, %g2 2879 stx %g2, [%g1 + KTR_PARM2] 2880 rdpr %cwp, %g2 2881 stx %g2, [%g1 + KTR_PARM3] 28829: 2883#endif 2884 mov %l0, %o0 2885 mov %l1, %o1 2886 call fork_exit 2887 mov %l2, %o2 2888 b,a %xcc, tl0_ret 2889 nop 2890END(fork_trampoline) 2891