exception.S revision 105012
1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2001 Jake Burkholder. 32 * All rights reserved. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 46 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 53 * SUCH DAMAGE. 54 * 55 * $FreeBSD: head/sys/sparc64/sparc64/exception.S 105012 2002-10-12 23:43:04Z jake $ 56 */ 57 58#include "opt_ddb.h" 59 60#include <machine/asi.h> 61#include <machine/asmacros.h> 62#include <machine/ktr.h> 63#include <machine/pstate.h> 64#include <machine/trap.h> 65#include <machine/tstate.h> 66#include <machine/wstate.h> 67 68#include "assym.s" 69 70#define TSB_KERNEL_MASK 0x0 71#define TSB_KERNEL 0x0 72 73 .register %g2,#ignore 74 .register %g3,#ignore 75 .register %g6,#ignore 76 .register %g7,#ignore 77 78/* 79 * Atomically set the reference bit in a tte. 80 */ 81#define TTE_SET_BIT(r1, r2, r3, bit) \ 82 add r1, TTE_DATA, r1 ; \ 83 ldx [r1], r2 ; \ 849: or r2, bit, r3 ; \ 85 casxa [r1] ASI_N, r2, r3 ; \ 86 cmp r2, r3 ; \ 87 bne,pn %xcc, 9b ; \ 88 mov r3, r2 89 90#define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF) 91#define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W) 92 93/* 94 * Macros for spilling and filling live windows. 95 * 96 * NOTE: These macros use exactly 16 instructions, and it is assumed that the 97 * handler will not use more than 24 instructions total, to leave room for 98 * resume vectors which occupy the last 8 instructions. 99 */ 100 101#define SPILL(storer, base, size, asi) \ 102 storer %l0, [base + (0 * size)] asi ; \ 103 storer %l1, [base + (1 * size)] asi ; \ 104 storer %l2, [base + (2 * size)] asi ; \ 105 storer %l3, [base + (3 * size)] asi ; \ 106 storer %l4, [base + (4 * size)] asi ; \ 107 storer %l5, [base + (5 * size)] asi ; \ 108 storer %l6, [base + (6 * size)] asi ; \ 109 storer %l7, [base + (7 * size)] asi ; \ 110 storer %i0, [base + (8 * size)] asi ; \ 111 storer %i1, [base + (9 * size)] asi ; \ 112 storer %i2, [base + (10 * size)] asi ; \ 113 storer %i3, [base + (11 * size)] asi ; \ 114 storer %i4, [base + (12 * size)] asi ; \ 115 storer %i5, [base + (13 * size)] asi ; \ 116 storer %i6, [base + (14 * size)] asi ; \ 117 storer %i7, [base + (15 * size)] asi 118 119#define FILL(loader, base, size, asi) \ 120 loader [base + (0 * size)] asi, %l0 ; \ 121 loader [base + (1 * size)] asi, %l1 ; \ 122 loader [base + (2 * size)] asi, %l2 ; \ 123 loader [base + (3 * size)] asi, %l3 ; \ 124 loader [base + (4 * size)] asi, %l4 ; \ 125 loader [base + (5 * size)] asi, %l5 ; \ 126 loader [base + (6 * size)] asi, %l6 ; \ 127 loader [base + (7 * size)] asi, %l7 ; \ 128 loader [base + (8 * size)] asi, %i0 ; \ 129 loader [base + (9 * size)] asi, %i1 ; \ 130 loader [base + (10 * size)] asi, %i2 ; \ 131 loader [base + (11 * size)] asi, %i3 ; \ 132 loader [base + (12 * size)] asi, %i4 ; \ 133 loader [base + (13 * size)] asi, %i5 ; \ 134 loader [base + (14 * size)] asi, %i6 ; \ 135 loader [base + (15 * size)] asi, %i7 136 137#define ERRATUM50(reg) mov reg, reg 138 139#define KSTACK_SLOP 1024 140 141/* 142 * Sanity check the kernel stack and bail out if its wrong. 143 * XXX: doesn't handle being on the panic stack. 144 */ 145#define KSTACK_CHECK \ 146 dec 16, ASP_REG ; \ 147 stx %g1, [ASP_REG + 0] ; \ 148 stx %g2, [ASP_REG + 8] ; \ 149 add %sp, SPOFF, %g1 ; \ 150 andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ 151 bnz,a %xcc, tl1_kstack_fault ; \ 152 inc 16, ASP_REG ; \ 153 ldx [PCPU(CURTHREAD)], %g2 ; \ 154 ldx [%g2 + TD_KSTACK], %g2 ; \ 155 add %g2, KSTACK_SLOP, %g2 ; \ 156 subcc %g1, %g2, %g1 ; \ 157 ble,a %xcc, tl1_kstack_fault ; \ 158 inc 16, ASP_REG ; \ 159 set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ 160 cmp %g1, %g2 ; \ 161 bgt,a %xcc, tl1_kstack_fault ; \ 162 inc 16, ASP_REG ; \ 163 ldx [ASP_REG + 8], %g2 ; \ 164 ldx [ASP_REG + 0], %g1 ; \ 165 inc 16, ASP_REG 166 167ENTRY(tl1_kstack_fault) 168 rdpr %tl, %g1 1691: cmp %g1, 2 170 be,a 2f 171 nop 172 173#if KTR_COMPILE & KTR_TRAP 174 CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" 175 , %g2, %g3, %g4, 7, 8, 9) 176 rdpr %tl, %g3 177 stx %g3, [%g2 + KTR_PARM1] 178 rdpr %tpc, %g3 179 stx %g3, [%g2 + KTR_PARM1] 180 rdpr %tnpc, %g3 181 stx %g3, [%g2 + KTR_PARM1] 1829: 183#endif 184 185 sub %g1, 1, %g1 186 wrpr %g1, 0, %tl 187 ba,a %xcc, 1b 188 nop 189 1902: 191#if KTR_COMPILE & KTR_TRAP 192 CATR(KTR_TRAP, 193 "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" 194 , %g1, %g2, %g3, 7, 8, 9) 195 add %sp, SPOFF, %g2 196 stx %g2, [%g1 + KTR_PARM1] 197 ldx [PCPU(CURTHREAD)], %g2 198 ldx [%g2 + TD_KSTACK], %g2 199 stx %g2, [%g1 + KTR_PARM2] 200 rdpr %canrestore, %g2 201 stx %g2, [%g1 + KTR_PARM3] 202 rdpr %cansave, %g2 203 stx %g2, [%g1 + KTR_PARM4] 204 rdpr %otherwin, %g2 205 stx %g2, [%g1 + KTR_PARM5] 206 rdpr %wstate, %g2 207 stx %g2, [%g1 + KTR_PARM6] 2089: 209#endif 210 211 wrpr %g0, 0, %canrestore 212 wrpr %g0, 6, %cansave 213 wrpr %g0, 0, %otherwin 214 wrpr %g0, WSTATE_KERNEL, %wstate 215 216 sub ASP_REG, SPOFF + CCFSZ, %sp 217 clr %fp 218 219 set trap, %o2 220 b %xcc, tl1_trap 221 mov T_KSTACK_FAULT | T_KERNEL, %o0 222END(tl1_kstack_fault) 223 224/* 225 * Magic to resume from a spill or fill trap. If we get an alignment or an 226 * mmu fault during a spill or a fill, this macro will detect the fault and 227 * resume at a set instruction offset in the trap handler. 228 * 229 * To check if the previous trap was a spill/fill we convert the trapped pc 230 * to a trap type and verify that it is in the range of spill/fill vectors. 231 * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the 232 * tl bit allows us to detect both ranges with one test. 233 * 234 * This is: 235 * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 236 * 237 * To calculate the new pc we take advantage of the xor feature of wrpr. 238 * Forcing all the low bits of the trapped pc on we can produce any offset 239 * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. 240 * 241 * 0x7f ^ 0x1f == 0x60 242 * 0x1f == (0x80 - 0x60) - 1 243 * 244 * Which are the offset and xor value used to resume from alignment faults. 245 */ 246 247/* 248 * Determine if we have trapped inside of a spill/fill vector, and if so resume 249 * at a fixed instruction offset in the trap vector. Must be called on 250 * alternate globals. 251 */ 252#define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ 253 dec 16, ASP_REG ; \ 254 stx %g1, [ASP_REG + 0] ; \ 255 stx %g2, [ASP_REG + 8] ; \ 256 rdpr %tpc, %g1 ; \ 257 ERRATUM50(%g1) ; \ 258 rdpr %tba, %g2 ; \ 259 sub %g1, %g2, %g2 ; \ 260 srlx %g2, 5, %g2 ; \ 261 andn %g2, 0x200, %g2 ; \ 262 cmp %g2, 0x80 ; \ 263 blu,pt %xcc, 9f ; \ 264 cmp %g2, 0x100 ; \ 265 bgeu,pt %xcc, 9f ; \ 266 or %g1, 0x7f, %g1 ; \ 267 wrpr %g1, xor, %tnpc ; \ 268 stxa_g0_sfsr ; \ 269 ldx [ASP_REG + 8], %g2 ; \ 270 ldx [ASP_REG + 0], %g1 ; \ 271 inc 16, ASP_REG ; \ 272 done ; \ 2739: ldx [ASP_REG + 8], %g2 ; \ 274 ldx [ASP_REG + 0], %g1 ; \ 275 inc 16, ASP_REG 276 277/* 278 * For certain faults we need to clear the sfsr mmu register before returning. 279 */ 280#define RSF_CLR_SFSR \ 281 wr %g0, ASI_DMMU, %asi ; \ 282 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 283 284#define RSF_XOR(off) ((0x80 - off) - 1) 285 286/* 287 * Instruction offsets in spill and fill trap handlers for handling certain 288 * nested traps, and corresponding xor constants for wrpr. 289 */ 290#define RSF_OFF_ALIGN 0x60 291#define RSF_OFF_MMU 0x70 292 293#define RESUME_SPILLFILL_ALIGN \ 294 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) 295#define RESUME_SPILLFILL_MMU \ 296 RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) 297#define RESUME_SPILLFILL_MMU_CLR_SFSR \ 298 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) 299 300/* 301 * Constant to add to %tnpc when taking a fill trap just before returning to 302 * user mode. 303 */ 304#define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill 305 306/* 307 * Retry a spill or fill with a different wstate due to an alignment fault. 308 * We may just be using the wrong stack offset. 309 */ 310#define RSF_ALIGN_RETRY(ws) \ 311 wrpr %g0, (ws), %wstate ; \ 312 retry ; \ 313 .align 16 314 315/* 316 * Generate a T_SPILL or T_FILL trap if the window operation fails. 317 */ 318#define RSF_TRAP(type) \ 319 b %xcc, tl0_sftrap ; \ 320 mov type, %g2 ; \ 321 .align 16 322 323/* 324 * Game over if the window operation fails. 325 */ 326#define RSF_FATAL(type) \ 327 b %xcc, rsf_fatal ; \ 328 mov type, %g2 ; \ 329 .align 16 330 331/* 332 * Magic to resume from a failed fill a few instructions after the corrsponding 333 * restore. This is used on return from the kernel to usermode. 334 */ 335#define RSF_FILL_MAGIC \ 336 rdpr %tnpc, %g1 ; \ 337 add %g1, RSF_FILL_INC, %g1 ; \ 338 wrpr %g1, 0, %tnpc ; \ 339 done ; \ 340 .align 16 341 342/* 343 * Spill to the pcb if a spill to the user stack in kernel mode fails. 344 */ 345#define RSF_SPILL_TOPCB \ 346 b,a %xcc, tl1_spill_topcb ; \ 347 nop ; \ 348 .align 16 349 350ENTRY(rsf_fatal) 351#if KTR_COMPILE & KTR_TRAP 352 CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" 353 , %g1, %g3, %g4, 7, 8, 9) 354 rdpr %tt, %g3 355 stx %g3, [%g1 + KTR_PARM1] 356 stx %g2, [%g1 + KTR_PARM2] 3579: 358#endif 359 360 KSTACK_CHECK 361 362 sir 363END(rsf_fatal) 364 365 .comm intrnames, IV_MAX * 8 366 .comm eintrnames, 0 367 368 .comm intrcnt, IV_MAX * 8 369 .comm eintrcnt, 0 370 371/* 372 * Trap table and associated macros 373 * 374 * Due to its size a trap table is an inherently hard thing to represent in 375 * code in a clean way. There are approximately 1024 vectors, of 8 or 32 376 * instructions each, many of which are identical. The way that this is 377 * layed out is the instructions (8 or 32) for the actual trap vector appear 378 * as an AS macro. In general this code branches to tl0_trap or tl1_trap, 379 * but if not supporting code can be placed just after the definition of the 380 * macro. The macros are then instantiated in a different section (.trap), 381 * which is setup to be placed by the linker at the beginning of .text, and the 382 * code around the macros is moved to the end of trap table. In this way the 383 * code that must be sequential in memory can be split up, and located near 384 * its supporting code so that it is easier to follow. 385 */ 386 387 /* 388 * Clean window traps occur when %cleanwin is zero to ensure that data 389 * is not leaked between address spaces in registers. 390 */ 391 .macro clean_window 392 clr %o0 393 clr %o1 394 clr %o2 395 clr %o3 396 clr %o4 397 clr %o5 398 clr %o6 399 clr %o7 400 clr %l0 401 clr %l1 402 clr %l2 403 clr %l3 404 clr %l4 405 clr %l5 406 clr %l6 407 rdpr %cleanwin, %l7 408 inc %l7 409 wrpr %l7, 0, %cleanwin 410 clr %l7 411 retry 412 .align 128 413 .endm 414 415 /* 416 * Stack fixups for entry from user mode. We are still running on the 417 * user stack, and with its live registers, so we must save soon. We 418 * are on alternate globals so we do have some registers. Set the 419 * transitional window state, and do the save. If this traps we 420 * we attempt to spill a window to the user stack. If this fails, 421 * we spill the window to the pcb and continue. Spilling to the pcb 422 * must not fail. 423 * 424 * NOTE: Must be called with alternate globals and clobbers %g1. 425 */ 426 427 .macro tl0_split 428 rdpr %wstate, %g1 429 wrpr %g1, WSTATE_TRANSITION, %wstate 430 save 431 .endm 432 433 .macro tl0_setup type 434 tl0_split 435 set trap, %o2 436 ba %xcc, tl0_utrap 437 mov \type, %o0 438 .endm 439 440 /* 441 * Generic trap type. Call trap() with the specified type. 442 */ 443 .macro tl0_gen type 444 tl0_setup \type 445 .align 32 446 .endm 447 448 /* 449 * This is used to suck up the massive swaths of reserved trap types. 450 * Generates count "reserved" trap vectors. 451 */ 452 .macro tl0_reserved count 453 .rept \count 454 tl0_gen T_RESERVED 455 .endr 456 .endm 457 458 .macro tl0_fp_restore 459 wr %g0, FPRS_FEF, %fprs 460 wr %g0, ASI_BLK_S, %asi 461 ldda [PCB_REG + PCB_FPSTATE + FP_FB0] %asi, %f0 462 ldda [PCB_REG + PCB_FPSTATE + FP_FB1] %asi, %f16 463 ldda [PCB_REG + PCB_FPSTATE + FP_FB2] %asi, %f32 464 ldda [PCB_REG + PCB_FPSTATE + FP_FB3] %asi, %f48 465 membar #Sync 466 done 467 .align 32 468 .endm 469 470 .macro tl0_insn_excptn 471 wrpr %g0, PSTATE_ALT, %pstate 472 wr %g0, ASI_IMMU, %asi 473 rdpr %tpc, %g3 474 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 475 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 476 membar #Sync 477 b %xcc, tl0_sfsr_trap 478 mov T_INSTRUCTION_EXCEPTION, %g2 479 .align 32 480 .endm 481 482 .macro tl0_data_excptn 483 wrpr %g0, PSTATE_ALT, %pstate 484 wr %g0, ASI_DMMU, %asi 485 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 486 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 487 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 488 membar #Sync 489 b %xcc, tl0_sfsr_trap 490 mov T_DATA_EXCEPTION, %g2 491 .align 32 492 .endm 493 494 .macro tl0_align 495 wr %g0, ASI_DMMU, %asi 496 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 497 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 498 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 499 membar #Sync 500 b %xcc, tl0_sfsr_trap 501 mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 502 .align 32 503 .endm 504 505ENTRY(tl0_sfsr_trap) 506 tl0_split 507 set trap, %o2 508 mov %g3, %o4 509 mov %g4, %o5 510 ba %xcc, tl0_utrap 511 mov %g2, %o0 512END(tl0_sfsr_trap) 513 514 .macro tl0_intr level, mask 515 tl0_split 516 set \mask, %o1 517 b %xcc, tl0_intr 518 mov \level, %o0 519 .align 32 520 .endm 521 522#define INTR(level, traplvl) \ 523 tl ## traplvl ## _intr level, 1 << level 524 525#define TICK(traplvl) \ 526 tl ## traplvl ## _intr PIL_TICK, 1 527 528#define INTR_LEVEL(tl) \ 529 INTR(1, tl) ; \ 530 INTR(2, tl) ; \ 531 INTR(3, tl) ; \ 532 INTR(4, tl) ; \ 533 INTR(5, tl) ; \ 534 INTR(6, tl) ; \ 535 INTR(7, tl) ; \ 536 INTR(8, tl) ; \ 537 INTR(9, tl) ; \ 538 INTR(10, tl) ; \ 539 INTR(11, tl) ; \ 540 INTR(12, tl) ; \ 541 INTR(13, tl) ; \ 542 TICK(tl) ; \ 543 INTR(15, tl) ; 544 545 .macro tl0_intr_level 546 INTR_LEVEL(0) 547 .endm 548 549 .macro intr_vector 550 ldxa [%g0] ASI_INTR_RECEIVE, %g1 551 andcc %g1, IRSR_BUSY, %g0 552 bnz,a,pt %xcc, intr_vector 553 nop 554 sir 555 .align 32 556 .endm 557 558 .macro immu_miss_user 559 /* 560 * Initialize the page size walker. 561 */ 562 mov TS_MIN, %g2 563 564 /* 565 * Loop over all supported page sizes. 566 */ 567 568 /* 569 * Compute the page shift for the page size we are currently looking 570 * for. 571 */ 5721: add %g2, %g2, %g3 573 add %g3, %g2, %g3 574 add %g3, PAGE_SHIFT, %g3 575 576 /* 577 * Extract the virtual page number from the contents of the tag 578 * access register. 579 */ 580 srlx %g1, %g3, %g3 581 582 /* 583 * Compute the tte bucket address. 584 */ 585 ldxa [%g0 + AA_IMMU_TSB] %asi, %g5 586 and %g3, TSB_BUCKET_MASK, %g4 587 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 588 add %g4, %g5, %g4 589 590 /* 591 * Compute the tte tag target. 592 */ 593 sllx %g3, TV_SIZE_BITS, %g3 594 or %g3, %g2, %g3 595 596 /* 597 * Loop over the ttes in this bucket 598 */ 599 600 /* 601 * Load the tte. Note that this instruction may fault, clobbering 602 * the contents of the tag access register, %g5, %g6, and %g7. We 603 * do not use %g5, and %g6 and %g7 are not used until this instruction 604 * completes successfully. 605 */ 6062: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 607 608 /* 609 * Check that its valid and executable and that the tte tags match. 610 */ 611 brgez,pn %g7, 3f 612 andcc %g7, TD_EXEC, %g0 613 bz,pn %xcc, 3f 614 cmp %g3, %g6 615 bne,pn %xcc, 3f 616 EMPTY 617 618 /* 619 * We matched a tte, load the tlb. 620 */ 621 622 /* 623 * Set the reference bit, if it's currently clear. 624 */ 625 andcc %g7, TD_REF, %g0 626 bz,a,pn %xcc, tl0_immu_miss_set_ref 627 nop 628 629 /* 630 * Load the tte tag and data into the tlb and retry the instruction. 631 */ 632 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 633 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 634 retry 635 636 /* 637 * Advance to the next tte in this bucket, and check the low bits 638 * of the bucket pointer to see if we've finished the bucket. 639 */ 6403: add %g4, 1 << TTE_SHIFT, %g4 641 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 642 bnz,pt %xcc, 2b 643 EMPTY 644 645 /* 646 * See if we just checked the largest page size, and advance to the 647 * next one if not. 648 */ 649 cmp %g2, TS_MAX 650 bne,pt %xcc, 1b 651 add %g2, 1, %g2 652 .endm 653 654 .macro tl0_immu_miss 655 /* 656 * Load the virtual page number and context from the tag access 657 * register. We ignore the context. 658 */ 659 wr %g0, ASI_IMMU, %asi 660 ldxa [%g0 + AA_IMMU_TAR] %asi, %g1 661 662 /* 663 * Try a fast inline lookup of the user tsb. 664 */ 665 immu_miss_user 666 667 /* 668 * Not in user tsb, call c code. 669 */ 670 ba,a %xcc, tl0_immu_miss_trap 671 .align 128 672 .endm 673 674ENTRY(tl0_immu_miss_set_ref) 675 /* 676 * Set the reference bit. 677 */ 678 TTE_SET_REF(%g4, %g2, %g3) 679 680 /* 681 * May have become invalid during casxa, in which case start over. 682 */ 683 brgez,pn %g2, 1f 684 nop 685 686 /* 687 * Load the tte tag and data into the tlb and retry the instruction. 688 */ 689 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 690 stxa %g2, [%g0] ASI_ITLB_DATA_IN_REG 6911: retry 692END(tl0_immu_miss_set_ref) 693 694ENTRY(tl0_immu_miss_trap) 695 /* 696 * Put back the contents of the tag access register, in case we 697 * faulted. 698 */ 699 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 700 membar #Sync 701 702 /* 703 * Switch to alternate globals. 704 */ 705 wrpr %g0, PSTATE_ALT, %pstate 706 707 /* 708 * Reload the tag access register. 709 */ 710 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 711 712 /* 713 * Save the tag access register, and call common trap code. 714 */ 715 tl0_split 716 set trap, %o2 717 mov %g2, %o3 718 b %xcc, tl0_trap 719 mov T_INSTRUCTION_MISS, %o0 720END(tl0_immu_miss_trap) 721 722 .macro dmmu_miss_user 723 /* 724 * Initialize the page size walker. 725 */ 726 mov TS_MIN, %g2 727 728 /* 729 * Loop over all supported page sizes. 730 */ 731 732 /* 733 * Compute the page shift for the page size we are currently looking 734 * for. 735 */ 7361: add %g2, %g2, %g3 737 add %g3, %g2, %g3 738 add %g3, PAGE_SHIFT, %g3 739 740 /* 741 * Extract the virtual page number from the contents of the tag 742 * access register. 743 */ 744 srlx %g1, %g3, %g3 745 746 /* 747 * Compute the tte bucket address. 748 */ 749 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 750 and %g3, TSB_BUCKET_MASK, %g4 751 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 752 add %g4, %g5, %g4 753 754 /* 755 * Compute the tte tag target. 756 */ 757 sllx %g3, TV_SIZE_BITS, %g3 758 or %g3, %g2, %g3 759 760 /* 761 * Loop over the ttes in this bucket 762 */ 763 764 /* 765 * Load the tte. Note that this instruction may fault, clobbering 766 * the contents of the tag access register, %g5, %g6, and %g7. We 767 * do not use %g5, and %g6 and %g7 are not used until this instruction 768 * completes successfully. 769 */ 7702: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 771 772 /* 773 * Check that its valid and that the virtual page numbers match. 774 */ 775 brgez,pn %g7, 3f 776 cmp %g3, %g6 777 bne,pn %xcc, 3f 778 EMPTY 779 780 /* 781 * We matched a tte, load the tlb. 782 */ 783 784 /* 785 * Set the reference bit, if it's currently clear. 786 */ 787 andcc %g7, TD_REF, %g0 788 bz,a,pn %xcc, dmmu_miss_user_set_ref 789 nop 790 791 /* 792 * Load the tte tag and data into the tlb and retry the instruction. 793 */ 794 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 795 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 796 retry 797 798 /* 799 * Advance to the next tte in this bucket, and check the low bits 800 * of the bucket pointer to see if we've finished the bucket. 801 */ 8023: add %g4, 1 << TTE_SHIFT, %g4 803 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 804 bnz,pt %xcc, 2b 805 EMPTY 806 807 /* 808 * See if we just checked the largest page size, and advance to the 809 * next one if not. 810 */ 811 cmp %g2, TS_MAX 812 bne,pt %xcc, 1b 813 add %g2, 1, %g2 814 .endm 815 816ENTRY(dmmu_miss_user_set_ref) 817 /* 818 * Set the reference bit. 819 */ 820 TTE_SET_REF(%g4, %g2, %g3) 821 822 /* 823 * May have become invalid during casxa, in which case start over. 824 */ 825 brgez,pn %g2, 1f 826 nop 827 828 /* 829 * Load the tte tag and data into the tlb and retry the instruction. 830 */ 831 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 832 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 8331: retry 834END(dmmu_miss_user_set_ref) 835 836 .macro tl0_dmmu_miss 837 /* 838 * Load the virtual page number and context from the tag access 839 * register. We ignore the context. 840 */ 841 wr %g0, ASI_DMMU, %asi 842 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 843 844 /* 845 * Try a fast inline lookup of the primary tsb. 846 */ 847 dmmu_miss_user 848 849 /* 850 * Not in user tsb, call c code. 851 */ 852 ba,a %xcc, tl0_dmmu_miss_trap 853 .align 128 854 .endm 855 856ENTRY(tl0_dmmu_miss_trap) 857 /* 858 * Put back the contents of the tag access register, in case we 859 * faulted. 860 */ 861 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 862 membar #Sync 863 864 /* 865 * Switch to alternate globals. 866 */ 867 wrpr %g0, PSTATE_ALT, %pstate 868 869 /* 870 * Reload the tag access register. 871 */ 872 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 873 874 /* 875 * Save the tag access register and call common trap code. 876 */ 877 tl0_split 878 set trap, %o2 879 mov %g2, %o3 880 b %xcc, tl0_trap 881 mov T_DATA_MISS, %o0 882END(tl0_dmmu_miss_trap) 883 884 .macro dmmu_prot_user 885 /* 886 * Initialize the page size walker. 887 */ 888 mov TS_MIN, %g2 889 890 /* 891 * Loop over all supported page sizes. 892 */ 893 894 /* 895 * Compute the page shift for the page size we are currently looking 896 * for. 897 */ 8981: add %g2, %g2, %g3 899 add %g3, %g2, %g3 900 add %g3, PAGE_SHIFT, %g3 901 902 /* 903 * Extract the virtual page number from the contents of the tag 904 * access register. 905 */ 906 srlx %g1, %g3, %g3 907 908 /* 909 * Compute the tte bucket address. 910 */ 911 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 912 and %g3, TSB_BUCKET_MASK, %g4 913 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 914 add %g4, %g5, %g4 915 916 /* 917 * Compute the tte tag target. 918 */ 919 sllx %g3, TV_SIZE_BITS, %g3 920 or %g3, %g2, %g3 921 922 /* 923 * Loop over the ttes in this bucket 924 */ 925 926 /* 927 * Load the tte. Note that this instruction may fault, clobbering 928 * the contents of the tag access register, %g5, %g6, and %g7. We 929 * do not use %g5, and %g6 and %g7 are not used until this instruction 930 * completes successfully. 931 */ 9322: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 933 934 /* 935 * Check that its valid and writable and that the virtual page 936 * numbers match. 937 */ 938 brgez,pn %g7, 4f 939 andcc %g7, TD_SW, %g0 940 bz,pn %xcc, 4f 941 cmp %g3, %g6 942 bne,pn %xcc, 4f 943 nop 944 945 /* 946 * Set the hardware write bit. 947 */ 948 TTE_SET_W(%g4, %g2, %g3) 949 950 /* 951 * Delete the old TLB entry and clear the sfsr. 952 */ 953 srlx %g1, PAGE_SHIFT, %g3 954 sllx %g3, PAGE_SHIFT, %g3 955 stxa %g0, [%g3] ASI_DMMU_DEMAP 956 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 957 membar #Sync 958 959 /* 960 * May have become invalid during casxa, in which case start over. 961 */ 962 brgez,pn %g2, 3f 963 or %g2, TD_W, %g2 964 965 /* 966 * Load the tte data into the tlb and retry the instruction. 967 */ 968 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 969 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 9703: retry 971 972 /* 973 * Check the low bits to see if we've finished the bucket. 974 */ 9754: add %g4, 1 << TTE_SHIFT, %g4 976 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 977 bnz,pt %xcc, 2b 978 EMPTY 979 980 /* 981 * See if we just checked the largest page size, and advance to the 982 * next one if not. 983 */ 984 cmp %g2, TS_MAX 985 bne,pt %xcc, 1b 986 add %g2, 1, %g2 987 .endm 988 989 .macro tl0_dmmu_prot 990 ba,a %xcc, tl0_dmmu_prot_1 991 nop 992 .align 128 993 .endm 994 995ENTRY(tl0_dmmu_prot_1) 996 /* 997 * Load the virtual page number and context from the tag access 998 * register. We ignore the context. 999 */ 1000 wr %g0, ASI_DMMU, %asi 1001 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 1002 1003 /* 1004 * Try a fast inline lookup of the tsb. 1005 */ 1006 dmmu_prot_user 1007 1008 /* 1009 * Not in user tsb, call c code. 1010 */ 1011 b,a %xcc, tl0_dmmu_prot_trap 1012 nop 1013END(tl0_dmmu_prot_1) 1014 1015ENTRY(tl0_dmmu_prot_trap) 1016 /* 1017 * Put back the contents of the tag access register, in case we 1018 * faulted. 1019 */ 1020 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1021 membar #Sync 1022 1023 /* 1024 * Switch to alternate globals. 1025 */ 1026 wrpr %g0, PSTATE_ALT, %pstate 1027 1028 /* 1029 * Load the tar, sfar and sfsr. 1030 */ 1031 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1032 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1033 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1034 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1035 membar #Sync 1036 1037 /* 1038 * Save the mmu registers and call common trap code. 1039 */ 1040 tl0_split 1041 set trap, %o2 1042 mov %g2, %o3 1043 mov %g3, %o4 1044 mov %g4, %o5 1045 ba %xcc, tl0_utrap 1046 mov T_DATA_PROTECTION, %o0 1047END(tl0_dmmu_prot_trap) 1048 1049 .macro tl0_spill_0_n 1050 wr %g0, ASI_AIUP, %asi 1051 SPILL(stxa, %sp + SPOFF, 8, %asi) 1052 saved 1053 retry 1054 .align 32 1055 RSF_TRAP(T_SPILL) 1056 RSF_TRAP(T_SPILL) 1057 .endm 1058 1059 .macro tl0_spill_1_n 1060 wr %g0, ASI_AIUP, %asi 1061 SPILL(stwa, %sp, 4, %asi) 1062 saved 1063 retry 1064 .align 32 1065 RSF_TRAP(T_SPILL) 1066 RSF_TRAP(T_SPILL) 1067 .endm 1068 1069 .macro tl0_fill_0_n 1070 wr %g0, ASI_AIUP, %asi 1071 FILL(ldxa, %sp + SPOFF, 8, %asi) 1072 restored 1073 retry 1074 .align 32 1075 RSF_TRAP(T_FILL) 1076 RSF_TRAP(T_FILL) 1077 .endm 1078 1079 .macro tl0_fill_1_n 1080 wr %g0, ASI_AIUP, %asi 1081 FILL(lduwa, %sp, 4, %asi) 1082 restored 1083 retry 1084 .align 32 1085 RSF_TRAP(T_FILL) 1086 RSF_TRAP(T_FILL) 1087 .endm 1088 1089ENTRY(tl0_sftrap) 1090 rdpr %tstate, %g1 1091 and %g1, TSTATE_CWP_MASK, %g1 1092 wrpr %g1, 0, %cwp 1093 tl0_split 1094 set trap, %o2 1095 b %xcc, tl0_trap 1096 mov %g2, %o0 1097END(tl0_sftrap) 1098 1099 .macro tl0_spill_bad count 1100 .rept \count 1101 sir 1102 .align 128 1103 .endr 1104 .endm 1105 1106 .macro tl0_fill_bad count 1107 .rept \count 1108 sir 1109 .align 128 1110 .endr 1111 .endm 1112 1113 .macro tl0_syscall 1114 tl0_split 1115 set syscall, %o2 1116 ba %xcc, tl0_trap 1117 mov T_SYSCALL, %o0 1118 .align 32 1119 .endm 1120 1121 .macro tl0_soft count 1122 .rept \count 1123 tl0_gen T_SOFT 1124 .endr 1125 .endm 1126 1127 .macro tl1_split 1128 rdpr %wstate, %g1 1129 wrpr %g1, WSTATE_NESTED, %wstate 1130 save %sp, -(CCFSZ + TF_SIZEOF), %sp 1131 .endm 1132 1133 .macro tl1_setup type 1134 tl1_split 1135 clr %o1 1136 set trap, %o2 1137 b %xcc, tl1_trap 1138 mov \type | T_KERNEL, %o0 1139 .endm 1140 1141 .macro tl1_gen type 1142 tl1_setup \type 1143 .align 32 1144 .endm 1145 1146 .macro tl1_reserved count 1147 .rept \count 1148 tl1_gen T_RESERVED 1149 .endr 1150 .endm 1151 1152 .macro tl1_insn_excptn 1153 wrpr %g0, PSTATE_ALT, %pstate 1154 wr %g0, ASI_IMMU, %asi 1155 rdpr %tpc, %g3 1156 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 1157 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 1158 membar #Sync 1159 b %xcc, tl1_insn_exceptn_trap 1160 mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 1161 .align 32 1162 .endm 1163 1164ENTRY(tl1_insn_exceptn_trap) 1165 tl1_split 1166 clr %o1 1167 set trap, %o2 1168 mov %g3, %o4 1169 mov %g4, %o5 1170 b %xcc, tl1_trap 1171 mov %g2, %o0 1172END(tl1_insn_exceptn_trap) 1173 1174 .macro tl1_data_excptn 1175 wrpr %g0, PSTATE_ALT, %pstate 1176 b,a %xcc, tl1_data_excptn_trap 1177 nop 1178 .align 32 1179 .endm 1180 1181ENTRY(tl1_data_excptn_trap) 1182 RESUME_SPILLFILL_MMU_CLR_SFSR 1183 b %xcc, tl1_sfsr_trap 1184 mov T_DATA_EXCEPTION | T_KERNEL, %g2 1185END(tl1_data_excptn_trap) 1186 1187 .macro tl1_align 1188 b,a %xcc, tl1_align_trap 1189 nop 1190 .align 32 1191 .endm 1192 1193ENTRY(tl1_align_trap) 1194 RESUME_SPILLFILL_ALIGN 1195 b %xcc, tl1_sfsr_trap 1196 mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 1197END(tl1_data_excptn_trap) 1198 1199ENTRY(tl1_sfsr_trap) 1200 wr %g0, ASI_DMMU, %asi 1201 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1202 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1203 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1204 membar #Sync 1205 1206 tl1_split 1207 clr %o1 1208 set trap, %o2 1209 mov %g3, %o4 1210 mov %g4, %o5 1211 b %xcc, tl1_trap 1212 mov %g2, %o0 1213END(tl1_sfsr_trap) 1214 1215 .macro tl1_intr level, mask 1216 tl1_split 1217 set \mask, %o1 1218 b %xcc, tl1_intr 1219 mov \level, %o0 1220 .align 32 1221 .endm 1222 1223 .macro tl1_intr_level 1224 INTR_LEVEL(1) 1225 .endm 1226 1227 .macro tl1_immu_miss 1228 /* 1229 * Load the context and the virtual page number from the tag access 1230 * register. We ignore the context. 1231 */ 1232 wr %g0, ASI_IMMU, %asi 1233 ldxa [%g0 + AA_IMMU_TAR] %asi, %g5 1234 1235 /* 1236 * Compute the address of the tte. The tsb mask and address of the 1237 * tsb are patched at startup. 1238 */ 1239 .globl tl1_immu_miss_patch_1 1240tl1_immu_miss_patch_1: 1241 sethi %hi(TSB_KERNEL_MASK), %g6 1242 or %g6, %lo(TSB_KERNEL_MASK), %g6 1243 sethi %hi(TSB_KERNEL), %g7 1244 1245 srlx %g5, TAR_VPN_SHIFT, %g5 1246 and %g5, %g6, %g6 1247 sllx %g6, TTE_SHIFT, %g6 1248 add %g6, %g7, %g6 1249 1250 /* 1251 * Load the tte. 1252 */ 1253 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1254 1255 /* 1256 * Check that its valid and executable and that the virtual page 1257 * numbers match. 1258 */ 1259 brgez,pn %g7, tl1_immu_miss_trap 1260 andcc %g7, TD_EXEC, %g0 1261 bz,pn %xcc, tl1_immu_miss_trap 1262 srlx %g6, TV_SIZE_BITS, %g6 1263 cmp %g5, %g6 1264 bne,pn %xcc, tl1_immu_miss_trap 1265 EMPTY 1266 1267 /* 1268 * Set the reference bit if its currently clear. 1269 */ 1270 andcc %g7, TD_REF, %g0 1271 bz,a,pn %xcc, tl1_immu_miss_set_ref 1272 nop 1273 1274 /* 1275 * Load the tte data into the TLB and retry the instruction. 1276 */ 1277 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 1278 retry 1279 .align 128 1280 .endm 1281 1282ENTRY(tl1_immu_miss_set_ref) 1283 /* 1284 * Recompute the tte address, which we clobbered loading the tte. The 1285 * tsb mask and address of the tsb are patched at startup. 1286 */ 1287 .globl tl1_immu_miss_patch_2 1288tl1_immu_miss_patch_2: 1289 sethi %hi(TSB_KERNEL_MASK), %g6 1290 or %g6, %lo(TSB_KERNEL_MASK), %g6 1291 sethi %hi(TSB_KERNEL), %g7 1292 1293 and %g5, %g6, %g5 1294 sllx %g5, TTE_SHIFT, %g5 1295 add %g5, %g7, %g5 1296 1297 /* 1298 * Set the reference bit. 1299 */ 1300 TTE_SET_REF(%g5, %g6, %g7) 1301 1302 /* 1303 * May have become invalid during casxa, in which case start over. 1304 */ 1305 brgez,pn %g6, 1f 1306 nop 1307 1308 /* 1309 * Load the tte data into the TLB and retry the instruction. 1310 */ 1311 stxa %g6, [%g0] ASI_ITLB_DATA_IN_REG 13121: retry 1313END(tl1_immu_miss_set_ref) 1314 1315ENTRY(tl1_immu_miss_trap) 1316 /* 1317 * Switch to alternate globals. 1318 */ 1319 wrpr %g0, PSTATE_ALT, %pstate 1320 1321 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 1322 1323 tl1_split 1324 clr %o1 1325 set trap, %o2 1326 mov %g2, %o3 1327 b %xcc, tl1_trap 1328 mov T_INSTRUCTION_MISS | T_KERNEL, %o0 1329END(tl1_immu_miss_trap) 1330 1331 .macro tl1_dmmu_miss 1332 /* 1333 * Load the context and the virtual page number from the tag access 1334 * register. 1335 */ 1336 wr %g0, ASI_DMMU, %asi 1337 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1338 1339 /* 1340 * Extract the context from the contents of the tag access register. 1341 * If its non-zero this is a fault on a user address. Note that the 1342 * faulting address is passed in %g2. 1343 */ 1344 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1345 brnz,a,pn %g6, tl1_dmmu_miss_user 1346 mov %g5, %g1 1347 1348 /* 1349 * Check for the direct mapped physical region. These addresses have 1350 * the high bit set so they are negative. 1351 */ 1352 brlz,pn %g5, tl1_dmmu_miss_direct 1353 EMPTY 1354 1355 /* 1356 * Compute the address of the tte. The tsb mask and address of the 1357 * tsb are patched at startup. 1358 */ 1359 .globl tl1_dmmu_miss_patch_1 1360tl1_dmmu_miss_patch_1: 1361 sethi %hi(TSB_KERNEL_MASK), %g6 1362 or %g6, %lo(TSB_KERNEL_MASK), %g6 1363 sethi %hi(TSB_KERNEL), %g7 1364 1365 srlx %g5, TAR_VPN_SHIFT, %g5 1366 and %g5, %g6, %g6 1367 sllx %g6, TTE_SHIFT, %g6 1368 add %g6, %g7, %g6 1369 1370 /* 1371 * Load the tte. 1372 */ 1373 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1374 1375 /* 1376 * Check that its valid and that the virtual page numbers match. 1377 */ 1378 brgez,pn %g7, tl1_dmmu_miss_trap 1379 srlx %g6, TV_SIZE_BITS, %g6 1380 cmp %g5, %g6 1381 bne,pn %xcc, tl1_dmmu_miss_trap 1382 EMPTY 1383 1384 /* 1385 * Set the reference bit if its currently clear. 1386 */ 1387 andcc %g7, TD_REF, %g0 1388 bz,a,pt %xcc, tl1_dmmu_miss_set_ref 1389 nop 1390 1391 /* 1392 * Load the tte data into the TLB and retry the instruction. 1393 */ 1394 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 1395 retry 1396 .align 128 1397 .endm 1398 1399ENTRY(tl1_dmmu_miss_set_ref) 1400 /* 1401 * Recompute the tte address, which we clobbered loading the tte. The 1402 * tsb mask and address of the tsb are patched at startup. 1403 */ 1404 .globl tl1_dmmu_miss_patch_2 1405tl1_dmmu_miss_patch_2: 1406 sethi %hi(TSB_KERNEL_MASK), %g6 1407 or %g6, %lo(TSB_KERNEL_MASK), %g6 1408 sethi %hi(TSB_KERNEL), %g7 1409 1410 and %g5, %g6, %g5 1411 sllx %g5, TTE_SHIFT, %g5 1412 add %g5, %g7, %g5 1413 1414 /* 1415 * Set the reference bit. 1416 */ 1417 TTE_SET_REF(%g5, %g6, %g7) 1418 1419 /* 1420 * May have become invalid during casxa, in which case start over. 1421 */ 1422 brgez,pn %g6, 1f 1423 nop 1424 1425 /* 1426 * Load the tte data into the TLB and retry the instruction. 1427 */ 1428 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 14291: retry 1430END(tl1_dmmu_miss_set_ref) 1431 1432ENTRY(tl1_dmmu_miss_trap) 1433 /* 1434 * Switch to alternate globals. 1435 */ 1436 wrpr %g0, PSTATE_ALT, %pstate 1437 1438 KSTACK_CHECK 1439 1440 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1441 1442 tl1_split 1443 clr %o1 1444 set trap, %o2 1445 mov %g2, %o3 1446 b %xcc, tl1_trap 1447 mov T_DATA_MISS | T_KERNEL, %o0 1448END(tl1_dmmu_miss_trap) 1449 1450ENTRY(tl1_dmmu_miss_direct) 1451 /* 1452 * Check the cache bits in the virtual address to see if this mapping 1453 * is virtually cacheable. We set this up so that the masks fit in 1454 * immediates... Note that the arithmetic shift sign extends, keeping 1455 * all the top bits set. 1456 */ 1457 srax %g5, TLB_DIRECT_SHIFT, %g5 1458 andcc %g5, TLB_DIRECT_UNCACHEABLE, %g0 1459 mov TD_CP | TD_CV | TD_W, %g6 1460 movnz %xcc, TD_CP | TD_W, %g6 1461 or %g5, %g6, %g5 1462 1463 /* 1464 * Mask off the high bits of the virtual address to get the physical 1465 * address, and or in the tte bits. The high bit is left set in the 1466 * physical address, which corresponds to the tte valid bit, so that 1467 * we don't have to include it in the tte bits. We ignore the cache 1468 * bits, since they get shifted into the soft tte bits anyway. 1469 */ 1470 setx TLB_DIRECT_MASK & ~TD_V, %g7, %g6 1471 andn %g5, %g6, %g5 1472 1473 /* 1474 * Load the tte data into the TLB and retry the instruction. 1475 */ 1476 stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG 1477 retry 1478END(tl1_dmmu_miss_direct) 1479 1480ENTRY(tl1_dmmu_miss_user) 1481 /* 1482 * Try a fast inline lookup of the user tsb. 1483 */ 1484 dmmu_miss_user 1485 1486 /* 1487 * Put back the contents of the tag access register, in case we 1488 * faulted. 1489 */ 1490 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1491 membar #Sync 1492 1493 /* 1494 * Switch to alternate globals. 1495 */ 1496 wrpr %g0, PSTATE_ALT, %pstate 1497 1498 /* 1499 * Handle faults during window spill/fill. 1500 */ 1501 RESUME_SPILLFILL_MMU 1502 1503 /* 1504 * Reload the tag access register. 1505 */ 1506 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1507 1508 tl1_split 1509 clr %o1 1510 set trap, %o2 1511 mov %g2, %o3 1512 b %xcc, tl1_trap 1513 mov T_DATA_MISS | T_KERNEL, %o0 1514END(tl1_dmmu_miss_user) 1515 1516 .macro tl1_dmmu_prot 1517 ba,a %xcc, tl1_dmmu_prot_1 1518 nop 1519 .align 128 1520 .endm 1521 1522ENTRY(tl1_dmmu_prot_1) 1523 /* 1524 * Load the context and the virtual page number from the tag access 1525 * register. 1526 */ 1527 wr %g0, ASI_DMMU, %asi 1528 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1529 1530 /* 1531 * Extract the context from the contents of the tag access register. 1532 * If its non-zero this is a fault on a user address, otherwise get 1533 * the virtual page number. 1534 */ 1535 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1536 brnz,a,pn %g6, tl1_dmmu_prot_user 1537 mov %g5, %g1 1538 1539 /* 1540 * Compute the address of the tte. The tsb mask and address of the 1541 * tsb are patched at startup. 1542 */ 1543 .globl tl1_dmmu_prot_patch_1 1544tl1_dmmu_prot_patch_1: 1545 sethi %hi(TSB_KERNEL_MASK), %g6 1546 or %g6, %lo(TSB_KERNEL_MASK), %g6 1547 sethi %hi(TSB_KERNEL), %g7 1548 1549 srlx %g5, TAR_VPN_SHIFT, %g5 1550 and %g5, %g6, %g6 1551 sllx %g6, TTE_SHIFT, %g6 1552 add %g6, %g7, %g6 1553 1554 /* 1555 * Load the tte. 1556 */ 1557 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1558 1559 /* 1560 * Check that its valid and writeable and that the virtual page 1561 * numbers match. 1562 */ 1563 brgez,pn %g7, tl1_dmmu_prot_trap 1564 andcc %g7, TD_SW, %g0 1565 bz,pn %xcc, tl1_dmmu_prot_trap 1566 srlx %g6, TV_SIZE_BITS, %g6 1567 cmp %g5, %g6 1568 bne,pn %xcc, tl1_dmmu_prot_trap 1569 EMPTY 1570 1571 /* 1572 * Delete the old TLB entry and clear the sfsr. 1573 */ 1574 sllx %g5, TAR_VPN_SHIFT, %g6 1575 or %g6, TLB_DEMAP_NUCLEUS, %g6 1576 stxa %g0, [%g6] ASI_DMMU_DEMAP 1577 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1578 membar #Sync 1579 1580 /* 1581 * Recompute the tte address, which we clobbered loading the tte. The 1582 * tsb mask and address of the tsb are patched at startup. 1583 */ 1584 .globl tl1_dmmu_prot_patch_2 1585tl1_dmmu_prot_patch_2: 1586 sethi %hi(TSB_KERNEL_MASK), %g6 1587 or %g6, %lo(TSB_KERNEL_MASK), %g6 1588 sethi %hi(TSB_KERNEL), %g7 1589 1590 and %g5, %g6, %g5 1591 sllx %g5, TTE_SHIFT, %g5 1592 add %g5, %g7, %g5 1593 1594 /* 1595 * Set the hardware write bit. 1596 */ 1597 TTE_SET_W(%g5, %g6, %g7) 1598 1599 /* 1600 * May have become invalid during casxa, in which case start over. 1601 */ 1602 brgez,pn %g6, 1f 1603 or %g6, TD_W, %g6 1604 1605 /* 1606 * Load the tte data into the TLB and retry the instruction. 1607 */ 1608 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 16091: retry 1610END(tl1_dmmu_prot_1) 1611 1612ENTRY(tl1_dmmu_prot_user) 1613 /* 1614 * Try a fast inline lookup of the user tsb. 1615 */ 1616 dmmu_prot_user 1617 1618 /* 1619 * Put back the contents of the tag access register, in case we 1620 * faulted. 1621 */ 1622 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1623 membar #Sync 1624 1625 /* 1626 * Switch to alternate globals. 1627 */ 1628 wrpr %g0, PSTATE_ALT, %pstate 1629 1630 /* Handle faults during window spill/fill. */ 1631 RESUME_SPILLFILL_MMU_CLR_SFSR 1632 1633 b,a %xcc, tl1_dmmu_prot_trap 1634 nop 1635END(tl1_dmmu_prot_user) 1636 1637ENTRY(tl1_dmmu_prot_trap) 1638 /* 1639 * Switch to alternate globals. 1640 */ 1641 wrpr %g0, PSTATE_ALT, %pstate 1642 1643 /* 1644 * Load the sfar, sfsr and tar. Clear the sfsr. 1645 */ 1646 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1647 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1648 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1649 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1650 membar #Sync 1651 1652 tl1_split 1653 clr %o1 1654 set trap, %o2 1655 mov %g2, %o3 1656 mov %g3, %o4 1657 mov %g4, %o5 1658 b %xcc, tl1_trap 1659 mov T_DATA_PROTECTION | T_KERNEL, %o0 1660END(tl1_dmmu_prot_trap) 1661 1662 .macro tl1_spill_0_n 1663 SPILL(stx, %sp + SPOFF, 8, EMPTY) 1664 saved 1665 retry 1666 .align 32 1667 RSF_FATAL(T_SPILL) 1668 RSF_FATAL(T_SPILL) 1669 .endm 1670 1671 .macro tl1_spill_2_n 1672 wr %g0, ASI_AIUP, %asi 1673 SPILL(stxa, %sp + SPOFF, 8, %asi) 1674 saved 1675 retry 1676 .align 32 1677 RSF_SPILL_TOPCB 1678 RSF_SPILL_TOPCB 1679 .endm 1680 1681 .macro tl1_spill_3_n 1682 wr %g0, ASI_AIUP, %asi 1683 SPILL(stwa, %sp, 4, %asi) 1684 saved 1685 retry 1686 .align 32 1687 RSF_SPILL_TOPCB 1688 RSF_SPILL_TOPCB 1689 .endm 1690 1691 .macro tl1_spill_0_o 1692 wr %g0, ASI_AIUP, %asi 1693 SPILL(stxa, %sp + SPOFF, 8, %asi) 1694 saved 1695 retry 1696 .align 32 1697 RSF_SPILL_TOPCB 1698 RSF_SPILL_TOPCB 1699 .endm 1700 1701 .macro tl1_spill_1_o 1702 wr %g0, ASI_AIUP, %asi 1703 SPILL(stwa, %sp, 4, %asi) 1704 saved 1705 retry 1706 .align 32 1707 RSF_SPILL_TOPCB 1708 RSF_SPILL_TOPCB 1709 .endm 1710 1711 .macro tl1_spill_2_o 1712 RSF_SPILL_TOPCB 1713 .align 128 1714 .endm 1715 1716 .macro tl1_fill_0_n 1717 FILL(ldx, %sp + SPOFF, 8, EMPTY) 1718 restored 1719 retry 1720 .align 32 1721 RSF_FATAL(T_FILL) 1722 RSF_FATAL(T_FILL) 1723 .endm 1724 1725 .macro tl1_fill_2_n 1726 wr %g0, ASI_AIUP, %asi 1727 FILL(ldxa, %sp + SPOFF, 8, %asi) 1728 restored 1729 retry 1730 .align 32 1731 RSF_FILL_MAGIC 1732 RSF_FILL_MAGIC 1733 .endm 1734 1735 .macro tl1_fill_3_n 1736 wr %g0, ASI_AIUP, %asi 1737 FILL(lduwa, %sp, 4, %asi) 1738 restored 1739 retry 1740 .align 32 1741 RSF_FILL_MAGIC 1742 RSF_FILL_MAGIC 1743 .endm 1744 1745/* 1746 * This is used to spill windows that are still occupied with user 1747 * data on kernel entry to the pcb. 1748 */ 1749ENTRY(tl1_spill_topcb) 1750 wrpr %g0, PSTATE_ALT, %pstate 1751 1752 /* Free some globals for our use. */ 1753 dec 24, ASP_REG 1754 stx %g1, [ASP_REG + 0] 1755 stx %g2, [ASP_REG + 8] 1756 stx %g3, [ASP_REG + 16] 1757 1758 ldx [PCB_REG + PCB_NSAVED], %g1 1759 1760 sllx %g1, PTR_SHIFT, %g2 1761 add %g2, PCB_REG, %g2 1762 stx %sp, [%g2 + PCB_RWSP] 1763 1764 sllx %g1, RW_SHIFT, %g2 1765 add %g2, PCB_REG, %g2 1766 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) 1767 1768 inc %g1 1769 stx %g1, [PCB_REG + PCB_NSAVED] 1770 1771#if KTR_COMPILE & KTR_TRAP 1772 CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" 1773 , %g1, %g2, %g3, 7, 8, 9) 1774 rdpr %tpc, %g2 1775 stx %g2, [%g1 + KTR_PARM1] 1776 rdpr %tnpc, %g2 1777 stx %g2, [%g1 + KTR_PARM2] 1778 stx %sp, [%g1 + KTR_PARM3] 1779 ldx [PCB_REG + PCB_NSAVED], %g2 1780 stx %g2, [%g1 + KTR_PARM4] 17819: 1782#endif 1783 1784 saved 1785 1786 ldx [ASP_REG + 16], %g3 1787 ldx [ASP_REG + 8], %g2 1788 ldx [ASP_REG + 0], %g1 1789 inc 24, ASP_REG 1790 retry 1791END(tl1_spill_topcb) 1792 1793 .macro tl1_spill_bad count 1794 .rept \count 1795 sir 1796 .align 128 1797 .endr 1798 .endm 1799 1800 .macro tl1_fill_bad count 1801 .rept \count 1802 sir 1803 .align 128 1804 .endr 1805 .endm 1806 1807 .macro tl1_soft count 1808 .rept \count 1809 tl1_gen T_SOFT | T_KERNEL 1810 .endr 1811 .endm 1812 1813 .sect .trap 1814 .align 0x8000 1815 .globl tl0_base 1816 1817tl0_base: 1818 tl0_reserved 8 ! 0x0-0x7 1819tl0_insn_excptn: 1820 tl0_insn_excptn ! 0x8 1821 tl0_reserved 1 ! 0x9 1822tl0_insn_error: 1823 tl0_gen T_INSTRUCTION_ERROR ! 0xa 1824 tl0_reserved 5 ! 0xb-0xf 1825tl0_insn_illegal: 1826 tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 1827tl0_priv_opcode: 1828 tl0_gen T_PRIVILEGED_OPCODE ! 0x11 1829 tl0_reserved 14 ! 0x12-0x1f 1830tl0_fp_disabled: 1831 tl0_gen T_FP_DISABLED ! 0x20 1832tl0_fp_ieee: 1833 tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 1834tl0_fp_other: 1835 tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 1836tl0_tag_ovflw: 1837 tl0_gen T_TAG_OFERFLOW ! 0x23 1838tl0_clean_window: 1839 clean_window ! 0x24 1840tl0_divide: 1841 tl0_gen T_DIVISION_BY_ZERO ! 0x28 1842 tl0_reserved 7 ! 0x29-0x2f 1843tl0_data_excptn: 1844 tl0_data_excptn ! 0x30 1845 tl0_reserved 1 ! 0x31 1846tl0_data_error: 1847 tl0_gen T_DATA_ERROR ! 0x32 1848 tl0_reserved 1 ! 0x33 1849tl0_align: 1850 tl0_align ! 0x34 1851tl0_align_lddf: 1852 tl0_gen T_RESERVED ! 0x35 1853tl0_align_stdf: 1854 tl0_gen T_RESERVED ! 0x36 1855tl0_priv_action: 1856 tl0_gen T_PRIVILEGED_ACTION ! 0x37 1857 tl0_reserved 9 ! 0x38-0x40 1858tl0_intr_level: 1859 tl0_intr_level ! 0x41-0x4f 1860 tl0_reserved 16 ! 0x50-0x5f 1861tl0_intr_vector: 1862 intr_vector ! 0x60 1863tl0_watch_phys: 1864 tl0_gen T_PA_WATCHPOINT ! 0x61 1865tl0_watch_virt: 1866 tl0_gen T_VA_WATCHPOINT ! 0x62 1867tl0_ecc: 1868 tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 1869tl0_immu_miss: 1870 tl0_immu_miss ! 0x64 1871tl0_dmmu_miss: 1872 tl0_dmmu_miss ! 0x68 1873tl0_dmmu_prot: 1874 tl0_dmmu_prot ! 0x6c 1875 tl0_reserved 16 ! 0x70-0x7f 1876tl0_spill_0_n: 1877 tl0_spill_0_n ! 0x80 1878tl0_spill_1_n: 1879 tl0_spill_1_n ! 0x84 1880 tl0_spill_bad 14 ! 0x88-0xbf 1881tl0_fill_0_n: 1882 tl0_fill_0_n ! 0xc0 1883tl0_fill_1_n: 1884 tl0_fill_1_n ! 0xc4 1885 tl0_fill_bad 14 ! 0xc8-0xff 1886tl0_soft: 1887 tl0_reserved 1 ! 0x100 1888 tl0_gen T_BREAKPOINT ! 0x101 1889 tl0_gen T_DIVISION_BY_ZERO ! 0x102 1890 tl0_reserved 1 ! 0x103 1891 tl0_gen T_CLEAN_WINDOW ! 0x104 1892 tl0_gen T_RANGE_CHECK ! 0x105 1893 tl0_gen T_FIX_ALIGNMENT ! 0x106 1894 tl0_gen T_INTEGER_OVERFLOW ! 0x107 1895 tl0_reserved 1 ! 0x108 1896 tl0_syscall ! 0x109 1897 tl0_fp_restore ! 0x10a 1898 tl0_reserved 5 ! 0x10b-0x10f 1899 tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 1900 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 1901 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 1902 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 1903 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 1904 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 1905 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 1906 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 1907 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 1908 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 1909 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a 1910 tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b 1911 tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c 1912 tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d 1913 tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e 1914 tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f 1915 tl0_reserved 224 ! 0x120-0x1ff 1916 1917tl1_base: 1918 tl1_reserved 8 ! 0x200-0x207 1919tl1_insn_excptn: 1920 tl1_insn_excptn ! 0x208 1921 tl1_reserved 1 ! 0x209 1922tl1_insn_error: 1923 tl1_gen T_INSTRUCTION_ERROR ! 0x20a 1924 tl1_reserved 5 ! 0x20b-0x20f 1925tl1_insn_illegal: 1926 tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 1927tl1_priv_opcode: 1928 tl1_gen T_PRIVILEGED_OPCODE ! 0x211 1929 tl1_reserved 14 ! 0x212-0x21f 1930tl1_fp_disabled: 1931 tl1_gen T_FP_DISABLED ! 0x220 1932tl1_fp_ieee: 1933 tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 1934tl1_fp_other: 1935 tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 1936tl1_tag_ovflw: 1937 tl1_gen T_TAG_OFERFLOW ! 0x223 1938tl1_clean_window: 1939 clean_window ! 0x224 1940tl1_divide: 1941 tl1_gen T_DIVISION_BY_ZERO ! 0x228 1942 tl1_reserved 7 ! 0x229-0x22f 1943tl1_data_excptn: 1944 tl1_data_excptn ! 0x230 1945 tl1_reserved 1 ! 0x231 1946tl1_data_error: 1947 tl1_gen T_DATA_ERROR ! 0x232 1948 tl1_reserved 1 ! 0x233 1949tl1_align: 1950 tl1_align ! 0x234 1951tl1_align_lddf: 1952 tl1_gen T_RESERVED ! 0x235 1953tl1_align_stdf: 1954 tl1_gen T_RESERVED ! 0x236 1955tl1_priv_action: 1956 tl1_gen T_PRIVILEGED_ACTION ! 0x237 1957 tl1_reserved 9 ! 0x238-0x240 1958tl1_intr_level: 1959 tl1_intr_level ! 0x241-0x24f 1960 tl1_reserved 16 ! 0x250-0x25f 1961tl1_intr_vector: 1962 intr_vector ! 0x260 1963tl1_watch_phys: 1964 tl1_gen T_PA_WATCHPOINT ! 0x261 1965tl1_watch_virt: 1966 tl1_gen T_VA_WATCHPOINT ! 0x262 1967tl1_ecc: 1968 tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 1969tl1_immu_miss: 1970 tl1_immu_miss ! 0x264 1971tl1_dmmu_miss: 1972 tl1_dmmu_miss ! 0x268 1973tl1_dmmu_prot: 1974 tl1_dmmu_prot ! 0x26c 1975 tl1_reserved 16 ! 0x270-0x27f 1976tl1_spill_0_n: 1977 tl1_spill_0_n ! 0x280 1978 tl1_spill_bad 1 ! 0x284 1979tl1_spill_2_n: 1980 tl1_spill_2_n ! 0x288 1981tl1_spill_3_n: 1982 tl1_spill_3_n ! 0x29c 1983 tl1_spill_bad 4 ! 0x290-0x29f 1984tl1_spill_0_o: 1985 tl1_spill_0_o ! 0x2a0 1986tl1_spill_1_o: 1987 tl1_spill_1_o ! 0x2a4 1988tl1_spill_2_o: 1989 tl1_spill_2_o ! 0x2a8 1990 tl1_spill_bad 5 ! 0x2ac-0x2bf 1991tl1_fill_0_n: 1992 tl1_fill_0_n ! 0x2c0 1993 tl1_fill_bad 1 ! 0x2c4 1994tl1_fill_2_n: 1995 tl1_fill_2_n ! 0x2d0 1996tl1_fill_3_n: 1997 tl1_fill_3_n ! 0x2d4 1998 tl1_fill_bad 12 ! 0x2d8-0x2ff 1999 tl1_reserved 1 ! 0x300 2000tl1_breakpoint: 2001 tl1_gen T_BREAKPOINT ! 0x301 2002 tl1_gen T_RSTRWP_PHYS ! 0x302 2003 tl1_gen T_RSTRWP_VIRT ! 0x303 2004 tl1_reserved 252 ! 0x304-0x3ff 2005 2006/* 2007 * User trap entry point. 2008 * 2009 * void tl0_utrap(u_long type, u_long o1, u_long o2, u_long tar, u_long sfar, 2010 * u_long sfsr) 2011 * 2012 * This handles redirecting a trap back to usermode as a user trap. The user 2013 * program must have first registered a trap handler with the kernel using 2014 * sysarch(SPARC_UTRAP_INSTALL). The trap handler is passed enough state 2015 * for it to return to the trapping code directly, it will not return through 2016 * the kernel. The trap type is passed in %o0, all out registers must be 2017 * passed through to tl0_trap or to usermode untouched. Note that the 2018 * parameters passed in out registers may be used by the user trap handler. 2019 * Do not change the registers they are passed in or you will break the ABI. 2020 * 2021 * If the trap type allows user traps, setup state to execute the user trap 2022 * handler and bounce back to usermode, otherwise branch to tl0_trap. 2023 */ 2024ENTRY(tl0_utrap) 2025 /* 2026 * Check if the trap type allows user traps. 2027 */ 2028 cmp %o0, UT_MAX 2029 bge,a,pt %xcc, tl0_trap 2030 nop 2031 2032 /* 2033 * Load the user trap handler from the utrap table. 2034 */ 2035 ldx [PCPU(CURTHREAD)], %l0 2036 ldx [%l0 + TD_PROC], %l0 2037 ldx [%l0 + P_MD + MD_UTRAP], %l0 2038 brz,pt %l0, tl0_trap 2039 sllx %o0, PTR_SHIFT, %l1 2040 ldx [%l0 + %l1], %l0 2041 brz,a,pt %l0, tl0_trap 2042 nop 2043 2044 /* 2045 * If the save we did on entry to the kernel had to spill a window 2046 * to the pcb, pretend we took a spill trap instead. Any windows 2047 * that are in the pcb must be copied out or the fill handler will 2048 * not be able to find them, since the user trap handler returns 2049 * directly to the trapping code. Note that we only support precise 2050 * user traps, which implies that the condition that caused the trap 2051 * in the first place is still valid, so it will occur again when we 2052 * re-execute the trapping instruction. 2053 */ 2054 ldx [PCB_REG + PCB_NSAVED], %l1 2055 brnz,a,pn %l1, tl0_trap 2056 mov T_SPILL, %o0 2057 2058 /* 2059 * Pass %fsr in %l4, %tstate in %l5, %tpc in %l6 and %tnpc in %l7. 2060 * The ABI specifies only %l6 and %l7, but we need to pass %fsr or 2061 * it may be clobbered by an interrupt before the user trap code 2062 * can read it, and we must pass %tstate in order to restore %ccr 2063 * and %asi. The %fsr must be stored to memory, so we use the 2064 * temporary stack for that. 2065 */ 2066 rd %fprs, %l1 2067 or %l1, FPRS_FEF, %l2 2068 wr %l2, 0, %fprs 2069 dec 8, ASP_REG 2070 stx %fsr, [ASP_REG] 2071 ldx [ASP_REG], %l4 2072 inc 8, ASP_REG 2073 wr %l1, 0, %fprs 2074 2075 rdpr %tstate, %l5 2076 rdpr %tpc, %l6 2077 rdpr %tnpc, %l7 2078 2079 /* 2080 * Setup %tnpc to return to. 2081 */ 2082 wrpr %l0, 0, %tnpc 2083 2084 /* 2085 * Setup %wstate for return, clear WSTATE_TRANSITION. 2086 */ 2087 rdpr %wstate, %l1 2088 and %l1, WSTATE_NORMAL_MASK, %l1 2089 wrpr %l1, 0, %wstate 2090 2091 /* 2092 * Setup %tstate for return, change the saved cwp to point to the 2093 * current window instead of the window at the time of the trap. 2094 */ 2095 andn %l5, TSTATE_CWP_MASK, %l1 2096 rdpr %cwp, %l2 2097 wrpr %l1, %l2, %tstate 2098 2099 /* 2100 * Setup %sp. Userland processes will crash if this is not setup. 2101 */ 2102 sub %fp, CCFSZ, %sp 2103 2104 /* 2105 * Execute the user trap handler. 2106 */ 2107 done 2108END(tl0_utrap) 2109 2110/* 2111 * (Real) User trap entry point. 2112 * 2113 * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2114 * u_int sfsr) 2115 * 2116 * The following setup has been performed: 2117 * - the windows have been split and the active user window has been saved 2118 * (maybe just to the pcb) 2119 * - we are on alternate globals and interrupts are disabled 2120 * 2121 * We switch to the kernel stack, build a trapframe, switch to normal 2122 * globals, enable interrupts and call trap. 2123 * 2124 * NOTE: We must be very careful setting up the per-cpu pointer. We know that 2125 * it has been pre-set in alternate globals, so we read it from there and setup 2126 * the normal %g7 *before* enabling interrupts. This avoids any possibility 2127 * of cpu migration and using the wrong pcpup. 2128 */ 2129ENTRY(tl0_trap) 2130 /* 2131 * Force kernel store order. 2132 */ 2133 wrpr %g0, PSTATE_ALT, %pstate 2134 2135 rdpr %tstate, %l0 2136 rdpr %tpc, %l1 2137 rdpr %tnpc, %l2 2138 rd %y, %l3 2139 rd %fprs, %l4 2140 rdpr %wstate, %l5 2141 2142#if KTR_COMPILE & KTR_TRAP 2143 CATR(KTR_TRAP, 2144 "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2145 , %g1, %g2, %g3, 7, 8, 9) 2146 ldx [PCPU(CURTHREAD)], %g2 2147 stx %g2, [%g1 + KTR_PARM1] 2148 stx %o0, [%g1 + KTR_PARM2] 2149 rdpr %pil, %g2 2150 stx %g2, [%g1 + KTR_PARM3] 2151 stx %l1, [%g1 + KTR_PARM4] 2152 stx %l2, [%g1 + KTR_PARM5] 2153 stx %i6, [%g1 + KTR_PARM6] 21549: 2155#endif 2156 21571: and %l5, WSTATE_NORMAL_MASK, %l5 2158 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2159 wrpr %l5, WSTATE_KERNEL, %wstate 2160 rdpr %canrestore, %l6 2161 wrpr %l6, 0, %otherwin 2162 wrpr %g0, 0, %canrestore 2163 2164 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2165 2166 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2167 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2168 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2169 stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2170 2171 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2172 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2173 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2174 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2175 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2176 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2177 2178 wr %g0, FPRS_FEF, %fprs 2179 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2180 wr %g0, 0, %fprs 2181 2182 mov PCB_REG, %l0 2183 mov PCPU_REG, %l1 2184 wrpr %g0, PSTATE_NORMAL, %pstate 2185 2186 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2187 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2188 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2189 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2190 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2191 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2192 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2193 2194 mov %l0, PCB_REG 2195 mov %l1, PCPU_REG 2196 wrpr %g0, PSTATE_KERNEL, %pstate 2197 2198 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2199 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2200 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2201 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2202 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2203 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2204 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2205 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2206 2207 set tl0_ret - 8, %o7 2208 jmpl %o2, %g0 2209 add %sp, CCFSZ + SPOFF, %o0 2210END(tl0_trap) 2211 2212/* 2213 * void tl0_intr(u_int level, u_int mask) 2214 */ 2215ENTRY(tl0_intr) 2216 /* 2217 * Force kernel store order. 2218 */ 2219 wrpr %g0, PSTATE_ALT, %pstate 2220 2221 rdpr %tstate, %l0 2222 rdpr %tpc, %l1 2223 rdpr %tnpc, %l2 2224 rd %y, %l3 2225 rd %fprs, %l4 2226 rdpr %wstate, %l5 2227 2228#if KTR_COMPILE & KTR_INTR 2229 CATR(KTR_INTR, 2230 "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2231 , %g1, %g2, %g3, 7, 8, 9) 2232 ldx [PCPU(CURTHREAD)], %g2 2233 stx %g2, [%g1 + KTR_PARM1] 2234 stx %o0, [%g1 + KTR_PARM2] 2235 rdpr %pil, %g2 2236 stx %g2, [%g1 + KTR_PARM3] 2237 stx %l1, [%g1 + KTR_PARM4] 2238 stx %l2, [%g1 + KTR_PARM5] 2239 stx %i6, [%g1 + KTR_PARM6] 22409: 2241#endif 2242 2243 wrpr %o0, 0, %pil 2244 wr %o1, 0, %asr21 2245 2246 and %l5, WSTATE_NORMAL_MASK, %l5 2247 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2248 wrpr %l5, WSTATE_KERNEL, %wstate 2249 rdpr %canrestore, %l6 2250 wrpr %l6, 0, %otherwin 2251 wrpr %g0, 0, %canrestore 2252 2253 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2254 2255 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2256 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2257 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2258 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2259 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2260 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2261 2262 wr %g0, FPRS_FEF, %fprs 2263 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2264 wr %g0, 0, %fprs 2265 2266 mov %o0, %l3 2267 mov T_INTERRUPT, %o1 2268 2269 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2270 stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2271 2272 mov PCB_REG, %l0 2273 mov PCPU_REG, %l1 2274 wrpr %g0, PSTATE_NORMAL, %pstate 2275 2276 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2277 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2278 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2279 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2280 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2281 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2282 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2283 2284 mov %l0, PCB_REG 2285 mov %l1, PCPU_REG 2286 wrpr %g0, PSTATE_KERNEL, %pstate 2287 2288 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2289 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2290 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2291 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2292 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2293 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2294 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2295 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2296 2297 call critical_enter 2298 nop 2299 2300 SET(cnt+V_INTR, %l1, %l0) 2301 ATOMIC_INC_INT(%l0, %l1, %l2) 2302 2303 SET(intr_handlers, %l1, %l0) 2304 sllx %l3, IH_SHIFT, %l1 2305 ldx [%l0 + %l1], %l1 2306 KASSERT(%l1, "tl0_intr: ih null") 2307 call %l1 2308 add %sp, CCFSZ + SPOFF, %o0 2309 2310 call critical_exit 2311 nop 2312 2313 b,a %xcc, tl0_ret 2314 nop 2315END(tl0_intr) 2316 2317ENTRY(tl0_ret) 2318 /* 2319 * Check for pending asts atomically with returning. We must raise 2320 * the pil before checking, and if no asts are found the pil must 2321 * remain raised until the retry is executed, or we risk missing asts 2322 * caused by interrupts occuring after the test. If the pil is lowered, 2323 * as it is when we call ast, the check must be re-executed. 2324 */ 2325 wrpr %g0, PIL_TICK, %pil 2326 ldx [PCPU(CURTHREAD)], %l0 2327 ldx [%l0 + TD_KSE], %l1 2328 lduw [%l1 + KE_FLAGS], %l2 2329 and %l2, KEF_ASTPENDING | KEF_NEEDRESCHED, %l2 2330 brz,a,pt %l2, 1f 2331 nop 2332 wrpr %g0, 0, %pil 2333 call ast 2334 add %sp, CCFSZ + SPOFF, %o0 2335 ba,a %xcc, tl0_ret 2336 nop 2337 2338 /* 2339 * Check for windows that were spilled to the pcb and need to be 2340 * copied out. This must be the last thing that is done before the 2341 * return to usermode. If there are still user windows in the cpu 2342 * and we call a nested function after this, which causes them to be 2343 * spilled to the pcb, they will not be copied out and the stack will 2344 * be inconsistent. 2345 */ 23461: ldx [PCB_REG + PCB_NSAVED], %l1 2347 brz,a,pt %l1, 2f 2348 nop 2349 wrpr %g0, 0, %pil 2350 mov T_SPILL, %o0 2351 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2352 call trap 2353 add %sp, SPOFF + CCFSZ, %o0 2354 ba,a %xcc, tl0_ret 2355 nop 2356 23572: ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 2358 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 2359 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 2360 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 2361 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 2362 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 2363 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 2364 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 2365 2366 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2367 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2368 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2369 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l3 2370 ldub [%sp + SPOFF + CCFSZ + TF_FPRS], %l4 2371 ldub [%sp + SPOFF + CCFSZ + TF_WSTATE], %l5 2372 2373 wrpr %g0, PSTATE_NORMAL, %pstate 2374 2375 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2376 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2377 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2378 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2379 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2380 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 2381 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 2382 2383 wrpr %g0, PSTATE_ALT, %pstate 2384 2385 wrpr %g0, 0, %pil 2386 wrpr %l1, 0, %tpc 2387 wrpr %l2, 0, %tnpc 2388 wr %l3, 0, %y 2389 2390 andn %l0, TSTATE_CWP_MASK, %g1 2391 mov %l4, %g2 2392 2393 srlx %l5, WSTATE_OTHER_SHIFT, %g3 2394 wrpr %g3, WSTATE_TRANSITION, %wstate 2395 rdpr %otherwin, %o0 2396 wrpr %o0, 0, %canrestore 2397 wrpr %g0, 0, %otherwin 2398 wrpr %o0, 0, %cleanwin 2399 2400 /* 2401 * If this instruction causes a fill trap which fails to fill a window 2402 * from the user stack, we will resume at tl0_ret_fill_end and call 2403 * back into the kernel. 2404 */ 2405 restore 2406tl0_ret_fill: 2407 2408 rdpr %cwp, %g4 2409 wrpr %g1, %g4, %tstate 2410 wr %g2, 0, %fprs 2411 wrpr %g3, 0, %wstate 2412 2413#if KTR_COMPILE & KTR_TRAP 2414 CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2415 , %g2, %g3, %g4, 7, 8, 9) 2416 ldx [PCPU(CURTHREAD)], %g3 2417 stx %g3, [%g2 + KTR_PARM1] 2418 rdpr %pil, %g3 2419 stx %g3, [%g2 + KTR_PARM2] 2420 rdpr %tpc, %g3 2421 stx %g3, [%g2 + KTR_PARM3] 2422 rdpr %tnpc, %g3 2423 stx %g3, [%g2 + KTR_PARM4] 2424 stx %sp, [%g2 + KTR_PARM5] 24259: 2426#endif 2427 2428 retry 2429tl0_ret_fill_end: 2430 2431#if KTR_COMPILE & KTR_TRAP 2432 CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" 2433 , %l0, %l1, %l2, 7, 8, 9) 2434 rdpr %pstate, %l1 2435 stx %l1, [%l0 + KTR_PARM1] 2436 stx %l5, [%l0 + KTR_PARM2] 2437 stx %sp, [%l0 + KTR_PARM3] 24389: 2439#endif 2440 2441 /* 2442 * The fill failed and magic has been performed. Call trap again, 2443 * which will copyin the window on the user's behalf. 2444 */ 2445 wrpr %l5, 0, %wstate 2446 wrpr %g0, PSTATE_ALT, %pstate 2447 mov PCB_REG, %o0 2448 mov PCPU_REG, %o1 2449 wrpr %g0, PSTATE_NORMAL, %pstate 2450 mov %o0, PCB_REG 2451 mov %o1, PCPU_REG 2452 wrpr %g0, PSTATE_KERNEL, %pstate 2453 mov T_FILL_RET, %o0 2454 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2455 call trap 2456 add %sp, SPOFF + CCFSZ, %o0 2457 ba,a %xcc, tl0_ret 2458 nop 2459END(tl0_ret) 2460 2461/* 2462 * Kernel trap entry point 2463 * 2464 * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2465 * u_int sfsr) 2466 * 2467 * This is easy because the stack is already setup and the windows don't need 2468 * to be split. We build a trapframe and call trap(), the same as above, but 2469 * the outs don't need to be saved. 2470 */ 2471ENTRY(tl1_trap) 2472 rdpr %tstate, %l0 2473 rdpr %tpc, %l1 2474 rdpr %tnpc, %l2 2475 rdpr %pil, %l3 2476 rd %y, %l4 2477 rdpr %wstate, %l5 2478 2479#if KTR_COMPILE & KTR_TRAP 2480 CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" 2481 , %g1, %g2, %g3, 7, 8, 9) 2482 ldx [PCPU(CURTHREAD)], %g2 2483 stx %g2, [%g1 + KTR_PARM1] 2484 stx %o0, [%g1 + KTR_PARM2] 2485 stx %l3, [%g1 + KTR_PARM3] 2486 stx %l1, [%g1 + KTR_PARM4] 2487 stx %i6, [%g1 + KTR_PARM5] 24889: 2489#endif 2490 2491 wrpr %g0, 1, %tl 2492 2493 and %l5, WSTATE_OTHER_MASK, %l5 2494 wrpr %l5, WSTATE_KERNEL, %wstate 2495 2496 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2497 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2498 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2499 stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2500 2501 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2502 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2503 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2504 stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2505 stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2506 2507 mov PCB_REG, %l0 2508 mov PCPU_REG, %l1 2509 wrpr %g0, PSTATE_NORMAL, %pstate 2510 2511 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2512 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2513 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2514 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2515 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2516 2517 mov %l0, PCB_REG 2518 mov %l1, PCPU_REG 2519 wrpr %g0, PSTATE_KERNEL, %pstate 2520 2521 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2522 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2523 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2524 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2525 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2526 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2527 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2528 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2529 2530 set tl1_ret - 8, %o7 2531 jmpl %o2, %g0 2532 add %sp, CCFSZ + SPOFF, %o0 2533END(tl1_trap) 2534 2535ENTRY(tl1_ret) 2536 ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 2537 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 2538 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 2539 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 2540 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 2541 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 2542 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 2543 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 2544 2545 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2546 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2547 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2548 ldub [%sp + SPOFF + CCFSZ + TF_PIL], %l3 2549 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 2550 2551 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2552 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2553 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2554 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2555 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2556 2557 wrpr %g0, PSTATE_ALT, %pstate 2558 2559 andn %l0, TSTATE_CWP_MASK, %g1 2560 mov %l1, %g2 2561 mov %l2, %g3 2562 2563 wrpr %l3, 0, %pil 2564 wr %l4, 0, %y 2565 2566 restore 2567 2568 wrpr %g0, 2, %tl 2569 2570 rdpr %cwp, %g4 2571 wrpr %g1, %g4, %tstate 2572 wrpr %g2, 0, %tpc 2573 wrpr %g3, 0, %tnpc 2574 2575#if KTR_COMPILE & KTR_TRAP 2576 CATR(KTR_TRAP, "tl1_ret: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2577 , %g2, %g3, %g4, 7, 8, 9) 2578 ldx [PCPU(CURTHREAD)], %g3 2579 stx %g3, [%g2 + KTR_PARM1] 2580 rdpr %pil, %g3 2581 stx %g3, [%g2 + KTR_PARM2] 2582 rdpr %tstate, %g3 2583 stx %g3, [%g2 + KTR_PARM3] 2584 rdpr %tpc, %g3 2585 stx %g3, [%g2 + KTR_PARM4] 2586 stx %sp, [%g2 + KTR_PARM5] 25879: 2588#endif 2589 2590 retry 2591END(tl1_ret) 2592 2593/* 2594 * void tl1_intr(u_int level, u_int mask) 2595 */ 2596ENTRY(tl1_intr) 2597 rdpr %tstate, %l0 2598 rdpr %tpc, %l1 2599 rdpr %tnpc, %l2 2600 rdpr %pil, %l3 2601 rd %y, %l4 2602 rdpr %wstate, %l5 2603 2604#if KTR_COMPILE & KTR_INTR 2605 CATR(KTR_INTR, 2606 "tl1_intr: td=%p level=%#lx pil=%#lx pc=%#lx sp=%#lx" 2607 , %g1, %g2, %g3, 7, 8, 9) 2608 ldx [PCPU(CURTHREAD)], %g2 2609 stx %g2, [%g1 + KTR_PARM1] 2610 stx %o0, [%g1 + KTR_PARM2] 2611 stx %l3, [%g1 + KTR_PARM3] 2612 stx %l1, [%g1 + KTR_PARM4] 2613 stx %i6, [%g1 + KTR_PARM5] 26149: 2615#endif 2616 2617 wrpr %o0, 0, %pil 2618 wr %o1, 0, %asr21 2619 2620 wrpr %g0, 1, %tl 2621 2622 and %l5, WSTATE_OTHER_MASK, %l5 2623 wrpr %l5, WSTATE_KERNEL, %wstate 2624 2625 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2626 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2627 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2628 stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2629 stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2630 2631 mov %o0, %l7 2632 mov T_INTERRUPT | T_KERNEL, %o1 2633 2634 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2635 stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2636 2637 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2638 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2639 2640 mov PCB_REG, %l4 2641 mov PCPU_REG, %l5 2642 wrpr %g0, PSTATE_NORMAL, %pstate 2643 2644 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2645 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2646 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2647 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2648 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2649 2650 mov %l4, PCB_REG 2651 mov %l5, PCPU_REG 2652 wrpr %g0, PSTATE_KERNEL, %pstate 2653 2654 call critical_enter 2655 nop 2656 2657 SET(cnt+V_INTR, %l5, %l4) 2658 ATOMIC_INC_INT(%l4, %l5, %l6) 2659 2660 SET(intr_handlers, %l5, %l4) 2661 sllx %l7, IH_SHIFT, %l5 2662 ldx [%l4 + %l5], %l5 2663 KASSERT(%l5, "tl1_intr: ih null") 2664 call %l5 2665 add %sp, CCFSZ + SPOFF, %o0 2666 2667 call critical_exit 2668 nop 2669 2670 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 2671 2672 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2673 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2674 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2675 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2676 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2677 2678 wrpr %g0, PSTATE_ALT, %pstate 2679 2680 andn %l0, TSTATE_CWP_MASK, %g1 2681 mov %l1, %g2 2682 mov %l2, %g3 2683 wrpr %l3, 0, %pil 2684 wr %l4, 0, %y 2685 2686 restore 2687 2688 wrpr %g0, 2, %tl 2689 2690 rdpr %cwp, %g4 2691 wrpr %g1, %g4, %tstate 2692 wrpr %g2, 0, %tpc 2693 wrpr %g3, 0, %tnpc 2694 2695#if KTR_COMPILE & KTR_INTR 2696 CATR(KTR_INTR, "tl1_intr: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2697 , %g2, %g3, %g4, 7, 8, 9) 2698 ldx [PCPU(CURTHREAD)], %g3 2699 stx %g3, [%g2 + KTR_PARM1] 2700 rdpr %pil, %g3 2701 stx %g3, [%g2 + KTR_PARM2] 2702 rdpr %tstate, %g3 2703 stx %g3, [%g2 + KTR_PARM3] 2704 rdpr %tpc, %g3 2705 stx %g3, [%g2 + KTR_PARM4] 2706 stx %sp, [%g2 + KTR_PARM5] 27079: 2708#endif 2709 2710 retry 2711END(tl1_intr) 2712 2713/* 2714 * Freshly forked processes come here when switched to for the first time. 2715 * The arguments to fork_exit() have been setup in the locals, we must move 2716 * them to the outs. 2717 */ 2718ENTRY(fork_trampoline) 2719#if KTR_COMPILE & KTR_PROC 2720 CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" 2721 , %g1, %g2, %g3, 7, 8, 9) 2722 ldx [PCPU(CURTHREAD)], %g2 2723 stx %g2, [%g1 + KTR_PARM1] 2724 ldx [%g2 + TD_PROC], %g2 2725 add %g2, P_COMM, %g2 2726 stx %g2, [%g1 + KTR_PARM2] 2727 rdpr %cwp, %g2 2728 stx %g2, [%g1 + KTR_PARM3] 27299: 2730#endif 2731 mov %l0, %o0 2732 mov %l1, %o1 2733 call fork_exit 2734 mov %l2, %o2 2735 b,a %xcc, tl0_ret 2736 nop 2737END(fork_trampoline) 2738