exception.S revision 102040
1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2001 Jake Burkholder. 32 * All rights reserved. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 46 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 53 * SUCH DAMAGE. 54 * 55 * $FreeBSD: head/sys/sparc64/sparc64/exception.S 102040 2002-08-18 02:09:27Z jake $ 56 */ 57 58#include "opt_ddb.h" 59 60#include <machine/asi.h> 61#include <machine/asmacros.h> 62#include <machine/ktr.h> 63#include <machine/pstate.h> 64#include <machine/trap.h> 65#include <machine/tstate.h> 66#include <machine/wstate.h> 67 68#include "assym.s" 69 70#define TSB_KERNEL_MASK 0x0 71#define TSB_KERNEL 0x0 72 73 .register %g2,#ignore 74 .register %g3,#ignore 75 .register %g6,#ignore 76 .register %g7,#ignore 77 78/* 79 * Atomically set the reference bit in a tte. 80 */ 81#define TTE_SET_BIT(r1, r2, r3, bit) \ 82 add r1, TTE_DATA, r1 ; \ 83 ldx [r1], r2 ; \ 849: or r2, bit, r3 ; \ 85 casxa [r1] ASI_N, r2, r3 ; \ 86 cmp r2, r3 ; \ 87 bne,pn %xcc, 9b ; \ 88 mov r3, r2 89 90#define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF) 91#define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W) 92 93/* 94 * Macros for spilling and filling live windows. 95 * 96 * NOTE: These macros use exactly 16 instructions, and it is assumed that the 97 * handler will not use more than 24 instructions total, to leave room for 98 * resume vectors which occupy the last 8 instructions. 99 */ 100 101#define SPILL(storer, base, size, asi) \ 102 storer %l0, [base + (0 * size)] asi ; \ 103 storer %l1, [base + (1 * size)] asi ; \ 104 storer %l2, [base + (2 * size)] asi ; \ 105 storer %l3, [base + (3 * size)] asi ; \ 106 storer %l4, [base + (4 * size)] asi ; \ 107 storer %l5, [base + (5 * size)] asi ; \ 108 storer %l6, [base + (6 * size)] asi ; \ 109 storer %l7, [base + (7 * size)] asi ; \ 110 storer %i0, [base + (8 * size)] asi ; \ 111 storer %i1, [base + (9 * size)] asi ; \ 112 storer %i2, [base + (10 * size)] asi ; \ 113 storer %i3, [base + (11 * size)] asi ; \ 114 storer %i4, [base + (12 * size)] asi ; \ 115 storer %i5, [base + (13 * size)] asi ; \ 116 storer %i6, [base + (14 * size)] asi ; \ 117 storer %i7, [base + (15 * size)] asi 118 119#define FILL(loader, base, size, asi) \ 120 loader [base + (0 * size)] asi, %l0 ; \ 121 loader [base + (1 * size)] asi, %l1 ; \ 122 loader [base + (2 * size)] asi, %l2 ; \ 123 loader [base + (3 * size)] asi, %l3 ; \ 124 loader [base + (4 * size)] asi, %l4 ; \ 125 loader [base + (5 * size)] asi, %l5 ; \ 126 loader [base + (6 * size)] asi, %l6 ; \ 127 loader [base + (7 * size)] asi, %l7 ; \ 128 loader [base + (8 * size)] asi, %i0 ; \ 129 loader [base + (9 * size)] asi, %i1 ; \ 130 loader [base + (10 * size)] asi, %i2 ; \ 131 loader [base + (11 * size)] asi, %i3 ; \ 132 loader [base + (12 * size)] asi, %i4 ; \ 133 loader [base + (13 * size)] asi, %i5 ; \ 134 loader [base + (14 * size)] asi, %i6 ; \ 135 loader [base + (15 * size)] asi, %i7 136 137#define ERRATUM50(reg) mov reg, reg 138 139#define KSTACK_SLOP 1024 140 141/* 142 * Sanity check the kernel stack and bail out if its wrong. 143 * XXX: doesn't handle being on the panic stack. 144 */ 145#define KSTACK_CHECK \ 146 dec 16, ASP_REG ; \ 147 stx %g1, [ASP_REG + 0] ; \ 148 stx %g2, [ASP_REG + 8] ; \ 149 add %sp, SPOFF, %g1 ; \ 150 andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ 151 bnz,a %xcc, tl1_kstack_fault ; \ 152 inc 16, ASP_REG ; \ 153 ldx [PCPU(CURTHREAD)], %g2 ; \ 154 ldx [%g2 + TD_KSTACK], %g2 ; \ 155 add %g2, KSTACK_SLOP, %g2 ; \ 156 subcc %g1, %g2, %g1 ; \ 157 ble,a %xcc, tl1_kstack_fault ; \ 158 inc 16, ASP_REG ; \ 159 set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ 160 cmp %g1, %g2 ; \ 161 bgt,a %xcc, tl1_kstack_fault ; \ 162 inc 16, ASP_REG ; \ 163 ldx [ASP_REG + 8], %g2 ; \ 164 ldx [ASP_REG + 0], %g1 ; \ 165 inc 16, ASP_REG 166 167ENTRY(tl1_kstack_fault) 168 rdpr %tl, %g1 1691: cmp %g1, 2 170 be,a 2f 171 nop 172 173#if KTR_COMPILE & KTR_TRAP 174 CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" 175 , %g2, %g3, %g4, 7, 8, 9) 176 rdpr %tl, %g3 177 stx %g3, [%g2 + KTR_PARM1] 178 rdpr %tpc, %g3 179 stx %g3, [%g2 + KTR_PARM1] 180 rdpr %tnpc, %g3 181 stx %g3, [%g2 + KTR_PARM1] 1829: 183#endif 184 185 sub %g1, 1, %g1 186 wrpr %g1, 0, %tl 187 ba,a %xcc, 1b 188 nop 189 1902: 191#if KTR_COMPILE & KTR_TRAP 192 CATR(KTR_TRAP, 193 "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" 194 , %g1, %g2, %g3, 7, 8, 9) 195 add %sp, SPOFF, %g2 196 stx %g2, [%g1 + KTR_PARM1] 197 ldx [PCPU(CURTHREAD)], %g2 198 ldx [%g2 + TD_KSTACK], %g2 199 stx %g2, [%g1 + KTR_PARM2] 200 rdpr %canrestore, %g2 201 stx %g2, [%g1 + KTR_PARM3] 202 rdpr %cansave, %g2 203 stx %g2, [%g1 + KTR_PARM4] 204 rdpr %otherwin, %g2 205 stx %g2, [%g1 + KTR_PARM5] 206 rdpr %wstate, %g2 207 stx %g2, [%g1 + KTR_PARM6] 2089: 209#endif 210 211 wrpr %g0, 0, %canrestore 212 wrpr %g0, 6, %cansave 213 wrpr %g0, 0, %otherwin 214 wrpr %g0, WSTATE_KERNEL, %wstate 215 216 sub ASP_REG, SPOFF + CCFSZ, %sp 217 clr %fp 218 219 b %xcc, tl1_trap 220 mov T_KSTACK_FAULT | T_KERNEL, %o0 221END(tl1_kstack_fault) 222 223/* 224 * Magic to resume from a spill or fill trap. If we get an alignment or an 225 * mmu fault during a spill or a fill, this macro will detect the fault and 226 * resume at a set instruction offset in the trap handler. 227 * 228 * To check if the previous trap was a spill/fill we convert the trapped pc 229 * to a trap type and verify that it is in the range of spill/fill vectors. 230 * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the 231 * tl bit allows us to detect both ranges with one test. 232 * 233 * This is: 234 * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 235 * 236 * To calculate the new pc we take advantage of the xor feature of wrpr. 237 * Forcing all the low bits of the trapped pc on we can produce any offset 238 * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. 239 * 240 * 0x7f ^ 0x1f == 0x60 241 * 0x1f == (0x80 - 0x60) - 1 242 * 243 * Which are the offset and xor value used to resume from alignment faults. 244 */ 245 246/* 247 * Determine if we have trapped inside of a spill/fill vector, and if so resume 248 * at a fixed instruction offset in the trap vector. Must be called on 249 * alternate globals. 250 */ 251#define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ 252 dec 16, ASP_REG ; \ 253 stx %g1, [ASP_REG + 0] ; \ 254 stx %g2, [ASP_REG + 8] ; \ 255 rdpr %tpc, %g1 ; \ 256 ERRATUM50(%g1) ; \ 257 rdpr %tba, %g2 ; \ 258 sub %g1, %g2, %g2 ; \ 259 srlx %g2, 5, %g2 ; \ 260 andn %g2, 0x200, %g2 ; \ 261 cmp %g2, 0x80 ; \ 262 blu,pt %xcc, 9f ; \ 263 cmp %g2, 0x100 ; \ 264 bgeu,pt %xcc, 9f ; \ 265 or %g1, 0x7f, %g1 ; \ 266 wrpr %g1, xor, %tnpc ; \ 267 stxa_g0_sfsr ; \ 268 ldx [ASP_REG + 8], %g2 ; \ 269 ldx [ASP_REG + 0], %g1 ; \ 270 inc 16, ASP_REG ; \ 271 done ; \ 2729: ldx [ASP_REG + 8], %g2 ; \ 273 ldx [ASP_REG + 0], %g1 ; \ 274 inc 16, ASP_REG 275 276/* 277 * For certain faults we need to clear the sfsr mmu register before returning. 278 */ 279#define RSF_CLR_SFSR \ 280 wr %g0, ASI_DMMU, %asi ; \ 281 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 282 283#define RSF_XOR(off) ((0x80 - off) - 1) 284 285/* 286 * Instruction offsets in spill and fill trap handlers for handling certain 287 * nested traps, and corresponding xor constants for wrpr. 288 */ 289#define RSF_OFF_ALIGN 0x60 290#define RSF_OFF_MMU 0x70 291 292#define RESUME_SPILLFILL_ALIGN \ 293 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) 294#define RESUME_SPILLFILL_MMU \ 295 RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) 296#define RESUME_SPILLFILL_MMU_CLR_SFSR \ 297 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) 298 299/* 300 * Constant to add to %tnpc when taking a fill trap just before returning to 301 * user mode. 302 */ 303#define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill 304 305/* 306 * Retry a spill or fill with a different wstate due to an alignment fault. 307 * We may just be using the wrong stack offset. 308 */ 309#define RSF_ALIGN_RETRY(ws) \ 310 wrpr %g0, (ws), %wstate ; \ 311 retry ; \ 312 .align 16 313 314/* 315 * Generate a T_SPILL or T_FILL trap if the window operation fails. 316 */ 317#define RSF_TRAP(type) \ 318 b %xcc, tl0_sftrap ; \ 319 mov type, %g2 ; \ 320 .align 16 321 322/* 323 * Game over if the window operation fails. 324 */ 325#define RSF_FATAL(type) \ 326 b %xcc, rsf_fatal ; \ 327 mov type, %g2 ; \ 328 .align 16 329 330/* 331 * Magic to resume from a failed fill a few instructions after the corrsponding 332 * restore. This is used on return from the kernel to usermode. 333 */ 334#define RSF_FILL_MAGIC \ 335 rdpr %tnpc, %g1 ; \ 336 add %g1, RSF_FILL_INC, %g1 ; \ 337 wrpr %g1, 0, %tnpc ; \ 338 done ; \ 339 .align 16 340 341/* 342 * Spill to the pcb if a spill to the user stack in kernel mode fails. 343 */ 344#define RSF_SPILL_TOPCB \ 345 b,a %xcc, tl1_spill_topcb ; \ 346 nop ; \ 347 .align 16 348 349ENTRY(rsf_fatal) 350#if KTR_COMPILE & KTR_TRAP 351 CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" 352 , %g1, %g3, %g4, 7, 8, 9) 353 rdpr %tt, %g3 354 stx %g3, [%g1 + KTR_PARM1] 355 stx %g2, [%g1 + KTR_PARM2] 3569: 357#endif 358 359 KSTACK_CHECK 360 361 sir 362END(rsf_fatal) 363 364 .comm intrnames, IV_MAX * 8 365 .comm eintrnames, 0 366 367 .comm intrcnt, IV_MAX * 8 368 .comm eintrcnt, 0 369 370/* 371 * Trap table and associated macros 372 * 373 * Due to its size a trap table is an inherently hard thing to represent in 374 * code in a clean way. There are approximately 1024 vectors, of 8 or 32 375 * instructions each, many of which are identical. The way that this is 376 * layed out is the instructions (8 or 32) for the actual trap vector appear 377 * as an AS macro. In general this code branches to tl0_trap or tl1_trap, 378 * but if not supporting code can be placed just after the definition of the 379 * macro. The macros are then instantiated in a different section (.trap), 380 * which is setup to be placed by the linker at the beginning of .text, and the 381 * code around the macros is moved to the end of trap table. In this way the 382 * code that must be sequential in memory can be split up, and located near 383 * its supporting code so that it is easier to follow. 384 */ 385 386 /* 387 * Clean window traps occur when %cleanwin is zero to ensure that data 388 * is not leaked between address spaces in registers. 389 */ 390 .macro clean_window 391 clr %o0 392 clr %o1 393 clr %o2 394 clr %o3 395 clr %o4 396 clr %o5 397 clr %o6 398 clr %o7 399 clr %l0 400 clr %l1 401 clr %l2 402 clr %l3 403 clr %l4 404 clr %l5 405 clr %l6 406 rdpr %cleanwin, %l7 407 inc %l7 408 wrpr %l7, 0, %cleanwin 409 clr %l7 410 retry 411 .align 128 412 .endm 413 414 /* 415 * Stack fixups for entry from user mode. We are still running on the 416 * user stack, and with its live registers, so we must save soon. We 417 * are on alternate globals so we do have some registers. Set the 418 * transitional window state, and do the save. If this traps we 419 * we attempt to spill a window to the user stack. If this fails, 420 * we spill the window to the pcb and continue. Spilling to the pcb 421 * must not fail. 422 * 423 * NOTE: Must be called with alternate globals and clobbers %g1. 424 */ 425 426 .macro tl0_split 427 rdpr %wstate, %g1 428 wrpr %g1, WSTATE_TRANSITION, %wstate 429 save 430 .endm 431 432 .macro tl0_setup type 433 tl0_split 434 b %xcc, tl0_trap 435 mov \type, %o0 436 .endm 437 438 /* 439 * Generic trap type. Call trap() with the specified type. 440 */ 441 .macro tl0_gen type 442 tl0_setup \type 443 .align 32 444 .endm 445 446 /* 447 * This is used to suck up the massive swaths of reserved trap types. 448 * Generates count "reserved" trap vectors. 449 */ 450 .macro tl0_reserved count 451 .rept \count 452 tl0_gen T_RESERVED 453 .endr 454 .endm 455 456 .macro tl0_fp_restore 457 wr %g0, FPRS_FEF, %fprs 458 wr %g0, ASI_BLK_S, %asi 459 ldda [PCB_REG + PCB_FPSTATE + FP_FB0] %asi, %f0 460 ldda [PCB_REG + PCB_FPSTATE + FP_FB1] %asi, %f16 461 ldda [PCB_REG + PCB_FPSTATE + FP_FB2] %asi, %f32 462 ldda [PCB_REG + PCB_FPSTATE + FP_FB3] %asi, %f48 463 membar #Sync 464 done 465 .align 32 466 .endm 467 468 .macro tl0_insn_excptn 469 wrpr %g0, PSTATE_ALT, %pstate 470 wr %g0, ASI_IMMU, %asi 471 rdpr %tpc, %g3 472 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 473 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 474 membar #Sync 475 b %xcc, tl0_sfsr_trap 476 mov T_INSTRUCTION_EXCEPTION, %g2 477 .align 32 478 .endm 479 480 .macro tl0_data_excptn 481 wrpr %g0, PSTATE_ALT, %pstate 482 wr %g0, ASI_DMMU, %asi 483 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 484 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 485 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 486 membar #Sync 487 b %xcc, tl0_sfsr_trap 488 mov T_DATA_EXCEPTION, %g2 489 .align 32 490 .endm 491 492 .macro tl0_align 493 wr %g0, ASI_DMMU, %asi 494 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 495 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 496 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 497 membar #Sync 498 b %xcc, tl0_sfsr_trap 499 mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 500 .align 32 501 .endm 502 503ENTRY(tl0_sfsr_trap) 504 tl0_split 505 mov %g3, %o4 506 mov %g4, %o5 507 b %xcc, tl0_trap 508 mov %g2, %o0 509END(tl0_sfsr_trap) 510 511 .macro tl0_intr level, mask 512 tl0_split 513 set \mask, %o1 514 b %xcc, tl0_intr 515 mov \level, %o0 516 .align 32 517 .endm 518 519#define INTR(level, traplvl) \ 520 tl ## traplvl ## _intr level, 1 << level 521 522#define TICK(traplvl) \ 523 tl ## traplvl ## _intr PIL_TICK, 1 524 525#define INTR_LEVEL(tl) \ 526 INTR(1, tl) ; \ 527 INTR(2, tl) ; \ 528 INTR(3, tl) ; \ 529 INTR(4, tl) ; \ 530 INTR(5, tl) ; \ 531 INTR(6, tl) ; \ 532 INTR(7, tl) ; \ 533 INTR(8, tl) ; \ 534 INTR(9, tl) ; \ 535 INTR(10, tl) ; \ 536 INTR(11, tl) ; \ 537 INTR(12, tl) ; \ 538 INTR(13, tl) ; \ 539 TICK(tl) ; \ 540 INTR(15, tl) ; 541 542 .macro tl0_intr_level 543 INTR_LEVEL(0) 544 .endm 545 546 .macro intr_vector 547 ldxa [%g0] ASI_INTR_RECEIVE, %g1 548 andcc %g1, IRSR_BUSY, %g0 549 bnz,a,pt %xcc, intr_enqueue 550 nop 551 sir 552 .align 32 553 .endm 554 555 .macro immu_miss_user 556 /* 557 * Initialize the page size walker. 558 */ 559 mov TS_MIN, %g2 560 561 /* 562 * Loop over all supported page sizes. 563 */ 564 565 /* 566 * Compute the page shift for the page size we are currently looking 567 * for. 568 */ 5691: add %g2, %g2, %g3 570 add %g3, %g2, %g3 571 add %g3, PAGE_SHIFT, %g3 572 573 /* 574 * Extract the virtual page number from the contents of the tag 575 * access register. 576 */ 577 srlx %g1, %g3, %g3 578 579 /* 580 * Compute the tte bucket address. 581 */ 582 ldxa [%g0 + AA_IMMU_TSB] %asi, %g5 583 and %g3, TSB_BUCKET_MASK, %g4 584 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 585 add %g4, %g5, %g4 586 587 /* 588 * Compute the tte tag target. 589 */ 590 sllx %g3, TV_SIZE_BITS, %g3 591 or %g3, %g2, %g3 592 593 /* 594 * Loop over the ttes in this bucket 595 */ 596 597 /* 598 * Load the tte. Note that this instruction may fault, clobbering 599 * the contents of the tag access register, %g5, %g6, and %g7. We 600 * do not use %g5, and %g6 and %g7 are not used until this instruction 601 * completes successfully. 602 */ 6032: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 604 605 /* 606 * Check that its valid and executable and that the tte tags match. 607 */ 608 brgez,pn %g7, 3f 609 andcc %g7, TD_EXEC, %g0 610 bz,pn %xcc, 3f 611 cmp %g3, %g6 612 bne,pn %xcc, 3f 613 EMPTY 614 615 /* 616 * We matched a tte, load the tlb. 617 */ 618 619 /* 620 * Set the reference bit, if it's currently clear. 621 */ 622 andcc %g7, TD_REF, %g0 623 bz,a,pn %xcc, tl0_immu_miss_set_ref 624 nop 625 626 /* 627 * Load the tte tag and data into the tlb and retry the instruction. 628 */ 629 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 630 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 631 retry 632 633 /* 634 * Advance to the next tte in this bucket, and check the low bits 635 * of the bucket pointer to see if we've finished the bucket. 636 */ 6373: add %g4, 1 << TTE_SHIFT, %g4 638 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 639 bnz,pt %xcc, 2b 640 EMPTY 641 642 /* 643 * See if we just checked the largest page size, and advance to the 644 * next one if not. 645 */ 646 cmp %g2, TS_MAX 647 bne,pt %xcc, 1b 648 add %g2, 1, %g2 649 .endm 650 651 .macro tl0_immu_miss 652 /* 653 * Load the virtual page number and context from the tag access 654 * register. We ignore the context. 655 */ 656 wr %g0, ASI_IMMU, %asi 657 ldxa [%g0 + AA_IMMU_TAR] %asi, %g1 658 659 /* 660 * Try a fast inline lookup of the user tsb. 661 */ 662 immu_miss_user 663 664 /* 665 * Not in user tsb, call c code. 666 */ 667 ba,a %xcc, tl0_immu_miss_trap 668 .align 128 669 .endm 670 671ENTRY(tl0_immu_miss_set_ref) 672 /* 673 * Set the reference bit. 674 */ 675 TTE_SET_REF(%g4, %g2, %g3) 676 677 /* 678 * May have become invalid during casxa, in which case start over. 679 */ 680 brgez,pn %g2, 1f 681 nop 682 683 /* 684 * Load the tte tag and data into the tlb and retry the instruction. 685 */ 686 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 687 stxa %g2, [%g0] ASI_ITLB_DATA_IN_REG 6881: retry 689END(tl0_immu_miss_set_ref) 690 691ENTRY(tl0_immu_miss_trap) 692 /* 693 * Put back the contents of the tag access register, in case we 694 * faulted. 695 */ 696 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 697 membar #Sync 698 699 /* 700 * Switch to alternate globals. 701 */ 702 wrpr %g0, PSTATE_ALT, %pstate 703 704 /* 705 * Reload the tag access register. 706 */ 707 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 708 709 /* 710 * Save the tag access register, and call common trap code. 711 */ 712 tl0_split 713 mov %g2, %o3 714 b %xcc, tl0_trap 715 mov T_INSTRUCTION_MISS, %o0 716END(tl0_immu_miss_trap) 717 718 .macro dmmu_miss_user 719 /* 720 * Initialize the page size walker. 721 */ 722 mov TS_MIN, %g2 723 724 /* 725 * Loop over all supported page sizes. 726 */ 727 728 /* 729 * Compute the page shift for the page size we are currently looking 730 * for. 731 */ 7321: add %g2, %g2, %g3 733 add %g3, %g2, %g3 734 add %g3, PAGE_SHIFT, %g3 735 736 /* 737 * Extract the virtual page number from the contents of the tag 738 * access register. 739 */ 740 srlx %g1, %g3, %g3 741 742 /* 743 * Compute the tte bucket address. 744 */ 745 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 746 and %g3, TSB_BUCKET_MASK, %g4 747 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 748 add %g4, %g5, %g4 749 750 /* 751 * Compute the tte tag target. 752 */ 753 sllx %g3, TV_SIZE_BITS, %g3 754 or %g3, %g2, %g3 755 756 /* 757 * Loop over the ttes in this bucket 758 */ 759 760 /* 761 * Load the tte. Note that this instruction may fault, clobbering 762 * the contents of the tag access register, %g5, %g6, and %g7. We 763 * do not use %g5, and %g6 and %g7 are not used until this instruction 764 * completes successfully. 765 */ 7662: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 767 768 /* 769 * Check that its valid and that the virtual page numbers match. 770 */ 771 brgez,pn %g7, 3f 772 cmp %g3, %g6 773 bne,pn %xcc, 3f 774 EMPTY 775 776 /* 777 * We matched a tte, load the tlb. 778 */ 779 780 /* 781 * Set the reference bit, if it's currently clear. 782 */ 783 andcc %g7, TD_REF, %g0 784 bz,a,pn %xcc, dmmu_miss_user_set_ref 785 nop 786 787 /* 788 * Load the tte tag and data into the tlb and retry the instruction. 789 */ 790 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 791 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 792 retry 793 794 /* 795 * Advance to the next tte in this bucket, and check the low bits 796 * of the bucket pointer to see if we've finished the bucket. 797 */ 7983: add %g4, 1 << TTE_SHIFT, %g4 799 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 800 bnz,pt %xcc, 2b 801 EMPTY 802 803 /* 804 * See if we just checked the largest page size, and advance to the 805 * next one if not. 806 */ 807 cmp %g2, TS_MAX 808 bne,pt %xcc, 1b 809 add %g2, 1, %g2 810 .endm 811 812ENTRY(dmmu_miss_user_set_ref) 813 /* 814 * Set the reference bit. 815 */ 816 TTE_SET_REF(%g4, %g2, %g3) 817 818 /* 819 * May have become invalid during casxa, in which case start over. 820 */ 821 brgez,pn %g2, 1f 822 nop 823 824 /* 825 * Load the tte tag and data into the tlb and retry the instruction. 826 */ 827 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 828 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 8291: retry 830END(dmmu_miss_user_set_ref) 831 832 .macro tl0_dmmu_miss 833 /* 834 * Load the virtual page number and context from the tag access 835 * register. We ignore the context. 836 */ 837 wr %g0, ASI_DMMU, %asi 838 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 839 840 /* 841 * Try a fast inline lookup of the primary tsb. 842 */ 843 dmmu_miss_user 844 845 /* 846 * Not in user tsb, call c code. 847 */ 848 ba,a %xcc, tl0_dmmu_miss_trap 849 .align 128 850 .endm 851 852ENTRY(tl0_dmmu_miss_trap) 853 /* 854 * Put back the contents of the tag access register, in case we 855 * faulted. 856 */ 857 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 858 membar #Sync 859 860 /* 861 * Switch to alternate globals. 862 */ 863 wrpr %g0, PSTATE_ALT, %pstate 864 865 /* 866 * Reload the tag access register. 867 */ 868 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 869 870 /* 871 * Save the tag access register and call common trap code. 872 */ 873 tl0_split 874 mov %g2, %o3 875 b %xcc, tl0_trap 876 mov T_DATA_MISS, %o0 877END(tl0_dmmu_miss_trap) 878 879 .macro dmmu_prot_user 880 /* 881 * Initialize the page size walker. 882 */ 883 mov TS_MIN, %g2 884 885 /* 886 * Loop over all supported page sizes. 887 */ 888 889 /* 890 * Compute the page shift for the page size we are currently looking 891 * for. 892 */ 8931: add %g2, %g2, %g3 894 add %g3, %g2, %g3 895 add %g3, PAGE_SHIFT, %g3 896 897 /* 898 * Extract the virtual page number from the contents of the tag 899 * access register. 900 */ 901 srlx %g1, %g3, %g3 902 903 /* 904 * Compute the tte bucket address. 905 */ 906 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 907 and %g3, TSB_BUCKET_MASK, %g4 908 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 909 add %g4, %g5, %g4 910 911 /* 912 * Compute the tte tag target. 913 */ 914 sllx %g3, TV_SIZE_BITS, %g3 915 or %g3, %g2, %g3 916 917 /* 918 * Loop over the ttes in this bucket 919 */ 920 921 /* 922 * Load the tte. Note that this instruction may fault, clobbering 923 * the contents of the tag access register, %g5, %g6, and %g7. We 924 * do not use %g5, and %g6 and %g7 are not used until this instruction 925 * completes successfully. 926 */ 9272: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 928 929 /* 930 * Check that its valid and writable and that the virtual page 931 * numbers match. 932 */ 933 brgez,pn %g7, 4f 934 andcc %g7, TD_SW, %g0 935 bz,pn %xcc, 4f 936 cmp %g3, %g6 937 bne,pn %xcc, 4f 938 nop 939 940 /* 941 * Set the hardware write bit. 942 */ 943 TTE_SET_W(%g4, %g2, %g3) 944 945 /* 946 * Delete the old TLB entry and clear the sfsr. 947 */ 948 srlx %g1, PAGE_SHIFT, %g3 949 sllx %g3, PAGE_SHIFT, %g3 950 stxa %g0, [%g3] ASI_DMMU_DEMAP 951 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 952 membar #Sync 953 954 /* 955 * May have become invalid during casxa, in which case start over. 956 */ 957 brgez,pn %g2, 3f 958 or %g2, TD_W, %g2 959 960 /* 961 * Load the tte data into the tlb and retry the instruction. 962 */ 963 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 964 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 9653: retry 966 967 /* 968 * Check the low bits to see if we've finished the bucket. 969 */ 9704: add %g4, 1 << TTE_SHIFT, %g4 971 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 972 bnz,pt %xcc, 2b 973 EMPTY 974 975 /* 976 * See if we just checked the largest page size, and advance to the 977 * next one if not. 978 */ 979 cmp %g2, TS_MAX 980 bne,pt %xcc, 1b 981 add %g2, 1, %g2 982 .endm 983 984 .macro tl0_dmmu_prot 985 ba,a %xcc, tl0_dmmu_prot_1 986 nop 987 .align 128 988 .endm 989 990ENTRY(tl0_dmmu_prot_1) 991 /* 992 * Load the virtual page number and context from the tag access 993 * register. We ignore the context. 994 */ 995 wr %g0, ASI_DMMU, %asi 996 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 997 998 /* 999 * Try a fast inline lookup of the tsb. 1000 */ 1001 dmmu_prot_user 1002 1003 /* 1004 * Not in user tsb, call c code. 1005 */ 1006 b,a %xcc, tl0_dmmu_prot_trap 1007 nop 1008END(tl0_dmmu_prot_1) 1009 1010ENTRY(tl0_dmmu_prot_trap) 1011 /* 1012 * Put back the contents of the tag access register, in case we 1013 * faulted. 1014 */ 1015 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1016 membar #Sync 1017 1018 /* 1019 * Switch to alternate globals. 1020 */ 1021 wrpr %g0, PSTATE_ALT, %pstate 1022 1023 /* 1024 * Load the tar, sfar and sfsr. 1025 */ 1026 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1027 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1028 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1029 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1030 membar #Sync 1031 1032 /* 1033 * Save the mmu registers and call common trap code. 1034 */ 1035 tl0_split 1036 mov %g2, %o3 1037 mov %g3, %o4 1038 mov %g4, %o5 1039 b %xcc, tl0_trap 1040 mov T_DATA_PROTECTION, %o0 1041END(tl0_dmmu_prot_trap) 1042 1043 .macro tl0_spill_0_n 1044 wr %g0, ASI_AIUP, %asi 1045 SPILL(stxa, %sp + SPOFF, 8, %asi) 1046 saved 1047 retry 1048 .align 32 1049 RSF_TRAP(T_SPILL) 1050 RSF_TRAP(T_SPILL) 1051 .endm 1052 1053 .macro tl0_spill_1_n 1054 wr %g0, ASI_AIUP, %asi 1055 SPILL(stwa, %sp, 4, %asi) 1056 saved 1057 retry 1058 .align 32 1059 RSF_TRAP(T_SPILL) 1060 RSF_TRAP(T_SPILL) 1061 .endm 1062 1063 .macro tl0_fill_0_n 1064 wr %g0, ASI_AIUP, %asi 1065 FILL(ldxa, %sp + SPOFF, 8, %asi) 1066 restored 1067 retry 1068 .align 32 1069 RSF_TRAP(T_FILL) 1070 RSF_TRAP(T_FILL) 1071 .endm 1072 1073 .macro tl0_fill_1_n 1074 wr %g0, ASI_AIUP, %asi 1075 FILL(lduwa, %sp, 4, %asi) 1076 restored 1077 retry 1078 .align 32 1079 RSF_TRAP(T_FILL) 1080 RSF_TRAP(T_FILL) 1081 .endm 1082 1083ENTRY(tl0_sftrap) 1084 rdpr %tstate, %g1 1085 and %g1, TSTATE_CWP_MASK, %g1 1086 wrpr %g1, 0, %cwp 1087 tl0_split 1088 b %xcc, tl0_trap 1089 mov %g2, %o0 1090END(tl0_sftrap) 1091 1092 .macro tl0_spill_bad count 1093 .rept \count 1094 sir 1095 .align 128 1096 .endr 1097 .endm 1098 1099 .macro tl0_fill_bad count 1100 .rept \count 1101 sir 1102 .align 128 1103 .endr 1104 .endm 1105 1106 .macro tl0_syscall 1107 tl0_split 1108 b %xcc, tl0_syscall 1109 mov T_SYSCALL, %o0 1110 .align 32 1111 .endm 1112 1113 .macro tl0_soft count 1114 .rept \count 1115 tl0_gen T_SOFT 1116 .endr 1117 .endm 1118 1119 .macro tl1_split 1120 rdpr %wstate, %g1 1121 wrpr %g1, WSTATE_NESTED, %wstate 1122 save %sp, -CCFSZ, %sp 1123 .endm 1124 1125 .macro tl1_setup type 1126 tl1_split 1127 b %xcc, tl1_trap 1128 mov \type | T_KERNEL, %o0 1129 .endm 1130 1131 .macro tl1_gen type 1132 tl1_setup \type 1133 .align 32 1134 .endm 1135 1136 .macro tl1_reserved count 1137 .rept \count 1138 tl1_gen T_RESERVED 1139 .endr 1140 .endm 1141 1142 .macro tl1_insn_excptn 1143 wrpr %g0, PSTATE_ALT, %pstate 1144 wr %g0, ASI_IMMU, %asi 1145 rdpr %tpc, %g3 1146 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 1147 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 1148 membar #Sync 1149 b %xcc, tl1_insn_exceptn_trap 1150 mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 1151 .align 32 1152 .endm 1153 1154ENTRY(tl1_insn_exceptn_trap) 1155 tl1_split 1156 mov %g3, %o4 1157 mov %g4, %o5 1158 b %xcc, tl1_trap 1159 mov %g2, %o0 1160END(tl1_insn_exceptn_trap) 1161 1162 .macro tl1_data_excptn 1163 wrpr %g0, PSTATE_ALT, %pstate 1164 b,a %xcc, tl1_data_excptn_trap 1165 nop 1166 .align 32 1167 .endm 1168 1169ENTRY(tl1_data_excptn_trap) 1170 RESUME_SPILLFILL_MMU_CLR_SFSR 1171 b %xcc, tl1_sfsr_trap 1172 mov T_DATA_EXCEPTION | T_KERNEL, %g2 1173END(tl1_data_excptn_trap) 1174 1175 .macro tl1_align 1176 b,a %xcc, tl1_align_trap 1177 nop 1178 .align 32 1179 .endm 1180 1181ENTRY(tl1_align_trap) 1182 RESUME_SPILLFILL_ALIGN 1183 b %xcc, tl1_sfsr_trap 1184 mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 1185END(tl1_data_excptn_trap) 1186 1187ENTRY(tl1_sfsr_trap) 1188 wr %g0, ASI_DMMU, %asi 1189 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1190 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1191 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1192 membar #Sync 1193 1194 tl1_split 1195 mov %g3, %o4 1196 mov %g4, %o5 1197 b %xcc, tl1_trap 1198 mov %g2, %o0 1199END(tl1_sfsr_trap) 1200 1201 .macro tl1_intr level, mask 1202 tl1_split 1203 set \mask, %o1 1204 b %xcc, tl1_intr 1205 mov \level, %o0 1206 .align 32 1207 .endm 1208 1209 .macro tl1_intr_level 1210 INTR_LEVEL(1) 1211 .endm 1212 1213ENTRY(intr_dequeue) 1214 save %sp, -CCFSZ, %sp 1215 12161: ldx [PCPU(IRHEAD)], %l0 1217 brnz,a,pt %l0, 2f 1218 nop 1219 1220 ret 1221 restore 1222 12232: wrpr %g0, PSTATE_NORMAL, %pstate 1224 1225 ldx [%l0 + IR_NEXT], %l1 1226 brnz,pt %l1, 3f 1227 stx %l1, [PCPU(IRHEAD)] 1228 PCPU_ADDR(IRHEAD, %l1) 1229 stx %l1, [PCPU(IRTAIL)] 1230 12313: ldx [%l0 + IR_FUNC], %o0 1232 ldx [%l0 + IR_ARG], %o1 1233 ldx [%l0 + IR_VEC], %o2 1234 1235 ldx [PCPU(IRFREE)], %l1 1236 stx %l1, [%l0 + IR_NEXT] 1237 stx %l0, [PCPU(IRFREE)] 1238 1239 wrpr %g0, PSTATE_KERNEL, %pstate 1240 1241 call %o0 1242 mov %o1, %o0 1243 ba,a %xcc, 1b 1244 nop 1245END(intr_dequeue) 1246 1247/* 1248 * Handle a vectored interrupt. 1249 * 1250 * This is either a data bearing mondo vector interrupt, or a cross trap 1251 * request from another cpu. In either case the hardware supplies an 1252 * interrupt packet, in the form of 3 data words which are read from internal 1253 * registers. A data bearing mondo vector packet consists of an interrupt 1254 * number in the first data word, and zero in 2nd and 3rd. We use the 1255 * interrupt number to find the function, argument and priority from the 1256 * intr_vector table, allocate and fill in an intr_request from the per-cpu 1257 * free list, link it onto the per-cpu active list and finally post a softint 1258 * at the desired priority. Cross trap requests come in 2 forms, direct 1259 * and queued. Direct requests are distinguished by the first data word 1260 * being zero. The 2nd data word carries a function to call and the 3rd 1261 * an argument to pass. The function is jumped to directly. It executes 1262 * in nucleus context on interrupt globals and with all interrupts disabled, 1263 * therefore it must be fast, and the things that it can do are limited. 1264 * Queued cross trap requests are handled much like mondo vectors, except 1265 * that the function, argument and priority are contained in the interrupt 1266 * packet itself. They are distinguished by the upper 4 bits of the data 1267 * word being non-zero, which specifies the priority of the softint to 1268 * deliver. 1269 * 1270 * Register usage: 1271 * %g1 - pointer to intr_request 1272 * %g2 - pointer to intr_vector, temp once required data is loaded 1273 * %g3 - interrupt number for mondo vectors, unused otherwise 1274 * %g4 - function, from the interrupt packet for cross traps, or 1275 * loaded from the interrupt registers for mondo vecors 1276 * %g5 - argument, as above for %g4 1277 * %g6 - softint priority 1278 */ 1279ENTRY(intr_enqueue) 1280 /* 1281 * Load the interrupt packet from the hardware. 1282 */ 1283 wr %g0, ASI_SDB_INTR_R, %asi 1284 ldxa [%g0 + AA_SDB_INTR_D0] %asi, %g3 1285 ldxa [%g0 + AA_SDB_INTR_D1] %asi, %g4 1286 ldxa [%g0 + AA_SDB_INTR_D2] %asi, %g5 1287 stxa %g0, [%g0] ASI_INTR_RECEIVE 1288 membar #Sync 1289 1290#if KTR_COMPILE & KTR_INTR 1291 CATR(KTR_INTR, "intr_enqueue: data=%#lx %#lx %#lx" 1292 , %g1, %g2, %g6, 7, 8, 9) 1293 stx %g3, [%g1 + KTR_PARM1] 1294 stx %g4, [%g1 + KTR_PARM2] 1295 stx %g5, [%g1 + KTR_PARM3] 12969: 1297#endif 1298 1299 /* 1300 * If the first data word is zero this is a direct cross trap request. 1301 * The 2nd word points to code to execute and the 3rd is an argument 1302 * to pass. Jump to it. 1303 */ 1304 brnz,a,pt %g3, 1f 1305 nop 1306 1307#if KTR_COMPILE & KTR_INTR 1308 CATR(KTR_INTR, "intr_enqueue: direct ipi func=%#lx arg=%#lx" 1309 , %g1, %g2, %g6, 7, 8, 9) 1310 stx %g4, [%g1 + KTR_PARM1] 1311 stx %g5, [%g1 + KTR_PARM2] 13129: 1313#endif 1314 1315 jmpl %g4, %g0 1316 nop 1317 /* NOTREACHED */ 1318 1319 /* 1320 * If the high 4 bits of the 1st data word are non-zero, this is a 1321 * queued cross trap request to be delivered as a softint. The high 1322 * 4 bits of the 1st data word specify a priority, and the 2nd and 1323 * 3rd a function and argument. 1324 */ 13251: srlx %g3, 60, %g6 1326 brnz,a,pn %g6, 2f 1327 clr %g3 1328 1329 /* 1330 * Find the function, argument and desired priority from the 1331 * intr_vector table. 1332 */ 1333 SET(intr_vectors, %g4, %g2) 1334 sllx %g3, IV_SHIFT, %g4 1335 add %g2, %g4, %g2 1336 1337#if KTR_COMPILE & KTR_INTR 1338 CATR(KTR_INTR, "intr_enqueue: mondo vector func=%#lx arg=%#lx pri=%#lx" 1339 , %g4, %g5, %g6, 7, 8, 9) 1340 ldx [%g2 + IV_FUNC], %g5 1341 stx %g5, [%g4 + KTR_PARM1] 1342 ldx [%g2 + IV_ARG], %g5 1343 stx %g5, [%g4 + KTR_PARM2] 1344 ldx [%g2 + IV_PRI], %g5 1345 stx %g5, [%g4 + KTR_PARM3] 13469: 1347#endif 1348 1349 ldx [%g2 + IV_FUNC], %g4 1350 ldx [%g2 + IV_ARG], %g5 1351 lduw [%g2 + IV_PRI], %g6 1352 1353 ba,a %xcc, 3f 1354 nop 1355 1356 /* 1357 * Get a intr_request from the free list. There should always be one 1358 * unless we are getting an interrupt storm from stray interrupts, in 1359 * which case the we will deference a NULL pointer and panic. 1360 */ 13612: 1362#if KTR_COMPILE & KTR_INTR 1363 CATR(KTR_INTR, "intr_enqueue: queued ipi func=%#lx arg=%#lx pri=%#lx" 1364 , %g1, %g2, %g3, 7, 8, 9) 1365 stx %g4, [%g1 + KTR_PARM1] 1366 stx %g5, [%g1 + KTR_PARM2] 1367 stx %g6, [%g1 + KTR_PARM3] 13689: 1369 clr %g3 1370#endif 1371 13723: 1373 ldx [PCPU(IRFREE)], %g1 1374 ldx [%g1 + IR_NEXT], %g2 1375 stx %g2, [PCPU(IRFREE)] 1376 1377 /* 1378 * Store the vector number, function, argument and priority. 1379 */ 1380 stw %g3, [%g1 + IR_VEC] 1381 stx %g4, [%g1 + IR_FUNC] 1382 stx %g5, [%g1 + IR_ARG] 1383 stw %g6, [%g1 + IR_PRI] 1384 1385 /* 1386 * Link it onto the end of the active list. 1387 */ 1388 stx %g0, [%g1 + IR_NEXT] 1389 ldx [PCPU(IRTAIL)], %g4 1390 stx %g1, [%g4] 1391 add %g1, IR_NEXT, %g1 1392 stx %g1, [PCPU(IRTAIL)] 1393 1394 /* 1395 * Trigger a softint at the level indicated by the priority. 1396 */ 1397 mov 1, %g1 1398 sllx %g1, %g6, %g1 1399 1400#if KTR_COMPILE & KTR_INTR 1401 CATR(KTR_INTR, "intr_enqueue: softint pil=%#lx pri=%#lx mask=%#lx" 1402 , %g2, %g3, %g4, 7, 8, 9) 1403 rdpr %pil, %g3 1404 stx %g3, [%g2 + KTR_PARM1] 1405 stx %g6, [%g2 + KTR_PARM2] 1406 stx %g1, [%g2 + KTR_PARM3] 14079: 1408#endif 1409 1410 wr %g1, 0, %asr20 1411 1412 /* 1413 * Done, retry the instruction. 1414 */ 1415 retry 1416END(intr_enqueue) 1417 1418 .macro tl1_immu_miss 1419 /* 1420 * Load the context and the virtual page number from the tag access 1421 * register. We ignore the context. 1422 */ 1423 wr %g0, ASI_IMMU, %asi 1424 ldxa [%g0 + AA_IMMU_TAR] %asi, %g5 1425 1426 /* 1427 * Compute the address of the tte. The tsb mask and address of the 1428 * tsb are patched at startup. 1429 */ 1430 .globl tl1_immu_miss_patch_1 1431tl1_immu_miss_patch_1: 1432 sethi %hi(TSB_KERNEL_MASK), %g6 1433 or %g6, %lo(TSB_KERNEL_MASK), %g6 1434 sethi %hi(TSB_KERNEL), %g7 1435 1436 srlx %g5, TAR_VPN_SHIFT, %g5 1437 and %g5, %g6, %g6 1438 sllx %g6, TTE_SHIFT, %g6 1439 add %g6, %g7, %g6 1440 1441 /* 1442 * Load the tte. 1443 */ 1444 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1445 1446 /* 1447 * Check that its valid and executable and that the virtual page 1448 * numbers match. 1449 */ 1450 brgez,pn %g7, tl1_immu_miss_trap 1451 andcc %g7, TD_EXEC, %g0 1452 bz,pn %xcc, tl1_immu_miss_trap 1453 srlx %g6, TV_SIZE_BITS, %g6 1454 cmp %g5, %g6 1455 bne,pn %xcc, tl1_immu_miss_trap 1456 EMPTY 1457 1458 /* 1459 * Set the reference bit if its currently clear. 1460 */ 1461 andcc %g7, TD_REF, %g0 1462 bz,a,pn %xcc, tl1_immu_miss_set_ref 1463 nop 1464 1465 /* 1466 * Load the tte data into the TLB and retry the instruction. 1467 */ 1468 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 1469 retry 1470 .align 128 1471 .endm 1472 1473ENTRY(tl1_immu_miss_set_ref) 1474 /* 1475 * Recompute the tte address, which we clobbered loading the tte. The 1476 * tsb mask and address of the tsb are patched at startup. 1477 */ 1478 .globl tl1_immu_miss_patch_2 1479tl1_immu_miss_patch_2: 1480 sethi %hi(TSB_KERNEL_MASK), %g6 1481 or %g6, %lo(TSB_KERNEL_MASK), %g6 1482 sethi %hi(TSB_KERNEL), %g7 1483 1484 and %g5, %g6, %g5 1485 sllx %g5, TTE_SHIFT, %g5 1486 add %g5, %g7, %g5 1487 1488 /* 1489 * Set the reference bit. 1490 */ 1491 TTE_SET_REF(%g5, %g6, %g7) 1492 1493 /* 1494 * May have become invalid during casxa, in which case start over. 1495 */ 1496 brgez,pn %g6, 1f 1497 nop 1498 1499 /* 1500 * Load the tte data into the TLB and retry the instruction. 1501 */ 1502 stxa %g6, [%g0] ASI_ITLB_DATA_IN_REG 15031: retry 1504END(tl1_immu_miss_set_ref) 1505 1506ENTRY(tl1_immu_miss_trap) 1507 /* 1508 * Switch to alternate globals. 1509 */ 1510 wrpr %g0, PSTATE_ALT, %pstate 1511 1512 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 1513 1514 tl1_split 1515 mov %g2, %o3 1516 b %xcc, tl1_trap 1517 mov T_INSTRUCTION_MISS | T_KERNEL, %o0 1518END(tl1_immu_miss_trap) 1519 1520 .macro tl1_dmmu_miss 1521 /* 1522 * Load the context and the virtual page number from the tag access 1523 * register. 1524 */ 1525 wr %g0, ASI_DMMU, %asi 1526 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1527 1528 /* 1529 * Extract the context from the contents of the tag access register. 1530 * If its non-zero this is a fault on a user address. Note that the 1531 * faulting address is passed in %g2. 1532 */ 1533 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1534 brnz,a,pn %g6, tl1_dmmu_miss_user 1535 mov %g5, %g1 1536 1537 /* 1538 * Check for the direct mapped physical region. These addresses have 1539 * the high bit set so they are negative. 1540 */ 1541 brlz,pn %g5, tl1_dmmu_miss_direct 1542 EMPTY 1543 1544 /* 1545 * Compute the address of the tte. The tsb mask and address of the 1546 * tsb are patched at startup. 1547 */ 1548 .globl tl1_dmmu_miss_patch_1 1549tl1_dmmu_miss_patch_1: 1550 sethi %hi(TSB_KERNEL_MASK), %g6 1551 or %g6, %lo(TSB_KERNEL_MASK), %g6 1552 sethi %hi(TSB_KERNEL), %g7 1553 1554 srlx %g5, TAR_VPN_SHIFT, %g5 1555 and %g5, %g6, %g6 1556 sllx %g6, TTE_SHIFT, %g6 1557 add %g6, %g7, %g6 1558 1559 /* 1560 * Load the tte. 1561 */ 1562 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1563 1564 /* 1565 * Check that its valid and that the virtual page numbers match. 1566 */ 1567 brgez,pn %g7, tl1_dmmu_miss_trap 1568 srlx %g6, TV_SIZE_BITS, %g6 1569 cmp %g5, %g6 1570 bne,pn %xcc, tl1_dmmu_miss_trap 1571 EMPTY 1572 1573 /* 1574 * Set the reference bit if its currently clear. 1575 */ 1576 andcc %g7, TD_REF, %g0 1577 bz,a,pt %xcc, tl1_dmmu_miss_set_ref 1578 nop 1579 1580 /* 1581 * Load the tte data into the TLB and retry the instruction. 1582 */ 1583 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 1584 retry 1585 .align 128 1586 .endm 1587 1588ENTRY(tl1_dmmu_miss_set_ref) 1589 /* 1590 * Recompute the tte address, which we clobbered loading the tte. The 1591 * tsb mask and address of the tsb are patched at startup. 1592 */ 1593 .globl tl1_dmmu_miss_patch_2 1594tl1_dmmu_miss_patch_2: 1595 sethi %hi(TSB_KERNEL_MASK), %g6 1596 or %g6, %lo(TSB_KERNEL_MASK), %g6 1597 sethi %hi(TSB_KERNEL), %g7 1598 1599 and %g5, %g6, %g5 1600 sllx %g5, TTE_SHIFT, %g5 1601 add %g5, %g7, %g5 1602 1603 /* 1604 * Set the reference bit. 1605 */ 1606 TTE_SET_REF(%g5, %g6, %g7) 1607 1608 /* 1609 * May have become invalid during casxa, in which case start over. 1610 */ 1611 brgez,pn %g6, 1f 1612 nop 1613 1614 /* 1615 * Load the tte data into the TLB and retry the instruction. 1616 */ 1617 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 16181: retry 1619END(tl1_dmmu_miss_set_ref) 1620 1621ENTRY(tl1_dmmu_miss_trap) 1622 /* 1623 * Switch to alternate globals. 1624 */ 1625 wrpr %g0, PSTATE_ALT, %pstate 1626 1627 KSTACK_CHECK 1628 1629 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1630 1631 tl1_split 1632 mov %g2, %o3 1633 b %xcc, tl1_trap 1634 mov T_DATA_MISS | T_KERNEL, %o0 1635END(tl1_dmmu_miss_trap) 1636 1637ENTRY(tl1_dmmu_miss_direct) 1638 /* 1639 * Check the cache bits in the virtual address to see if this mapping 1640 * is virtually cacheable. We set this up so that the masks fit in 1641 * immediates... Note that the arithmetic shift sign extends, keeping 1642 * all the top bits set. 1643 */ 1644 srax %g5, TLB_DIRECT_SHIFT, %g5 1645 andcc %g5, TLB_DIRECT_UNCACHEABLE, %g0 1646 mov TD_CP | TD_CV | TD_W, %g6 1647 movnz %xcc, TD_CP | TD_W, %g6 1648 or %g5, %g6, %g5 1649 1650 /* 1651 * Mask off the high bits of the virtual address to get the physical 1652 * address, and or in the tte bits. The high bit is left set in the 1653 * physical address, which corresponds to the tte valid bit, so that 1654 * we don't have to include it in the tte bits. We ignore the cache 1655 * bits, since they get shifted into the soft tte bits anyway. 1656 */ 1657 setx TLB_DIRECT_MASK & ~TD_V, %g7, %g6 1658 andn %g5, %g6, %g5 1659 1660 /* 1661 * Load the tte data into the TLB and retry the instruction. 1662 */ 1663 stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG 1664 retry 1665END(tl1_dmmu_miss_direct) 1666 1667ENTRY(tl1_dmmu_miss_user) 1668 /* 1669 * Try a fast inline lookup of the user tsb. 1670 */ 1671 dmmu_miss_user 1672 1673 /* 1674 * Put back the contents of the tag access register, in case we 1675 * faulted. 1676 */ 1677 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1678 membar #Sync 1679 1680 /* 1681 * Switch to alternate globals. 1682 */ 1683 wrpr %g0, PSTATE_ALT, %pstate 1684 1685 /* 1686 * Handle faults during window spill/fill. 1687 */ 1688 RESUME_SPILLFILL_MMU 1689 1690 /* 1691 * Reload the tag access register. 1692 */ 1693 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1694 1695 tl1_split 1696 mov %g2, %o3 1697 b %xcc, tl1_trap 1698 mov T_DATA_MISS | T_KERNEL, %o0 1699END(tl1_dmmu_miss_user) 1700 1701 .macro tl1_dmmu_prot 1702 ba,a %xcc, tl1_dmmu_prot_1 1703 nop 1704 .align 128 1705 .endm 1706 1707ENTRY(tl1_dmmu_prot_1) 1708 /* 1709 * Load the context and the virtual page number from the tag access 1710 * register. 1711 */ 1712 wr %g0, ASI_DMMU, %asi 1713 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1714 1715 /* 1716 * Extract the context from the contents of the tag access register. 1717 * If its non-zero this is a fault on a user address, otherwise get 1718 * the virtual page number. 1719 */ 1720 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1721 brnz,a,pn %g6, tl1_dmmu_prot_user 1722 mov %g5, %g1 1723 1724 /* 1725 * Compute the address of the tte. The tsb mask and address of the 1726 * tsb are patched at startup. 1727 */ 1728 .globl tl1_dmmu_prot_patch_1 1729tl1_dmmu_prot_patch_1: 1730 sethi %hi(TSB_KERNEL_MASK), %g6 1731 or %g6, %lo(TSB_KERNEL_MASK), %g6 1732 sethi %hi(TSB_KERNEL), %g7 1733 1734 srlx %g5, TAR_VPN_SHIFT, %g5 1735 and %g5, %g6, %g6 1736 sllx %g6, TTE_SHIFT, %g6 1737 add %g6, %g7, %g6 1738 1739 /* 1740 * Load the tte. 1741 */ 1742 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1743 1744 /* 1745 * Check that its valid and writeable and that the virtual page 1746 * numbers match. 1747 */ 1748 brgez,pn %g7, tl1_dmmu_prot_trap 1749 andcc %g7, TD_SW, %g0 1750 bz,pn %xcc, tl1_dmmu_prot_trap 1751 srlx %g6, TV_SIZE_BITS, %g6 1752 cmp %g5, %g6 1753 bne,pn %xcc, tl1_dmmu_prot_trap 1754 EMPTY 1755 1756 /* 1757 * Delete the old TLB entry and clear the sfsr. 1758 */ 1759 sllx %g5, TAR_VPN_SHIFT, %g6 1760 or %g6, TLB_DEMAP_NUCLEUS, %g6 1761 stxa %g0, [%g6] ASI_DMMU_DEMAP 1762 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1763 membar #Sync 1764 1765 /* 1766 * Recompute the tte address, which we clobbered loading the tte. The 1767 * tsb mask and address of the tsb are patched at startup. 1768 */ 1769 .globl tl1_dmmu_prot_patch_2 1770tl1_dmmu_prot_patch_2: 1771 sethi %hi(TSB_KERNEL_MASK), %g6 1772 or %g6, %lo(TSB_KERNEL_MASK), %g6 1773 sethi %hi(TSB_KERNEL), %g7 1774 1775 and %g5, %g6, %g5 1776 sllx %g5, TTE_SHIFT, %g5 1777 add %g5, %g7, %g5 1778 1779 /* 1780 * Set the hardware write bit. 1781 */ 1782 TTE_SET_W(%g5, %g6, %g7) 1783 1784 /* 1785 * May have become invalid during casxa, in which case start over. 1786 */ 1787 brgez,pn %g6, 1f 1788 or %g6, TD_W, %g6 1789 1790 /* 1791 * Load the tte data into the TLB and retry the instruction. 1792 */ 1793 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 17941: retry 1795END(tl1_dmmu_prot_1) 1796 1797ENTRY(tl1_dmmu_prot_user) 1798 /* 1799 * Try a fast inline lookup of the user tsb. 1800 */ 1801 dmmu_prot_user 1802 1803 /* 1804 * Put back the contents of the tag access register, in case we 1805 * faulted. 1806 */ 1807 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1808 membar #Sync 1809 1810 /* 1811 * Switch to alternate globals. 1812 */ 1813 wrpr %g0, PSTATE_ALT, %pstate 1814 1815 /* Handle faults during window spill/fill. */ 1816 RESUME_SPILLFILL_MMU_CLR_SFSR 1817 1818 b,a %xcc, tl1_dmmu_prot_trap 1819 nop 1820END(tl1_dmmu_prot_user) 1821 1822ENTRY(tl1_dmmu_prot_trap) 1823 /* 1824 * Switch to alternate globals. 1825 */ 1826 wrpr %g0, PSTATE_ALT, %pstate 1827 1828 /* 1829 * Load the sfar, sfsr and tar. Clear the sfsr. 1830 */ 1831 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1832 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1833 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1834 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1835 membar #Sync 1836 1837 tl1_split 1838 mov %g2, %o3 1839 mov %g3, %o4 1840 mov %g4, %o5 1841 b %xcc, tl1_trap 1842 mov T_DATA_PROTECTION | T_KERNEL, %o0 1843END(tl1_dmmu_prot_trap) 1844 1845 .macro tl1_spill_0_n 1846 SPILL(stx, %sp + SPOFF, 8, EMPTY) 1847 saved 1848 retry 1849 .align 32 1850 RSF_FATAL(T_SPILL) 1851 RSF_FATAL(T_SPILL) 1852 .endm 1853 1854 .macro tl1_spill_2_n 1855 wr %g0, ASI_AIUP, %asi 1856 SPILL(stxa, %sp + SPOFF, 8, %asi) 1857 saved 1858 retry 1859 .align 32 1860 RSF_SPILL_TOPCB 1861 RSF_SPILL_TOPCB 1862 .endm 1863 1864 .macro tl1_spill_3_n 1865 wr %g0, ASI_AIUP, %asi 1866 SPILL(stwa, %sp, 4, %asi) 1867 saved 1868 retry 1869 .align 32 1870 RSF_SPILL_TOPCB 1871 RSF_SPILL_TOPCB 1872 .endm 1873 1874 .macro tl1_spill_0_o 1875 wr %g0, ASI_AIUP, %asi 1876 SPILL(stxa, %sp + SPOFF, 8, %asi) 1877 saved 1878 retry 1879 .align 32 1880 RSF_SPILL_TOPCB 1881 RSF_SPILL_TOPCB 1882 .endm 1883 1884 .macro tl1_spill_1_o 1885 wr %g0, ASI_AIUP, %asi 1886 SPILL(stwa, %sp, 4, %asi) 1887 saved 1888 retry 1889 .align 32 1890 RSF_SPILL_TOPCB 1891 RSF_SPILL_TOPCB 1892 .endm 1893 1894 .macro tl1_spill_2_o 1895 RSF_SPILL_TOPCB 1896 .align 128 1897 .endm 1898 1899 .macro tl1_fill_0_n 1900 FILL(ldx, %sp + SPOFF, 8, EMPTY) 1901 restored 1902 retry 1903 .align 32 1904 RSF_FATAL(T_FILL) 1905 RSF_FATAL(T_FILL) 1906 .endm 1907 1908 .macro tl1_fill_2_n 1909 wr %g0, ASI_AIUP, %asi 1910 FILL(ldxa, %sp + SPOFF, 8, %asi) 1911 restored 1912 retry 1913 .align 32 1914 RSF_FILL_MAGIC 1915 RSF_FILL_MAGIC 1916 .endm 1917 1918 .macro tl1_fill_3_n 1919 wr %g0, ASI_AIUP, %asi 1920 FILL(lduwa, %sp, 4, %asi) 1921 restored 1922 retry 1923 .align 32 1924 RSF_FILL_MAGIC 1925 RSF_FILL_MAGIC 1926 .endm 1927 1928/* 1929 * This is used to spill windows that are still occupied with user 1930 * data on kernel entry to the pcb. 1931 */ 1932ENTRY(tl1_spill_topcb) 1933 wrpr %g0, PSTATE_ALT, %pstate 1934 1935 /* Free some globals for our use. */ 1936 dec 24, ASP_REG 1937 stx %g1, [ASP_REG + 0] 1938 stx %g2, [ASP_REG + 8] 1939 stx %g3, [ASP_REG + 16] 1940 1941 ldx [PCB_REG + PCB_NSAVED], %g1 1942 1943 sllx %g1, PTR_SHIFT, %g2 1944 add %g2, PCB_REG, %g2 1945 stx %sp, [%g2 + PCB_RWSP] 1946 1947 sllx %g1, RW_SHIFT, %g2 1948 add %g2, PCB_REG, %g2 1949 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) 1950 1951 inc %g1 1952 stx %g1, [PCB_REG + PCB_NSAVED] 1953 1954#if KTR_COMPILE & KTR_TRAP 1955 CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" 1956 , %g1, %g2, %g3, 7, 8, 9) 1957 rdpr %tpc, %g2 1958 stx %g2, [%g1 + KTR_PARM1] 1959 rdpr %tnpc, %g2 1960 stx %g2, [%g1 + KTR_PARM2] 1961 stx %sp, [%g1 + KTR_PARM3] 1962 ldx [PCB_REG + PCB_NSAVED], %g2 1963 stx %g2, [%g1 + KTR_PARM4] 19649: 1965#endif 1966 1967 saved 1968 1969 ldx [ASP_REG + 16], %g3 1970 ldx [ASP_REG + 8], %g2 1971 ldx [ASP_REG + 0], %g1 1972 inc 24, ASP_REG 1973 retry 1974END(tl1_spill_topcb) 1975 1976 .macro tl1_spill_bad count 1977 .rept \count 1978 sir 1979 .align 128 1980 .endr 1981 .endm 1982 1983 .macro tl1_fill_bad count 1984 .rept \count 1985 sir 1986 .align 128 1987 .endr 1988 .endm 1989 1990 .macro tl1_soft count 1991 .rept \count 1992 tl1_gen T_SOFT | T_KERNEL 1993 .endr 1994 .endm 1995 1996 .sect .trap 1997 .align 0x8000 1998 .globl tl0_base 1999 2000tl0_base: 2001 tl0_reserved 8 ! 0x0-0x7 2002tl0_insn_excptn: 2003 tl0_insn_excptn ! 0x8 2004 tl0_reserved 1 ! 0x9 2005tl0_insn_error: 2006 tl0_gen T_INSTRUCTION_ERROR ! 0xa 2007 tl0_reserved 5 ! 0xb-0xf 2008tl0_insn_illegal: 2009 tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 2010tl0_priv_opcode: 2011 tl0_gen T_PRIVILEGED_OPCODE ! 0x11 2012 tl0_reserved 14 ! 0x12-0x1f 2013tl0_fp_disabled: 2014 tl0_gen T_FP_DISABLED ! 0x20 2015tl0_fp_ieee: 2016 tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 2017tl0_fp_other: 2018 tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 2019tl0_tag_ovflw: 2020 tl0_gen T_TAG_OFERFLOW ! 0x23 2021tl0_clean_window: 2022 clean_window ! 0x24 2023tl0_divide: 2024 tl0_gen T_DIVISION_BY_ZERO ! 0x28 2025 tl0_reserved 7 ! 0x29-0x2f 2026tl0_data_excptn: 2027 tl0_data_excptn ! 0x30 2028 tl0_reserved 1 ! 0x31 2029tl0_data_error: 2030 tl0_gen T_DATA_ERROR ! 0x32 2031 tl0_reserved 1 ! 0x33 2032tl0_align: 2033 tl0_align ! 0x34 2034tl0_align_lddf: 2035 tl0_gen T_RESERVED ! 0x35 2036tl0_align_stdf: 2037 tl0_gen T_RESERVED ! 0x36 2038tl0_priv_action: 2039 tl0_gen T_PRIVILEGED_ACTION ! 0x37 2040 tl0_reserved 9 ! 0x38-0x40 2041tl0_intr_level: 2042 tl0_intr_level ! 0x41-0x4f 2043 tl0_reserved 16 ! 0x50-0x5f 2044tl0_intr_vector: 2045 intr_vector ! 0x60 2046tl0_watch_phys: 2047 tl0_gen T_PA_WATCHPOINT ! 0x61 2048tl0_watch_virt: 2049 tl0_gen T_VA_WATCHPOINT ! 0x62 2050tl0_ecc: 2051 tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 2052tl0_immu_miss: 2053 tl0_immu_miss ! 0x64 2054tl0_dmmu_miss: 2055 tl0_dmmu_miss ! 0x68 2056tl0_dmmu_prot: 2057 tl0_dmmu_prot ! 0x6c 2058 tl0_reserved 16 ! 0x70-0x7f 2059tl0_spill_0_n: 2060 tl0_spill_0_n ! 0x80 2061tl0_spill_1_n: 2062 tl0_spill_1_n ! 0x84 2063 tl0_spill_bad 14 ! 0x88-0xbf 2064tl0_fill_0_n: 2065 tl0_fill_0_n ! 0xc0 2066tl0_fill_1_n: 2067 tl0_fill_1_n ! 0xc4 2068 tl0_fill_bad 14 ! 0xc8-0xff 2069tl0_soft: 2070 tl0_reserved 1 ! 0x100 2071 tl0_gen T_BREAKPOINT ! 0x101 2072 tl0_gen T_DIVISION_BY_ZERO ! 0x102 2073 tl0_reserved 1 ! 0x103 2074 tl0_gen T_CLEAN_WINDOW ! 0x104 2075 tl0_gen T_RANGE_CHECK ! 0x105 2076 tl0_gen T_FIX_ALIGNMENT ! 0x106 2077 tl0_gen T_INTEGER_OVERFLOW ! 0x107 2078 tl0_reserved 1 ! 0x108 2079 tl0_syscall ! 0x109 2080 tl0_fp_restore ! 0x10a 2081 tl0_reserved 5 ! 0x10b-0x10f 2082 tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 2083 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 2084 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 2085 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 2086 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 2087 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 2088 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 2089 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 2090 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 2091 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 2092 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a 2093 tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b 2094 tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c 2095 tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d 2096 tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e 2097 tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f 2098 tl0_reserved 224 ! 0x120-0x1ff 2099 2100tl1_base: 2101 tl1_reserved 8 ! 0x200-0x207 2102tl1_insn_excptn: 2103 tl1_insn_excptn ! 0x208 2104 tl1_reserved 1 ! 0x209 2105tl1_insn_error: 2106 tl1_gen T_INSTRUCTION_ERROR ! 0x20a 2107 tl1_reserved 5 ! 0x20b-0x20f 2108tl1_insn_illegal: 2109 tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 2110tl1_priv_opcode: 2111 tl1_gen T_PRIVILEGED_OPCODE ! 0x211 2112 tl1_reserved 14 ! 0x212-0x21f 2113tl1_fp_disabled: 2114 tl1_gen T_FP_DISABLED ! 0x220 2115tl1_fp_ieee: 2116 tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 2117tl1_fp_other: 2118 tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 2119tl1_tag_ovflw: 2120 tl1_gen T_TAG_OFERFLOW ! 0x223 2121tl1_clean_window: 2122 clean_window ! 0x224 2123tl1_divide: 2124 tl1_gen T_DIVISION_BY_ZERO ! 0x228 2125 tl1_reserved 7 ! 0x229-0x22f 2126tl1_data_excptn: 2127 tl1_data_excptn ! 0x230 2128 tl1_reserved 1 ! 0x231 2129tl1_data_error: 2130 tl1_gen T_DATA_ERROR ! 0x232 2131 tl1_reserved 1 ! 0x233 2132tl1_align: 2133 tl1_align ! 0x234 2134tl1_align_lddf: 2135 tl1_gen T_RESERVED ! 0x235 2136tl1_align_stdf: 2137 tl1_gen T_RESERVED ! 0x236 2138tl1_priv_action: 2139 tl1_gen T_PRIVILEGED_ACTION ! 0x237 2140 tl1_reserved 9 ! 0x238-0x240 2141tl1_intr_level: 2142 tl1_intr_level ! 0x241-0x24f 2143 tl1_reserved 16 ! 0x250-0x25f 2144tl1_intr_vector: 2145 intr_vector ! 0x260 2146tl1_watch_phys: 2147 tl1_gen T_PA_WATCHPOINT ! 0x261 2148tl1_watch_virt: 2149 tl1_gen T_VA_WATCHPOINT ! 0x262 2150tl1_ecc: 2151 tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 2152tl1_immu_miss: 2153 tl1_immu_miss ! 0x264 2154tl1_dmmu_miss: 2155 tl1_dmmu_miss ! 0x268 2156tl1_dmmu_prot: 2157 tl1_dmmu_prot ! 0x26c 2158 tl1_reserved 16 ! 0x270-0x27f 2159tl1_spill_0_n: 2160 tl1_spill_0_n ! 0x280 2161 tl1_spill_bad 1 ! 0x284 2162tl1_spill_2_n: 2163 tl1_spill_2_n ! 0x288 2164tl1_spill_3_n: 2165 tl1_spill_3_n ! 0x29c 2166 tl1_spill_bad 4 ! 0x290-0x29f 2167tl1_spill_0_o: 2168 tl1_spill_0_o ! 0x2a0 2169tl1_spill_1_o: 2170 tl1_spill_1_o ! 0x2a4 2171tl1_spill_2_o: 2172 tl1_spill_2_o ! 0x2a8 2173 tl1_spill_bad 5 ! 0x2ac-0x2bf 2174tl1_fill_0_n: 2175 tl1_fill_0_n ! 0x2c0 2176 tl1_fill_bad 1 ! 0x2c4 2177tl1_fill_2_n: 2178 tl1_fill_2_n ! 0x2d0 2179tl1_fill_3_n: 2180 tl1_fill_3_n ! 0x2d4 2181 tl1_fill_bad 12 ! 0x2d8-0x2ff 2182 tl1_reserved 1 ! 0x300 2183tl1_breakpoint: 2184 tl1_gen T_BREAKPOINT ! 0x301 2185 tl1_gen T_RSTRWP_PHYS ! 0x302 2186 tl1_gen T_RSTRWP_VIRT ! 0x303 2187 tl1_reserved 252 ! 0x304-0x3ff 2188 2189/* 2190 * User trap entry point. 2191 * 2192 * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2193 * u_int sfsr) 2194 * 2195 * The following setup has been performed: 2196 * - the windows have been split and the active user window has been saved 2197 * (maybe just to the pcb) 2198 * - we are on alternate globals and interrupts are disabled 2199 * 2200 * We switch to the kernel stack, build a trapframe, switch to normal 2201 * globals, enable interrupts and call trap. 2202 * 2203 * NOTE: We must be very careful setting up the per-cpu pointer. We know that 2204 * it has been pre-set in alternate globals, so we read it from there and setup 2205 * the normal %g7 *before* enabling interrupts. This avoids any possibility 2206 * of cpu migration and using the wrong pcpup. 2207 */ 2208ENTRY(tl0_trap) 2209 /* 2210 * Force kernel store order. 2211 */ 2212 wrpr %g0, PSTATE_ALT, %pstate 2213 2214 rdpr %tstate, %l0 2215 rdpr %tpc, %l1 2216 rdpr %tnpc, %l2 2217 rd %y, %l3 2218 rd %fprs, %l4 2219 rdpr %wstate, %l5 2220 2221#if KTR_COMPILE & KTR_TRAP 2222 CATR(KTR_TRAP, 2223 "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2224 , %g1, %g2, %g3, 7, 8, 9) 2225 ldx [PCPU(CURTHREAD)], %g2 2226 stx %g2, [%g1 + KTR_PARM1] 2227 stx %o0, [%g1 + KTR_PARM2] 2228 rdpr %pil, %g2 2229 stx %g2, [%g1 + KTR_PARM3] 2230 stx %l1, [%g1 + KTR_PARM4] 2231 stx %l2, [%g1 + KTR_PARM5] 2232 stx %i6, [%g1 + KTR_PARM6] 22339: 2234#endif 2235 2236 and %l5, WSTATE_NORMAL_MASK, %l5 2237 2238 cmp %o0, UT_MAX 2239 bge,a,pt %xcc, 2f 2240 nop 2241 2242 ldx [PCPU(CURTHREAD)], %l6 2243 ldx [%l6 + TD_PROC], %l6 2244 ldx [%l6 + P_MD + MD_UTRAP], %l6 2245 brz,pt %l6, 2f 2246 sllx %o0, PTR_SHIFT, %l7 2247 ldx [%l6 + %l7], %l6 2248 brz,pt %l6, 2f 2249 andn %l0, TSTATE_CWP_MASK, %l7 2250 2251 ldx [PCB_REG + PCB_NSAVED], %g1 2252 brnz,a,pn %g1, 1f 2253 mov T_SPILL, %o0 2254 2255#if KTR_COMPILE & KTR_TRAP 2256 CATR(KTR_TRAP, "tl0_trap: user trap npc=%#lx" 2257 , %g1, %g2, %g3, 7, 8, 9) 2258 stx %l6, [%g1 + KTR_PARM1] 22599: 2260#endif 2261 2262 wrpr %l5, %wstate 2263 wrpr %l6, %tnpc 2264 rdpr %cwp, %l6 2265 wrpr %l6, %l7, %tstate 2266 2267 sub %fp, CCFSZ, %sp 2268 2269 /* 2270 * Need to store %fsr to pass it to the user trap handler. Otherwise, 2271 * the ftt field might be zeoed out in a store in another trap or 2272 * interrupt. Use the temporary stack for that. 2273 */ 2274 rd %fprs, %l3 2275 or %l3, FPRS_FEF, %l4 2276 wr %l4, 0, %fprs 2277 dec 8, ASP_REG 2278 stx %fsr, [ASP_REG] 2279 ldx [ASP_REG], %l4 2280 inc 8, ASP_REG 2281 wr %l3, 0, %fprs 2282 2283 mov %l0, %l5 2284 mov %l1, %l6 2285 mov %l2, %l7 2286 2287 done 2288 22891: 2290#if KTR_COMPILE & KTR_TRAP 2291 CATR(KTR_TRAP, "tl0_trap: defer user trap npc=%#lx nsaved=%#lx" 2292 , %g1, %g2, %g3, 7, 8, 9) 2293 stx %l6, [%g1 + KTR_PARM1] 2294 ldx [PCB_REG + PCB_NSAVED], %g2 2295 stx %g2, [%g1 + KTR_PARM2] 22969: 2297#endif 2298 22992: sllx %l5, WSTATE_OTHER_SHIFT, %l5 2300 wrpr %l5, WSTATE_KERNEL, %wstate 2301 rdpr %canrestore, %l6 2302 wrpr %l6, 0, %otherwin 2303 wrpr %g0, 0, %canrestore 2304 2305 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2306 2307 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2308 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2309 stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2310 2311 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2312 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2313 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2314 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2315 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2316 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2317 2318 wr %g0, FPRS_FEF, %fprs 2319 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2320 wr %g0, 0, %fprs 2321 2322 mov PCB_REG, %l0 2323 mov PCPU_REG, %l1 2324 wrpr %g0, PSTATE_NORMAL, %pstate 2325 2326 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2327 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2328 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2329 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2330 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2331 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2332 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2333 2334 mov %l0, PCB_REG 2335 mov %l1, PCPU_REG 2336 wrpr %g0, PSTATE_KERNEL, %pstate 2337 2338 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2339 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2340 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2341 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2342 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2343 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2344 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2345 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2346 2347.Ltl0_trap_reenter: 2348 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2349 call trap 2350 add %sp, CCFSZ + SPOFF, %o0 2351 b,a %xcc, tl0_ret 2352 nop 2353END(tl0_trap) 2354 2355/* 2356 * void tl0_syscall(u_int type) 2357 */ 2358ENTRY(tl0_syscall) 2359 /* 2360 * Force kernel store order. 2361 */ 2362 wrpr %g0, PSTATE_ALT, %pstate 2363 2364 rdpr %tstate, %l0 2365 rdpr %tpc, %l1 2366 rdpr %tnpc, %l2 2367 rd %y, %l3 2368 rd %fprs, %l4 2369 rdpr %wstate, %l5 2370 2371#if KTR_COMPILE & KTR_SYSC 2372 CATR(KTR_SYSC, 2373 "tl0_syscall: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2374 , %g1, %g2, %g3, 7, 8, 9) 2375 ldx [PCPU(CURTHREAD)], %g2 2376 stx %g2, [%g1 + KTR_PARM1] 2377 stx %o0, [%g1 + KTR_PARM2] 2378 rdpr %pil, %g2 2379 stx %g2, [%g1 + KTR_PARM3] 2380 stx %l1, [%g1 + KTR_PARM4] 2381 stx %l2, [%g1 + KTR_PARM5] 2382 stx %i6, [%g1 + KTR_PARM6] 23839: 2384#endif 2385 2386 and %l5, WSTATE_NORMAL_MASK, %l5 2387 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2388 wrpr %l5, WSTATE_KERNEL, %wstate 2389 rdpr %canrestore, %l6 2390 wrpr %l6, 0, %otherwin 2391 wrpr %g0, 0, %canrestore 2392 2393 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2394 2395 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2396 2397 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2398 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2399 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2400 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2401 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2402 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2403 2404 wr %g0, FPRS_FEF, %fprs 2405 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2406 wr %g0, 0, %fprs 2407 2408 mov PCB_REG, %l0 2409 mov PCPU_REG, %l1 2410 wrpr %g0, PSTATE_NORMAL, %pstate 2411 2412 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2413 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2414 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2415 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2416 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2417 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2418 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2419 2420 mov %l0, PCB_REG 2421 mov %l1, PCPU_REG 2422 wrpr %g0, PSTATE_KERNEL, %pstate 2423 2424 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2425 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2426 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2427 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2428 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2429 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2430 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2431 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2432 2433 call syscall 2434 add %sp, CCFSZ + SPOFF, %o0 2435 b,a %xcc, tl0_ret 2436 nop 2437END(tl0_syscall) 2438 2439/* 2440 * void tl0_intr(u_int level, u_int mask) 2441 */ 2442ENTRY(tl0_intr) 2443 /* 2444 * Force kernel store order. 2445 */ 2446 wrpr %g0, PSTATE_ALT, %pstate 2447 2448 rdpr %tstate, %l0 2449 rdpr %tpc, %l1 2450 rdpr %tnpc, %l2 2451 rd %y, %l3 2452 rd %fprs, %l4 2453 rdpr %wstate, %l5 2454 2455#if KTR_COMPILE & KTR_INTR 2456 CATR(KTR_INTR, 2457 "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2458 , %g1, %g2, %g3, 7, 8, 9) 2459 ldx [PCPU(CURTHREAD)], %g2 2460 stx %g2, [%g1 + KTR_PARM1] 2461 stx %o0, [%g1 + KTR_PARM2] 2462 rdpr %pil, %g2 2463 stx %g2, [%g1 + KTR_PARM3] 2464 stx %l1, [%g1 + KTR_PARM4] 2465 stx %l2, [%g1 + KTR_PARM5] 2466 stx %i6, [%g1 + KTR_PARM6] 24679: 2468#endif 2469 2470 wrpr %o0, 0, %pil 2471 wr %o1, 0, %asr21 2472 2473 and %l5, WSTATE_NORMAL_MASK, %l5 2474 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2475 wrpr %l5, WSTATE_KERNEL, %wstate 2476 rdpr %canrestore, %l6 2477 wrpr %l6, 0, %otherwin 2478 wrpr %g0, 0, %canrestore 2479 2480 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2481 2482 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2483 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2484 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2485 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2486 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2487 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2488 2489 wr %g0, FPRS_FEF, %fprs 2490 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2491 wr %g0, 0, %fprs 2492 2493 mov %o0, %l3 2494 mov T_INTERRUPT, %o1 2495 2496 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2497 stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2498 2499 mov PCB_REG, %l0 2500 mov PCPU_REG, %l1 2501 wrpr %g0, PSTATE_NORMAL, %pstate 2502 2503 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2504 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2505 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2506 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2507 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2508 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2509 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2510 2511 mov %l0, PCB_REG 2512 mov %l1, PCPU_REG 2513 wrpr %g0, PSTATE_KERNEL, %pstate 2514 2515 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2516 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2517 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2518 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2519 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2520 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2521 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2522 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2523 2524 call critical_enter 2525 nop 2526 2527 SET(cnt+V_INTR, %l1, %l0) 2528 ATOMIC_INC_INT(%l0, %l1, %l2) 2529 2530 SET(intr_handlers, %l1, %l0) 2531 sllx %l3, IH_SHIFT, %l1 2532 ldx [%l0 + %l1], %l1 2533 KASSERT(%l1, "tl0_intr: ih null") 2534 call %l1 2535 add %sp, CCFSZ + SPOFF, %o0 2536 2537 call critical_exit 2538 nop 2539 2540 b,a %xcc, tl0_ret 2541 nop 2542END(tl0_intr) 2543 2544ENTRY(tl0_ret) 2545#if KTR_COMPILE & KTR_TRAP 2546 CATR(KTR_TRAP, "tl0_ret: check ast td=%p (%s) pil=%#lx sflag=%#x" 2547 , %g1, %g2, %g3, 7, 8, 9) 2548 ldx [PCPU(CURTHREAD)], %g2 2549 stx %g2, [%g1 + KTR_PARM1] 2550 ldx [%g2 + TD_PROC], %g2 2551 add %g2, P_COMM, %g3 2552 stx %g3, [%g1 + KTR_PARM2] 2553 rdpr %pil, %g3 2554 stx %g3, [%g1 + KTR_PARM3] 2555 lduw [%g2 + P_SFLAG], %g3 2556 stx %g3, [%g1 + KTR_PARM4] 25579: 2558#endif 2559 2560 /* 2561 * Check for pending asts atomically with returning. We must raise 2562 * the pil before checking, and if no asts are found the pil must 2563 * remain raised until the retry is executed, or we risk missing asts 2564 * caused by interrupts occuring after the test. If the pil is lowered, 2565 * as it is when we call ast, the check must be re-executed. 2566 */ 25671: wrpr %g0, PIL_TICK, %pil 2568 ldx [PCPU(CURTHREAD)], %l0 2569 ldx [%l0 + TD_KSE], %l1 2570 lduw [%l1 + KE_FLAGS], %l2 2571 and %l2, KEF_ASTPENDING | KEF_NEEDRESCHED, %l2 2572 brz,a,pt %l2, 2f 2573 nop 2574 wrpr %g0, 0, %pil 2575 call ast 2576 add %sp, CCFSZ + SPOFF, %o0 2577 ba,a %xcc, 1b 2578 nop 2579 2580 /* 2581 * Check for windows that were spilled to the pcb and need to be 2582 * copied out. This must be the last thing that is done before the 2583 * return to usermode. If there are still user windows in the cpu 2584 * and we call a nested function after this, which causes them to be 2585 * spilled to the pcb, they will not be copied out and the stack will 2586 * be inconsistent. 2587 */ 25882: ldx [PCB_REG + PCB_NSAVED], %l1 2589 mov T_SPILL, %o0 2590 brnz,a,pn %l1, .Ltl0_trap_reenter 2591 wrpr %g0, 0, %pil 2592 2593 ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 2594 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 2595 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 2596 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 2597 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 2598 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 2599 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 2600 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 2601 2602 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2603 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2604 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2605 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l3 2606 ldub [%sp + SPOFF + CCFSZ + TF_FPRS], %l4 2607 ldub [%sp + SPOFF + CCFSZ + TF_WSTATE], %l5 2608 2609 wrpr %g0, PSTATE_NORMAL, %pstate 2610 2611 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2612 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2613 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2614 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2615 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2616 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 2617 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 2618 2619 wrpr %g0, PSTATE_ALT, %pstate 2620 2621 wrpr %g0, 0, %pil 2622 wrpr %l1, 0, %tpc 2623 wrpr %l2, 0, %tnpc 2624 wr %l3, 0, %y 2625 2626 andn %l0, TSTATE_CWP_MASK, %g1 2627 mov %l4, %g2 2628 2629 srlx %l5, WSTATE_OTHER_SHIFT, %g3 2630 wrpr %g3, WSTATE_TRANSITION, %wstate 2631 rdpr %otherwin, %o0 2632 wrpr %o0, 0, %canrestore 2633 wrpr %g0, 0, %otherwin 2634 wrpr %o0, 0, %cleanwin 2635 2636 /* 2637 * If this instruction causes a fill trap which fails to fill a window 2638 * from the user stack, we will resume at tl0_ret_fill_end and call 2639 * back into the kernel. 2640 */ 2641 restore 2642tl0_ret_fill: 2643 2644 rdpr %cwp, %g4 2645 wrpr %g1, %g4, %tstate 2646 wr %g2, 0, %fprs 2647 wrpr %g3, 0, %wstate 2648 2649#if KTR_COMPILE & KTR_TRAP 2650 CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2651 , %g2, %g3, %g4, 7, 8, 9) 2652 ldx [PCPU(CURTHREAD)], %g3 2653 stx %g3, [%g2 + KTR_PARM1] 2654 rdpr %pil, %g3 2655 stx %g3, [%g2 + KTR_PARM2] 2656 rdpr %tpc, %g3 2657 stx %g3, [%g2 + KTR_PARM3] 2658 rdpr %tnpc, %g3 2659 stx %g3, [%g2 + KTR_PARM4] 2660 stx %sp, [%g2 + KTR_PARM5] 26619: 2662#endif 2663 2664 retry 2665tl0_ret_fill_end: 2666 2667#if KTR_COMPILE & KTR_TRAP 2668 CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" 2669 , %l0, %l1, %l2, 7, 8, 9) 2670 rdpr %pstate, %l1 2671 stx %l1, [%l0 + KTR_PARM1] 2672 stx %l5, [%l0 + KTR_PARM2] 2673 stx %sp, [%l0 + KTR_PARM3] 26749: 2675#endif 2676 2677 /* 2678 * The fill failed and magic has been performed. Call trap again, 2679 * which will copyin the window on the user's behalf. 2680 */ 2681 wrpr %l5, 0, %wstate 2682 wrpr %g0, PSTATE_ALT, %pstate 2683 mov PCB_REG, %o0 2684 mov PCPU_REG, %o1 2685 wrpr %g0, PSTATE_NORMAL, %pstate 2686 mov %o0, PCB_REG 2687 mov %o1, PCPU_REG 2688 wrpr %g0, PSTATE_KERNEL, %pstate 2689 b %xcc, .Ltl0_trap_reenter 2690 mov T_FILL_RET, %o0 2691END(tl0_ret) 2692 2693/* 2694 * Kernel trap entry point 2695 * 2696 * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2697 * u_int sfsr) 2698 * 2699 * This is easy because the stack is already setup and the windows don't need 2700 * to be split. We build a trapframe and call trap(), the same as above, but 2701 * the outs don't need to be saved. 2702 */ 2703ENTRY(tl1_trap) 2704 sub %sp, TF_SIZEOF, %sp 2705 2706 rdpr %tstate, %l0 2707 rdpr %tpc, %l1 2708 rdpr %tnpc, %l2 2709 rdpr %pil, %l3 2710 rd %y, %l4 2711 rdpr %wstate, %l5 2712 2713#if KTR_COMPILE & KTR_TRAP 2714 CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" 2715 , %g1, %g2, %g3, 7, 8, 9) 2716 ldx [PCPU(CURTHREAD)], %g2 2717 stx %g2, [%g1 + KTR_PARM1] 2718 stx %o0, [%g1 + KTR_PARM2] 2719 stx %l3, [%g1 + KTR_PARM3] 2720 stx %l1, [%g1 + KTR_PARM4] 2721 stx %i6, [%g1 + KTR_PARM5] 27229: 2723#endif 2724 2725 wrpr %g0, 1, %tl 2726 2727 and %l5, WSTATE_OTHER_MASK, %l5 2728 wrpr %l5, WSTATE_KERNEL, %wstate 2729 2730 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2731 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2732 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2733 stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2734 stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2735 2736 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2737 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2738 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2739 stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2740 2741 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2742 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2743 2744 mov PCB_REG, %l4 2745 mov PCPU_REG, %l5 2746 wrpr %g0, PSTATE_NORMAL, %pstate 2747 2748 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2749 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2750 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2751 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2752 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2753 2754 mov %l4, PCB_REG 2755 mov %l5, PCPU_REG 2756 wrpr %g0, PSTATE_KERNEL, %pstate 2757 2758 call trap 2759 add %sp, CCFSZ + SPOFF, %o0 2760 2761 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2762 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2763 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2764 ldub [%sp + SPOFF + CCFSZ + TF_PIL], %l3 2765 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 2766 2767 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2768 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2769 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2770 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2771 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2772 2773 wrpr %g0, PSTATE_ALT, %pstate 2774 2775 andn %l0, TSTATE_CWP_MASK, %g1 2776 mov %l1, %g2 2777 mov %l2, %g3 2778 2779 wrpr %l3, 0, %pil 2780 wr %l4, 0, %y 2781 2782 restore 2783 2784 wrpr %g0, 2, %tl 2785 2786 rdpr %cwp, %g4 2787 wrpr %g1, %g4, %tstate 2788 wrpr %g2, 0, %tpc 2789 wrpr %g3, 0, %tnpc 2790 2791#if KTR_COMPILE & KTR_TRAP 2792 CATR(KTR_TRAP, "tl1_trap: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2793 , %g2, %g3, %g4, 7, 8, 9) 2794 ldx [PCPU(CURTHREAD)], %g3 2795 stx %g3, [%g2 + KTR_PARM1] 2796 rdpr %pil, %g3 2797 stx %g3, [%g2 + KTR_PARM2] 2798 rdpr %tstate, %g3 2799 stx %g3, [%g2 + KTR_PARM3] 2800 rdpr %tpc, %g3 2801 stx %g3, [%g2 + KTR_PARM4] 2802 stx %sp, [%g2 + KTR_PARM5] 28039: 2804#endif 2805 2806 retry 2807END(tl1_trap) 2808 2809/* 2810 * void tl1_intr(u_int level, u_int mask) 2811 */ 2812ENTRY(tl1_intr) 2813 sub %sp, TF_SIZEOF, %sp 2814 2815 rdpr %tstate, %l0 2816 rdpr %tpc, %l1 2817 rdpr %tnpc, %l2 2818 rdpr %pil, %l3 2819 rd %y, %l4 2820 rdpr %wstate, %l5 2821 2822#if KTR_COMPILE & KTR_INTR 2823 CATR(KTR_INTR, 2824 "tl1_intr: td=%p level=%#lx pil=%#lx pc=%#lx sp=%#lx" 2825 , %g1, %g2, %g3, 7, 8, 9) 2826 ldx [PCPU(CURTHREAD)], %g2 2827 stx %g2, [%g1 + KTR_PARM1] 2828 stx %o0, [%g1 + KTR_PARM2] 2829 stx %l3, [%g1 + KTR_PARM3] 2830 stx %l1, [%g1 + KTR_PARM4] 2831 stx %i6, [%g1 + KTR_PARM5] 28329: 2833#endif 2834 2835 wrpr %o0, 0, %pil 2836 wr %o1, 0, %asr21 2837 2838 wrpr %g0, 1, %tl 2839 2840 and %l5, WSTATE_OTHER_MASK, %l5 2841 wrpr %l5, WSTATE_KERNEL, %wstate 2842 2843 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2844 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2845 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2846 stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2847 stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2848 2849 mov %o0, %l7 2850 mov T_INTERRUPT | T_KERNEL, %o1 2851 2852 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2853 stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2854 2855 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2856 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2857 2858 mov PCB_REG, %l4 2859 mov PCPU_REG, %l5 2860 wrpr %g0, PSTATE_NORMAL, %pstate 2861 2862 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2863 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2864 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2865 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2866 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2867 2868 mov %l4, PCB_REG 2869 mov %l5, PCPU_REG 2870 wrpr %g0, PSTATE_KERNEL, %pstate 2871 2872 call critical_enter 2873 nop 2874 2875 SET(cnt+V_INTR, %l5, %l4) 2876 ATOMIC_INC_INT(%l4, %l5, %l6) 2877 2878 SET(intr_handlers, %l5, %l4) 2879 sllx %l7, IH_SHIFT, %l5 2880 ldx [%l4 + %l5], %l5 2881 KASSERT(%l5, "tl1_intr: ih null") 2882 call %l5 2883 add %sp, CCFSZ + SPOFF, %o0 2884 2885 call critical_exit 2886 nop 2887 2888 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 2889 2890 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2891 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2892 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2893 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2894 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2895 2896 wrpr %g0, PSTATE_ALT, %pstate 2897 2898 andn %l0, TSTATE_CWP_MASK, %g1 2899 mov %l1, %g2 2900 mov %l2, %g3 2901 wrpr %l3, 0, %pil 2902 wr %l4, 0, %y 2903 2904 restore 2905 2906 wrpr %g0, 2, %tl 2907 2908 rdpr %cwp, %g4 2909 wrpr %g1, %g4, %tstate 2910 wrpr %g2, 0, %tpc 2911 wrpr %g3, 0, %tnpc 2912 2913#if KTR_COMPILE & KTR_INTR 2914 CATR(KTR_INTR, "tl1_intr: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2915 , %g2, %g3, %g4, 7, 8, 9) 2916 ldx [PCPU(CURTHREAD)], %g3 2917 stx %g3, [%g2 + KTR_PARM1] 2918 rdpr %pil, %g3 2919 stx %g3, [%g2 + KTR_PARM2] 2920 rdpr %tstate, %g3 2921 stx %g3, [%g2 + KTR_PARM3] 2922 rdpr %tpc, %g3 2923 stx %g3, [%g2 + KTR_PARM4] 2924 stx %sp, [%g2 + KTR_PARM5] 29259: 2926#endif 2927 2928 retry 2929END(tl1_intr) 2930 2931/* 2932 * Freshly forked processes come here when switched to for the first time. 2933 * The arguments to fork_exit() have been setup in the locals, we must move 2934 * them to the outs. 2935 */ 2936ENTRY(fork_trampoline) 2937#if KTR_COMPILE & KTR_PROC 2938 CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" 2939 , %g1, %g2, %g3, 7, 8, 9) 2940 ldx [PCPU(CURTHREAD)], %g2 2941 stx %g2, [%g1 + KTR_PARM1] 2942 ldx [%g2 + TD_PROC], %g2 2943 add %g2, P_COMM, %g2 2944 stx %g2, [%g1 + KTR_PARM2] 2945 rdpr %cwp, %g2 2946 stx %g2, [%g1 + KTR_PARM3] 29479: 2948#endif 2949 mov %l0, %o0 2950 mov %l1, %o1 2951 call fork_exit 2952 mov %l2, %o2 2953 b,a %xcc, tl0_ret 2954 nop 2955END(fork_trampoline) 2956