exception.S revision 103784
11556Srgrimes/*- 21556Srgrimes * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 31556Srgrimes * 41556Srgrimes * Redistribution and use in source and binary forms, with or without 51556Srgrimes * modification, are permitted provided that the following conditions 61556Srgrimes * are met: 71556Srgrimes * 1. Redistributions of source code must retain the above copyright 81556Srgrimes * notice, this list of conditions and the following disclaimer. 91556Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 101556Srgrimes * notice, this list of conditions and the following disclaimer in the 111556Srgrimes * documentation and/or other materials provided with the distribution. 121556Srgrimes * 3. Berkeley Software Design Inc's name may not be used to endorse or 131556Srgrimes * promote products derived from this software without specific prior 141556Srgrimes * written permission. 151556Srgrimes * 161556Srgrimes * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 171556Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 181556Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 191556Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 201556Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 211556Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 221556Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 231556Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 241556Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 251556Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 261556Srgrimes * SUCH DAMAGE. 271556Srgrimes * 281556Srgrimes * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 291556Srgrimes */ 301556Srgrimes/*- 311556Srgrimes * Copyright (c) 2001 Jake Burkholder. 321556Srgrimes * All rights reserved. 331556Srgrimes * 341556Srgrimes * Redistribution and use in source and binary forms, with or without 351556Srgrimes * modification, are permitted provided that the following conditions 361556Srgrimes * are met: 371556Srgrimes * 1. Redistributions of source code must retain the above copyright 3820412Ssteve * notice, this list of conditions and the following disclaimer. 391556Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 401556Srgrimes * notice, this list of conditions and the following disclaimer in the 411556Srgrimes * documentation and/or other materials provided with the distribution. 421556Srgrimes * 431556Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 4435772Scharnier * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4520412Ssteve * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4635772Scharnier * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 4735772Scharnier * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 4835772Scharnier * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 491556Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 501556Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 511556Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 521556Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 531556Srgrimes * SUCH DAMAGE. 541556Srgrimes * 551556Srgrimes * $FreeBSD: head/sys/sparc64/sparc64/exception.S 103784 2002-09-22 06:51:41Z jake $ 561556Srgrimes */ 5718578Sache 581556Srgrimes#include "opt_ddb.h" 591556Srgrimes 601556Srgrimes#include <machine/asi.h> 611556Srgrimes#include <machine/asmacros.h> 621556Srgrimes#include <machine/ktr.h> 631556Srgrimes#include <machine/pstate.h> 641556Srgrimes#include <machine/trap.h> 651556Srgrimes#include <machine/tstate.h> 661556Srgrimes#include <machine/wstate.h> 671556Srgrimes 681556Srgrimes#include "assym.s" 691556Srgrimes 701556Srgrimes#define TSB_KERNEL_MASK 0x0 711556Srgrimes#define TSB_KERNEL 0x0 721556Srgrimes 731556Srgrimes .register %g2,#ignore 741556Srgrimes .register %g3,#ignore 751556Srgrimes .register %g6,#ignore 761556Srgrimes .register %g7,#ignore 771556Srgrimes 781556Srgrimes/* 7918578Sache * Atomically set the reference bit in a tte. 8018578Sache */ 8124348Simp#define TTE_SET_BIT(r1, r2, r3, bit) \ 821556Srgrimes add r1, TTE_DATA, r1 ; \ 831556Srgrimes ldx [r1], r2 ; \ 841556Srgrimes9: or r2, bit, r3 ; \ 851556Srgrimes casxa [r1] ASI_N, r2, r3 ; \ 861556Srgrimes cmp r2, r3 ; \ 871556Srgrimes bne,pn %xcc, 9b ; \ 881556Srgrimes mov r3, r2 891556Srgrimes 901556Srgrimes#define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF) 911556Srgrimes#define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W) 921556Srgrimes 931556Srgrimes/* 941556Srgrimes * Macros for spilling and filling live windows. 951556Srgrimes * 961556Srgrimes * NOTE: These macros use exactly 16 instructions, and it is assumed that the 971556Srgrimes * handler will not use more than 24 instructions total, to leave room for 981556Srgrimes * resume vectors which occupy the last 8 instructions. 991556Srgrimes */ 1001556Srgrimes 1011556Srgrimes#define SPILL(storer, base, size, asi) \ 1021556Srgrimes storer %l0, [base + (0 * size)] asi ; \ 1031556Srgrimes storer %l1, [base + (1 * size)] asi ; \ 10418546Simp storer %l2, [base + (2 * size)] asi ; \ 1051556Srgrimes storer %l3, [base + (3 * size)] asi ; \ 1061556Srgrimes storer %l4, [base + (4 * size)] asi ; \ 1071556Srgrimes storer %l5, [base + (5 * size)] asi ; \ 1081556Srgrimes storer %l6, [base + (6 * size)] asi ; \ 1091556Srgrimes storer %l7, [base + (7 * size)] asi ; \ 1101556Srgrimes storer %i0, [base + (8 * size)] asi ; \ 1111556Srgrimes storer %i1, [base + (9 * size)] asi ; \ 1121556Srgrimes storer %i2, [base + (10 * size)] asi ; \ 1131556Srgrimes storer %i3, [base + (11 * size)] asi ; \ 1141556Srgrimes storer %i4, [base + (12 * size)] asi ; \ 1151556Srgrimes storer %i5, [base + (13 * size)] asi ; \ 1161556Srgrimes storer %i6, [base + (14 * size)] asi ; \ 1171556Srgrimes storer %i7, [base + (15 * size)] asi 1181556Srgrimes 1191556Srgrimes#define FILL(loader, base, size, asi) \ 1201556Srgrimes loader [base + (0 * size)] asi, %l0 ; \ 1211556Srgrimes loader [base + (1 * size)] asi, %l1 ; \ 1221556Srgrimes loader [base + (2 * size)] asi, %l2 ; \ 1231556Srgrimes loader [base + (3 * size)] asi, %l3 ; \ 1241556Srgrimes loader [base + (4 * size)] asi, %l4 ; \ 1251556Srgrimes loader [base + (5 * size)] asi, %l5 ; \ 1261556Srgrimes loader [base + (6 * size)] asi, %l6 ; \ 1271556Srgrimes loader [base + (7 * size)] asi, %l7 ; \ 1281556Srgrimes loader [base + (8 * size)] asi, %i0 ; \ 1291556Srgrimes loader [base + (9 * size)] asi, %i1 ; \ 1301556Srgrimes loader [base + (10 * size)] asi, %i2 ; \ 1311556Srgrimes loader [base + (11 * size)] asi, %i3 ; \ 1321556Srgrimes loader [base + (12 * size)] asi, %i4 ; \ 1331556Srgrimes loader [base + (13 * size)] asi, %i5 ; \ 13411145Sbde loader [base + (14 * size)] asi, %i6 ; \ 1351556Srgrimes loader [base + (15 * size)] asi, %i7 1361556Srgrimes 1371556Srgrimes#define ERRATUM50(reg) mov reg, reg 1381556Srgrimes 1391556Srgrimes#define KSTACK_SLOP 1024 1401556Srgrimes 1411556Srgrimes/* 1421556Srgrimes * Sanity check the kernel stack and bail out if its wrong. 1431556Srgrimes * XXX: doesn't handle being on the panic stack. 1441556Srgrimes */ 1451556Srgrimes#define KSTACK_CHECK \ 1461556Srgrimes dec 16, ASP_REG ; \ 1471556Srgrimes stx %g1, [ASP_REG + 0] ; \ 1481556Srgrimes stx %g2, [ASP_REG + 8] ; \ 1491556Srgrimes add %sp, SPOFF, %g1 ; \ 1501556Srgrimes andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ 1511556Srgrimes bnz,a %xcc, tl1_kstack_fault ; \ 1521556Srgrimes inc 16, ASP_REG ; \ 1531556Srgrimes ldx [PCPU(CURTHREAD)], %g2 ; \ 1541556Srgrimes ldx [%g2 + TD_KSTACK], %g2 ; \ 1551556Srgrimes add %g2, KSTACK_SLOP, %g2 ; \ 1561556Srgrimes subcc %g1, %g2, %g1 ; \ 1571556Srgrimes ble,a %xcc, tl1_kstack_fault ; \ 1581556Srgrimes inc 16, ASP_REG ; \ 1591556Srgrimes set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ 1601556Srgrimes cmp %g1, %g2 ; \ 1611556Srgrimes bgt,a %xcc, tl1_kstack_fault ; \ 1621556Srgrimes inc 16, ASP_REG ; \ 1631556Srgrimes ldx [ASP_REG + 8], %g2 ; \ 1641556Srgrimes ldx [ASP_REG + 0], %g1 ; \ 1651556Srgrimes inc 16, ASP_REG 1661556Srgrimes 1671556SrgrimesENTRY(tl1_kstack_fault) 1681556Srgrimes rdpr %tl, %g1 1691556Srgrimes1: cmp %g1, 2 1701556Srgrimes be,a 2f 1711556Srgrimes nop 1721556Srgrimes 1731556Srgrimes#if KTR_COMPILE & KTR_TRAP 1741556Srgrimes CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" 1751556Srgrimes , %g2, %g3, %g4, 7, 8, 9) 1761556Srgrimes rdpr %tl, %g3 1771556Srgrimes stx %g3, [%g2 + KTR_PARM1] 1781556Srgrimes rdpr %tpc, %g3 1791556Srgrimes stx %g3, [%g2 + KTR_PARM1] 1801556Srgrimes rdpr %tnpc, %g3 1811556Srgrimes stx %g3, [%g2 + KTR_PARM1] 1821556Srgrimes9: 1831556Srgrimes#endif 1841556Srgrimes 18518578Sache sub %g1, 1, %g1 1861556Srgrimes wrpr %g1, 0, %tl 1871556Srgrimes ba,a %xcc, 1b 1881556Srgrimes nop 1891556Srgrimes 1901556Srgrimes2: 1911556Srgrimes#if KTR_COMPILE & KTR_TRAP 1921556Srgrimes CATR(KTR_TRAP, 1931556Srgrimes "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" 1941556Srgrimes , %g1, %g2, %g3, 7, 8, 9) 1951556Srgrimes add %sp, SPOFF, %g2 1961556Srgrimes stx %g2, [%g1 + KTR_PARM1] 1971556Srgrimes ldx [PCPU(CURTHREAD)], %g2 1981556Srgrimes ldx [%g2 + TD_KSTACK], %g2 1991556Srgrimes stx %g2, [%g1 + KTR_PARM2] 2001556Srgrimes rdpr %canrestore, %g2 2011556Srgrimes stx %g2, [%g1 + KTR_PARM3] 2021556Srgrimes rdpr %cansave, %g2 20311145Sbde stx %g2, [%g1 + KTR_PARM4] 2041556Srgrimes rdpr %otherwin, %g2 2051556Srgrimes stx %g2, [%g1 + KTR_PARM5] 2061556Srgrimes rdpr %wstate, %g2 2071556Srgrimes stx %g2, [%g1 + KTR_PARM6] 2081556Srgrimes9: 2091556Srgrimes#endif 2101556Srgrimes 2111556Srgrimes wrpr %g0, 0, %canrestore 2121556Srgrimes wrpr %g0, 6, %cansave 2131556Srgrimes wrpr %g0, 0, %otherwin 2141556Srgrimes wrpr %g0, WSTATE_KERNEL, %wstate 2151556Srgrimes 2161556Srgrimes sub ASP_REG, SPOFF + CCFSZ, %sp 2171556Srgrimes clr %fp 2181556Srgrimes 2191556Srgrimes b %xcc, tl1_trap 2201556Srgrimes mov T_KSTACK_FAULT | T_KERNEL, %o0 2211556SrgrimesEND(tl1_kstack_fault) 2221556Srgrimes 2231556Srgrimes/* 22411145Sbde * Magic to resume from a spill or fill trap. If we get an alignment or an 2251556Srgrimes * mmu fault during a spill or a fill, this macro will detect the fault and 2261556Srgrimes * resume at a set instruction offset in the trap handler. 2271556Srgrimes * 2281556Srgrimes * To check if the previous trap was a spill/fill we convert the trapped pc 2291556Srgrimes * to a trap type and verify that it is in the range of spill/fill vectors. 2301556Srgrimes * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the 2311556Srgrimes * tl bit allows us to detect both ranges with one test. 2321556Srgrimes * 2331556Srgrimes * This is: 2341556Srgrimes * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 2351556Srgrimes * 2361556Srgrimes * To calculate the new pc we take advantage of the xor feature of wrpr. 2371556Srgrimes * Forcing all the low bits of the trapped pc on we can produce any offset 2381556Srgrimes * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. 2391556Srgrimes * 2401556Srgrimes * 0x7f ^ 0x1f == 0x60 2411556Srgrimes * 0x1f == (0x80 - 0x60) - 1 2421556Srgrimes * 2431556Srgrimes * Which are the offset and xor value used to resume from alignment faults. 2441556Srgrimes */ 2451556Srgrimes 2461556Srgrimes/* 2471556Srgrimes * Determine if we have trapped inside of a spill/fill vector, and if so resume 2481556Srgrimes * at a fixed instruction offset in the trap vector. Must be called on 2491556Srgrimes * alternate globals. 2501556Srgrimes */ 25118546Simp#define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ 2521556Srgrimes dec 16, ASP_REG ; \ 2531556Srgrimes stx %g1, [ASP_REG + 0] ; \ 2541556Srgrimes stx %g2, [ASP_REG + 8] ; \ 2551556Srgrimes rdpr %tpc, %g1 ; \ 2561556Srgrimes ERRATUM50(%g1) ; \ 25711145Sbde rdpr %tba, %g2 ; \ 2581556Srgrimes sub %g1, %g2, %g2 ; \ 25911145Sbde srlx %g2, 5, %g2 ; \ 26011145Sbde andn %g2, 0x200, %g2 ; \ 2611556Srgrimes cmp %g2, 0x80 ; \ 262 blu,pt %xcc, 9f ; \ 263 cmp %g2, 0x100 ; \ 264 bgeu,pt %xcc, 9f ; \ 265 or %g1, 0x7f, %g1 ; \ 266 wrpr %g1, xor, %tnpc ; \ 267 stxa_g0_sfsr ; \ 268 ldx [ASP_REG + 8], %g2 ; \ 269 ldx [ASP_REG + 0], %g1 ; \ 270 inc 16, ASP_REG ; \ 271 done ; \ 2729: ldx [ASP_REG + 8], %g2 ; \ 273 ldx [ASP_REG + 0], %g1 ; \ 274 inc 16, ASP_REG 275 276/* 277 * For certain faults we need to clear the sfsr mmu register before returning. 278 */ 279#define RSF_CLR_SFSR \ 280 wr %g0, ASI_DMMU, %asi ; \ 281 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 282 283#define RSF_XOR(off) ((0x80 - off) - 1) 284 285/* 286 * Instruction offsets in spill and fill trap handlers for handling certain 287 * nested traps, and corresponding xor constants for wrpr. 288 */ 289#define RSF_OFF_ALIGN 0x60 290#define RSF_OFF_MMU 0x70 291 292#define RESUME_SPILLFILL_ALIGN \ 293 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) 294#define RESUME_SPILLFILL_MMU \ 295 RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) 296#define RESUME_SPILLFILL_MMU_CLR_SFSR \ 297 RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) 298 299/* 300 * Constant to add to %tnpc when taking a fill trap just before returning to 301 * user mode. 302 */ 303#define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill 304 305/* 306 * Retry a spill or fill with a different wstate due to an alignment fault. 307 * We may just be using the wrong stack offset. 308 */ 309#define RSF_ALIGN_RETRY(ws) \ 310 wrpr %g0, (ws), %wstate ; \ 311 retry ; \ 312 .align 16 313 314/* 315 * Generate a T_SPILL or T_FILL trap if the window operation fails. 316 */ 317#define RSF_TRAP(type) \ 318 b %xcc, tl0_sftrap ; \ 319 mov type, %g2 ; \ 320 .align 16 321 322/* 323 * Game over if the window operation fails. 324 */ 325#define RSF_FATAL(type) \ 326 b %xcc, rsf_fatal ; \ 327 mov type, %g2 ; \ 328 .align 16 329 330/* 331 * Magic to resume from a failed fill a few instructions after the corrsponding 332 * restore. This is used on return from the kernel to usermode. 333 */ 334#define RSF_FILL_MAGIC \ 335 rdpr %tnpc, %g1 ; \ 336 add %g1, RSF_FILL_INC, %g1 ; \ 337 wrpr %g1, 0, %tnpc ; \ 338 done ; \ 339 .align 16 340 341/* 342 * Spill to the pcb if a spill to the user stack in kernel mode fails. 343 */ 344#define RSF_SPILL_TOPCB \ 345 b,a %xcc, tl1_spill_topcb ; \ 346 nop ; \ 347 .align 16 348 349ENTRY(rsf_fatal) 350#if KTR_COMPILE & KTR_TRAP 351 CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" 352 , %g1, %g3, %g4, 7, 8, 9) 353 rdpr %tt, %g3 354 stx %g3, [%g1 + KTR_PARM1] 355 stx %g2, [%g1 + KTR_PARM2] 3569: 357#endif 358 359 KSTACK_CHECK 360 361 sir 362END(rsf_fatal) 363 364 .comm intrnames, IV_MAX * 8 365 .comm eintrnames, 0 366 367 .comm intrcnt, IV_MAX * 8 368 .comm eintrcnt, 0 369 370/* 371 * Trap table and associated macros 372 * 373 * Due to its size a trap table is an inherently hard thing to represent in 374 * code in a clean way. There are approximately 1024 vectors, of 8 or 32 375 * instructions each, many of which are identical. The way that this is 376 * layed out is the instructions (8 or 32) for the actual trap vector appear 377 * as an AS macro. In general this code branches to tl0_trap or tl1_trap, 378 * but if not supporting code can be placed just after the definition of the 379 * macro. The macros are then instantiated in a different section (.trap), 380 * which is setup to be placed by the linker at the beginning of .text, and the 381 * code around the macros is moved to the end of trap table. In this way the 382 * code that must be sequential in memory can be split up, and located near 383 * its supporting code so that it is easier to follow. 384 */ 385 386 /* 387 * Clean window traps occur when %cleanwin is zero to ensure that data 388 * is not leaked between address spaces in registers. 389 */ 390 .macro clean_window 391 clr %o0 392 clr %o1 393 clr %o2 394 clr %o3 395 clr %o4 396 clr %o5 397 clr %o6 398 clr %o7 399 clr %l0 400 clr %l1 401 clr %l2 402 clr %l3 403 clr %l4 404 clr %l5 405 clr %l6 406 rdpr %cleanwin, %l7 407 inc %l7 408 wrpr %l7, 0, %cleanwin 409 clr %l7 410 retry 411 .align 128 412 .endm 413 414 /* 415 * Stack fixups for entry from user mode. We are still running on the 416 * user stack, and with its live registers, so we must save soon. We 417 * are on alternate globals so we do have some registers. Set the 418 * transitional window state, and do the save. If this traps we 419 * we attempt to spill a window to the user stack. If this fails, 420 * we spill the window to the pcb and continue. Spilling to the pcb 421 * must not fail. 422 * 423 * NOTE: Must be called with alternate globals and clobbers %g1. 424 */ 425 426 .macro tl0_split 427 rdpr %wstate, %g1 428 wrpr %g1, WSTATE_TRANSITION, %wstate 429 save 430 .endm 431 432 .macro tl0_setup type 433 tl0_split 434 b %xcc, tl0_trap 435 mov \type, %o0 436 .endm 437 438 /* 439 * Generic trap type. Call trap() with the specified type. 440 */ 441 .macro tl0_gen type 442 tl0_setup \type 443 .align 32 444 .endm 445 446 /* 447 * This is used to suck up the massive swaths of reserved trap types. 448 * Generates count "reserved" trap vectors. 449 */ 450 .macro tl0_reserved count 451 .rept \count 452 tl0_gen T_RESERVED 453 .endr 454 .endm 455 456 .macro tl0_fp_restore 457 wr %g0, FPRS_FEF, %fprs 458 wr %g0, ASI_BLK_S, %asi 459 ldda [PCB_REG + PCB_FPSTATE + FP_FB0] %asi, %f0 460 ldda [PCB_REG + PCB_FPSTATE + FP_FB1] %asi, %f16 461 ldda [PCB_REG + PCB_FPSTATE + FP_FB2] %asi, %f32 462 ldda [PCB_REG + PCB_FPSTATE + FP_FB3] %asi, %f48 463 membar #Sync 464 done 465 .align 32 466 .endm 467 468 .macro tl0_insn_excptn 469 wrpr %g0, PSTATE_ALT, %pstate 470 wr %g0, ASI_IMMU, %asi 471 rdpr %tpc, %g3 472 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 473 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 474 membar #Sync 475 b %xcc, tl0_sfsr_trap 476 mov T_INSTRUCTION_EXCEPTION, %g2 477 .align 32 478 .endm 479 480 .macro tl0_data_excptn 481 wrpr %g0, PSTATE_ALT, %pstate 482 wr %g0, ASI_DMMU, %asi 483 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 484 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 485 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 486 membar #Sync 487 b %xcc, tl0_sfsr_trap 488 mov T_DATA_EXCEPTION, %g2 489 .align 32 490 .endm 491 492 .macro tl0_align 493 wr %g0, ASI_DMMU, %asi 494 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 495 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 496 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 497 membar #Sync 498 b %xcc, tl0_sfsr_trap 499 mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 500 .align 32 501 .endm 502 503ENTRY(tl0_sfsr_trap) 504 tl0_split 505 mov %g3, %o4 506 mov %g4, %o5 507 b %xcc, tl0_trap 508 mov %g2, %o0 509END(tl0_sfsr_trap) 510 511 .macro tl0_intr level, mask 512 tl0_split 513 set \mask, %o1 514 b %xcc, tl0_intr 515 mov \level, %o0 516 .align 32 517 .endm 518 519#define INTR(level, traplvl) \ 520 tl ## traplvl ## _intr level, 1 << level 521 522#define TICK(traplvl) \ 523 tl ## traplvl ## _intr PIL_TICK, 1 524 525#define INTR_LEVEL(tl) \ 526 INTR(1, tl) ; \ 527 INTR(2, tl) ; \ 528 INTR(3, tl) ; \ 529 INTR(4, tl) ; \ 530 INTR(5, tl) ; \ 531 INTR(6, tl) ; \ 532 INTR(7, tl) ; \ 533 INTR(8, tl) ; \ 534 INTR(9, tl) ; \ 535 INTR(10, tl) ; \ 536 INTR(11, tl) ; \ 537 INTR(12, tl) ; \ 538 INTR(13, tl) ; \ 539 TICK(tl) ; \ 540 INTR(15, tl) ; 541 542 .macro tl0_intr_level 543 INTR_LEVEL(0) 544 .endm 545 546 .macro intr_vector 547 ldxa [%g0] ASI_INTR_RECEIVE, %g1 548 andcc %g1, IRSR_BUSY, %g0 549 bnz,a,pt %xcc, intr_enqueue 550 nop 551 sir 552 .align 32 553 .endm 554 555 .macro immu_miss_user 556 /* 557 * Initialize the page size walker. 558 */ 559 mov TS_MIN, %g2 560 561 /* 562 * Loop over all supported page sizes. 563 */ 564 565 /* 566 * Compute the page shift for the page size we are currently looking 567 * for. 568 */ 5691: add %g2, %g2, %g3 570 add %g3, %g2, %g3 571 add %g3, PAGE_SHIFT, %g3 572 573 /* 574 * Extract the virtual page number from the contents of the tag 575 * access register. 576 */ 577 srlx %g1, %g3, %g3 578 579 /* 580 * Compute the tte bucket address. 581 */ 582 ldxa [%g0 + AA_IMMU_TSB] %asi, %g5 583 and %g3, TSB_BUCKET_MASK, %g4 584 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 585 add %g4, %g5, %g4 586 587 /* 588 * Compute the tte tag target. 589 */ 590 sllx %g3, TV_SIZE_BITS, %g3 591 or %g3, %g2, %g3 592 593 /* 594 * Loop over the ttes in this bucket 595 */ 596 597 /* 598 * Load the tte. Note that this instruction may fault, clobbering 599 * the contents of the tag access register, %g5, %g6, and %g7. We 600 * do not use %g5, and %g6 and %g7 are not used until this instruction 601 * completes successfully. 602 */ 6032: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 604 605 /* 606 * Check that its valid and executable and that the tte tags match. 607 */ 608 brgez,pn %g7, 3f 609 andcc %g7, TD_EXEC, %g0 610 bz,pn %xcc, 3f 611 cmp %g3, %g6 612 bne,pn %xcc, 3f 613 EMPTY 614 615 /* 616 * We matched a tte, load the tlb. 617 */ 618 619 /* 620 * Set the reference bit, if it's currently clear. 621 */ 622 andcc %g7, TD_REF, %g0 623 bz,a,pn %xcc, tl0_immu_miss_set_ref 624 nop 625 626 /* 627 * Load the tte tag and data into the tlb and retry the instruction. 628 */ 629 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 630 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 631 retry 632 633 /* 634 * Advance to the next tte in this bucket, and check the low bits 635 * of the bucket pointer to see if we've finished the bucket. 636 */ 6373: add %g4, 1 << TTE_SHIFT, %g4 638 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 639 bnz,pt %xcc, 2b 640 EMPTY 641 642 /* 643 * See if we just checked the largest page size, and advance to the 644 * next one if not. 645 */ 646 cmp %g2, TS_MAX 647 bne,pt %xcc, 1b 648 add %g2, 1, %g2 649 .endm 650 651 .macro tl0_immu_miss 652 /* 653 * Load the virtual page number and context from the tag access 654 * register. We ignore the context. 655 */ 656 wr %g0, ASI_IMMU, %asi 657 ldxa [%g0 + AA_IMMU_TAR] %asi, %g1 658 659 /* 660 * Try a fast inline lookup of the user tsb. 661 */ 662 immu_miss_user 663 664 /* 665 * Not in user tsb, call c code. 666 */ 667 ba,a %xcc, tl0_immu_miss_trap 668 .align 128 669 .endm 670 671ENTRY(tl0_immu_miss_set_ref) 672 /* 673 * Set the reference bit. 674 */ 675 TTE_SET_REF(%g4, %g2, %g3) 676 677 /* 678 * May have become invalid during casxa, in which case start over. 679 */ 680 brgez,pn %g2, 1f 681 nop 682 683 /* 684 * Load the tte tag and data into the tlb and retry the instruction. 685 */ 686 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 687 stxa %g2, [%g0] ASI_ITLB_DATA_IN_REG 6881: retry 689END(tl0_immu_miss_set_ref) 690 691ENTRY(tl0_immu_miss_trap) 692 /* 693 * Put back the contents of the tag access register, in case we 694 * faulted. 695 */ 696 stxa %g1, [%g0 + AA_IMMU_TAR] %asi 697 membar #Sync 698 699 /* 700 * Switch to alternate globals. 701 */ 702 wrpr %g0, PSTATE_ALT, %pstate 703 704 /* 705 * Reload the tag access register. 706 */ 707 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 708 709 /* 710 * Save the tag access register, and call common trap code. 711 */ 712 tl0_split 713 mov %g2, %o3 714 b %xcc, tl0_trap 715 mov T_INSTRUCTION_MISS, %o0 716END(tl0_immu_miss_trap) 717 718 .macro dmmu_miss_user 719 /* 720 * Initialize the page size walker. 721 */ 722 mov TS_MIN, %g2 723 724 /* 725 * Loop over all supported page sizes. 726 */ 727 728 /* 729 * Compute the page shift for the page size we are currently looking 730 * for. 731 */ 7321: add %g2, %g2, %g3 733 add %g3, %g2, %g3 734 add %g3, PAGE_SHIFT, %g3 735 736 /* 737 * Extract the virtual page number from the contents of the tag 738 * access register. 739 */ 740 srlx %g1, %g3, %g3 741 742 /* 743 * Compute the tte bucket address. 744 */ 745 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 746 and %g3, TSB_BUCKET_MASK, %g4 747 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 748 add %g4, %g5, %g4 749 750 /* 751 * Compute the tte tag target. 752 */ 753 sllx %g3, TV_SIZE_BITS, %g3 754 or %g3, %g2, %g3 755 756 /* 757 * Loop over the ttes in this bucket 758 */ 759 760 /* 761 * Load the tte. Note that this instruction may fault, clobbering 762 * the contents of the tag access register, %g5, %g6, and %g7. We 763 * do not use %g5, and %g6 and %g7 are not used until this instruction 764 * completes successfully. 765 */ 7662: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 767 768 /* 769 * Check that its valid and that the virtual page numbers match. 770 */ 771 brgez,pn %g7, 3f 772 cmp %g3, %g6 773 bne,pn %xcc, 3f 774 EMPTY 775 776 /* 777 * We matched a tte, load the tlb. 778 */ 779 780 /* 781 * Set the reference bit, if it's currently clear. 782 */ 783 andcc %g7, TD_REF, %g0 784 bz,a,pn %xcc, dmmu_miss_user_set_ref 785 nop 786 787 /* 788 * Load the tte tag and data into the tlb and retry the instruction. 789 */ 790 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 791 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 792 retry 793 794 /* 795 * Advance to the next tte in this bucket, and check the low bits 796 * of the bucket pointer to see if we've finished the bucket. 797 */ 7983: add %g4, 1 << TTE_SHIFT, %g4 799 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 800 bnz,pt %xcc, 2b 801 EMPTY 802 803 /* 804 * See if we just checked the largest page size, and advance to the 805 * next one if not. 806 */ 807 cmp %g2, TS_MAX 808 bne,pt %xcc, 1b 809 add %g2, 1, %g2 810 .endm 811 812ENTRY(dmmu_miss_user_set_ref) 813 /* 814 * Set the reference bit. 815 */ 816 TTE_SET_REF(%g4, %g2, %g3) 817 818 /* 819 * May have become invalid during casxa, in which case start over. 820 */ 821 brgez,pn %g2, 1f 822 nop 823 824 /* 825 * Load the tte tag and data into the tlb and retry the instruction. 826 */ 827 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 828 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 8291: retry 830END(dmmu_miss_user_set_ref) 831 832 .macro tl0_dmmu_miss 833 /* 834 * Load the virtual page number and context from the tag access 835 * register. We ignore the context. 836 */ 837 wr %g0, ASI_DMMU, %asi 838 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 839 840 /* 841 * Try a fast inline lookup of the primary tsb. 842 */ 843 dmmu_miss_user 844 845 /* 846 * Not in user tsb, call c code. 847 */ 848 ba,a %xcc, tl0_dmmu_miss_trap 849 .align 128 850 .endm 851 852ENTRY(tl0_dmmu_miss_trap) 853 /* 854 * Put back the contents of the tag access register, in case we 855 * faulted. 856 */ 857 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 858 membar #Sync 859 860 /* 861 * Switch to alternate globals. 862 */ 863 wrpr %g0, PSTATE_ALT, %pstate 864 865 /* 866 * Reload the tag access register. 867 */ 868 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 869 870 /* 871 * Save the tag access register and call common trap code. 872 */ 873 tl0_split 874 mov %g2, %o3 875 b %xcc, tl0_trap 876 mov T_DATA_MISS, %o0 877END(tl0_dmmu_miss_trap) 878 879 .macro dmmu_prot_user 880 /* 881 * Initialize the page size walker. 882 */ 883 mov TS_MIN, %g2 884 885 /* 886 * Loop over all supported page sizes. 887 */ 888 889 /* 890 * Compute the page shift for the page size we are currently looking 891 * for. 892 */ 8931: add %g2, %g2, %g3 894 add %g3, %g2, %g3 895 add %g3, PAGE_SHIFT, %g3 896 897 /* 898 * Extract the virtual page number from the contents of the tag 899 * access register. 900 */ 901 srlx %g1, %g3, %g3 902 903 /* 904 * Compute the tte bucket address. 905 */ 906 ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 907 and %g3, TSB_BUCKET_MASK, %g4 908 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 909 add %g4, %g5, %g4 910 911 /* 912 * Compute the tte tag target. 913 */ 914 sllx %g3, TV_SIZE_BITS, %g3 915 or %g3, %g2, %g3 916 917 /* 918 * Loop over the ttes in this bucket 919 */ 920 921 /* 922 * Load the tte. Note that this instruction may fault, clobbering 923 * the contents of the tag access register, %g5, %g6, and %g7. We 924 * do not use %g5, and %g6 and %g7 are not used until this instruction 925 * completes successfully. 926 */ 9272: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 928 929 /* 930 * Check that its valid and writable and that the virtual page 931 * numbers match. 932 */ 933 brgez,pn %g7, 4f 934 andcc %g7, TD_SW, %g0 935 bz,pn %xcc, 4f 936 cmp %g3, %g6 937 bne,pn %xcc, 4f 938 nop 939 940 /* 941 * Set the hardware write bit. 942 */ 943 TTE_SET_W(%g4, %g2, %g3) 944 945 /* 946 * Delete the old TLB entry and clear the sfsr. 947 */ 948 srlx %g1, PAGE_SHIFT, %g3 949 sllx %g3, PAGE_SHIFT, %g3 950 stxa %g0, [%g3] ASI_DMMU_DEMAP 951 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 952 membar #Sync 953 954 /* 955 * May have become invalid during casxa, in which case start over. 956 */ 957 brgez,pn %g2, 3f 958 or %g2, TD_W, %g2 959 960 /* 961 * Load the tte data into the tlb and retry the instruction. 962 */ 963 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 964 stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 9653: retry 966 967 /* 968 * Check the low bits to see if we've finished the bucket. 969 */ 9704: add %g4, 1 << TTE_SHIFT, %g4 971 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 972 bnz,pt %xcc, 2b 973 EMPTY 974 975 /* 976 * See if we just checked the largest page size, and advance to the 977 * next one if not. 978 */ 979 cmp %g2, TS_MAX 980 bne,pt %xcc, 1b 981 add %g2, 1, %g2 982 .endm 983 984 .macro tl0_dmmu_prot 985 ba,a %xcc, tl0_dmmu_prot_1 986 nop 987 .align 128 988 .endm 989 990ENTRY(tl0_dmmu_prot_1) 991 /* 992 * Load the virtual page number and context from the tag access 993 * register. We ignore the context. 994 */ 995 wr %g0, ASI_DMMU, %asi 996 ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 997 998 /* 999 * Try a fast inline lookup of the tsb. 1000 */ 1001 dmmu_prot_user 1002 1003 /* 1004 * Not in user tsb, call c code. 1005 */ 1006 b,a %xcc, tl0_dmmu_prot_trap 1007 nop 1008END(tl0_dmmu_prot_1) 1009 1010ENTRY(tl0_dmmu_prot_trap) 1011 /* 1012 * Put back the contents of the tag access register, in case we 1013 * faulted. 1014 */ 1015 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1016 membar #Sync 1017 1018 /* 1019 * Switch to alternate globals. 1020 */ 1021 wrpr %g0, PSTATE_ALT, %pstate 1022 1023 /* 1024 * Load the tar, sfar and sfsr. 1025 */ 1026 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1027 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1028 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1029 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1030 membar #Sync 1031 1032 /* 1033 * Save the mmu registers and call common trap code. 1034 */ 1035 tl0_split 1036 mov %g2, %o3 1037 mov %g3, %o4 1038 mov %g4, %o5 1039 b %xcc, tl0_trap 1040 mov T_DATA_PROTECTION, %o0 1041END(tl0_dmmu_prot_trap) 1042 1043 .macro tl0_spill_0_n 1044 wr %g0, ASI_AIUP, %asi 1045 SPILL(stxa, %sp + SPOFF, 8, %asi) 1046 saved 1047 retry 1048 .align 32 1049 RSF_TRAP(T_SPILL) 1050 RSF_TRAP(T_SPILL) 1051 .endm 1052 1053 .macro tl0_spill_1_n 1054 wr %g0, ASI_AIUP, %asi 1055 SPILL(stwa, %sp, 4, %asi) 1056 saved 1057 retry 1058 .align 32 1059 RSF_TRAP(T_SPILL) 1060 RSF_TRAP(T_SPILL) 1061 .endm 1062 1063 .macro tl0_fill_0_n 1064 wr %g0, ASI_AIUP, %asi 1065 FILL(ldxa, %sp + SPOFF, 8, %asi) 1066 restored 1067 retry 1068 .align 32 1069 RSF_TRAP(T_FILL) 1070 RSF_TRAP(T_FILL) 1071 .endm 1072 1073 .macro tl0_fill_1_n 1074 wr %g0, ASI_AIUP, %asi 1075 FILL(lduwa, %sp, 4, %asi) 1076 restored 1077 retry 1078 .align 32 1079 RSF_TRAP(T_FILL) 1080 RSF_TRAP(T_FILL) 1081 .endm 1082 1083ENTRY(tl0_sftrap) 1084 rdpr %tstate, %g1 1085 and %g1, TSTATE_CWP_MASK, %g1 1086 wrpr %g1, 0, %cwp 1087 tl0_split 1088 b %xcc, tl0_trap 1089 mov %g2, %o0 1090END(tl0_sftrap) 1091 1092 .macro tl0_spill_bad count 1093 .rept \count 1094 sir 1095 .align 128 1096 .endr 1097 .endm 1098 1099 .macro tl0_fill_bad count 1100 .rept \count 1101 sir 1102 .align 128 1103 .endr 1104 .endm 1105 1106 .macro tl0_syscall 1107 tl0_split 1108 b %xcc, tl0_syscall 1109 mov T_SYSCALL, %o0 1110 .align 32 1111 .endm 1112 1113 .macro tl0_soft count 1114 .rept \count 1115 tl0_gen T_SOFT 1116 .endr 1117 .endm 1118 1119 .macro tl1_split 1120 rdpr %wstate, %g1 1121 wrpr %g1, WSTATE_NESTED, %wstate 1122 save %sp, -CCFSZ, %sp 1123 .endm 1124 1125 .macro tl1_setup type 1126 tl1_split 1127 b %xcc, tl1_trap 1128 mov \type | T_KERNEL, %o0 1129 .endm 1130 1131 .macro tl1_gen type 1132 tl1_setup \type 1133 .align 32 1134 .endm 1135 1136 .macro tl1_reserved count 1137 .rept \count 1138 tl1_gen T_RESERVED 1139 .endr 1140 .endm 1141 1142 .macro tl1_insn_excptn 1143 wrpr %g0, PSTATE_ALT, %pstate 1144 wr %g0, ASI_IMMU, %asi 1145 rdpr %tpc, %g3 1146 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 1147 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi 1148 membar #Sync 1149 b %xcc, tl1_insn_exceptn_trap 1150 mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 1151 .align 32 1152 .endm 1153 1154ENTRY(tl1_insn_exceptn_trap) 1155 tl1_split 1156 mov %g3, %o4 1157 mov %g4, %o5 1158 b %xcc, tl1_trap 1159 mov %g2, %o0 1160END(tl1_insn_exceptn_trap) 1161 1162 .macro tl1_data_excptn 1163 wrpr %g0, PSTATE_ALT, %pstate 1164 b,a %xcc, tl1_data_excptn_trap 1165 nop 1166 .align 32 1167 .endm 1168 1169ENTRY(tl1_data_excptn_trap) 1170 RESUME_SPILLFILL_MMU_CLR_SFSR 1171 b %xcc, tl1_sfsr_trap 1172 mov T_DATA_EXCEPTION | T_KERNEL, %g2 1173END(tl1_data_excptn_trap) 1174 1175 .macro tl1_align 1176 b,a %xcc, tl1_align_trap 1177 nop 1178 .align 32 1179 .endm 1180 1181ENTRY(tl1_align_trap) 1182 RESUME_SPILLFILL_ALIGN 1183 b %xcc, tl1_sfsr_trap 1184 mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 1185END(tl1_data_excptn_trap) 1186 1187ENTRY(tl1_sfsr_trap) 1188 wr %g0, ASI_DMMU, %asi 1189 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1190 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1191 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1192 membar #Sync 1193 1194 tl1_split 1195 mov %g3, %o4 1196 mov %g4, %o5 1197 b %xcc, tl1_trap 1198 mov %g2, %o0 1199END(tl1_sfsr_trap) 1200 1201 .macro tl1_intr level, mask 1202 tl1_split 1203 set \mask, %o1 1204 b %xcc, tl1_intr 1205 mov \level, %o0 1206 .align 32 1207 .endm 1208 1209 .macro tl1_intr_level 1210 INTR_LEVEL(1) 1211 .endm 1212 1213ENTRY(intr_dequeue) 1214 save %sp, -CCFSZ, %sp 1215 12161: ldx [PCPU(IRHEAD)], %l0 1217 brnz,a,pt %l0, 2f 1218 nop 1219 1220 ret 1221 restore 1222 12232: wrpr %g0, PSTATE_NORMAL, %pstate 1224 1225 ldx [%l0 + IR_NEXT], %l1 1226 brnz,pt %l1, 3f 1227 stx %l1, [PCPU(IRHEAD)] 1228 PCPU_ADDR(IRHEAD, %l1) 1229 stx %l1, [PCPU(IRTAIL)] 1230 12313: ldx [%l0 + IR_FUNC], %o0 1232 ldx [%l0 + IR_ARG], %o1 1233 ldx [%l0 + IR_VEC], %o2 1234 1235 ldx [PCPU(IRFREE)], %l1 1236 stx %l1, [%l0 + IR_NEXT] 1237 stx %l0, [PCPU(IRFREE)] 1238 1239 wrpr %g0, PSTATE_KERNEL, %pstate 1240 1241 call %o0 1242 mov %o1, %o0 1243 ba,a %xcc, 1b 1244 nop 1245END(intr_dequeue) 1246 1247/* 1248 * Handle a vectored interrupt. 1249 * 1250 * This is either a data bearing mondo vector interrupt, or a cross trap 1251 * request from another cpu. In either case the hardware supplies an 1252 * interrupt packet, in the form of 3 data words which are read from internal 1253 * registers. A data bearing mondo vector packet consists of an interrupt 1254 * number in the first data word, and zero in 2nd and 3rd. We use the 1255 * interrupt number to find the function, argument and priority from the 1256 * intr_vector table, allocate and fill in an intr_request from the per-cpu 1257 * free list, link it onto the per-cpu active list and finally post a softint 1258 * at the desired priority. Cross trap requests come in 2 forms, direct 1259 * and queued. Direct requests are distinguished by the first data word 1260 * being zero. The 2nd data word carries a function to call and the 3rd 1261 * an argument to pass. The function is jumped to directly. It executes 1262 * in nucleus context on interrupt globals and with all interrupts disabled, 1263 * therefore it must be fast, and the things that it can do are limited. 1264 * Queued cross trap requests are handled much like mondo vectors, except 1265 * that the function, argument and priority are contained in the interrupt 1266 * packet itself. They are distinguished by the upper 4 bits of the data 1267 * word being non-zero, which specifies the priority of the softint to 1268 * deliver. 1269 * 1270 * Register usage: 1271 * %g1 - pointer to intr_request 1272 * %g2 - pointer to intr_vector, temp once required data is loaded 1273 * %g3 - interrupt number for mondo vectors, unused otherwise 1274 * %g4 - function, from the interrupt packet for cross traps, or 1275 * loaded from the interrupt registers for mondo vecors 1276 * %g5 - argument, as above for %g4 1277 * %g6 - softint priority 1278 */ 1279ENTRY(intr_enqueue) 1280 /* 1281 * Load the interrupt packet from the hardware. 1282 */ 1283 wr %g0, ASI_SDB_INTR_R, %asi 1284 ldxa [%g0 + AA_SDB_INTR_D0] %asi, %g3 1285 ldxa [%g0 + AA_SDB_INTR_D1] %asi, %g4 1286 ldxa [%g0 + AA_SDB_INTR_D2] %asi, %g5 1287 stxa %g0, [%g0] ASI_INTR_RECEIVE 1288 membar #Sync 1289 1290#if KTR_COMPILE & KTR_INTR 1291 CATR(KTR_INTR, "intr_enqueue: data=%#lx %#lx %#lx" 1292 , %g1, %g2, %g6, 7, 8, 9) 1293 stx %g3, [%g1 + KTR_PARM1] 1294 stx %g4, [%g1 + KTR_PARM2] 1295 stx %g5, [%g1 + KTR_PARM3] 12969: 1297#endif 1298 1299 /* 1300 * If the first data word is zero this is a direct cross trap request. 1301 * The 2nd word points to code to execute and the 3rd is an argument 1302 * to pass. Jump to it. 1303 */ 1304 brnz,a,pt %g3, 1f 1305 nop 1306 1307#if KTR_COMPILE & KTR_INTR 1308 CATR(KTR_INTR, "intr_enqueue: direct ipi func=%#lx arg=%#lx" 1309 , %g1, %g2, %g6, 7, 8, 9) 1310 stx %g4, [%g1 + KTR_PARM1] 1311 stx %g5, [%g1 + KTR_PARM2] 13129: 1313#endif 1314 1315 jmpl %g4, %g0 1316 nop 1317 /* NOTREACHED */ 1318 1319 /* 1320 * If the high 4 bits of the 1st data word are non-zero, this is a 1321 * queued cross trap request to be delivered as a softint. The high 1322 * 4 bits of the 1st data word specify a priority, and the 2nd and 1323 * 3rd a function and argument. 1324 */ 13251: srlx %g3, 60, %g6 1326 brnz,a,pn %g6, 2f 1327 clr %g3 1328 1329 /* 1330 * Find the function, argument and desired priority from the 1331 * intr_vector table. 1332 */ 1333 SET(intr_vectors, %g4, %g2) 1334 sllx %g3, IV_SHIFT, %g4 1335 add %g2, %g4, %g2 1336 1337#if KTR_COMPILE & KTR_INTR 1338 CATR(KTR_INTR, "intr_enqueue: mondo vector func=%#lx arg=%#lx pri=%#lx" 1339 , %g4, %g5, %g6, 7, 8, 9) 1340 ldx [%g2 + IV_FUNC], %g5 1341 stx %g5, [%g4 + KTR_PARM1] 1342 ldx [%g2 + IV_ARG], %g5 1343 stx %g5, [%g4 + KTR_PARM2] 1344 ldx [%g2 + IV_PRI], %g5 1345 stx %g5, [%g4 + KTR_PARM3] 13469: 1347#endif 1348 1349 ldx [%g2 + IV_FUNC], %g4 1350 ldx [%g2 + IV_ARG], %g5 1351 lduw [%g2 + IV_PRI], %g6 1352 1353 ba,a %xcc, 3f 1354 nop 1355 1356 /* 1357 * Get a intr_request from the free list. There should always be one 1358 * unless we are getting an interrupt storm from stray interrupts, in 1359 * which case the we will deference a NULL pointer and panic. 1360 */ 13612: 1362#if KTR_COMPILE & KTR_INTR 1363 CATR(KTR_INTR, "intr_enqueue: queued ipi func=%#lx arg=%#lx pri=%#lx" 1364 , %g1, %g2, %g3, 7, 8, 9) 1365 stx %g4, [%g1 + KTR_PARM1] 1366 stx %g5, [%g1 + KTR_PARM2] 1367 stx %g6, [%g1 + KTR_PARM3] 13689: 1369 clr %g3 1370#endif 1371 13723: 1373 ldx [PCPU(IRFREE)], %g1 1374 ldx [%g1 + IR_NEXT], %g2 1375 stx %g2, [PCPU(IRFREE)] 1376 1377 /* 1378 * Store the vector number, function, argument and priority. 1379 */ 1380 stw %g3, [%g1 + IR_VEC] 1381 stx %g4, [%g1 + IR_FUNC] 1382 stx %g5, [%g1 + IR_ARG] 1383 stw %g6, [%g1 + IR_PRI] 1384 1385 /* 1386 * Link it onto the end of the active list. 1387 */ 1388 stx %g0, [%g1 + IR_NEXT] 1389 ldx [PCPU(IRTAIL)], %g4 1390 stx %g1, [%g4] 1391 add %g1, IR_NEXT, %g1 1392 stx %g1, [PCPU(IRTAIL)] 1393 1394 /* 1395 * Trigger a softint at the level indicated by the priority. 1396 */ 1397 mov 1, %g1 1398 sllx %g1, %g6, %g1 1399 1400#if KTR_COMPILE & KTR_INTR 1401 CATR(KTR_INTR, "intr_enqueue: softint pil=%#lx pri=%#lx mask=%#lx" 1402 , %g2, %g3, %g4, 7, 8, 9) 1403 rdpr %pil, %g3 1404 stx %g3, [%g2 + KTR_PARM1] 1405 stx %g6, [%g2 + KTR_PARM2] 1406 stx %g1, [%g2 + KTR_PARM3] 14079: 1408#endif 1409 1410 wr %g1, 0, %asr20 1411 1412 /* 1413 * Done, retry the instruction. 1414 */ 1415 retry 1416END(intr_enqueue) 1417 1418 .macro tl1_immu_miss 1419 /* 1420 * Load the context and the virtual page number from the tag access 1421 * register. We ignore the context. 1422 */ 1423 wr %g0, ASI_IMMU, %asi 1424 ldxa [%g0 + AA_IMMU_TAR] %asi, %g5 1425 1426 /* 1427 * Compute the address of the tte. The tsb mask and address of the 1428 * tsb are patched at startup. 1429 */ 1430 .globl tl1_immu_miss_patch_1 1431tl1_immu_miss_patch_1: 1432 sethi %hi(TSB_KERNEL_MASK), %g6 1433 or %g6, %lo(TSB_KERNEL_MASK), %g6 1434 sethi %hi(TSB_KERNEL), %g7 1435 1436 srlx %g5, TAR_VPN_SHIFT, %g5 1437 and %g5, %g6, %g6 1438 sllx %g6, TTE_SHIFT, %g6 1439 add %g6, %g7, %g6 1440 1441 /* 1442 * Load the tte. 1443 */ 1444 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1445 1446 /* 1447 * Check that its valid and executable and that the virtual page 1448 * numbers match. 1449 */ 1450 brgez,pn %g7, tl1_immu_miss_trap 1451 andcc %g7, TD_EXEC, %g0 1452 bz,pn %xcc, tl1_immu_miss_trap 1453 srlx %g6, TV_SIZE_BITS, %g6 1454 cmp %g5, %g6 1455 bne,pn %xcc, tl1_immu_miss_trap 1456 EMPTY 1457 1458 /* 1459 * Set the reference bit if its currently clear. 1460 */ 1461 andcc %g7, TD_REF, %g0 1462 bz,a,pn %xcc, tl1_immu_miss_set_ref 1463 nop 1464 1465 /* 1466 * Load the tte data into the TLB and retry the instruction. 1467 */ 1468 stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG 1469 retry 1470 .align 128 1471 .endm 1472 1473ENTRY(tl1_immu_miss_set_ref) 1474 /* 1475 * Recompute the tte address, which we clobbered loading the tte. The 1476 * tsb mask and address of the tsb are patched at startup. 1477 */ 1478 .globl tl1_immu_miss_patch_2 1479tl1_immu_miss_patch_2: 1480 sethi %hi(TSB_KERNEL_MASK), %g6 1481 or %g6, %lo(TSB_KERNEL_MASK), %g6 1482 sethi %hi(TSB_KERNEL), %g7 1483 1484 and %g5, %g6, %g5 1485 sllx %g5, TTE_SHIFT, %g5 1486 add %g5, %g7, %g5 1487 1488 /* 1489 * Set the reference bit. 1490 */ 1491 TTE_SET_REF(%g5, %g6, %g7) 1492 1493 /* 1494 * May have become invalid during casxa, in which case start over. 1495 */ 1496 brgez,pn %g6, 1f 1497 nop 1498 1499 /* 1500 * Load the tte data into the TLB and retry the instruction. 1501 */ 1502 stxa %g6, [%g0] ASI_ITLB_DATA_IN_REG 15031: retry 1504END(tl1_immu_miss_set_ref) 1505 1506ENTRY(tl1_immu_miss_trap) 1507 /* 1508 * Switch to alternate globals. 1509 */ 1510 wrpr %g0, PSTATE_ALT, %pstate 1511 1512 ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 1513 1514 tl1_split 1515 mov %g2, %o3 1516 b %xcc, tl1_trap 1517 mov T_INSTRUCTION_MISS | T_KERNEL, %o0 1518END(tl1_immu_miss_trap) 1519 1520 .macro tl1_dmmu_miss 1521 /* 1522 * Load the context and the virtual page number from the tag access 1523 * register. 1524 */ 1525 wr %g0, ASI_DMMU, %asi 1526 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1527 1528 /* 1529 * Extract the context from the contents of the tag access register. 1530 * If its non-zero this is a fault on a user address. Note that the 1531 * faulting address is passed in %g2. 1532 */ 1533 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1534 brnz,a,pn %g6, tl1_dmmu_miss_user 1535 mov %g5, %g1 1536 1537 /* 1538 * Check for the direct mapped physical region. These addresses have 1539 * the high bit set so they are negative. 1540 */ 1541 brlz,pn %g5, tl1_dmmu_miss_direct 1542 EMPTY 1543 1544 /* 1545 * Compute the address of the tte. The tsb mask and address of the 1546 * tsb are patched at startup. 1547 */ 1548 .globl tl1_dmmu_miss_patch_1 1549tl1_dmmu_miss_patch_1: 1550 sethi %hi(TSB_KERNEL_MASK), %g6 1551 or %g6, %lo(TSB_KERNEL_MASK), %g6 1552 sethi %hi(TSB_KERNEL), %g7 1553 1554 srlx %g5, TAR_VPN_SHIFT, %g5 1555 and %g5, %g6, %g6 1556 sllx %g6, TTE_SHIFT, %g6 1557 add %g6, %g7, %g6 1558 1559 /* 1560 * Load the tte. 1561 */ 1562 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1563 1564 /* 1565 * Check that its valid and that the virtual page numbers match. 1566 */ 1567 brgez,pn %g7, tl1_dmmu_miss_trap 1568 srlx %g6, TV_SIZE_BITS, %g6 1569 cmp %g5, %g6 1570 bne,pn %xcc, tl1_dmmu_miss_trap 1571 EMPTY 1572 1573 /* 1574 * Set the reference bit if its currently clear. 1575 */ 1576 andcc %g7, TD_REF, %g0 1577 bz,a,pt %xcc, tl1_dmmu_miss_set_ref 1578 nop 1579 1580 /* 1581 * Load the tte data into the TLB and retry the instruction. 1582 */ 1583 stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG 1584 retry 1585 .align 128 1586 .endm 1587 1588ENTRY(tl1_dmmu_miss_set_ref) 1589 /* 1590 * Recompute the tte address, which we clobbered loading the tte. The 1591 * tsb mask and address of the tsb are patched at startup. 1592 */ 1593 .globl tl1_dmmu_miss_patch_2 1594tl1_dmmu_miss_patch_2: 1595 sethi %hi(TSB_KERNEL_MASK), %g6 1596 or %g6, %lo(TSB_KERNEL_MASK), %g6 1597 sethi %hi(TSB_KERNEL), %g7 1598 1599 and %g5, %g6, %g5 1600 sllx %g5, TTE_SHIFT, %g5 1601 add %g5, %g7, %g5 1602 1603 /* 1604 * Set the reference bit. 1605 */ 1606 TTE_SET_REF(%g5, %g6, %g7) 1607 1608 /* 1609 * May have become invalid during casxa, in which case start over. 1610 */ 1611 brgez,pn %g6, 1f 1612 nop 1613 1614 /* 1615 * Load the tte data into the TLB and retry the instruction. 1616 */ 1617 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 16181: retry 1619END(tl1_dmmu_miss_set_ref) 1620 1621ENTRY(tl1_dmmu_miss_trap) 1622 /* 1623 * Switch to alternate globals. 1624 */ 1625 wrpr %g0, PSTATE_ALT, %pstate 1626 1627 KSTACK_CHECK 1628 1629 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1630 1631 tl1_split 1632 mov %g2, %o3 1633 b %xcc, tl1_trap 1634 mov T_DATA_MISS | T_KERNEL, %o0 1635END(tl1_dmmu_miss_trap) 1636 1637ENTRY(tl1_dmmu_miss_direct) 1638 /* 1639 * Check the cache bits in the virtual address to see if this mapping 1640 * is virtually cacheable. We set this up so that the masks fit in 1641 * immediates... Note that the arithmetic shift sign extends, keeping 1642 * all the top bits set. 1643 */ 1644 srax %g5, TLB_DIRECT_SHIFT, %g5 1645 andcc %g5, TLB_DIRECT_UNCACHEABLE, %g0 1646 mov TD_CP | TD_CV | TD_W, %g6 1647 movnz %xcc, TD_CP | TD_W, %g6 1648 or %g5, %g6, %g5 1649 1650 /* 1651 * Mask off the high bits of the virtual address to get the physical 1652 * address, and or in the tte bits. The high bit is left set in the 1653 * physical address, which corresponds to the tte valid bit, so that 1654 * we don't have to include it in the tte bits. We ignore the cache 1655 * bits, since they get shifted into the soft tte bits anyway. 1656 */ 1657 setx TLB_DIRECT_MASK & ~TD_V, %g7, %g6 1658 andn %g5, %g6, %g5 1659 1660 /* 1661 * Load the tte data into the TLB and retry the instruction. 1662 */ 1663 stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG 1664 retry 1665END(tl1_dmmu_miss_direct) 1666 1667ENTRY(tl1_dmmu_miss_user) 1668 /* 1669 * Try a fast inline lookup of the user tsb. 1670 */ 1671 dmmu_miss_user 1672 1673 /* 1674 * Put back the contents of the tag access register, in case we 1675 * faulted. 1676 */ 1677 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1678 membar #Sync 1679 1680 /* 1681 * Switch to alternate globals. 1682 */ 1683 wrpr %g0, PSTATE_ALT, %pstate 1684 1685 /* 1686 * Handle faults during window spill/fill. 1687 */ 1688 RESUME_SPILLFILL_MMU 1689 1690 /* 1691 * Reload the tag access register. 1692 */ 1693 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1694 1695 tl1_split 1696 mov %g2, %o3 1697 b %xcc, tl1_trap 1698 mov T_DATA_MISS | T_KERNEL, %o0 1699END(tl1_dmmu_miss_user) 1700 1701 .macro tl1_dmmu_prot 1702 ba,a %xcc, tl1_dmmu_prot_1 1703 nop 1704 .align 128 1705 .endm 1706 1707ENTRY(tl1_dmmu_prot_1) 1708 /* 1709 * Load the context and the virtual page number from the tag access 1710 * register. 1711 */ 1712 wr %g0, ASI_DMMU, %asi 1713 ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 1714 1715 /* 1716 * Extract the context from the contents of the tag access register. 1717 * If its non-zero this is a fault on a user address, otherwise get 1718 * the virtual page number. 1719 */ 1720 sllx %g5, 64 - TAR_VPN_SHIFT, %g6 1721 brnz,a,pn %g6, tl1_dmmu_prot_user 1722 mov %g5, %g1 1723 1724 /* 1725 * Compute the address of the tte. The tsb mask and address of the 1726 * tsb are patched at startup. 1727 */ 1728 .globl tl1_dmmu_prot_patch_1 1729tl1_dmmu_prot_patch_1: 1730 sethi %hi(TSB_KERNEL_MASK), %g6 1731 or %g6, %lo(TSB_KERNEL_MASK), %g6 1732 sethi %hi(TSB_KERNEL), %g7 1733 1734 srlx %g5, TAR_VPN_SHIFT, %g5 1735 and %g5, %g6, %g6 1736 sllx %g6, TTE_SHIFT, %g6 1737 add %g6, %g7, %g6 1738 1739 /* 1740 * Load the tte. 1741 */ 1742 ldda [%g6] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ 1743 1744 /* 1745 * Check that its valid and writeable and that the virtual page 1746 * numbers match. 1747 */ 1748 brgez,pn %g7, tl1_dmmu_prot_trap 1749 andcc %g7, TD_SW, %g0 1750 bz,pn %xcc, tl1_dmmu_prot_trap 1751 srlx %g6, TV_SIZE_BITS, %g6 1752 cmp %g5, %g6 1753 bne,pn %xcc, tl1_dmmu_prot_trap 1754 EMPTY 1755 1756 /* 1757 * Delete the old TLB entry and clear the sfsr. 1758 */ 1759 sllx %g5, TAR_VPN_SHIFT, %g6 1760 or %g6, TLB_DEMAP_NUCLEUS, %g6 1761 stxa %g0, [%g6] ASI_DMMU_DEMAP 1762 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1763 membar #Sync 1764 1765 /* 1766 * Recompute the tte address, which we clobbered loading the tte. The 1767 * tsb mask and address of the tsb are patched at startup. 1768 */ 1769 .globl tl1_dmmu_prot_patch_2 1770tl1_dmmu_prot_patch_2: 1771 sethi %hi(TSB_KERNEL_MASK), %g6 1772 or %g6, %lo(TSB_KERNEL_MASK), %g6 1773 sethi %hi(TSB_KERNEL), %g7 1774 1775 and %g5, %g6, %g5 1776 sllx %g5, TTE_SHIFT, %g5 1777 add %g5, %g7, %g5 1778 1779 /* 1780 * Set the hardware write bit. 1781 */ 1782 TTE_SET_W(%g5, %g6, %g7) 1783 1784 /* 1785 * May have become invalid during casxa, in which case start over. 1786 */ 1787 brgez,pn %g6, 1f 1788 or %g6, TD_W, %g6 1789 1790 /* 1791 * Load the tte data into the TLB and retry the instruction. 1792 */ 1793 stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 17941: retry 1795END(tl1_dmmu_prot_1) 1796 1797ENTRY(tl1_dmmu_prot_user) 1798 /* 1799 * Try a fast inline lookup of the user tsb. 1800 */ 1801 dmmu_prot_user 1802 1803 /* 1804 * Put back the contents of the tag access register, in case we 1805 * faulted. 1806 */ 1807 stxa %g1, [%g0 + AA_DMMU_TAR] %asi 1808 membar #Sync 1809 1810 /* 1811 * Switch to alternate globals. 1812 */ 1813 wrpr %g0, PSTATE_ALT, %pstate 1814 1815 /* Handle faults during window spill/fill. */ 1816 RESUME_SPILLFILL_MMU_CLR_SFSR 1817 1818 b,a %xcc, tl1_dmmu_prot_trap 1819 nop 1820END(tl1_dmmu_prot_user) 1821 1822ENTRY(tl1_dmmu_prot_trap) 1823 /* 1824 * Switch to alternate globals. 1825 */ 1826 wrpr %g0, PSTATE_ALT, %pstate 1827 1828 /* 1829 * Load the sfar, sfsr and tar. Clear the sfsr. 1830 */ 1831 ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 1832 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 1833 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 1834 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi 1835 membar #Sync 1836 1837 tl1_split 1838 mov %g2, %o3 1839 mov %g3, %o4 1840 mov %g4, %o5 1841 b %xcc, tl1_trap 1842 mov T_DATA_PROTECTION | T_KERNEL, %o0 1843END(tl1_dmmu_prot_trap) 1844 1845 .macro tl1_spill_0_n 1846 SPILL(stx, %sp + SPOFF, 8, EMPTY) 1847 saved 1848 retry 1849 .align 32 1850 RSF_FATAL(T_SPILL) 1851 RSF_FATAL(T_SPILL) 1852 .endm 1853 1854 .macro tl1_spill_2_n 1855 wr %g0, ASI_AIUP, %asi 1856 SPILL(stxa, %sp + SPOFF, 8, %asi) 1857 saved 1858 retry 1859 .align 32 1860 RSF_SPILL_TOPCB 1861 RSF_SPILL_TOPCB 1862 .endm 1863 1864 .macro tl1_spill_3_n 1865 wr %g0, ASI_AIUP, %asi 1866 SPILL(stwa, %sp, 4, %asi) 1867 saved 1868 retry 1869 .align 32 1870 RSF_SPILL_TOPCB 1871 RSF_SPILL_TOPCB 1872 .endm 1873 1874 .macro tl1_spill_0_o 1875 wr %g0, ASI_AIUP, %asi 1876 SPILL(stxa, %sp + SPOFF, 8, %asi) 1877 saved 1878 retry 1879 .align 32 1880 RSF_SPILL_TOPCB 1881 RSF_SPILL_TOPCB 1882 .endm 1883 1884 .macro tl1_spill_1_o 1885 wr %g0, ASI_AIUP, %asi 1886 SPILL(stwa, %sp, 4, %asi) 1887 saved 1888 retry 1889 .align 32 1890 RSF_SPILL_TOPCB 1891 RSF_SPILL_TOPCB 1892 .endm 1893 1894 .macro tl1_spill_2_o 1895 RSF_SPILL_TOPCB 1896 .align 128 1897 .endm 1898 1899 .macro tl1_fill_0_n 1900 FILL(ldx, %sp + SPOFF, 8, EMPTY) 1901 restored 1902 retry 1903 .align 32 1904 RSF_FATAL(T_FILL) 1905 RSF_FATAL(T_FILL) 1906 .endm 1907 1908 .macro tl1_fill_2_n 1909 wr %g0, ASI_AIUP, %asi 1910 FILL(ldxa, %sp + SPOFF, 8, %asi) 1911 restored 1912 retry 1913 .align 32 1914 RSF_FILL_MAGIC 1915 RSF_FILL_MAGIC 1916 .endm 1917 1918 .macro tl1_fill_3_n 1919 wr %g0, ASI_AIUP, %asi 1920 FILL(lduwa, %sp, 4, %asi) 1921 restored 1922 retry 1923 .align 32 1924 RSF_FILL_MAGIC 1925 RSF_FILL_MAGIC 1926 .endm 1927 1928/* 1929 * This is used to spill windows that are still occupied with user 1930 * data on kernel entry to the pcb. 1931 */ 1932ENTRY(tl1_spill_topcb) 1933 wrpr %g0, PSTATE_ALT, %pstate 1934 1935 /* Free some globals for our use. */ 1936 dec 24, ASP_REG 1937 stx %g1, [ASP_REG + 0] 1938 stx %g2, [ASP_REG + 8] 1939 stx %g3, [ASP_REG + 16] 1940 1941 ldx [PCB_REG + PCB_NSAVED], %g1 1942 1943 sllx %g1, PTR_SHIFT, %g2 1944 add %g2, PCB_REG, %g2 1945 stx %sp, [%g2 + PCB_RWSP] 1946 1947 sllx %g1, RW_SHIFT, %g2 1948 add %g2, PCB_REG, %g2 1949 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) 1950 1951 inc %g1 1952 stx %g1, [PCB_REG + PCB_NSAVED] 1953 1954#if KTR_COMPILE & KTR_TRAP 1955 CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" 1956 , %g1, %g2, %g3, 7, 8, 9) 1957 rdpr %tpc, %g2 1958 stx %g2, [%g1 + KTR_PARM1] 1959 rdpr %tnpc, %g2 1960 stx %g2, [%g1 + KTR_PARM2] 1961 stx %sp, [%g1 + KTR_PARM3] 1962 ldx [PCB_REG + PCB_NSAVED], %g2 1963 stx %g2, [%g1 + KTR_PARM4] 19649: 1965#endif 1966 1967 saved 1968 1969 ldx [ASP_REG + 16], %g3 1970 ldx [ASP_REG + 8], %g2 1971 ldx [ASP_REG + 0], %g1 1972 inc 24, ASP_REG 1973 retry 1974END(tl1_spill_topcb) 1975 1976 .macro tl1_spill_bad count 1977 .rept \count 1978 sir 1979 .align 128 1980 .endr 1981 .endm 1982 1983 .macro tl1_fill_bad count 1984 .rept \count 1985 sir 1986 .align 128 1987 .endr 1988 .endm 1989 1990 .macro tl1_soft count 1991 .rept \count 1992 tl1_gen T_SOFT | T_KERNEL 1993 .endr 1994 .endm 1995 1996 .sect .trap 1997 .align 0x8000 1998 .globl tl0_base 1999 2000tl0_base: 2001 tl0_reserved 8 ! 0x0-0x7 2002tl0_insn_excptn: 2003 tl0_insn_excptn ! 0x8 2004 tl0_reserved 1 ! 0x9 2005tl0_insn_error: 2006 tl0_gen T_INSTRUCTION_ERROR ! 0xa 2007 tl0_reserved 5 ! 0xb-0xf 2008tl0_insn_illegal: 2009 tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 2010tl0_priv_opcode: 2011 tl0_gen T_PRIVILEGED_OPCODE ! 0x11 2012 tl0_reserved 14 ! 0x12-0x1f 2013tl0_fp_disabled: 2014 tl0_gen T_FP_DISABLED ! 0x20 2015tl0_fp_ieee: 2016 tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 2017tl0_fp_other: 2018 tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 2019tl0_tag_ovflw: 2020 tl0_gen T_TAG_OFERFLOW ! 0x23 2021tl0_clean_window: 2022 clean_window ! 0x24 2023tl0_divide: 2024 tl0_gen T_DIVISION_BY_ZERO ! 0x28 2025 tl0_reserved 7 ! 0x29-0x2f 2026tl0_data_excptn: 2027 tl0_data_excptn ! 0x30 2028 tl0_reserved 1 ! 0x31 2029tl0_data_error: 2030 tl0_gen T_DATA_ERROR ! 0x32 2031 tl0_reserved 1 ! 0x33 2032tl0_align: 2033 tl0_align ! 0x34 2034tl0_align_lddf: 2035 tl0_gen T_RESERVED ! 0x35 2036tl0_align_stdf: 2037 tl0_gen T_RESERVED ! 0x36 2038tl0_priv_action: 2039 tl0_gen T_PRIVILEGED_ACTION ! 0x37 2040 tl0_reserved 9 ! 0x38-0x40 2041tl0_intr_level: 2042 tl0_intr_level ! 0x41-0x4f 2043 tl0_reserved 16 ! 0x50-0x5f 2044tl0_intr_vector: 2045 intr_vector ! 0x60 2046tl0_watch_phys: 2047 tl0_gen T_PA_WATCHPOINT ! 0x61 2048tl0_watch_virt: 2049 tl0_gen T_VA_WATCHPOINT ! 0x62 2050tl0_ecc: 2051 tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 2052tl0_immu_miss: 2053 tl0_immu_miss ! 0x64 2054tl0_dmmu_miss: 2055 tl0_dmmu_miss ! 0x68 2056tl0_dmmu_prot: 2057 tl0_dmmu_prot ! 0x6c 2058 tl0_reserved 16 ! 0x70-0x7f 2059tl0_spill_0_n: 2060 tl0_spill_0_n ! 0x80 2061tl0_spill_1_n: 2062 tl0_spill_1_n ! 0x84 2063 tl0_spill_bad 14 ! 0x88-0xbf 2064tl0_fill_0_n: 2065 tl0_fill_0_n ! 0xc0 2066tl0_fill_1_n: 2067 tl0_fill_1_n ! 0xc4 2068 tl0_fill_bad 14 ! 0xc8-0xff 2069tl0_soft: 2070 tl0_reserved 1 ! 0x100 2071 tl0_gen T_BREAKPOINT ! 0x101 2072 tl0_gen T_DIVISION_BY_ZERO ! 0x102 2073 tl0_reserved 1 ! 0x103 2074 tl0_gen T_CLEAN_WINDOW ! 0x104 2075 tl0_gen T_RANGE_CHECK ! 0x105 2076 tl0_gen T_FIX_ALIGNMENT ! 0x106 2077 tl0_gen T_INTEGER_OVERFLOW ! 0x107 2078 tl0_reserved 1 ! 0x108 2079 tl0_syscall ! 0x109 2080 tl0_fp_restore ! 0x10a 2081 tl0_reserved 5 ! 0x10b-0x10f 2082 tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 2083 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 2084 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 2085 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 2086 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 2087 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 2088 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 2089 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 2090 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 2091 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 2092 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a 2093 tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b 2094 tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c 2095 tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d 2096 tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e 2097 tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f 2098 tl0_reserved 224 ! 0x120-0x1ff 2099 2100tl1_base: 2101 tl1_reserved 8 ! 0x200-0x207 2102tl1_insn_excptn: 2103 tl1_insn_excptn ! 0x208 2104 tl1_reserved 1 ! 0x209 2105tl1_insn_error: 2106 tl1_gen T_INSTRUCTION_ERROR ! 0x20a 2107 tl1_reserved 5 ! 0x20b-0x20f 2108tl1_insn_illegal: 2109 tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 2110tl1_priv_opcode: 2111 tl1_gen T_PRIVILEGED_OPCODE ! 0x211 2112 tl1_reserved 14 ! 0x212-0x21f 2113tl1_fp_disabled: 2114 tl1_gen T_FP_DISABLED ! 0x220 2115tl1_fp_ieee: 2116 tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 2117tl1_fp_other: 2118 tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 2119tl1_tag_ovflw: 2120 tl1_gen T_TAG_OFERFLOW ! 0x223 2121tl1_clean_window: 2122 clean_window ! 0x224 2123tl1_divide: 2124 tl1_gen T_DIVISION_BY_ZERO ! 0x228 2125 tl1_reserved 7 ! 0x229-0x22f 2126tl1_data_excptn: 2127 tl1_data_excptn ! 0x230 2128 tl1_reserved 1 ! 0x231 2129tl1_data_error: 2130 tl1_gen T_DATA_ERROR ! 0x232 2131 tl1_reserved 1 ! 0x233 2132tl1_align: 2133 tl1_align ! 0x234 2134tl1_align_lddf: 2135 tl1_gen T_RESERVED ! 0x235 2136tl1_align_stdf: 2137 tl1_gen T_RESERVED ! 0x236 2138tl1_priv_action: 2139 tl1_gen T_PRIVILEGED_ACTION ! 0x237 2140 tl1_reserved 9 ! 0x238-0x240 2141tl1_intr_level: 2142 tl1_intr_level ! 0x241-0x24f 2143 tl1_reserved 16 ! 0x250-0x25f 2144tl1_intr_vector: 2145 intr_vector ! 0x260 2146tl1_watch_phys: 2147 tl1_gen T_PA_WATCHPOINT ! 0x261 2148tl1_watch_virt: 2149 tl1_gen T_VA_WATCHPOINT ! 0x262 2150tl1_ecc: 2151 tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 2152tl1_immu_miss: 2153 tl1_immu_miss ! 0x264 2154tl1_dmmu_miss: 2155 tl1_dmmu_miss ! 0x268 2156tl1_dmmu_prot: 2157 tl1_dmmu_prot ! 0x26c 2158 tl1_reserved 16 ! 0x270-0x27f 2159tl1_spill_0_n: 2160 tl1_spill_0_n ! 0x280 2161 tl1_spill_bad 1 ! 0x284 2162tl1_spill_2_n: 2163 tl1_spill_2_n ! 0x288 2164tl1_spill_3_n: 2165 tl1_spill_3_n ! 0x29c 2166 tl1_spill_bad 4 ! 0x290-0x29f 2167tl1_spill_0_o: 2168 tl1_spill_0_o ! 0x2a0 2169tl1_spill_1_o: 2170 tl1_spill_1_o ! 0x2a4 2171tl1_spill_2_o: 2172 tl1_spill_2_o ! 0x2a8 2173 tl1_spill_bad 5 ! 0x2ac-0x2bf 2174tl1_fill_0_n: 2175 tl1_fill_0_n ! 0x2c0 2176 tl1_fill_bad 1 ! 0x2c4 2177tl1_fill_2_n: 2178 tl1_fill_2_n ! 0x2d0 2179tl1_fill_3_n: 2180 tl1_fill_3_n ! 0x2d4 2181 tl1_fill_bad 12 ! 0x2d8-0x2ff 2182 tl1_reserved 1 ! 0x300 2183tl1_breakpoint: 2184 tl1_gen T_BREAKPOINT ! 0x301 2185 tl1_gen T_RSTRWP_PHYS ! 0x302 2186 tl1_gen T_RSTRWP_VIRT ! 0x303 2187 tl1_reserved 252 ! 0x304-0x3ff 2188 2189/* 2190 * User trap entry point. 2191 * 2192 * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2193 * u_int sfsr) 2194 * 2195 * The following setup has been performed: 2196 * - the windows have been split and the active user window has been saved 2197 * (maybe just to the pcb) 2198 * - we are on alternate globals and interrupts are disabled 2199 * 2200 * We switch to the kernel stack, build a trapframe, switch to normal 2201 * globals, enable interrupts and call trap. 2202 * 2203 * NOTE: We must be very careful setting up the per-cpu pointer. We know that 2204 * it has been pre-set in alternate globals, so we read it from there and setup 2205 * the normal %g7 *before* enabling interrupts. This avoids any possibility 2206 * of cpu migration and using the wrong pcpup. 2207 */ 2208ENTRY(tl0_trap) 2209 /* 2210 * Force kernel store order. 2211 */ 2212 wrpr %g0, PSTATE_ALT, %pstate 2213 2214 rdpr %tstate, %l0 2215 rdpr %tpc, %l1 2216 rdpr %tnpc, %l2 2217 rd %y, %l3 2218 rd %fprs, %l4 2219 rdpr %wstate, %l5 2220 2221#if KTR_COMPILE & KTR_TRAP 2222 CATR(KTR_TRAP, 2223 "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2224 , %g1, %g2, %g3, 7, 8, 9) 2225 ldx [PCPU(CURTHREAD)], %g2 2226 stx %g2, [%g1 + KTR_PARM1] 2227 stx %o0, [%g1 + KTR_PARM2] 2228 rdpr %pil, %g2 2229 stx %g2, [%g1 + KTR_PARM3] 2230 stx %l1, [%g1 + KTR_PARM4] 2231 stx %l2, [%g1 + KTR_PARM5] 2232 stx %i6, [%g1 + KTR_PARM6] 22339: 2234#endif 2235 2236 and %l5, WSTATE_NORMAL_MASK, %l5 2237 2238 cmp %o0, UT_MAX 2239 bge,a,pt %xcc, 2f 2240 nop 2241 2242 ldx [PCPU(CURTHREAD)], %l6 2243 ldx [%l6 + TD_PROC], %l6 2244 ldx [%l6 + P_MD + MD_UTRAP], %l6 2245 brz,pt %l6, 2f 2246 sllx %o0, PTR_SHIFT, %l7 2247 ldx [%l6 + %l7], %l6 2248 brz,pt %l6, 2f 2249 andn %l0, TSTATE_CWP_MASK, %l7 2250 2251 ldx [PCB_REG + PCB_NSAVED], %g1 2252 brnz,a,pn %g1, 1f 2253 mov T_SPILL, %o0 2254 2255#if KTR_COMPILE & KTR_TRAP 2256 CATR(KTR_TRAP, "tl0_trap: user trap npc=%#lx" 2257 , %g1, %g2, %g3, 7, 8, 9) 2258 stx %l6, [%g1 + KTR_PARM1] 22599: 2260#endif 2261 2262 wrpr %l5, %wstate 2263 wrpr %l6, %tnpc 2264 rdpr %cwp, %l6 2265 wrpr %l6, %l7, %tstate 2266 2267 sub %fp, CCFSZ, %sp 2268 2269 /* 2270 * Need to store %fsr to pass it to the user trap handler. Otherwise, 2271 * the ftt field might be zeoed out in a store in another trap or 2272 * interrupt. Use the temporary stack for that. 2273 */ 2274 rd %fprs, %l3 2275 or %l3, FPRS_FEF, %l4 2276 wr %l4, 0, %fprs 2277 dec 8, ASP_REG 2278 stx %fsr, [ASP_REG] 2279 ldx [ASP_REG], %l4 2280 inc 8, ASP_REG 2281 wr %l3, 0, %fprs 2282 2283 mov %l0, %l5 2284 mov %l1, %l6 2285 mov %l2, %l7 2286 2287 done 2288 22891: 2290#if KTR_COMPILE & KTR_TRAP 2291 CATR(KTR_TRAP, "tl0_trap: defer user trap npc=%#lx nsaved=%#lx" 2292 , %g1, %g2, %g3, 7, 8, 9) 2293 stx %l6, [%g1 + KTR_PARM1] 2294 ldx [PCB_REG + PCB_NSAVED], %g2 2295 stx %g2, [%g1 + KTR_PARM2] 22969: 2297#endif 2298 22992: sllx %l5, WSTATE_OTHER_SHIFT, %l5 2300 wrpr %l5, WSTATE_KERNEL, %wstate 2301 rdpr %canrestore, %l6 2302 wrpr %l6, 0, %otherwin 2303 wrpr %g0, 0, %canrestore 2304 2305 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2306 2307 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2308 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2309 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2310 stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2311 2312 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2313 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2314 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2315 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2316 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2317 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2318 2319 wr %g0, FPRS_FEF, %fprs 2320 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2321 wr %g0, 0, %fprs 2322 2323 mov PCB_REG, %l0 2324 mov PCPU_REG, %l1 2325 wrpr %g0, PSTATE_NORMAL, %pstate 2326 2327 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2328 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2329 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2330 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2331 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2332 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2333 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2334 2335 mov %l0, PCB_REG 2336 mov %l1, PCPU_REG 2337 wrpr %g0, PSTATE_KERNEL, %pstate 2338 2339 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2340 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2341 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2342 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2343 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2344 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2345 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2346 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2347 2348 call trap 2349 add %sp, CCFSZ + SPOFF, %o0 2350 b,a %xcc, tl0_ret 2351 nop 2352END(tl0_trap) 2353 2354/* 2355 * void tl0_syscall(u_int type) 2356 */ 2357ENTRY(tl0_syscall) 2358 /* 2359 * Force kernel store order. 2360 */ 2361 wrpr %g0, PSTATE_ALT, %pstate 2362 2363 rdpr %tstate, %l0 2364 rdpr %tpc, %l1 2365 rdpr %tnpc, %l2 2366 rd %y, %l3 2367 rd %fprs, %l4 2368 rdpr %wstate, %l5 2369 2370#if KTR_COMPILE & KTR_SYSC 2371 CATR(KTR_SYSC, 2372 "tl0_syscall: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2373 , %g1, %g2, %g3, 7, 8, 9) 2374 ldx [PCPU(CURTHREAD)], %g2 2375 stx %g2, [%g1 + KTR_PARM1] 2376 stx %o0, [%g1 + KTR_PARM2] 2377 rdpr %pil, %g2 2378 stx %g2, [%g1 + KTR_PARM3] 2379 stx %l1, [%g1 + KTR_PARM4] 2380 stx %l2, [%g1 + KTR_PARM5] 2381 stx %i6, [%g1 + KTR_PARM6] 23829: 2383#endif 2384 2385 and %l5, WSTATE_NORMAL_MASK, %l5 2386 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2387 wrpr %l5, WSTATE_KERNEL, %wstate 2388 rdpr %canrestore, %l6 2389 wrpr %l6, 0, %otherwin 2390 wrpr %g0, 0, %canrestore 2391 2392 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2393 2394 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2395 2396 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2397 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2398 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2399 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2400 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2401 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2402 2403 wr %g0, FPRS_FEF, %fprs 2404 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2405 wr %g0, 0, %fprs 2406 2407 mov PCB_REG, %l0 2408 mov PCPU_REG, %l1 2409 wrpr %g0, PSTATE_NORMAL, %pstate 2410 2411 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2412 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2413 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2414 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2415 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2416 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2417 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2418 2419 mov %l0, PCB_REG 2420 mov %l1, PCPU_REG 2421 wrpr %g0, PSTATE_KERNEL, %pstate 2422 2423 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2424 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2425 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2426 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2427 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2428 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2429 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2430 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2431 2432 call syscall 2433 add %sp, CCFSZ + SPOFF, %o0 2434 b,a %xcc, tl0_ret 2435 nop 2436END(tl0_syscall) 2437 2438/* 2439 * void tl0_intr(u_int level, u_int mask) 2440 */ 2441ENTRY(tl0_intr) 2442 /* 2443 * Force kernel store order. 2444 */ 2445 wrpr %g0, PSTATE_ALT, %pstate 2446 2447 rdpr %tstate, %l0 2448 rdpr %tpc, %l1 2449 rdpr %tnpc, %l2 2450 rd %y, %l3 2451 rd %fprs, %l4 2452 rdpr %wstate, %l5 2453 2454#if KTR_COMPILE & KTR_INTR 2455 CATR(KTR_INTR, 2456 "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2457 , %g1, %g2, %g3, 7, 8, 9) 2458 ldx [PCPU(CURTHREAD)], %g2 2459 stx %g2, [%g1 + KTR_PARM1] 2460 stx %o0, [%g1 + KTR_PARM2] 2461 rdpr %pil, %g2 2462 stx %g2, [%g1 + KTR_PARM3] 2463 stx %l1, [%g1 + KTR_PARM4] 2464 stx %l2, [%g1 + KTR_PARM5] 2465 stx %i6, [%g1 + KTR_PARM6] 24669: 2467#endif 2468 2469 wrpr %o0, 0, %pil 2470 wr %o1, 0, %asr21 2471 2472 and %l5, WSTATE_NORMAL_MASK, %l5 2473 sllx %l5, WSTATE_OTHER_SHIFT, %l5 2474 wrpr %l5, WSTATE_KERNEL, %wstate 2475 rdpr %canrestore, %l6 2476 wrpr %l6, 0, %otherwin 2477 wrpr %g0, 0, %canrestore 2478 2479 sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp 2480 2481 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2482 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2483 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2484 stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] 2485 stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] 2486 stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] 2487 2488 wr %g0, FPRS_FEF, %fprs 2489 stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] 2490 wr %g0, 0, %fprs 2491 2492 mov %o0, %l3 2493 mov T_INTERRUPT, %o1 2494 2495 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2496 stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2497 2498 mov PCB_REG, %l0 2499 mov PCPU_REG, %l1 2500 wrpr %g0, PSTATE_NORMAL, %pstate 2501 2502 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2503 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2504 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2505 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2506 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2507 stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] 2508 stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] 2509 2510 mov %l0, PCB_REG 2511 mov %l1, PCPU_REG 2512 wrpr %g0, PSTATE_KERNEL, %pstate 2513 2514 stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] 2515 stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] 2516 stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] 2517 stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] 2518 stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] 2519 stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] 2520 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2521 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2522 2523 call critical_enter 2524 nop 2525 2526 SET(cnt+V_INTR, %l1, %l0) 2527 ATOMIC_INC_INT(%l0, %l1, %l2) 2528 2529 SET(intr_handlers, %l1, %l0) 2530 sllx %l3, IH_SHIFT, %l1 2531 ldx [%l0 + %l1], %l1 2532 KASSERT(%l1, "tl0_intr: ih null") 2533 call %l1 2534 add %sp, CCFSZ + SPOFF, %o0 2535 2536 call critical_exit 2537 nop 2538 2539 b,a %xcc, tl0_ret 2540 nop 2541END(tl0_intr) 2542 2543ENTRY(tl0_ret) 2544 /* 2545 * Check for pending asts atomically with returning. We must raise 2546 * the pil before checking, and if no asts are found the pil must 2547 * remain raised until the retry is executed, or we risk missing asts 2548 * caused by interrupts occuring after the test. If the pil is lowered, 2549 * as it is when we call ast, the check must be re-executed. 2550 */ 2551 wrpr %g0, PIL_TICK, %pil 2552 ldx [PCPU(CURTHREAD)], %l0 2553 ldx [%l0 + TD_KSE], %l1 2554 lduw [%l1 + KE_FLAGS], %l2 2555 and %l2, KEF_ASTPENDING | KEF_NEEDRESCHED, %l2 2556 brz,a,pt %l2, 1f 2557 nop 2558 wrpr %g0, 0, %pil 2559 call ast 2560 add %sp, CCFSZ + SPOFF, %o0 2561 ba,a %xcc, tl0_ret 2562 nop 2563 2564 /* 2565 * Check for windows that were spilled to the pcb and need to be 2566 * copied out. This must be the last thing that is done before the 2567 * return to usermode. If there are still user windows in the cpu 2568 * and we call a nested function after this, which causes them to be 2569 * spilled to the pcb, they will not be copied out and the stack will 2570 * be inconsistent. 2571 */ 25721: ldx [PCB_REG + PCB_NSAVED], %l1 2573 brz,a,pt %l1, 2f 2574 nop 2575 wrpr %g0, 0, %pil 2576 mov T_SPILL, %o0 2577 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2578 call trap 2579 add %sp, SPOFF + CCFSZ, %o0 2580 ba,a %xcc, tl0_ret 2581 nop 2582 25832: ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 2584 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 2585 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 2586 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 2587 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 2588 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 2589 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 2590 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 2591 2592 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2593 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2594 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2595 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l3 2596 ldub [%sp + SPOFF + CCFSZ + TF_FPRS], %l4 2597 ldub [%sp + SPOFF + CCFSZ + TF_WSTATE], %l5 2598 2599 wrpr %g0, PSTATE_NORMAL, %pstate 2600 2601 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2602 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2603 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2604 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2605 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2606 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 2607 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 2608 2609 wrpr %g0, PSTATE_ALT, %pstate 2610 2611 wrpr %g0, 0, %pil 2612 wrpr %l1, 0, %tpc 2613 wrpr %l2, 0, %tnpc 2614 wr %l3, 0, %y 2615 2616 andn %l0, TSTATE_CWP_MASK, %g1 2617 mov %l4, %g2 2618 2619 srlx %l5, WSTATE_OTHER_SHIFT, %g3 2620 wrpr %g3, WSTATE_TRANSITION, %wstate 2621 rdpr %otherwin, %o0 2622 wrpr %o0, 0, %canrestore 2623 wrpr %g0, 0, %otherwin 2624 wrpr %o0, 0, %cleanwin 2625 2626 /* 2627 * If this instruction causes a fill trap which fails to fill a window 2628 * from the user stack, we will resume at tl0_ret_fill_end and call 2629 * back into the kernel. 2630 */ 2631 restore 2632tl0_ret_fill: 2633 2634 rdpr %cwp, %g4 2635 wrpr %g1, %g4, %tstate 2636 wr %g2, 0, %fprs 2637 wrpr %g3, 0, %wstate 2638 2639#if KTR_COMPILE & KTR_TRAP 2640 CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" 2641 , %g2, %g3, %g4, 7, 8, 9) 2642 ldx [PCPU(CURTHREAD)], %g3 2643 stx %g3, [%g2 + KTR_PARM1] 2644 rdpr %pil, %g3 2645 stx %g3, [%g2 + KTR_PARM2] 2646 rdpr %tpc, %g3 2647 stx %g3, [%g2 + KTR_PARM3] 2648 rdpr %tnpc, %g3 2649 stx %g3, [%g2 + KTR_PARM4] 2650 stx %sp, [%g2 + KTR_PARM5] 26519: 2652#endif 2653 2654 retry 2655tl0_ret_fill_end: 2656 2657#if KTR_COMPILE & KTR_TRAP 2658 CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" 2659 , %l0, %l1, %l2, 7, 8, 9) 2660 rdpr %pstate, %l1 2661 stx %l1, [%l0 + KTR_PARM1] 2662 stx %l5, [%l0 + KTR_PARM2] 2663 stx %sp, [%l0 + KTR_PARM3] 26649: 2665#endif 2666 2667 /* 2668 * The fill failed and magic has been performed. Call trap again, 2669 * which will copyin the window on the user's behalf. 2670 */ 2671 wrpr %l5, 0, %wstate 2672 wrpr %g0, PSTATE_ALT, %pstate 2673 mov PCB_REG, %o0 2674 mov PCPU_REG, %o1 2675 wrpr %g0, PSTATE_NORMAL, %pstate 2676 mov %o0, PCB_REG 2677 mov %o1, PCPU_REG 2678 wrpr %g0, PSTATE_KERNEL, %pstate 2679 mov T_FILL_RET, %o0 2680 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2681 call trap 2682 add %sp, SPOFF + CCFSZ, %o0 2683 ba,a %xcc, tl0_ret 2684 nop 2685END(tl0_ret) 2686 2687/* 2688 * Kernel trap entry point 2689 * 2690 * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, 2691 * u_int sfsr) 2692 * 2693 * This is easy because the stack is already setup and the windows don't need 2694 * to be split. We build a trapframe and call trap(), the same as above, but 2695 * the outs don't need to be saved. 2696 */ 2697ENTRY(tl1_trap) 2698 sub %sp, TF_SIZEOF, %sp 2699 2700 rdpr %tstate, %l0 2701 rdpr %tpc, %l1 2702 rdpr %tnpc, %l2 2703 rdpr %pil, %l3 2704 rd %y, %l4 2705 rdpr %wstate, %l5 2706 2707#if KTR_COMPILE & KTR_TRAP 2708 CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" 2709 , %g1, %g2, %g3, 7, 8, 9) 2710 ldx [PCPU(CURTHREAD)], %g2 2711 stx %g2, [%g1 + KTR_PARM1] 2712 stx %o0, [%g1 + KTR_PARM2] 2713 stx %l3, [%g1 + KTR_PARM3] 2714 stx %l1, [%g1 + KTR_PARM4] 2715 stx %i6, [%g1 + KTR_PARM5] 27169: 2717#endif 2718 2719 wrpr %g0, 1, %tl 2720 2721 and %l5, WSTATE_OTHER_MASK, %l5 2722 wrpr %l5, WSTATE_KERNEL, %wstate 2723 2724 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2725 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2726 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2727 stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2728 stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2729 2730 stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] 2731 stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] 2732 stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] 2733 stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] 2734 2735 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2736 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2737 2738 mov PCB_REG, %l4 2739 mov PCPU_REG, %l5 2740 wrpr %g0, PSTATE_NORMAL, %pstate 2741 2742 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2743 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2744 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2745 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2746 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2747 2748 mov %l4, PCB_REG 2749 mov %l5, PCPU_REG 2750 wrpr %g0, PSTATE_KERNEL, %pstate 2751 2752 call trap 2753 add %sp, CCFSZ + SPOFF, %o0 2754 2755 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 2756 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 2757 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 2758 ldub [%sp + SPOFF + CCFSZ + TF_PIL], %l3 2759 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 2760 2761 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2762 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2763 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2764 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2765 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2766 2767 wrpr %g0, PSTATE_ALT, %pstate 2768 2769 andn %l0, TSTATE_CWP_MASK, %g1 2770 mov %l1, %g2 2771 mov %l2, %g3 2772 2773 wrpr %l3, 0, %pil 2774 wr %l4, 0, %y 2775 2776 restore 2777 2778 wrpr %g0, 2, %tl 2779 2780 rdpr %cwp, %g4 2781 wrpr %g1, %g4, %tstate 2782 wrpr %g2, 0, %tpc 2783 wrpr %g3, 0, %tnpc 2784 2785#if KTR_COMPILE & KTR_TRAP 2786 CATR(KTR_TRAP, "tl1_trap: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2787 , %g2, %g3, %g4, 7, 8, 9) 2788 ldx [PCPU(CURTHREAD)], %g3 2789 stx %g3, [%g2 + KTR_PARM1] 2790 rdpr %pil, %g3 2791 stx %g3, [%g2 + KTR_PARM2] 2792 rdpr %tstate, %g3 2793 stx %g3, [%g2 + KTR_PARM3] 2794 rdpr %tpc, %g3 2795 stx %g3, [%g2 + KTR_PARM4] 2796 stx %sp, [%g2 + KTR_PARM5] 27979: 2798#endif 2799 2800 retry 2801END(tl1_trap) 2802 2803/* 2804 * void tl1_intr(u_int level, u_int mask) 2805 */ 2806ENTRY(tl1_intr) 2807 sub %sp, TF_SIZEOF, %sp 2808 2809 rdpr %tstate, %l0 2810 rdpr %tpc, %l1 2811 rdpr %tnpc, %l2 2812 rdpr %pil, %l3 2813 rd %y, %l4 2814 rdpr %wstate, %l5 2815 2816#if KTR_COMPILE & KTR_INTR 2817 CATR(KTR_INTR, 2818 "tl1_intr: td=%p level=%#lx pil=%#lx pc=%#lx sp=%#lx" 2819 , %g1, %g2, %g3, 7, 8, 9) 2820 ldx [PCPU(CURTHREAD)], %g2 2821 stx %g2, [%g1 + KTR_PARM1] 2822 stx %o0, [%g1 + KTR_PARM2] 2823 stx %l3, [%g1 + KTR_PARM3] 2824 stx %l1, [%g1 + KTR_PARM4] 2825 stx %i6, [%g1 + KTR_PARM5] 28269: 2827#endif 2828 2829 wrpr %o0, 0, %pil 2830 wr %o1, 0, %asr21 2831 2832 wrpr %g0, 1, %tl 2833 2834 and %l5, WSTATE_OTHER_MASK, %l5 2835 wrpr %l5, WSTATE_KERNEL, %wstate 2836 2837 stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] 2838 stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] 2839 stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] 2840 stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] 2841 stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] 2842 2843 mov %o0, %l7 2844 mov T_INTERRUPT | T_KERNEL, %o1 2845 2846 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] 2847 stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] 2848 2849 stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] 2850 stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] 2851 2852 mov PCB_REG, %l4 2853 mov PCPU_REG, %l5 2854 wrpr %g0, PSTATE_NORMAL, %pstate 2855 2856 stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] 2857 stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] 2858 stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] 2859 stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] 2860 stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] 2861 2862 mov %l4, PCB_REG 2863 mov %l5, PCPU_REG 2864 wrpr %g0, PSTATE_KERNEL, %pstate 2865 2866 call critical_enter 2867 nop 2868 2869 SET(cnt+V_INTR, %l5, %l4) 2870 ATOMIC_INC_INT(%l4, %l5, %l6) 2871 2872 SET(intr_handlers, %l5, %l4) 2873 sllx %l7, IH_SHIFT, %l5 2874 ldx [%l4 + %l5], %l5 2875 KASSERT(%l5, "tl1_intr: ih null") 2876 call %l5 2877 add %sp, CCFSZ + SPOFF, %o0 2878 2879 call critical_exit 2880 nop 2881 2882 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 2883 2884 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 2885 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 2886 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 2887 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 2888 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 2889 2890 wrpr %g0, PSTATE_ALT, %pstate 2891 2892 andn %l0, TSTATE_CWP_MASK, %g1 2893 mov %l1, %g2 2894 mov %l2, %g3 2895 wrpr %l3, 0, %pil 2896 wr %l4, 0, %y 2897 2898 restore 2899 2900 wrpr %g0, 2, %tl 2901 2902 rdpr %cwp, %g4 2903 wrpr %g1, %g4, %tstate 2904 wrpr %g2, 0, %tpc 2905 wrpr %g3, 0, %tnpc 2906 2907#if KTR_COMPILE & KTR_INTR 2908 CATR(KTR_INTR, "tl1_intr: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" 2909 , %g2, %g3, %g4, 7, 8, 9) 2910 ldx [PCPU(CURTHREAD)], %g3 2911 stx %g3, [%g2 + KTR_PARM1] 2912 rdpr %pil, %g3 2913 stx %g3, [%g2 + KTR_PARM2] 2914 rdpr %tstate, %g3 2915 stx %g3, [%g2 + KTR_PARM3] 2916 rdpr %tpc, %g3 2917 stx %g3, [%g2 + KTR_PARM4] 2918 stx %sp, [%g2 + KTR_PARM5] 29199: 2920#endif 2921 2922 retry 2923END(tl1_intr) 2924 2925/* 2926 * Freshly forked processes come here when switched to for the first time. 2927 * The arguments to fork_exit() have been setup in the locals, we must move 2928 * them to the outs. 2929 */ 2930ENTRY(fork_trampoline) 2931#if KTR_COMPILE & KTR_PROC 2932 CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" 2933 , %g1, %g2, %g3, 7, 8, 9) 2934 ldx [PCPU(CURTHREAD)], %g2 2935 stx %g2, [%g1 + KTR_PARM1] 2936 ldx [%g2 + TD_PROC], %g2 2937 add %g2, P_COMM, %g2 2938 stx %g2, [%g1 + KTR_PARM2] 2939 rdpr %cwp, %g2 2940 stx %g2, [%g1 + KTR_PARM3] 29419: 2942#endif 2943 mov %l0, %o0 2944 mov %l1, %o1 2945 call fork_exit 2946 mov %l2, %o2 2947 b,a %xcc, tl0_ret 2948 nop 2949END(fork_trampoline) 2950