exception.S revision 115291
1/*- 2 * Copyright (c) 2003 Marcel Moolenaar 3 * Copyright (c) 2000 Doug Rabson 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD: head/sys/ia64/ia64/exception.S 115291 2003-05-24 20:51:55Z marcel $ 28 */ 29 30#include <machine/asm.h> 31#include <machine/pte.h> 32#include <assym.s> 33 34/* 35 * ar.k7 = kernel memory stack 36 * ar.k6 = kernel register stack 37 * ar.k5 = EPC gateway page 38 * ar.k4 = PCPU data 39 */ 40 41 .text 42 43/* 44 * exception_save: save interrupted state 45 * 46 * Arguments: 47 * r16 address of bundle that contains the branch. The 48 * return address will be the next bundle. 49 * r17 the value to save as ifa in the trapframe. This 50 * normally is cr.ifa, but some interruptions set 51 * set cr.iim and not cr.ifa. 52 * 53 * Returns: 54 * p15 interrupted from user stack 55 * p14 interrupted from kernel stack 56 * p13 interrupted from user backing store 57 * p12 interrupted from kernel backing store 58 * p11 interrupts were enabled 59 * p10 interrupts were disabled 60 */ 61ENTRY(exception_save, 0) 62{ .mii 63 mov r20=ar.unat 64 extr.u r31=sp,61,3 65 mov r18=pr 66 ;; 67} 68{ .mmi 69 cmp.le p14,p15=5,r31 70 ;; 71(p15) mov r23=ar.k7 // kernel memory stack 72(p14) mov r23=sp 73 ;; 74} 75{ .mii 76 mov r21=ar.rsc 77 add r30=-SIZEOF_TRAPFRAME,r23 78 ;; 79 dep r30=0,r30,0,10 80 ;; 81} 82{ .mmi 83 mov ar.rsc=0 84 sub r19=r23,r30 85 add r31=8,r30 86 ;; 87} 88{ .mlx 89 mov r22=cr.iip 90 movl r26=exception_save_restart 91 ;; 92} 93 94 /* 95 * We have a 1KB aligned trapframe, pointed to by sp. If we write 96 * to the trapframe, we may trigger a data nested TLB fault. By 97 * aligning the trapframe on a 1KB boundary, we guarantee that if 98 * we get a data nested TLB fault, it will be on the very first 99 * write. Since the data nested TLB fault does not preserve any 100 * state, we have to be careful what we clobber. Consequently, we 101 * have to be careful what we use here. Below a list of registers 102 * that are currently alive: 103 * r16,r17=arguments 104 * r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS 105 * r26=restart point 106 * r30,r31=trapframe pointers 107 * p14,p15=memory stack switch 108 */ 109exception_save_restart: 110{ .mmi 111 st8 [r30]=r19,16 // length 112 st8 [r31]=r0,16 // flags 113 add r19=16,r19 114 ;; 115} 116{ .mmi 117 st8.spill [r30]=sp,16 // sp 118 st8 [r31]=r20,16 // unat 119 sub sp=r23,r19 120 ;; 121} 122{ .mmi 123 mov r19=ar.rnat 124 mov r20=ar.bspstore 125 mov r23=rp 126 ;; 127} 128 // r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp 129{ .mmi 130 st8 [r30]=r23,16 // rp 131 st8 [r31]=r18,16 // pr 132 mov r24=ar.pfs 133 ;; 134} 135{ .mmb 136 st8 [r30]=r24,16 // pfs 137 st8 [r31]=r20,16 // bspstore 138 cover 139 ;; 140} 141{ .mmi 142 mov r18=ar.fpsr 143 mov r23=cr.ipsr 144 extr.u r24=r20,61,3 145 ;; 146} 147 // r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr 148{ .mmi 149 st8 [r30]=r19,16 // rnat 150 st8 [r31]=r0,16 // __spare 151 cmp.le p12,p13=5,r24 152 ;; 153} 154{ .mmi 155 st8.spill [r30]=r13,16 // tp 156 st8 [r31]=r21,16 // rsc 157 tbit.nz p11,p10=r23,14 // p11=interrupts enabled 158 ;; 159} 160{ .mmi 161 st8 [r30]=r18,16 // fpsr 162(p13) mov r20=ar.k6 // kernel register stack 163 nop 0 164 ;; 165} 166 // r20=bspstore, r22=iip, r23=ipsr 167{ .mmi 168 st8 [r31]=r23,16 // psr 169(p13) mov ar.bspstore=r20 170 nop 0 171 ;; 172} 173{ .mmi 174 mov r18=ar.bsp 175 ;; 176 mov r19=cr.ifs 177 sub r18=r18,r20 178 ;; 179} 180{ .mmi 181 st8.spill [r30]=gp,16 // gp 182 st8 [r31]=r18,16 // ndirty 183 nop 0 184 ;; 185} 186 // r19=ifs, r22=iip 187{ .mmi 188 st8 [r30]=r19,16 // cfm 189 st8 [r31]=r22,16 // iip 190 nop 0 191 ;; 192} 193{ .mmi 194 st8 [r30]=r17 // ifa 195 mov r18=cr.isr 196 add r29=16,r30 197 ;; 198} 199{ .mmi 200 st8 [r31]=r18 // isr 201 add r30=8,r29 202 add r31=16,r29 203 ;; 204} 205{ .mmi 206 .mem.offset 0,0 207 st8.spill [r30]=r2,16 // r2 208 .mem.offset 8,0 209 st8.spill [r31]=r3,16 // r3 210 add r2=9*8,r29 211 ;; 212} 213{ .mmi 214 .mem.offset 0,0 215 st8.spill [r30]=r8,16 // r8 216 .mem.offset 8,0 217 st8.spill [r31]=r9,16 // r9 218 add r3=8,r2 219 ;; 220} 221{ .mmi 222 .mem.offset 0,0 223 st8.spill [r30]=r10,16 // r10 224 .mem.offset 8,0 225 st8.spill [r31]=r11,16 // r11 226 add r8=16,r16 227 ;; 228} 229{ .mmi 230 .mem.offset 0,0 231 st8.spill [r30]=r14 // r14 232 .mem.offset 8,0 233 st8.spill [r31]=r15 // r15 234 mov r9=r29 235} 236{ .mmb 237 mov r10=ar.csd 238 mov r11=ar.ssd 239 bsw.1 240 ;; 241} 242{ .mmi 243 .mem.offset 0,0 244 st8.spill [r2]=r16,16 // r16 245 .mem.offset 8,0 246 st8.spill [r3]=r17,16 // r17 247 mov r14=b6 248 ;; 249} 250{ .mmi 251 .mem.offset 0,0 252 st8.spill [r2]=r18,16 // r18 253 .mem.offset 8,0 254 st8.spill [r3]=r19,16 // r19 255 mov r15=b7 256 ;; 257} 258{ .mmi 259 .mem.offset 0,0 260 st8.spill [r2]=r20,16 // r20 261 .mem.offset 8,0 262 st8.spill [r3]=r21,16 // r21 263 mov b7=r8 264 ;; 265} 266{ .mmi 267 .mem.offset 0,0 268 st8.spill [r2]=r22,16 // r22 269 .mem.offset 8,0 270 st8.spill [r3]=r23,16 // r23 271 ;; 272} 273 274 .mem.offset 0,0 275 st8.spill [r2]=r24,16 // r24 276 .mem.offset 8,0 277 st8.spill [r3]=r25,16 // r25 278 ;; 279 .mem.offset 0,0 280 st8.spill [r2]=r26,16 // r26 281 .mem.offset 8,0 282 st8.spill [r3]=r27,16 // r27 283 ;; 284 .mem.offset 0,0 285 st8.spill [r2]=r28,16 // r28 286 .mem.offset 8,0 287 st8.spill [r3]=r29,16 // r29 288 ;; 289 .mem.offset 0,0 290 st8.spill [r2]=r30,16 // r30 291 .mem.offset 8,0 292 st8.spill [r3]=r31,16 // r31 293 ;; 294 295{ .mmi 296 st8 [r2]=r14,16 // b6 297 mov r17=ar.unat 298 nop 0 299 ;; 300} 301{ .mmi 302 st8 [r3]=r15,16 // b7 303 mov r16=ar.ccv 304 nop 0 305 ;; 306} 307{ .mmi 308 st8 [r2]=r16,16 // ccv 309 st8 [r3]=r10,16 // csd 310 nop 0 311 ;; 312} 313{ .mmi 314 st8 [r2]=r11,24 // ssd 315 st8 [r9]=r17 316 nop 0 317 ;; 318} 319 320 stf.spill [r3]=f6,32 // f6 321 stf.spill [r2]=f7,32 // f7 322 ;; 323 stf.spill [r3]=f8,32 // f8 324 stf.spill [r2]=f9,32 // f9 325 ;; 326 stf.spill [r3]=f10,32 // f10 327 stf.spill [r2]=f11,32 // f11 328 ;; 329 stf.spill [r3]=f12,32 // f12 330 stf.spill [r2]=f13,32 // f13 331 ;; 332 stf.spill [r3]=f14 // f14 333 stf.spill [r2]=f15 // f15 334 ;; 335{ .mmi 336 mov ar.rsc=3 337 mov r13=ar.k4 338 nop 0 339 ;; 340} 341{ .mlx 342 ssm psr.ic|psr.dfh 343 movl gp=__gp 344 ;; 345} 346{ .mfb 347 srlz.d 348 nop 0 349 br.sptk b7 350 ;; 351} 352END(exception_save) 353 354/* 355 * exception_restore: restore interrupted state 356 * 357 * Arguments: 358 * sp+16 trapframe pointer 359 */ 360ENTRY(exception_restore, 0) 361{ .mmi 362 rsm psr.i 363 add sp=16,sp 364 nop 0 365 ;; 366} 367{ .mmi 368 add r3=SIZEOF_TRAPFRAME-32,sp 369 add r2=SIZEOF_TRAPFRAME-16,sp 370 add r8=SIZEOF_SPECIAL+16,sp 371 ;; 372} 373 // The next load can trap. Let it be... 374 ldf.fill f15=[r2],-32 // f15 375 ldf.fill f14=[r3],-32 // f14 376 ;; 377 ldf.fill f13=[r2],-32 // f13 378 ldf.fill f12=[r3],-32 // f12 379 ;; 380 ldf.fill f11=[r2],-32 // f11 381 ldf.fill f10=[r3],-32 // f10 382 ;; 383 ldf.fill f9=[r2],-32 // f9 384 ldf.fill f8=[r3],-32 // f8 385 ;; 386 ldf.fill f7=[r2],-24 // f7 387 ldf.fill f6=[r3],-16 // f6 388 ;; 389 390{ .mmi 391 ld8 r8=[r8] // unat (after) 392 ;; 393 mov ar.unat=r8 394 nop 0 395 ;; 396} 397 398 ld8 r10=[r2],-16 // ssd 399 ld8 r11=[r3],-16 // csd 400 ;; 401 mov ar.ssd=r10 402 mov ar.csd=r11 403 404 ld8 r14=[r2],-16 // ccv 405 ld8 r15=[r3],-16 // b7 406 ;; 407 408{ .mmi 409 mov ar.ccv=r14 410 ld8 r8=[r2],-16 // b6 411 mov b7=r15 412 ;; 413} 414{ .mmi 415 ld8.fill r31=[r3],-16 // r31 416 ld8.fill r30=[r2],-16 // r30 417 mov b6=r8 418 ;; 419} 420 421 ld8.fill r29=[r3],-16 // r29 422 ld8.fill r28=[r2],-16 // r28 423 ;; 424 ld8.fill r27=[r3],-16 // r27 425 ld8.fill r26=[r2],-16 // r26 426 ;; 427 ld8.fill r25=[r3],-16 // r25 428 ld8.fill r24=[r2],-16 // r24 429 ;; 430 ld8.fill r23=[r3],-16 // r23 431 ld8.fill r22=[r2],-16 // r22 432 ;; 433 ld8.fill r21=[r3],-16 // r21 434 ld8.fill r20=[r2],-16 // r20 435 ;; 436 ld8.fill r19=[r3],-16 // r19 437 ld8.fill r18=[r2],-16 // r18 438 ;; 439 440{ .mmb 441 ld8.fill r17=[r3],-16 // r17 442 ld8.fill r16=[r2],-16 // r16 443 bsw.0 444 ;; 445} 446{ .mmi 447 ld8.fill r15=[r3],-16 // r15 448 ld8.fill r14=[r2],-16 // r14 449 add r31=16,sp 450 ;; 451} 452{ .mmi 453 ld8 r16=[sp] // tf_length 454 ld8.fill r11=[r3],-16 // r11 455 add r30=24,sp 456 ;; 457} 458{ .mmi 459 ld8.fill r10=[r2],-16 // r10 460 ld8.fill r9=[r3],-16 // r9 461 add r16=r16,sp // ar.k7 462 ;; 463} 464{ .mmi 465 ld8.fill r8=[r2],-16 // r8 466 ld8.fill r3=[r3] // r3 467 ;; 468} 469 // We want nested TLB faults from here on... 470 rsm psr.ic|psr.i 471 ld8.fill r2=[r2] // r2 472 nop 0 473 ;; 474 srlz.d 475 ld8.fill sp=[r31],16 // sp 476 nop 0 477 ;; 478 479 ld8 r17=[r30],16 // unat 480 ld8 r29=[r31],16 // rp 481 ;; 482 ld8 r18=[r30],16 // pr 483 ld8 r28=[r31],16 // pfs 484 mov rp=r29 485 ;; 486 ld8 r20=[r30],24 // bspstore 487 ld8 r21=[r31],24 // rnat 488 mov ar.pfs=r28 489 ;; 490 ld8.fill r29=[r30],16 // tp 491 ld8 r22=[r31],16 // rsc 492 ;; 493{ .mmi 494 ld8 r23=[r30],16 // fpsr 495 ld8 r24=[r31],16 // psr 496 extr.u r28=r20,61,3 497 ;; 498} 499{ .mmi 500 ld8.fill r1=[r30],16 // gp 501 ld8 r25=[r31],16 // ndirty 502 cmp.le p14,p15=5,r28 503 ;; 504} 505{ .mmb 506 ld8 r26=[r30] // cfm 507 ld8 r19=[r31] // ip 508 nop 0 509 ;; 510} 511{ .mib 512 // Switch register stack 513 alloc r30=ar.pfs,0,0,0,0 // discard current frame 514 shl r31=r25,16 // value for ar.rsc 515 nop 0 516 ;; 517} 518 // The loadrs can fault if the backing store is not currently 519 // mapped. We assured forward progress by getting everything we 520 // need from the trapframe so that we don't care if the CPU 521 // purges that translation when it needs to insert a new one for 522 // the backing store. 523{ .mmi 524 mov ar.rsc=r31 // setup for loadrs 525 mov ar.k7=r16 526(p15) mov r13=r29 527 ;; 528} 529exception_restore_restart: 530{ .mmi 531 mov r30=ar.bspstore 532 ;; 533 loadrs // load user regs 534 nop 0 535 ;; 536} 537{ .mmi 538(p15) mov r31=ar.bspstore 539 ;; 540 mov ar.bspstore=r20 541(p15) dep r31=0,r31,0,9 542 ;; 543} 544{ .mmb 545 mov ar.k6=r31 546 mov ar.rnat=r21 547 nop 0 548 ;; 549} 550{ .mmb 551 mov ar.unat=r17 552 mov cr.iip=r19 553 nop 0 554} 555{ .mmi 556 mov cr.ipsr=r24 557 mov cr.ifs=r26 558 mov pr=r18,0x1fffe 559 ;; 560} 561{ .mmb 562 mov ar.rsc=r22 563 mov ar.fpsr=r23 564 rfi 565 ;; 566} 567END(exception_restore) 568 569/* 570 * Call exception_save_regs to preserve the interrupted state in a 571 * trapframe. Note that we don't use a call instruction because we 572 * must be careful not to lose track of the RSE state. We then call 573 * trap() with the value of _n_ as an argument to handle the 574 * exception. We arrange for trap() to return to exception_restore 575 * which will restore the interrupted state before executing an rfi to 576 * resume it. 577 */ 578#define TRAP(_n_, _ifa_) \ 579{ .mib ; \ 580 mov r17=_ifa_ ; \ 581 mov r16=ip ; \ 582 br.sptk exception_save ; \ 583} ; \ 584{ .mmi ; \ 585(p11) ssm psr.i ;; \ 586 alloc r15=ar.pfs,0,0,2,0 ; \ 587 mov out0=_n_ ;; \ 588} ; \ 589{ .mfb ; \ 590 add out1=16,sp ; \ 591 nop 0 ; \ 592 br.call.sptk rp=trap ; \ 593} ; \ 594{ .mfb ; \ 595 nop 0 ; \ 596 nop 0 ; \ 597 br.sptk exception_restore ; \ 598} 599 600#define IVT_ENTRY(name, offset) \ 601 .org ia64_vector_table + offset; \ 602 .global ivt_##name; \ 603 .proc ivt_##name; \ 604 .prologue; \ 605 .unwabi @svr4, 'I'; \ 606 .save rp, r0; \ 607 .body; \ 608ivt_##name: 609 610#define IVT_END(name) \ 611 .endp ivt_##name; \ 612 .align 0x100 613 614/* 615 * The IA64 Interrupt Vector Table (IVT) contains 20 slots with 64 616 * bundles per vector and 48 slots with 16 bundles per vector. 617 */ 618 619 .section .text.ivt,"ax" 620 621 .align 32768 622 .global ia64_vector_table 623 .size ia64_vector_table, 32768 624ia64_vector_table: 625 626IVT_ENTRY(VHPT_Translation, 0x0000) 627 TRAP(0, cr.ifa) 628IVT_END(VHPT_Translation) 629 630IVT_ENTRY(Instruction_TLB, 0x0400) 631 mov r16=cr.ifa 632 mov r17=pr 633 ;; 634 thash r18=r16 635 ttag r19=r16 636 ;; 637 add r21=16,r18 // tag 638 add r20=24,r18 // collision chain 639 ;; 640 ld8 r21=[r21] // check VHPT tag 641 ;; 642 cmp.ne p15,p0=r21,r19 643(p15) br.dpnt.few 1f 644 ;; 645 ld8 r21=[r18] // read pte 646 ;; 647 itc.i r21 // insert pte 648 ;; 649 mov pr=r17,0x1ffff 650 rfi // done 651 ;; 6521: ld8 r20=[r20] // first entry 653 ;; 654 rsm psr.dt // turn off data translations 655 ;; 656 srlz.d // serialize 657 ;; 6582: cmp.eq p15,p0=r0,r20 // done? 659(p15) br.cond.spnt.few 9f // bail if done 660 ;; 661 add r21=16,r20 // tag location 662 ;; 663 ld8 r21=[r21] // read tag 664 ;; 665 cmp.ne p15,p0=r21,r19 // compare tags 666(p15) br.cond.sptk.few 3f // if not, read next in chain 667 ;; 668 ld8 r21=[r20],8 // read pte 669 ;; 670 ld8 r22=[r20] // read rest of pte 671 ;; 672 dep r18=0,r18,61,3 // convert vhpt ptr to physical 673 ;; 674 add r20=16,r18 // address of tag 675 ;; 676 ld8.acq r23=[r20] // read old tag 677 ;; 678 dep r23=-1,r23,63,1 // set ti bit 679 ;; 680 st8.rel [r20]=r23 // store old tag + ti 681 ;; 682 mf // make sure everyone sees 683 ;; 684 st8 [r18]=r21,8 // store pte 685 ;; 686 st8 [r18]=r22,8 687 ;; 688 st8.rel [r18]=r19 // store new tag 689 ;; 690 itc.i r21 // and place in TLB 691 ;; 692 mov pr=r17,0x1ffff // restore predicates 693 rfi 694 6953: add r20=24,r20 // next in chain 696 ;; 697 ld8 r20=[r20] // read chain 698 br.cond.sptk.few 2b // loop 699 7009: mov pr=r17,0x1ffff // restore predicates 701 ssm psr.dt 702 ;; 703 srlz.d 704 ;; 705 TRAP(20, cr.ifa) // Page Not Present trap 706IVT_END(Instruction_TLB) 707 708IVT_ENTRY(Data_TLB, 0x0800) 709 mov r16=cr.ifa 710 mov r17=pr 711 ;; 712 thash r18=r16 713 ttag r19=r16 714 ;; 715 add r21=16,r18 // tag 716 add r20=24,r18 // collision chain 717 ;; 718 ld8 r21=[r21] // check VHPT tag 719 ;; 720 cmp.ne p15,p0=r21,r19 721(p15) br.dpnt.few 1f 722 ;; 723 ld8 r21=[r18] // read pte 724 ;; 725 itc.d r21 // insert pte 726 ;; 727 mov pr=r17,0x1ffff 728 rfi // done 729 ;; 7301: ld8 r20=[r20] // first entry 731 ;; 732 rsm psr.dt // turn off data translations 733 ;; 734 srlz.d // serialize 735 ;; 7362: cmp.eq p15,p0=r0,r20 // done? 737(p15) br.cond.spnt.few 9f // bail if done 738 ;; 739 add r21=16,r20 // tag location 740 ;; 741 ld8 r21=[r21] // read tag 742 ;; 743 cmp.ne p15,p0=r21,r19 // compare tags 744(p15) br.cond.sptk.few 3f // if not, read next in chain 745 ;; 746 ld8 r21=[r20],8 // read pte 747 ;; 748 ld8 r22=[r20] // read rest of pte 749 ;; 750 dep r18=0,r18,61,3 // convert vhpt ptr to physical 751 ;; 752 add r20=16,r18 // address of tag 753 ;; 754 ld8.acq r23=[r20] // read old tag 755 ;; 756 dep r23=-1,r23,63,1 // set ti bit 757 ;; 758 st8.rel [r20]=r23 // store old tag + ti 759 ;; 760 mf // make sure everyone sees 761 ;; 762 st8 [r18]=r21,8 // store pte 763 ;; 764 st8 [r18]=r22,8 765 ;; 766 st8.rel [r18]=r19 // store new tag 767 ;; 768 itc.d r21 // and place in TLB 769 ;; 770 mov pr=r17,0x1ffff // restore predicates 771 rfi 772 7733: add r20=24,r20 // next in chain 774 ;; 775 ld8 r20=[r20] // read chain 776 br.cond.sptk.few 2b // loop 777 7789: mov pr=r17,0x1ffff // restore predicates 779 ssm psr.dt 780 ;; 781 srlz.d 782 ;; 783 TRAP(20, cr.ifa) // Page Not Present trap 784IVT_END(Data_TLB) 785 786IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) 787 mov r16=cr.ifa // where did it happen 788 mov r18=pr // save predicates 789 ;; 790 extr.u r17=r16,61,3 // get region number 791 ;; 792 cmp.ge p13,p0=5,r17 // RR0-RR5? 793 cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 794(p13) br.spnt 9f 795 ;; 796(p15) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX 797(p14) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX 798 ;; 799 dep r16=0,r16,50,14 // clear bits above PPN 800 ;; 801 dep r16=r17,r16,0,12 // put pte bits in 0..11 802 ;; 803 itc.i r16 804 mov pr=r18,0x1ffff // restore predicates 805 ;; 806 rfi 8079: mov pr=r18,0x1ffff // restore predicates 808 TRAP(3, cr.ifa) 809IVT_END(Alternate_Instruction_TLB) 810 811IVT_ENTRY(Alternate_Data_TLB, 0x1000) 812 mov r16=cr.ifa // where did it happen 813 mov r18=pr // save predicates 814 ;; 815 extr.u r17=r16,61,3 // get region number 816 ;; 817 cmp.ge p13,p0=5,r17 // RR0-RR5? 818 cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 819(p13) br.spnt 9f 820 ;; 821(p15) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW 822(p14) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW 823 ;; 824 dep r16=0,r16,50,14 // clear bits above PPN 825 ;; 826 dep r16=r17,r16,0,12 // put pte bits in 0..11 827 ;; 828 itc.d r16 829 mov pr=r18,0x1ffff // restore predicates 830 ;; 831 rfi 8329: mov pr=r18,0x1ffff // restore predicates 833 TRAP(4, cr.ifa) 834IVT_END(Alternate_Data_TLB) 835 836IVT_ENTRY(Data_Nested_TLB, 0x1400) 837 // See exception_save_restart and exception_restore_restart for the 838 // contexts that may cause a data nested TLB. We can only use the 839 // banked general registers and predicates, but don't use: 840 // p14 & p15 - Set in exception save 841 // r16 & r17 - Arguments to exception save 842 // r30 - Faulting address (modulo page size) 843 // We assume r30 has the virtual addresses that relate to the data 844 // nested TLB fault. The address does not have to be exact, as long 845 // as it's in the same page. We use physical addressing to avoid 846 // double nested faults. Since all virtual addresses we encounter 847 // here are direct mapped region 7 addresses, we have no problem 848 // constructing physical addresses. 849{ .mlx 850 rsm psr.dt 851 movl r27=ia64_kptdir 852 ;; 853} 854{ .mii 855 srlz.d 856 dep r27=0,r27,61,3 857 extr.u r28=r30,PAGE_SHIFT,61-PAGE_SHIFT 858 ;; 859} 860{ .mii 861 ld8 r27=[r27] 862 shr.u r29=r28,PAGE_SHIFT-5 // dir index 863 extr.u r28=r28,0,PAGE_SHIFT-5 // pte index 864 ;; 865} 866{ .mmi 867 shladd r27=r29,3,r27 868 ;; 869 mov cr.ifa=r30 870 dep r27=0,r27,61,3 871 ;; 872} 873{ .mmi 874 ld8 r27=[r27] 875 mov r29=rr[r30] 876 shl r28=r28,5 877 ;; 878} 879{ .mii 880 add r27=r27,r28 // address of pte 881 dep r29=0,r29,0,2 882 ;; 883 dep r27=0,r27,61,3 884 ;; 885} 886{ .mmi 887 ld8 r28=[r27] 888 ;; 889 mov cr.itir=r29 890 or r28=PTE_D|PTE_A,r28 891 ;; 892} 893{ .mlx 894 st8 [r27]=r28 895 movl r29=exception_save_restart 896 ;; 897} 898{ .mmi 899 itc.d r28 900 ;; 901 ssm psr.dt 902 cmp.eq p12,p13=r26,r29 903 ;; 904} 905{ .mbb 906 srlz.d 907(p12) br.sptk exception_save_restart 908(p13) br.sptk exception_restore_restart 909 ;; 910} 911IVT_END(Data_Nested_TLB) 912 913IVT_ENTRY(Instruction_Key_Miss, 0x1800) 914 TRAP(6, cr.ifa) 915IVT_END(Instruction_Key_Miss) 916 917IVT_ENTRY(Data_Key_Miss, 0x1c00) 918 TRAP(7, cr.ifa) 919IVT_END(Data_Key_Miss) 920 921IVT_ENTRY(Dirty_Bit, 0x2000) 922 mov r16=cr.ifa 923 mov r17=pr 924 ;; 925 thash r18=r16 926 ;; 927 ttag r19=r16 928 add r20=24,r18 // collision chain 929 ;; 930 ld8 r20=[r20] // first entry 931 ;; 932 rsm psr.dt // turn off data translations 933 ;; 934 srlz.d // serialize 935 ;; 9361: cmp.eq p15,p0=r0,r20 // done? 937(p15) br.cond.spnt.few 9f // bail if done 938 ;; 939 add r21=16,r20 // tag location 940 ;; 941 ld8 r21=[r21] // read tag 942 ;; 943 cmp.ne p15,p0=r21,r19 // compare tags 944(p15) br.cond.sptk.few 2f // if not, read next in chain 945 ;; 946 ld8 r21=[r20] // read pte 947 mov r22=PTE_D|PTE_A 948 ;; 949 or r21=r22,r21 // set dirty & access bit 950 ;; 951 st8 [r20]=r21,8 // store back 952 ;; 953 ld8 r22=[r20] // read rest of pte 954 ;; 955 dep r18=0,r18,61,3 // convert vhpt ptr to physical 956 ;; 957 add r20=16,r18 // address of tag 958 ;; 959 ld8.acq r23=[r20] // read old tag 960 ;; 961 dep r23=-1,r23,63,1 // set ti bit 962 ;; 963 st8.rel [r20]=r23 // store old tag + ti 964 ;; 965 mf // make sure everyone sees 966 ;; 967 st8 [r18]=r21,8 // store pte 968 ;; 969 st8 [r18]=r22,8 970 ;; 971 st8.rel [r18]=r19 // store new tag 972 ;; 973 itc.d r21 // and place in TLB 974 ;; 975 mov pr=r17,0x1ffff // restore predicates 976 rfi 977 9782: add r20=24,r20 // next in chain 979 ;; 980 ld8 r20=[r20] // read chain 981 br.cond.sptk.few 1b // loop 982 9839: mov pr=r17,0x1ffff // restore predicates 984 TRAP(8, cr.ifa) // die horribly 985IVT_END(Dirty_Bit) 986 987IVT_ENTRY(Instruction_Access_Bit, 0x2400) 988 mov r16=cr.ifa 989 mov r17=pr 990 ;; 991 thash r18=r16 992 ;; 993 ttag r19=r16 994 add r20=24,r18 // collision chain 995 ;; 996 ld8 r20=[r20] // first entry 997 ;; 998 rsm psr.dt // turn off data translations 999 ;; 1000 srlz.d // serialize 1001 ;; 10021: cmp.eq p15,p0=r0,r20 // done? 1003(p15) br.cond.spnt.few 9f // bail if done 1004 ;; 1005 add r21=16,r20 // tag location 1006 ;; 1007 ld8 r21=[r21] // read tag 1008 ;; 1009 cmp.ne p15,p0=r21,r19 // compare tags 1010(p15) br.cond.sptk.few 2f // if not, read next in chain 1011 ;; 1012 ld8 r21=[r20] // read pte 1013 mov r22=PTE_A 1014 ;; 1015 or r21=r22,r21 // set accessed bit 1016 ;; 1017 st8 [r20]=r21,8 // store back 1018 ;; 1019 ld8 r22=[r20] // read rest of pte 1020 ;; 1021 dep r18=0,r18,61,3 // convert vhpt ptr to physical 1022 ;; 1023 add r20=16,r18 // address of tag 1024 ;; 1025 ld8.acq r23=[r20] // read old tag 1026 ;; 1027 dep r23=-1,r23,63,1 // set ti bit 1028 ;; 1029 st8.rel [r20]=r23 // store old tag + ti 1030 ;; 1031 mf // make sure everyone sees 1032 ;; 1033 st8 [r18]=r21,8 // store pte 1034 ;; 1035 st8 [r18]=r22,8 1036 ;; 1037 st8.rel [r18]=r19 // store new tag 1038 ;; 1039 itc.i r21 // and place in TLB 1040 ;; 1041 mov pr=r17,0x1ffff // restore predicates 1042 rfi // walker will retry the access 1043 10442: add r20=24,r20 // next in chain 1045 ;; 1046 ld8 r20=[r20] // read chain 1047 br.cond.sptk.few 1b // loop 1048 10499: mov pr=r17,0x1ffff // restore predicates 1050 TRAP(9, cr.ifa) 1051IVT_END(Instruction_Access_Bit) 1052 1053IVT_ENTRY(Data_Access_Bit, 0x2800) 1054 mov r16=cr.ifa 1055 mov r17=pr 1056 ;; 1057 thash r18=r16 1058 ;; 1059 ttag r19=r16 1060 add r20=24,r18 // collision chain 1061 ;; 1062 ld8 r20=[r20] // first entry 1063 ;; 1064 rsm psr.dt // turn off data translations 1065 ;; 1066 srlz.d // serialize 1067 ;; 10681: cmp.eq p15,p0=r0,r20 // done? 1069(p15) br.cond.spnt.few 9f // bail if done 1070 ;; 1071 add r21=16,r20 // tag location 1072 ;; 1073 ld8 r21=[r21] // read tag 1074 ;; 1075 cmp.ne p15,p0=r21,r19 // compare tags 1076(p15) br.cond.sptk.few 2f // if not, read next in chain 1077 ;; 1078 ld8 r21=[r20] // read pte 1079 mov r22=PTE_A 1080 ;; 1081 or r21=r22,r21 // set accessed bit 1082 ;; 1083 st8 [r20]=r21,8 // store back 1084 ;; 1085 ld8 r22=[r20] // read rest of pte 1086 ;; 1087 dep r18=0,r18,61,3 // convert vhpt ptr to physical 1088 ;; 1089 add r20=16,r18 // address of tag 1090 ;; 1091 ld8.acq r23=[r20] // read old tag 1092 ;; 1093 dep r23=-1,r23,63,1 // set ti bit 1094 ;; 1095 st8.rel [r20]=r23 // store old tag + ti 1096 ;; 1097 mf // make sure everyone sees 1098 ;; 1099 st8 [r18]=r21,8 // store pte 1100 ;; 1101 st8 [r18]=r22,8 1102 ;; 1103 st8.rel [r18]=r19 // store new tag 1104 ;; 1105 itc.d r21 // and place in TLB 1106 ;; 1107 mov pr=r17,0x1ffff // restore predicates 1108 rfi // walker will retry the access 1109 11102: add r20=24,r20 // next in chain 1111 ;; 1112 ld8 r20=[r20] // read chain 1113 br.cond.sptk.few 1b // loop 1114 11159: mov pr=r17,0x1ffff // restore predicates 1116 TRAP(10, cr.ifa) 1117IVT_END(Data_Access_Bit) 1118 1119IVT_ENTRY(Break_Instruction, 0x2c00) 1120{ .mib 1121 mov r17=cr.iim 1122 mov r16=ip 1123 br.sptk exception_save 1124 ;; 1125} 1126{ .mmi 1127 alloc r15=ar.pfs,0,0,2,0 1128 flushrs 1129 mov out0=11 1130 ;; 1131} 1132{ .mib 1133(p11) ssm psr.i 1134 add out1=16,sp 1135 br.call.sptk rp=trap 1136 ;; 1137} 1138{ .mfb 1139 nop 0 1140 nop 0 1141 br.sptk exception_restore 1142 ;; 1143} 1144IVT_END(Break_Instruction) 1145 1146IVT_ENTRY(External_Interrupt, 0x3000) 1147{ .mib 1148 mov r17=cr.lid // cr.iim and cr.ifa are undefined. 1149 mov r16=ip 1150 br.sptk exception_save 1151 ;; 1152} 1153{ .mmb 11542: alloc r14=ar.pfs,0,0,2,0 // make a frame for calling with 1155 add out1=16,sp 1156 nop 0 1157} 1158 11593: mov out0=cr.ivr // find interrupt vector 1160 ;; 1161 cmp.eq p15,p0=15,out0 // check for spurious vector number 1162(p15) br.dpnt.few exception_restore // if spurious, we are done 1163 ;; 1164 ssm psr.i // re-enable interrupts 1165 br.call.sptk.many rp=interrupt // call high-level handler 1166 ;; 1167 rsm psr.i // disable interrupts 1168 ;; 1169 srlz.d 1170 mov cr.eoi=r0 // and ack the interrupt 1171 ;; 1172 srlz.d 1173 br.sptk.few 3b // loop for more 1174 ;; 1175IVT_END(External_Interrupt) 1176 1177IVT_ENTRY(Reserved_3400, 0x3400) 1178 TRAP(13, cr.ifa) 1179IVT_END(Reserved_3400) 1180 1181IVT_ENTRY(Reserved_3800, 0x3800) 1182 TRAP(14, cr.ifa) 1183IVT_END(Reserved_3800) 1184 1185IVT_ENTRY(Reserved_3c00, 0x3c00) 1186 TRAP(15, cr.ifa) 1187IVT_END(Reserved_3c00) 1188 1189IVT_ENTRY(Reserved_4000, 0x4000) 1190 TRAP(16, cr.ifa) 1191IVT_END(Reserved_4000) 1192 1193IVT_ENTRY(Reserved_4400, 0x4400) 1194 TRAP(17, cr.ifa) 1195IVT_END(Reserved_4400) 1196 1197IVT_ENTRY(Reserved_4800, 0x4800) 1198 TRAP(18, cr.ifa) 1199IVT_END(Reserved_4800) 1200 1201IVT_ENTRY(Reserved_4c00, 0x4c00) 1202 TRAP(19, cr.ifa) 1203IVT_END(Reserved_4c00) 1204 1205IVT_ENTRY(Page_Not_Present, 0x5000) 1206 TRAP(20, cr.ifa) 1207IVT_END(Page_Not_Present) 1208 1209IVT_ENTRY(Key_Permission, 0x5100) 1210 TRAP(21, cr.ifa) 1211IVT_END(Key_Permission) 1212 1213IVT_ENTRY(Instruction_Access_Rights, 0x5200) 1214 TRAP(22, cr.ifa) 1215IVT_END(Instruction_Access_Rights) 1216 1217IVT_ENTRY(Data_Access_Rights, 0x5300) 1218 TRAP(23, cr.ifa) 1219IVT_END(Data_Access_Rights) 1220 1221IVT_ENTRY(General_Exception, 0x5400) 1222 TRAP(24, cr.ifa) 1223IVT_END(General_Exception) 1224 1225IVT_ENTRY(Disabled_FP_Register, 0x5500) 1226 TRAP(25, cr.ifa) 1227IVT_END(Disabled_FP_Register) 1228 1229IVT_ENTRY(NaT_Consumption, 0x5600) 1230 TRAP(26, cr.ifa) 1231IVT_END(NaT_Consumption) 1232 1233IVT_ENTRY(Speculation, 0x5700) 1234 TRAP(27, cr.iim) 1235IVT_END(Speculation) 1236 1237IVT_ENTRY(Reserved_5800, 0x5800) 1238 TRAP(28, cr.ifa) 1239IVT_END(Reserved_5800) 1240 1241IVT_ENTRY(Debug, 0x5900) 1242 TRAP(29, cr.ifa) 1243IVT_END(Debug) 1244 1245IVT_ENTRY(Unaligned_Reference, 0x5a00) 1246 TRAP(30, cr.ifa) 1247IVT_END(Unaligned_Reference) 1248 1249IVT_ENTRY(Unsupported_Data_Reference, 0x5b00) 1250 TRAP(31, cr.ifa) 1251IVT_END(Unsupported_Data_Reference) 1252 1253IVT_ENTRY(Floating_Point_Fault, 0x5c00) 1254 TRAP(32, cr.ifa) 1255IVT_END(Floating_Point_Fault) 1256 1257IVT_ENTRY(Floating_Point_Trap, 0x5d00) 1258 TRAP(33, cr.ifa) 1259IVT_END(Floating_Point_Trap) 1260 1261IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00) 1262 TRAP(34, cr.ifa) 1263IVT_END(Lower_Privilege_Transfer_Trap) 1264 1265IVT_ENTRY(Taken_Branch_Trap, 0x5f00) 1266 TRAP(35, cr.ifa) 1267IVT_END(Taken_Branch_Trap) 1268 1269IVT_ENTRY(Single_Step_Trap, 0x6000) 1270 TRAP(36, cr.ifa) 1271IVT_END(Single_Step_Trap) 1272 1273IVT_ENTRY(Reserved_6100, 0x6100) 1274 TRAP(37, cr.ifa) 1275IVT_END(Reserved_6100) 1276 1277IVT_ENTRY(Reserved_6200, 0x6200) 1278 TRAP(38, cr.ifa) 1279IVT_END(Reserved_6200) 1280 1281IVT_ENTRY(Reserved_6300, 0x6300) 1282 TRAP(39, cr.ifa) 1283IVT_END(Reserved_6300) 1284 1285IVT_ENTRY(Reserved_6400, 0x6400) 1286 TRAP(40, cr.ifa) 1287IVT_END(Reserved_6400) 1288 1289IVT_ENTRY(Reserved_6500, 0x6500) 1290 TRAP(41, cr.ifa) 1291IVT_END(Reserved_6500) 1292 1293IVT_ENTRY(Reserved_6600, 0x6600) 1294 TRAP(42, cr.ifa) 1295IVT_END(Reserved_6600) 1296 1297IVT_ENTRY(Reserved_6700, 0x6700) 1298 TRAP(43, cr.ifa) 1299IVT_END(Reserved_6700) 1300 1301IVT_ENTRY(Reserved_6800, 0x6800) 1302 TRAP(44, cr.ifa) 1303IVT_END(Reserved_6800) 1304 1305IVT_ENTRY(IA_32_Exception, 0x6900) 1306 TRAP(45, cr.ifa) 1307IVT_END(IA_32_Exception) 1308 1309IVT_ENTRY(IA_32_Intercept, 0x6a00) 1310 TRAP(46, cr.iim) 1311IVT_END(IA_32_Intercept) 1312 1313IVT_ENTRY(IA_32_Interrupt, 0x6b00) 1314 TRAP(47, cr.ifa) 1315IVT_END(IA_32_Interrupt) 1316 1317IVT_ENTRY(Reserved_6c00, 0x6c00) 1318 TRAP(48, cr.ifa) 1319IVT_END(Reserved_6c00) 1320 1321IVT_ENTRY(Reserved_6d00, 0x6d00) 1322 TRAP(49, cr.ifa) 1323IVT_END(Reserved_6d00) 1324 1325IVT_ENTRY(Reserved_6e00, 0x6e00) 1326 TRAP(50, cr.ifa) 1327IVT_END(Reserved_6e00) 1328 1329IVT_ENTRY(Reserved_6f00, 0x6f00) 1330 TRAP(51, cr.ifa) 1331IVT_END(Reserved_6f00) 1332 1333IVT_ENTRY(Reserved_7000, 0x7000) 1334 TRAP(52, cr.ifa) 1335IVT_END(Reserved_7000) 1336 1337IVT_ENTRY(Reserved_7100, 0x7100) 1338 TRAP(53, cr.ifa) 1339IVT_END(Reserved_7100) 1340 1341IVT_ENTRY(Reserved_7200, 0x7200) 1342 TRAP(54, cr.ifa) 1343IVT_END(Reserved_7200) 1344 1345IVT_ENTRY(Reserved_7300, 0x7300) 1346 TRAP(55, cr.ifa) 1347IVT_END(Reserved_7300) 1348 1349IVT_ENTRY(Reserved_7400, 0x7400) 1350 TRAP(56, cr.ifa) 1351IVT_END(Reserved_7400) 1352 1353IVT_ENTRY(Reserved_7500, 0x7500) 1354 TRAP(57, cr.ifa) 1355IVT_END(Reserved_7500) 1356 1357IVT_ENTRY(Reserved_7600, 0x7600) 1358 TRAP(58, cr.ifa) 1359IVT_END(Reserved_7600) 1360 1361IVT_ENTRY(Reserved_7700, 0x7700) 1362 TRAP(59, cr.ifa) 1363IVT_END(Reserved_7700) 1364 1365IVT_ENTRY(Reserved_7800, 0x7800) 1366 TRAP(60, cr.ifa) 1367IVT_END(Reserved_7800) 1368 1369IVT_ENTRY(Reserved_7900, 0x7900) 1370 TRAP(61, cr.ifa) 1371IVT_END(Reserved_7900) 1372 1373IVT_ENTRY(Reserved_7a00, 0x7a00) 1374 TRAP(62, cr.ifa) 1375IVT_END(Reserved_7a00) 1376 1377IVT_ENTRY(Reserved_7b00, 0x7b00) 1378 TRAP(63, cr.ifa) 1379IVT_END(Reserved_7b00) 1380 1381IVT_ENTRY(Reserved_7c00, 0x7c00) 1382 TRAP(64, cr.ifa) 1383IVT_END(Reserved_7c00) 1384 1385IVT_ENTRY(Reserved_7d00, 0x7d00) 1386 TRAP(65, cr.ifa) 1387IVT_END(Reserved_7d00) 1388 1389IVT_ENTRY(Reserved_7e00, 0x7e00) 1390 TRAP(66, cr.ifa) 1391IVT_END(Reserved_7e00) 1392 1393IVT_ENTRY(Reserved_7f00, 0x7f00) 1394 TRAP(67, cr.ifa) 1395IVT_END(Reserved_7f00) 1396