apic_vector.s revision 69578
1/* 2 * from: vector.s, 386BSD 0.1 unknown origin 3 * $FreeBSD: head/sys/i386/i386/apic_vector.s 69578 2000-12-04 21:15:14Z peter $ 4 */ 5 6 7#include <machine/apic.h> 8#include <machine/smp.h> 9 10#include "i386/isa/intr_machdep.h" 11 12/* convert an absolute IRQ# into a bitmask */ 13#define IRQ_BIT(irq_num) (1 << (irq_num)) 14 15/* make an index into the IO APIC from the IRQ# */ 16#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) 17 18/* 19 * 20 */ 21#define PUSH_FRAME \ 22 pushl $0 ; /* dummy error code */ \ 23 pushl $0 ; /* dummy trap type */ \ 24 pushal ; \ 25 pushl %ds ; /* save data and extra segments ... */ \ 26 pushl %es ; \ 27 pushl %fs 28 29#define POP_FRAME \ 30 popl %fs ; \ 31 popl %es ; \ 32 popl %ds ; \ 33 popal ; \ 34 addl $4+4,%esp 35 36/* 37 * Macros for interrupt entry, call to handler, and exit. 38 */ 39 40#define FAST_INTR(irq_num, vec_name) \ 41 .text ; \ 42 SUPERALIGN_TEXT ; \ 43IDTVEC(vec_name) ; \ 44 PUSH_FRAME ; \ 45 movl $KDSEL,%eax ; \ 46 mov %ax,%ds ; \ 47 mov %ax,%es ; \ 48 movl $KPSEL,%eax ; \ 49 mov %ax,%fs ; \ 50 FAKE_MCOUNT(13*4(%esp)) ; \ 51 incb _intr_nesting_level ; \ 52 pushl _intr_unit + (irq_num) * 4 ; \ 53 call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ 54 addl $4, %esp ; \ 55 movl $0, lapic_eoi ; \ 56 lock ; \ 57 incl _cnt+V_INTR ; /* book-keeping can wait */ \ 58 movl _intr_countp + (irq_num) * 4, %eax ; \ 59 lock ; \ 60 incl (%eax) ; \ 61 MEXITCOUNT ; \ 62 jmp _doreti 63 64#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8 65#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12 66 67#define MASK_IRQ(irq_num) \ 68 IMASK_LOCK ; /* into critical reg */ \ 69 testl $IRQ_BIT(irq_num), _apic_imen ; \ 70 jne 7f ; /* masked, don't mask */ \ 71 orl $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */ \ 72 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ 73 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 74 movl %eax, (%ecx) ; /* write the index */ \ 75 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ 76 orl $IOART_INTMASK, %eax ; /* set the mask */ \ 77 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ 787: ; /* already masked */ \ 79 IMASK_UNLOCK 80/* 81 * Test to see whether we are handling an edge or level triggered INT. 82 * Level-triggered INTs must still be masked as we don't clear the source, 83 * and the EOI cycle would cause redundant INTs to occur. 84 */ 85#define MASK_LEVEL_IRQ(irq_num) \ 86 testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \ 87 jz 9f ; /* edge, don't mask */ \ 88 MASK_IRQ(irq_num) ; \ 899: 90 91 92#ifdef APIC_INTR_REORDER 93#define EOI_IRQ(irq_num) \ 94 movl _apic_isrbit_location + 8 * (irq_num), %eax ; \ 95 movl (%eax), %eax ; \ 96 testl _apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \ 97 jz 9f ; /* not active */ \ 98 movl $0, lapic_eoi ; \ 99 APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \ 1009: 101 102#else 103#define EOI_IRQ(irq_num) \ 104 testl $IRQ_BIT(irq_num), lapic_isr1; \ 105 jz 9f ; /* not active */ \ 106 movl $0, lapic_eoi; \ 107 APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \ 1089: 109#endif 110 111 112/* 113 * Test to see if the source is currently masked, clear if so. 114 */ 115#define UNMASK_IRQ(irq_num) \ 116 IMASK_LOCK ; /* into critical reg */ \ 117 testl $IRQ_BIT(irq_num), _apic_imen ; \ 118 je 7f ; /* bit clear, not masked */ \ 119 andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \ 120 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ 121 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 122 movl %eax, (%ecx) ; /* write the index */ \ 123 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ 124 andl $~IOART_INTMASK, %eax ; /* clear the mask */ \ 125 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ 1267: ; /* already unmasked */ \ 127 IMASK_UNLOCK 128 129#ifdef APIC_INTR_DIAGNOSTIC 130#ifdef APIC_INTR_DIAGNOSTIC_IRQ 131log_intr_event: 132 pushf 133 cli 134 pushl $CNAME(apic_itrace_debuglock) 135 call CNAME(s_lock_np) 136 addl $4, %esp 137 movl CNAME(apic_itrace_debugbuffer_idx), %ecx 138 andl $32767, %ecx 139 movl _cpuid, %eax 140 shll $8, %eax 141 orl 8(%esp), %eax 142 movw %ax, CNAME(apic_itrace_debugbuffer)(,%ecx,2) 143 incl %ecx 144 andl $32767, %ecx 145 movl %ecx, CNAME(apic_itrace_debugbuffer_idx) 146 pushl $CNAME(apic_itrace_debuglock) 147 call CNAME(s_unlock_np) 148 addl $4, %esp 149 popf 150 ret 151 152 153#define APIC_ITRACE(name, irq_num, id) \ 154 lock ; /* MP-safe */ \ 155 incl CNAME(name) + (irq_num) * 4 ; \ 156 pushl %eax ; \ 157 pushl %ecx ; \ 158 pushl %edx ; \ 159 movl $(irq_num), %eax ; \ 160 cmpl $APIC_INTR_DIAGNOSTIC_IRQ, %eax ; \ 161 jne 7f ; \ 162 pushl $id ; \ 163 call log_intr_event ; \ 164 addl $4, %esp ; \ 1657: ; \ 166 popl %edx ; \ 167 popl %ecx ; \ 168 popl %eax 169#else 170#define APIC_ITRACE(name, irq_num, id) \ 171 lock ; /* MP-safe */ \ 172 incl CNAME(name) + (irq_num) * 4 173#endif 174 175#define APIC_ITRACE_ENTER 1 176#define APIC_ITRACE_EOI 2 177#define APIC_ITRACE_TRYISRLOCK 3 178#define APIC_ITRACE_GOTISRLOCK 4 179#define APIC_ITRACE_ENTER2 5 180#define APIC_ITRACE_LEAVE 6 181#define APIC_ITRACE_UNMASK 7 182#define APIC_ITRACE_ACTIVE 8 183#define APIC_ITRACE_MASKED 9 184#define APIC_ITRACE_NOISRLOCK 10 185#define APIC_ITRACE_MASKED2 11 186#define APIC_ITRACE_SPLZ 12 187#define APIC_ITRACE_DORETI 13 188 189#else 190#define APIC_ITRACE(name, irq_num, id) 191#endif 192 193/* 194 * Slow, threaded interrupts. 195 * 196 * XXX Most of the parameters here are obsolete. Fix this when we're 197 * done. 198 * XXX we really shouldn't return via doreti if we just schedule the 199 * interrupt handler and don't run anything. We could just do an 200 * iret. FIXME. 201 */ 202#define INTR(irq_num, vec_name, maybe_extra_ipending) \ 203 .text ; \ 204 SUPERALIGN_TEXT ; \ 205/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \ 206IDTVEC(vec_name) ; \ 207 PUSH_FRAME ; \ 208 movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ 209 mov %ax, %ds ; \ 210 mov %ax, %es ; \ 211 movl $KPSEL, %eax ; \ 212 mov %ax, %fs ; \ 213; \ 214 maybe_extra_ipending ; \ 215; \ 216 APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \ 217; \ 218 MASK_LEVEL_IRQ(irq_num) ; \ 219 EOI_IRQ(irq_num) ; \ 2200: ; \ 221 incb _intr_nesting_level ; \ 222; \ 223 /* entry point used by doreti_unpend for HWIs. */ \ 224__CONCAT(Xresume,irq_num): ; \ 225 FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ 226 pushl $irq_num; /* pass the IRQ */ \ 227 APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \ 228 sti ; \ 229 call _sched_ithd ; \ 230 addl $4, %esp ; /* discard the parameter */ \ 231 APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \ 232; \ 233 MEXITCOUNT ; \ 234 jmp _doreti 235 236/* 237 * Handle "spurious INTerrupts". 238 * Notes: 239 * This is different than the "spurious INTerrupt" generated by an 240 * 8259 PIC for missing INTs. See the APIC documentation for details. 241 * This routine should NOT do an 'EOI' cycle. 242 */ 243 .text 244 SUPERALIGN_TEXT 245 .globl _Xspuriousint 246_Xspuriousint: 247 248 /* No EOI cycle used here */ 249 250 iret 251 252 253/* 254 * Handle TLB shootdowns. 255 */ 256 .text 257 SUPERALIGN_TEXT 258 .globl _Xinvltlb 259_Xinvltlb: 260 pushl %eax 261 262#ifdef COUNT_XINVLTLB_HITS 263 pushl %fs 264 movl $KPSEL, %eax 265 mov %ax, %fs 266 movl _cpuid, %eax 267 popl %fs 268 ss 269 incl _xhits(,%eax,4) 270#endif /* COUNT_XINVLTLB_HITS */ 271 272 movl %cr3, %eax /* invalidate the TLB */ 273 movl %eax, %cr3 274 275 ss /* stack segment, avoid %ds load */ 276 movl $0, lapic_eoi /* End Of Interrupt to APIC */ 277 278 popl %eax 279 iret 280 281 282#ifdef BETTER_CLOCK 283 284/* 285 * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU, 286 * 287 * - Stores current cpu state in checkstate_cpustate[cpuid] 288 * 0 == user, 1 == sys, 2 == intr 289 * - Stores current process in checkstate_curproc[cpuid] 290 * 291 * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. 292 * 293 * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags 294 */ 295 296 .text 297 SUPERALIGN_TEXT 298 .globl _Xcpucheckstate 299 .globl _checkstate_cpustate 300 .globl _checkstate_curproc 301 .globl _checkstate_pc 302_Xcpucheckstate: 303 pushl %eax 304 pushl %ebx 305 pushl %ds /* save current data segment */ 306 pushl %fs 307 308 movl $KDSEL, %eax 309 mov %ax, %ds /* use KERNEL data segment */ 310 movl $KPSEL, %eax 311 mov %ax, %fs 312 313 movl $0, lapic_eoi /* End Of Interrupt to APIC */ 314 315 movl $0, %ebx 316 movl 20(%esp), %eax 317 andl $3, %eax 318 cmpl $3, %eax 319 je 1f 320 testl $PSL_VM, 24(%esp) 321 jne 1f 322 incl %ebx /* system or interrupt */ 3231: 324 movl _cpuid, %eax 325 movl %ebx, _checkstate_cpustate(,%eax,4) 326 movl _curproc, %ebx 327 movl %ebx, _checkstate_curproc(,%eax,4) 328 movl 16(%esp), %ebx 329 movl %ebx, _checkstate_pc(,%eax,4) 330 331 lock /* checkstate_probed_cpus |= (1<<id) */ 332 btsl %eax, _checkstate_probed_cpus 333 334 popl %fs 335 popl %ds /* restore previous data segment */ 336 popl %ebx 337 popl %eax 338 iret 339 340#endif /* BETTER_CLOCK */ 341 342/* 343 * Executed by a CPU when it receives an Xcpuast IPI from another CPU, 344 * 345 * - Signals its receipt by clearing bit cpuid in checkstate_need_ast. 346 * 347 * - We need a better method of triggering asts on other cpus. 348 */ 349 350 .text 351 SUPERALIGN_TEXT 352 .globl _Xcpuast 353_Xcpuast: 354 PUSH_FRAME 355 movl $KDSEL, %eax 356 mov %ax, %ds /* use KERNEL data segment */ 357 mov %ax, %es 358 movl $KPSEL, %eax 359 mov %ax, %fs 360 361 movl _cpuid, %eax 362 lock /* checkstate_need_ast &= ~(1<<id) */ 363 btrl %eax, _checkstate_need_ast 364 movl $0, lapic_eoi /* End Of Interrupt to APIC */ 365 366 lock 367 btsl %eax, _checkstate_pending_ast 368 jc 1f 369 370 FAKE_MCOUNT(13*4(%esp)) 371 372 orl $AST_PENDING, _astpending /* XXX */ 373 incb _intr_nesting_level 374 sti 375 376 movl _cpuid, %eax 377 lock 378 btrl %eax, _checkstate_pending_ast 379 lock 380 btrl %eax, CNAME(resched_cpus) 381 jnc 2f 382 orl $AST_PENDING+AST_RESCHED, _astpending 383 lock 384 incl CNAME(want_resched_cnt) 3852: 386 lock 387 incl CNAME(cpuast_cnt) 388 MEXITCOUNT 389 jmp _doreti 3901: 391 /* We are already in the process of delivering an ast for this CPU */ 392 POP_FRAME 393 iret 394 395 396/* 397 * Executed by a CPU when it receives an XFORWARD_IRQ IPI. 398 */ 399 400 .text 401 SUPERALIGN_TEXT 402 .globl _Xforward_irq 403_Xforward_irq: 404 PUSH_FRAME 405 movl $KDSEL, %eax 406 mov %ax, %ds /* use KERNEL data segment */ 407 mov %ax, %es 408 movl $KPSEL, %eax 409 mov %ax, %fs 410 411 movl $0, lapic_eoi /* End Of Interrupt to APIC */ 412 413 FAKE_MCOUNT(13*4(%esp)) 414 415 lock 416 incl CNAME(forward_irq_hitcnt) 417 cmpb $4, _intr_nesting_level 418 jae 1f 419 420 incb _intr_nesting_level 421 sti 422 423 MEXITCOUNT 424 jmp doreti_next /* Handle forwarded interrupt */ 4251: 426 lock 427 incl CNAME(forward_irq_toodeepcnt) 428 MEXITCOUNT 429 POP_FRAME 430 iret 431 432#if 0 433/* 434 * 435 */ 436forward_irq: 437 MCOUNT 438 cmpl $0,_invltlb_ok 439 jz 4f 440 441 cmpl $0, CNAME(forward_irq_enabled) 442 jz 4f 443 444/* XXX - this is broken now, because mp_lock doesn't exist 445 movl _mp_lock,%eax 446 cmpl $FREE_LOCK,%eax 447 jne 1f 448 */ 449 movl $0, %eax /* Pick CPU #0 if noone has lock */ 4501: 451 shrl $24,%eax 452 movl _cpu_num_to_apic_id(,%eax,4),%ecx 453 shll $24,%ecx 454 movl lapic_icr_hi, %eax 455 andl $~APIC_ID_MASK, %eax 456 orl %ecx, %eax 457 movl %eax, lapic_icr_hi 458 4592: 460 movl lapic_icr_lo, %eax 461 andl $APIC_DELSTAT_MASK,%eax 462 jnz 2b 463 movl lapic_icr_lo, %eax 464 andl $APIC_RESV2_MASK, %eax 465 orl $(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax 466 movl %eax, lapic_icr_lo 4673: 468 movl lapic_icr_lo, %eax 469 andl $APIC_DELSTAT_MASK,%eax 470 jnz 3b 4714: 472 ret 473#endif 474 475/* 476 * Executed by a CPU when it receives an Xcpustop IPI from another CPU, 477 * 478 * - Signals its receipt. 479 * - Waits for permission to restart. 480 * - Signals its restart. 481 */ 482 483 .text 484 SUPERALIGN_TEXT 485 .globl _Xcpustop 486_Xcpustop: 487 pushl %ebp 488 movl %esp, %ebp 489 pushl %eax 490 pushl %ecx 491 pushl %edx 492 pushl %ds /* save current data segment */ 493 pushl %fs 494 495 movl $KDSEL, %eax 496 mov %ax, %ds /* use KERNEL data segment */ 497 movl $KPSEL, %eax 498 mov %ax, %fs 499 500 movl $0, lapic_eoi /* End Of Interrupt to APIC */ 501 502 movl _cpuid, %eax 503 imull $PCB_SIZE, %eax 504 leal CNAME(stoppcbs)(%eax), %eax 505 pushl %eax 506 call CNAME(savectx) /* Save process context */ 507 addl $4, %esp 508 509 510 movl _cpuid, %eax 511 512 lock 513 btsl %eax, _stopped_cpus /* stopped_cpus |= (1<<id) */ 5141: 515 btl %eax, _started_cpus /* while (!(started_cpus & (1<<id))) */ 516 jnc 1b 517 518 lock 519 btrl %eax, _started_cpus /* started_cpus &= ~(1<<id) */ 520 lock 521 btrl %eax, _stopped_cpus /* stopped_cpus &= ~(1<<id) */ 522 523 test %eax, %eax 524 jnz 2f 525 526 movl CNAME(cpustop_restartfunc), %eax 527 test %eax, %eax 528 jz 2f 529 movl $0, CNAME(cpustop_restartfunc) /* One-shot */ 530 531 call *%eax 5322: 533 popl %fs 534 popl %ds /* restore previous data segment */ 535 popl %edx 536 popl %ecx 537 popl %eax 538 movl %ebp, %esp 539 popl %ebp 540 iret 541 542 543MCOUNT_LABEL(bintr) 544 FAST_INTR(0,fastintr0) 545 FAST_INTR(1,fastintr1) 546 FAST_INTR(2,fastintr2) 547 FAST_INTR(3,fastintr3) 548 FAST_INTR(4,fastintr4) 549 FAST_INTR(5,fastintr5) 550 FAST_INTR(6,fastintr6) 551 FAST_INTR(7,fastintr7) 552 FAST_INTR(8,fastintr8) 553 FAST_INTR(9,fastintr9) 554 FAST_INTR(10,fastintr10) 555 FAST_INTR(11,fastintr11) 556 FAST_INTR(12,fastintr12) 557 FAST_INTR(13,fastintr13) 558 FAST_INTR(14,fastintr14) 559 FAST_INTR(15,fastintr15) 560 FAST_INTR(16,fastintr16) 561 FAST_INTR(17,fastintr17) 562 FAST_INTR(18,fastintr18) 563 FAST_INTR(19,fastintr19) 564 FAST_INTR(20,fastintr20) 565 FAST_INTR(21,fastintr21) 566 FAST_INTR(22,fastintr22) 567 FAST_INTR(23,fastintr23) 568 FAST_INTR(24,fastintr24) 569 FAST_INTR(25,fastintr25) 570 FAST_INTR(26,fastintr26) 571 FAST_INTR(27,fastintr27) 572 FAST_INTR(28,fastintr28) 573 FAST_INTR(29,fastintr29) 574 FAST_INTR(30,fastintr30) 575 FAST_INTR(31,fastintr31) 576#define CLKINTR_PENDING movl $1,CNAME(clkintr_pending) 577/* Threaded interrupts */ 578 INTR(0,intr0, CLKINTR_PENDING) 579 INTR(1,intr1,) 580 INTR(2,intr2,) 581 INTR(3,intr3,) 582 INTR(4,intr4,) 583 INTR(5,intr5,) 584 INTR(6,intr6,) 585 INTR(7,intr7,) 586 INTR(8,intr8,) 587 INTR(9,intr9,) 588 INTR(10,intr10,) 589 INTR(11,intr11,) 590 INTR(12,intr12,) 591 INTR(13,intr13,) 592 INTR(14,intr14,) 593 INTR(15,intr15,) 594 INTR(16,intr16,) 595 INTR(17,intr17,) 596 INTR(18,intr18,) 597 INTR(19,intr19,) 598 INTR(20,intr20,) 599 INTR(21,intr21,) 600 INTR(22,intr22,) 601 INTR(23,intr23,) 602 INTR(24,intr24,) 603 INTR(25,intr25,) 604 INTR(26,intr26,) 605 INTR(27,intr27,) 606 INTR(28,intr28,) 607 INTR(29,intr29,) 608 INTR(30,intr30,) 609 INTR(31,intr31,) 610MCOUNT_LABEL(eintr) 611 612/* 613 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 614 * 615 * - Calls the generic rendezvous action function. 616 */ 617 .text 618 SUPERALIGN_TEXT 619 .globl _Xrendezvous 620_Xrendezvous: 621 PUSH_FRAME 622 movl $KDSEL, %eax 623 mov %ax, %ds /* use KERNEL data segment */ 624 mov %ax, %es 625 movl $KPSEL, %eax 626 mov %ax, %fs 627 628 call _smp_rendezvous_action 629 630 movl $0, lapic_eoi /* End Of Interrupt to APIC */ 631 POP_FRAME 632 iret 633 634 635 .data 636#if 0 637/* active flag for lazy masking */ 638iactive: 639 .long 0 640#endif 641 642#ifdef COUNT_XINVLTLB_HITS 643 .globl _xhits 644_xhits: 645 .space (NCPU * 4), 0 646#endif /* COUNT_XINVLTLB_HITS */ 647 648/* variables used by stop_cpus()/restart_cpus()/Xcpustop */ 649 .globl _stopped_cpus, _started_cpus 650_stopped_cpus: 651 .long 0 652_started_cpus: 653 .long 0 654 655#ifdef BETTER_CLOCK 656 .globl _checkstate_probed_cpus 657_checkstate_probed_cpus: 658 .long 0 659#endif /* BETTER_CLOCK */ 660 .globl _checkstate_need_ast 661_checkstate_need_ast: 662 .long 0 663_checkstate_pending_ast: 664 .long 0 665 .globl CNAME(forward_irq_misscnt) 666 .globl CNAME(forward_irq_toodeepcnt) 667 .globl CNAME(forward_irq_hitcnt) 668 .globl CNAME(resched_cpus) 669 .globl CNAME(want_resched_cnt) 670 .globl CNAME(cpuast_cnt) 671 .globl CNAME(cpustop_restartfunc) 672CNAME(forward_irq_misscnt): 673 .long 0 674CNAME(forward_irq_hitcnt): 675 .long 0 676CNAME(forward_irq_toodeepcnt): 677 .long 0 678CNAME(resched_cpus): 679 .long 0 680CNAME(want_resched_cnt): 681 .long 0 682CNAME(cpuast_cnt): 683 .long 0 684CNAME(cpustop_restartfunc): 685 .long 0 686 687 .globl _apic_pin_trigger 688_apic_pin_trigger: 689 .long 0 690 691 .text 692