apic_vector.s revision 71532
1/* 2 * from: vector.s, 386BSD 0.1 unknown origin 3 * $FreeBSD: head/sys/i386/i386/apic_vector.s 71532 2001-01-24 10:00:07Z jhb $ 4 */ 5 6 7#include <machine/apic.h> 8#include <machine/smp.h> 9 10#include "i386/isa/intr_machdep.h" 11 12/* convert an absolute IRQ# into a bitmask */ 13#define IRQ_BIT(irq_num) (1 << (irq_num)) 14 15/* make an index into the IO APIC from the IRQ# */ 16#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) 17 18/* 19 * 20 */ 21#define PUSH_FRAME \ 22 pushl $0 ; /* dummy error code */ \ 23 pushl $0 ; /* dummy trap type */ \ 24 pushal ; \ 25 pushl %ds ; /* save data and extra segments ... */ \ 26 pushl %es ; \ 27 pushl %fs 28 29#define POP_FRAME \ 30 popl %fs ; \ 31 popl %es ; \ 32 popl %ds ; \ 33 popal ; \ 34 addl $4+4,%esp 35 36/* 37 * Macros for interrupt entry, call to handler, and exit. 38 */ 39 40#define FAST_INTR(irq_num, vec_name) \ 41 .text ; \ 42 SUPERALIGN_TEXT ; \ 43IDTVEC(vec_name) ; \ 44 PUSH_FRAME ; \ 45 movl $KDSEL,%eax ; \ 46 mov %ax,%ds ; \ 47 mov %ax,%es ; \ 48 movl $KPSEL,%eax ; \ 49 mov %ax,%fs ; \ 50 FAKE_MCOUNT(13*4(%esp)) ; \ 51 movl PCPU(CURPROC),%ebx ; \ 52 incl P_INTR_NESTING_LEVEL(%ebx) ; \ 53 pushl _intr_unit + (irq_num) * 4 ; \ 54 call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ 55 addl $4, %esp ; \ 56 movl $0, _lapic+LA_EOI ; \ 57 lock ; \ 58 incl _cnt+V_INTR ; /* book-keeping can wait */ \ 59 movl _intr_countp + (irq_num) * 4, %eax ; \ 60 lock ; \ 61 incl (%eax) ; \ 62 decl P_INTR_NESTING_LEVEL(%ebx) ; \ 63 MEXITCOUNT ; \ 64 jmp _doreti 65 66#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8 67#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12 68 69#define MASK_IRQ(irq_num) \ 70 IMASK_LOCK ; /* into critical reg */ \ 71 testl $IRQ_BIT(irq_num), _apic_imen ; \ 72 jne 7f ; /* masked, don't mask */ \ 73 orl $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */ \ 74 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ 75 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 76 movl %eax, (%ecx) ; /* write the index */ \ 77 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ 78 orl $IOART_INTMASK, %eax ; /* set the mask */ \ 79 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ 807: ; /* already masked */ \ 81 IMASK_UNLOCK 82/* 83 * Test to see whether we are handling an edge or level triggered INT. 84 * Level-triggered INTs must still be masked as we don't clear the source, 85 * and the EOI cycle would cause redundant INTs to occur. 86 */ 87#define MASK_LEVEL_IRQ(irq_num) \ 88 testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \ 89 jz 9f ; /* edge, don't mask */ \ 90 MASK_IRQ(irq_num) ; \ 919: 92 93 94#ifdef APIC_INTR_REORDER 95#define EOI_IRQ(irq_num) \ 96 movl _apic_isrbit_location + 8 * (irq_num), %eax ; \ 97 movl (%eax), %eax ; \ 98 testl _apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \ 99 jz 9f ; /* not active */ \ 100 movl $0, _lapic+LA_EOI ; \ 1019: 102 103#else 104#define EOI_IRQ(irq_num) \ 105 testl $IRQ_BIT(irq_num), _lapic+LA_ISR1; \ 106 jz 9f ; /* not active */ \ 107 movl $0, _lapic+LA_EOI; \ 1089: 109#endif 110 111 112/* 113 * Test to see if the source is currently masked, clear if so. 114 */ 115#define UNMASK_IRQ(irq_num) \ 116 IMASK_LOCK ; /* into critical reg */ \ 117 testl $IRQ_BIT(irq_num), _apic_imen ; \ 118 je 7f ; /* bit clear, not masked */ \ 119 andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \ 120 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ 121 movl REDIRIDX(irq_num), %eax ; /* get the index */ \ 122 movl %eax, (%ecx) ; /* write the index */ \ 123 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ 124 andl $~IOART_INTMASK, %eax ; /* clear the mask */ \ 125 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ 1267: ; /* already unmasked */ \ 127 IMASK_UNLOCK 128 129/* 130 * Slow, threaded interrupts. 131 * 132 * XXX Most of the parameters here are obsolete. Fix this when we're 133 * done. 134 * XXX we really shouldn't return via doreti if we just schedule the 135 * interrupt handler and don't run anything. We could just do an 136 * iret. FIXME. 137 */ 138#define INTR(irq_num, vec_name, maybe_extra_ipending) \ 139 .text ; \ 140 SUPERALIGN_TEXT ; \ 141/* _XintrNN: entry point used by IDT/HWIs via _vec[]. */ \ 142IDTVEC(vec_name) ; \ 143 PUSH_FRAME ; \ 144 movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ 145 mov %ax, %ds ; \ 146 mov %ax, %es ; \ 147 movl $KPSEL, %eax ; \ 148 mov %ax, %fs ; \ 149; \ 150 maybe_extra_ipending ; \ 151; \ 152 MASK_LEVEL_IRQ(irq_num) ; \ 153 EOI_IRQ(irq_num) ; \ 1540: ; \ 155 movl PCPU(CURPROC),%ebx ; \ 156 incl P_INTR_NESTING_LEVEL(%ebx) ; \ 157; \ 158 /* entry point used by doreti_unpend for HWIs. */ \ 159__CONCAT(Xresume,irq_num): ; \ 160 FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ 161 pushl $irq_num; /* pass the IRQ */ \ 162 sti ; \ 163 call _sched_ithd ; \ 164 addl $4, %esp ; /* discard the parameter */ \ 165; \ 166 decl P_INTR_NESTING_LEVEL(%ebx) ; \ 167 MEXITCOUNT ; \ 168 jmp _doreti 169 170/* 171 * Handle "spurious INTerrupts". 172 * Notes: 173 * This is different than the "spurious INTerrupt" generated by an 174 * 8259 PIC for missing INTs. See the APIC documentation for details. 175 * This routine should NOT do an 'EOI' cycle. 176 */ 177 .text 178 SUPERALIGN_TEXT 179 .globl _Xspuriousint 180_Xspuriousint: 181 182 /* No EOI cycle used here */ 183 184 iret 185 186 187/* 188 * Handle TLB shootdowns. 189 */ 190 .text 191 SUPERALIGN_TEXT 192 .globl _Xinvltlb 193_Xinvltlb: 194 pushl %eax 195 196#ifdef COUNT_XINVLTLB_HITS 197 pushl %fs 198 movl $KPSEL, %eax 199 mov %ax, %fs 200 movl PCPU(CPUID), %eax 201 popl %fs 202 ss 203 incl _xhits(,%eax,4) 204#endif /* COUNT_XINVLTLB_HITS */ 205 206 movl %cr3, %eax /* invalidate the TLB */ 207 movl %eax, %cr3 208 209 ss /* stack segment, avoid %ds load */ 210 movl $0, _lapic+LA_EOI /* End Of Interrupt to APIC */ 211 212 popl %eax 213 iret 214 215 216#ifdef BETTER_CLOCK 217 218/* 219 * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU, 220 * 221 * - Stores current cpu state in checkstate_cpustate[cpuid] 222 * 0 == user, 1 == sys, 2 == intr 223 * - Stores current process in checkstate_curproc[cpuid] 224 * 225 * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. 226 * 227 * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags 228 */ 229 230 .text 231 SUPERALIGN_TEXT 232 .globl _Xcpucheckstate 233 .globl _checkstate_cpustate 234 .globl _checkstate_curproc 235 .globl _checkstate_pc 236_Xcpucheckstate: 237 pushl %eax 238 pushl %ebx 239 pushl %ds /* save current data segment */ 240 pushl %fs 241 242 movl $KDSEL, %eax 243 mov %ax, %ds /* use KERNEL data segment */ 244 movl $KPSEL, %eax 245 mov %ax, %fs 246 247 movl $0, _lapic+LA_EOI /* End Of Interrupt to APIC */ 248 249 movl $0, %ebx 250 movl 20(%esp), %eax 251 andl $3, %eax 252 cmpl $3, %eax 253 je 1f 254 testl $PSL_VM, 24(%esp) 255 jne 1f 256 incl %ebx /* system or interrupt */ 2571: 258 movl PCPU(CPUID), %eax 259 movl %ebx, _checkstate_cpustate(,%eax,4) 260 movl PCPU(CURPROC), %ebx 261 movl %ebx, _checkstate_curproc(,%eax,4) 262 263 movl 16(%esp), %ebx 264 movl %ebx, _checkstate_pc(,%eax,4) 265 266 lock /* checkstate_probed_cpus |= (1<<id) */ 267 btsl %eax, _checkstate_probed_cpus 268 269 popl %fs 270 popl %ds /* restore previous data segment */ 271 popl %ebx 272 popl %eax 273 iret 274 275#endif /* BETTER_CLOCK */ 276 277/* 278 * Executed by a CPU when it receives an Xcpuast IPI from another CPU, 279 * 280 * - Signals its receipt by clearing bit cpuid in checkstate_need_ast. 281 * 282 * - We need a better method of triggering asts on other cpus. 283 */ 284 285 .text 286 SUPERALIGN_TEXT 287 .globl _Xcpuast 288_Xcpuast: 289 PUSH_FRAME 290 movl $KDSEL, %eax 291 mov %ax, %ds /* use KERNEL data segment */ 292 mov %ax, %es 293 movl $KPSEL, %eax 294 mov %ax, %fs 295 296 movl PCPU(CPUID), %eax 297 lock /* checkstate_need_ast &= ~(1<<id) */ 298 btrl %eax, _checkstate_need_ast 299 movl $0, _lapic+LA_EOI /* End Of Interrupt to APIC */ 300 301 lock 302 btsl %eax, _checkstate_pending_ast 303 jc 1f 304 305 FAKE_MCOUNT(13*4(%esp)) 306 307 orl $AST_PENDING, PCPU(ASTPENDING) /* XXX */ 308 movl PCPU(CURPROC),%ebx 309 incl P_INTR_NESTING_LEVEL(%ebx) 310 sti 311 312 movl PCPU(CPUID), %eax 313 lock 314 btrl %eax, _checkstate_pending_ast 315 lock 316 btrl %eax, CNAME(resched_cpus) 317 jnc 2f 318 orl $AST_PENDING+AST_RESCHED, PCPU(ASTPENDING) 319 lock 320 incl CNAME(want_resched_cnt) 3212: 322 lock 323 incl CNAME(cpuast_cnt) 324 decl P_INTR_NESTING_LEVEL(%ebx) 325 MEXITCOUNT 326 jmp _doreti 3271: 328 /* We are already in the process of delivering an ast for this CPU */ 329 POP_FRAME 330 iret 331 332/* 333 * Executed by a CPU when it receives an Xcpustop IPI from another CPU, 334 * 335 * - Signals its receipt. 336 * - Waits for permission to restart. 337 * - Signals its restart. 338 */ 339 340 .text 341 SUPERALIGN_TEXT 342 .globl _Xcpustop 343_Xcpustop: 344 pushl %ebp 345 movl %esp, %ebp 346 pushl %eax 347 pushl %ecx 348 pushl %edx 349 pushl %ds /* save current data segment */ 350 pushl %fs 351 352 movl $KDSEL, %eax 353 mov %ax, %ds /* use KERNEL data segment */ 354 movl $KPSEL, %eax 355 mov %ax, %fs 356 357 movl $0, _lapic+LA_EOI /* End Of Interrupt to APIC */ 358 359 movl PCPU(CPUID), %eax 360 imull $PCB_SIZE, %eax 361 leal CNAME(stoppcbs)(%eax), %eax 362 pushl %eax 363 call CNAME(savectx) /* Save process context */ 364 addl $4, %esp 365 366 367 movl PCPU(CPUID), %eax 368 369 lock 370 btsl %eax, _stopped_cpus /* stopped_cpus |= (1<<id) */ 3711: 372 btl %eax, _started_cpus /* while (!(started_cpus & (1<<id))) */ 373 jnc 1b 374 375 lock 376 btrl %eax, _started_cpus /* started_cpus &= ~(1<<id) */ 377 lock 378 btrl %eax, _stopped_cpus /* stopped_cpus &= ~(1<<id) */ 379 380 test %eax, %eax 381 jnz 2f 382 383 movl CNAME(cpustop_restartfunc), %eax 384 test %eax, %eax 385 jz 2f 386 movl $0, CNAME(cpustop_restartfunc) /* One-shot */ 387 388 call *%eax 3892: 390 popl %fs 391 popl %ds /* restore previous data segment */ 392 popl %edx 393 popl %ecx 394 popl %eax 395 movl %ebp, %esp 396 popl %ebp 397 iret 398 399 400MCOUNT_LABEL(bintr) 401 FAST_INTR(0,fastintr0) 402 FAST_INTR(1,fastintr1) 403 FAST_INTR(2,fastintr2) 404 FAST_INTR(3,fastintr3) 405 FAST_INTR(4,fastintr4) 406 FAST_INTR(5,fastintr5) 407 FAST_INTR(6,fastintr6) 408 FAST_INTR(7,fastintr7) 409 FAST_INTR(8,fastintr8) 410 FAST_INTR(9,fastintr9) 411 FAST_INTR(10,fastintr10) 412 FAST_INTR(11,fastintr11) 413 FAST_INTR(12,fastintr12) 414 FAST_INTR(13,fastintr13) 415 FAST_INTR(14,fastintr14) 416 FAST_INTR(15,fastintr15) 417 FAST_INTR(16,fastintr16) 418 FAST_INTR(17,fastintr17) 419 FAST_INTR(18,fastintr18) 420 FAST_INTR(19,fastintr19) 421 FAST_INTR(20,fastintr20) 422 FAST_INTR(21,fastintr21) 423 FAST_INTR(22,fastintr22) 424 FAST_INTR(23,fastintr23) 425 FAST_INTR(24,fastintr24) 426 FAST_INTR(25,fastintr25) 427 FAST_INTR(26,fastintr26) 428 FAST_INTR(27,fastintr27) 429 FAST_INTR(28,fastintr28) 430 FAST_INTR(29,fastintr29) 431 FAST_INTR(30,fastintr30) 432 FAST_INTR(31,fastintr31) 433#define CLKINTR_PENDING movl $1,CNAME(clkintr_pending) 434/* Threaded interrupts */ 435 INTR(0,intr0, CLKINTR_PENDING) 436 INTR(1,intr1,) 437 INTR(2,intr2,) 438 INTR(3,intr3,) 439 INTR(4,intr4,) 440 INTR(5,intr5,) 441 INTR(6,intr6,) 442 INTR(7,intr7,) 443 INTR(8,intr8,) 444 INTR(9,intr9,) 445 INTR(10,intr10,) 446 INTR(11,intr11,) 447 INTR(12,intr12,) 448 INTR(13,intr13,) 449 INTR(14,intr14,) 450 INTR(15,intr15,) 451 INTR(16,intr16,) 452 INTR(17,intr17,) 453 INTR(18,intr18,) 454 INTR(19,intr19,) 455 INTR(20,intr20,) 456 INTR(21,intr21,) 457 INTR(22,intr22,) 458 INTR(23,intr23,) 459 INTR(24,intr24,) 460 INTR(25,intr25,) 461 INTR(26,intr26,) 462 INTR(27,intr27,) 463 INTR(28,intr28,) 464 INTR(29,intr29,) 465 INTR(30,intr30,) 466 INTR(31,intr31,) 467MCOUNT_LABEL(eintr) 468 469/* 470 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 471 * 472 * - Calls the generic rendezvous action function. 473 */ 474 .text 475 SUPERALIGN_TEXT 476 .globl _Xrendezvous 477_Xrendezvous: 478 PUSH_FRAME 479 movl $KDSEL, %eax 480 mov %ax, %ds /* use KERNEL data segment */ 481 mov %ax, %es 482 movl $KPSEL, %eax 483 mov %ax, %fs 484 485 call _smp_rendezvous_action 486 487 movl $0, _lapic+LA_EOI /* End Of Interrupt to APIC */ 488 POP_FRAME 489 iret 490 491 492 .data 493 494#ifdef COUNT_XINVLTLB_HITS 495 .globl _xhits 496_xhits: 497 .space (NCPU * 4), 0 498#endif /* COUNT_XINVLTLB_HITS */ 499 500/* variables used by stop_cpus()/restart_cpus()/Xcpustop */ 501 .globl _stopped_cpus, _started_cpus 502_stopped_cpus: 503 .long 0 504_started_cpus: 505 .long 0 506 507#ifdef BETTER_CLOCK 508 .globl _checkstate_probed_cpus 509_checkstate_probed_cpus: 510 .long 0 511#endif /* BETTER_CLOCK */ 512 .globl _checkstate_need_ast 513_checkstate_need_ast: 514 .long 0 515_checkstate_pending_ast: 516 .long 0 517 .globl CNAME(forward_irq_misscnt) 518 .globl CNAME(forward_irq_toodeepcnt) 519 .globl CNAME(forward_irq_hitcnt) 520 .globl CNAME(resched_cpus) 521 .globl CNAME(want_resched_cnt) 522 .globl CNAME(cpuast_cnt) 523 .globl CNAME(cpustop_restartfunc) 524CNAME(forward_irq_misscnt): 525 .long 0 526CNAME(forward_irq_hitcnt): 527 .long 0 528CNAME(forward_irq_toodeepcnt): 529 .long 0 530CNAME(resched_cpus): 531 .long 0 532CNAME(want_resched_cnt): 533 .long 0 534CNAME(cpuast_cnt): 535 .long 0 536CNAME(cpustop_restartfunc): 537 .long 0 538 539 .globl _apic_pin_trigger 540_apic_pin_trigger: 541 .long 0 542 543 .text 544