apic_vector.s revision 122572
1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by the University of 17 * California, Berkeley and its contributors. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * from: vector.s, 386BSD 0.1 unknown origin 35 * $FreeBSD: head/sys/i386/i386/apic_vector.s 122572 2003-11-12 18:13:57Z jhb $ 36 */ 37 38/* 39 * Interrupt entry points for external interrupts triggered by I/O APICs 40 * as well as IPI handlers. 41 */ 42 43#include <machine/asmacros.h> 44#include <machine/apicreg.h> 45#include <machine/smptests.h> 46 47#include "assym.s" 48 49/* 50 * Macros to create and destroy a trap frame. 51 */ 52#define PUSH_FRAME \ 53 pushl $0 ; /* dummy error code */ \ 54 pushl $0 ; /* dummy trap type */ \ 55 pushal ; /* 8 ints */ \ 56 pushl %ds ; /* save data and extra segments ... */ \ 57 pushl %es ; \ 58 pushl %fs 59 60#define POP_FRAME \ 61 popl %fs ; \ 62 popl %es ; \ 63 popl %ds ; \ 64 popal ; \ 65 addl $4+4,%esp 66 67/* 68 * I/O Interrupt Entry Point. Rather than having one entry point for 69 * each interrupt source, we use one entry point for each 32-bit word 70 * in the ISR. The handler determines the highest bit set in the ISR, 71 * translates that into a vector, and passes the vector to the 72 * lapic_handle_intr() function. 73 */ 74#define ISR_VEC(index, vec_name) \ 75 .text ; \ 76 SUPERALIGN_TEXT ; \ 77IDTVEC(vec_name) ; \ 78 PUSH_FRAME ; \ 79 movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ 80 mov %ax, %ds ; \ 81 mov %ax, %es ; \ 82 movl $KPSEL, %eax ; /* reload with per-CPU data segment */ \ 83 mov %ax, %fs ; \ 84 movl lapic, %edx ; /* pointer to local APIC */ \ 85 movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \ 86 bsrl %eax, %eax ; /* index of highset set bit in ISR */ \ 87 jz 2f ; \ 88 addl $(32 * index),%eax ; \ 891: ; \ 90 FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid double count */ \ 91 pushl %eax ; /* pass the IRQ */ \ 92 call lapic_handle_intr ; \ 93 addl $4, %esp ; /* discard parameter */ \ 94 MEXITCOUNT ; \ 95 jmp doreti ; \ 962: movl $-1, %eax ; /* send a vector of -1 */ \ 97 jmp 1b 98 99/* 100 * Handle "spurious INTerrupts". 101 * Notes: 102 * This is different than the "spurious INTerrupt" generated by an 103 * 8259 PIC for missing INTs. See the APIC documentation for details. 104 * This routine should NOT do an 'EOI' cycle. 105 */ 106 .text 107 SUPERALIGN_TEXT 108IDTVEC(spuriousint) 109 110 /* No EOI cycle used here */ 111 112 iret 113 114MCOUNT_LABEL(bintr2) 115 ISR_VEC(1,apic_isr1) 116 ISR_VEC(2,apic_isr2) 117 ISR_VEC(3,apic_isr3) 118 ISR_VEC(4,apic_isr4) 119 ISR_VEC(5,apic_isr5) 120MCOUNT_LABEL(eintr2) 121 122#ifdef SMP 123/* 124 * Global address space TLB shootdown. 125 */ 126 .text 127 SUPERALIGN_TEXT 128IDTVEC(invltlb) 129 pushl %eax 130 pushl %ds 131 movl $KDSEL, %eax /* Kernel data selector */ 132 mov %ax, %ds 133 134#ifdef COUNT_XINVLTLB_HITS 135 pushl %fs 136 movl $KPSEL, %eax /* Private space selector */ 137 mov %ax, %fs 138 movl PCPU(CPUID), %eax 139 popl %fs 140 incl xhits_gbl(,%eax,4) 141#endif /* COUNT_XINVLTLB_HITS */ 142 143 movl %cr3, %eax /* invalidate the TLB */ 144 movl %eax, %cr3 145 146 movl lapic, %eax 147 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 148 149 lock 150 incl smp_tlb_wait 151 152 popl %ds 153 popl %eax 154 iret 155 156/* 157 * Single page TLB shootdown 158 */ 159 .text 160 SUPERALIGN_TEXT 161IDTVEC(invlpg) 162 pushl %eax 163 pushl %ds 164 movl $KDSEL, %eax /* Kernel data selector */ 165 mov %ax, %ds 166 167#ifdef COUNT_XINVLTLB_HITS 168 pushl %fs 169 movl $KPSEL, %eax /* Private space selector */ 170 mov %ax, %fs 171 movl PCPU(CPUID), %eax 172 popl %fs 173 incl xhits_pg(,%eax,4) 174#endif /* COUNT_XINVLTLB_HITS */ 175 176 movl smp_tlb_addr1, %eax 177 invlpg (%eax) /* invalidate single page */ 178 179 movl lapic, %eax 180 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 181 182 lock 183 incl smp_tlb_wait 184 185 popl %ds 186 popl %eax 187 iret 188 189/* 190 * Page range TLB shootdown. 191 */ 192 .text 193 SUPERALIGN_TEXT 194IDTVEC(invlrng) 195 pushl %eax 196 pushl %edx 197 pushl %ds 198 movl $KDSEL, %eax /* Kernel data selector */ 199 mov %ax, %ds 200 201#ifdef COUNT_XINVLTLB_HITS 202 pushl %fs 203 movl $KPSEL, %eax /* Private space selector */ 204 mov %ax, %fs 205 movl PCPU(CPUID), %eax 206 popl %fs 207 incl xhits_rng(,%eax,4) 208#endif /* COUNT_XINVLTLB_HITS */ 209 210 movl smp_tlb_addr1, %edx 211 movl smp_tlb_addr2, %eax 2121: invlpg (%edx) /* invalidate single page */ 213 addl $PAGE_SIZE, %edx 214 cmpl %eax, %edx 215 jb 1b 216 217 movl lapic, %eax 218 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 219 220 lock 221 incl smp_tlb_wait 222 223 popl %ds 224 popl %edx 225 popl %eax 226 iret 227 228/* 229 * Forward hardclock to another CPU. Pushes a clockframe and calls 230 * forwarded_hardclock(). 231 */ 232 .text 233 SUPERALIGN_TEXT 234IDTVEC(hardclock) 235 PUSH_FRAME 236 movl $KDSEL, %eax /* reload with kernel's data segment */ 237 mov %ax, %ds 238 mov %ax, %es 239 movl $KPSEL, %eax 240 mov %ax, %fs 241 242 movl lapic, %edx 243 movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ 244 245 pushl $0 /* XXX convert trapframe to clockframe */ 246 call forwarded_hardclock 247 addl $4, %esp /* XXX convert clockframe to trapframe */ 248 MEXITCOUNT 249 jmp doreti 250 251/* 252 * Forward statclock to another CPU. Pushes a clockframe and calls 253 * forwarded_statclock(). 254 */ 255 .text 256 SUPERALIGN_TEXT 257IDTVEC(statclock) 258 PUSH_FRAME 259 movl $KDSEL, %eax /* reload with kernel's data segment */ 260 mov %ax, %ds 261 mov %ax, %es 262 movl $KPSEL, %eax 263 mov %ax, %fs 264 265 movl lapic, %edx 266 movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ 267 268 FAKE_MCOUNT(13*4(%esp)) 269 270 pushl $0 /* XXX convert trapframe to clockframe */ 271 call forwarded_statclock 272 addl $4, %esp /* XXX convert clockframe to trapframe */ 273 MEXITCOUNT 274 jmp doreti 275 276/* 277 * Executed by a CPU when it receives an Xcpuast IPI from another CPU, 278 * 279 * The other CPU has already executed aston() or need_resched() on our 280 * current process, so we simply need to ack the interrupt and return 281 * via doreti to run ast(). 282 */ 283 284 .text 285 SUPERALIGN_TEXT 286IDTVEC(cpuast) 287 PUSH_FRAME 288 movl $KDSEL, %eax 289 mov %ax, %ds /* use KERNEL data segment */ 290 mov %ax, %es 291 movl $KPSEL, %eax 292 mov %ax, %fs 293 294 movl lapic, %edx 295 movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ 296 297 FAKE_MCOUNT(13*4(%esp)) 298 299 MEXITCOUNT 300 jmp doreti 301 302/* 303 * Executed by a CPU when it receives an Xcpustop IPI from another CPU, 304 * 305 * - Signals its receipt. 306 * - Waits for permission to restart. 307 * - Signals its restart. 308 */ 309 .text 310 SUPERALIGN_TEXT 311IDTVEC(cpustop) 312 pushl %ebp 313 movl %esp, %ebp 314 pushl %eax 315 pushl %ecx 316 pushl %edx 317 pushl %ds /* save current data segment */ 318 pushl %es 319 pushl %fs 320 321 movl $KDSEL, %eax 322 mov %ax, %ds /* use KERNEL data segment */ 323 mov %ax, %es 324 movl $KPSEL, %eax 325 mov %ax, %fs 326 327 movl lapic, %eax 328 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 329 330 movl PCPU(CPUID), %eax 331 imull $PCB_SIZE, %eax 332 leal CNAME(stoppcbs)(%eax), %eax 333 pushl %eax 334 call CNAME(savectx) /* Save process context */ 335 addl $4, %esp 336 337 movl PCPU(CPUID), %eax 338 339 lock 340 btsl %eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */ 3411: 342 btl %eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */ 343 jnc 1b 344 345 lock 346 btrl %eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */ 347 lock 348 btrl %eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */ 349 350 test %eax, %eax 351 jnz 2f 352 353 movl CNAME(cpustop_restartfunc), %eax 354 test %eax, %eax 355 jz 2f 356 movl $0, CNAME(cpustop_restartfunc) /* One-shot */ 357 358 call *%eax 3592: 360 popl %fs 361 popl %es 362 popl %ds /* restore previous data segment */ 363 popl %edx 364 popl %ecx 365 popl %eax 366 movl %ebp, %esp 367 popl %ebp 368 iret 369 370/* 371 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 372 * 373 * - Calls the generic rendezvous action function. 374 */ 375 .text 376 SUPERALIGN_TEXT 377IDTVEC(rendezvous) 378 PUSH_FRAME 379 movl $KDSEL, %eax 380 mov %ax, %ds /* use KERNEL data segment */ 381 mov %ax, %es 382 movl $KPSEL, %eax 383 mov %ax, %fs 384 385 call smp_rendezvous_action 386 387 movl lapic, %eax 388 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 389 POP_FRAME 390 iret 391 392/* 393 * Clean up when we lose out on the lazy context switch optimization. 394 * ie: when we are about to release a PTD but a cpu is still borrowing it. 395 */ 396 SUPERALIGN_TEXT 397IDTVEC(lazypmap) 398 PUSH_FRAME 399 movl $KDSEL, %eax 400 mov %ax, %ds /* use KERNEL data segment */ 401 mov %ax, %es 402 movl $KPSEL, %eax 403 mov %ax, %fs 404 405 call pmap_lazyfix_action 406 407 movl lapic, %eax 408 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 409 POP_FRAME 410 iret 411#endif /* SMP */ 412