1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: vector.s, 386BSD 0.1 unknown origin 31 * $FreeBSD: releng/10.2/sys/amd64/amd64/apic_vector.S 283280 2015-05-22 09:03:55Z whu $ 32 */ 33 34/* 35 * Interrupt entry points for external interrupts triggered by I/O APICs 36 * as well as IPI handlers. 37 */ 38 39#include "opt_smp.h" 40 41#include <machine/asmacros.h> 42#include <x86/apicreg.h> 43 44#include "assym.s" 45 46#ifdef SMP 47#define LK lock ; 48#else 49#define LK 50#endif 51 52/* 53 * I/O Interrupt Entry Point. Rather than having one entry point for 54 * each interrupt source, we use one entry point for each 32-bit word 55 * in the ISR. The handler determines the highest bit set in the ISR, 56 * translates that into a vector, and passes the vector to the 57 * lapic_handle_intr() function. 58 */ 59#define ISR_VEC(index, vec_name) \ 60 .text ; \ 61 SUPERALIGN_TEXT ; \ 62IDTVEC(vec_name) ; \ 63 PUSH_FRAME ; \ 64 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 65 movq lapic, %rdx ; /* pointer to local APIC */ \ 66 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \ 67 bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ 68 jz 1f ; \ 69 addl $(32 * index),%eax ; \ 70 movq %rsp, %rsi ; \ 71 movl %eax, %edi ; /* pass the IRQ */ \ 72 call lapic_handle_intr ; \ 731: ; \ 74 MEXITCOUNT ; \ 75 jmp doreti 76 77/* 78 * Handle "spurious INTerrupts". 79 * Notes: 80 * This is different than the "spurious INTerrupt" generated by an 81 * 8259 PIC for missing INTs. See the APIC documentation for details. 82 * This routine should NOT do an 'EOI' cycle. 83 */ 84 .text 85 SUPERALIGN_TEXT 86IDTVEC(spuriousint) 87 88 /* No EOI cycle used here */ 89 90 jmp doreti_iret 91 92 ISR_VEC(1, apic_isr1) 93 ISR_VEC(2, apic_isr2) 94 ISR_VEC(3, apic_isr3) 95 ISR_VEC(4, apic_isr4) 96 ISR_VEC(5, apic_isr5) 97 ISR_VEC(6, apic_isr6) 98 ISR_VEC(7, apic_isr7) 99 100/* 101 * Local APIC periodic timer handler. 102 */ 103 .text 104 SUPERALIGN_TEXT 105IDTVEC(timerint) 106 PUSH_FRAME 107 FAKE_MCOUNT(TF_RIP(%rsp)) 108 movq %rsp, %rdi 109 call lapic_handle_timer 110 MEXITCOUNT 111 jmp doreti 112 113/* 114 * Local APIC CMCI handler. 115 */ 116 .text 117 SUPERALIGN_TEXT 118IDTVEC(cmcint) 119 PUSH_FRAME 120 FAKE_MCOUNT(TF_RIP(%rsp)) 121 call lapic_handle_cmc 122 MEXITCOUNT 123 jmp doreti 124 125/* 126 * Local APIC error interrupt handler. 127 */ 128 .text 129 SUPERALIGN_TEXT 130IDTVEC(errorint) 131 PUSH_FRAME 132 FAKE_MCOUNT(TF_RIP(%rsp)) 133 call lapic_handle_error 134 MEXITCOUNT 135 jmp doreti 136 137#ifdef XENHVM 138/* 139 * Xen event channel upcall interrupt handler. 140 * Only used when the hypervisor supports direct vector callbacks. 141 */ 142 .text 143 SUPERALIGN_TEXT 144IDTVEC(xen_intr_upcall) 145 PUSH_FRAME 146 FAKE_MCOUNT(TF_RIP(%rsp)) 147 movq %rsp, %rdi 148 call xen_intr_handle_upcall 149 MEXITCOUNT 150 jmp doreti 151#endif 152 153#ifdef HYPERV 154/* 155 * This is the Hyper-V vmbus channel direct callback interrupt. 156 * Only used when it is running on Hyper-V. 157 */ 158 .text 159 SUPERALIGN_TEXT 160IDTVEC(hv_vmbus_callback) 161 PUSH_FRAME 162 FAKE_MCOUNT(TF_RIP(%rsp)) 163 movq %rsp, %rdi 164 call hv_vector_handler 165 MEXITCOUNT 166 jmp doreti 167#endif 168 169#ifdef SMP 170/* 171 * Global address space TLB shootdown. 172 */ 173 .text 174 175 SUPERALIGN_TEXT 176invltlb_ret: 177 movq lapic, %rax 178 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 179 POP_FRAME 180 jmp doreti_iret 181 182 SUPERALIGN_TEXT 183IDTVEC(invltlb_pcid) 184 PUSH_FRAME 185 186 call invltlb_pcid_handler 187 jmp invltlb_ret 188 189 190 SUPERALIGN_TEXT 191IDTVEC(invltlb) 192 PUSH_FRAME 193 194 call invltlb_handler 195 jmp invltlb_ret 196 197/* 198 * Single page TLB shootdown 199 */ 200 .text 201 SUPERALIGN_TEXT 202IDTVEC(invlpg_pcid) 203 PUSH_FRAME 204 205 call invlpg_pcid_handler 206 jmp invltlb_ret 207 208 SUPERALIGN_TEXT 209IDTVEC(invlpg) 210 PUSH_FRAME 211 212 call invlpg_handler 213 jmp invltlb_ret 214 215/* 216 * Page range TLB shootdown. 217 */ 218 .text 219 SUPERALIGN_TEXT 220IDTVEC(invlrng) 221 PUSH_FRAME 222 223 call invlrng_handler 224 jmp invltlb_ret 225 226/* 227 * Invalidate cache. 228 */ 229 .text 230 SUPERALIGN_TEXT 231IDTVEC(invlcache) 232 PUSH_FRAME 233 234 call invlcache_handler 235 jmp invltlb_ret 236 237/* 238 * Handler for IPIs sent via the per-cpu IPI bitmap. 239 */ 240 .text 241 SUPERALIGN_TEXT 242IDTVEC(ipi_intr_bitmap_handler) 243 PUSH_FRAME 244 245 movq lapic, %rdx 246 movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */ 247 248 FAKE_MCOUNT(TF_RIP(%rsp)) 249 250 call ipi_bitmap_handler 251 MEXITCOUNT 252 jmp doreti 253 254/* 255 * Executed by a CPU when it receives an IPI_STOP from another CPU. 256 */ 257 .text 258 SUPERALIGN_TEXT 259IDTVEC(cpustop) 260 PUSH_FRAME 261 262 movq lapic, %rax 263 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 264 265 call cpustop_handler 266 jmp doreti 267 268/* 269 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 270 */ 271 .text 272 SUPERALIGN_TEXT 273IDTVEC(cpususpend) 274 PUSH_FRAME 275 276 call cpususpend_handler 277 movq lapic, %rax 278 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 279 jmp doreti 280 281/* 282 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 283 * 284 * - Calls the generic rendezvous action function. 285 */ 286 .text 287 SUPERALIGN_TEXT 288IDTVEC(rendezvous) 289 PUSH_FRAME 290#ifdef COUNT_IPIS 291 movl PCPU(CPUID), %eax 292 movq ipi_rendezvous_counts(,%rax,8), %rax 293 incq (%rax) 294#endif 295 call smp_rendezvous_action 296 movq lapic, %rax 297 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ 298 jmp doreti 299#endif /* SMP */ 300