1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: vector.s, 386BSD 0.1 unknown origin 31 * $FreeBSD: releng/11.0/sys/amd64/amd64/apic_vector.S 298022 2016-04-15 02:20:18Z sephe $ 32 */ 33 34/* 35 * Interrupt entry points for external interrupts triggered by I/O APICs 36 * as well as IPI handlers. 37 */ 38 39#include "opt_smp.h" 40 41#include <machine/asmacros.h> 42#include <machine/specialreg.h> 43#include <x86/apicreg.h> 44 45#include "assym.s" 46 47#ifdef SMP 48#define LK lock ; 49#else 50#define LK 51#endif 52 53 .text 54 SUPERALIGN_TEXT 55 /* End Of Interrupt to APIC */ 56as_lapic_eoi: 57 cmpl $0,x2apic_mode 58 jne 1f 59 movq lapic_map,%rax 60 movl $0,LA_EOI(%rax) 61 ret 621: 63 movl $MSR_APIC_EOI,%ecx 64 xorl %eax,%eax 65 xorl %edx,%edx 66 wrmsr 67 ret 68 69/* 70 * I/O Interrupt Entry Point. Rather than having one entry point for 71 * each interrupt source, we use one entry point for each 32-bit word 72 * in the ISR. The handler determines the highest bit set in the ISR, 73 * translates that into a vector, and passes the vector to the 74 * lapic_handle_intr() function. 75 */ 76#define ISR_VEC(index, vec_name) \ 77 .text ; \ 78 SUPERALIGN_TEXT ; \ 79IDTVEC(vec_name) ; \ 80 PUSH_FRAME ; \ 81 FAKE_MCOUNT(TF_RIP(%rsp)) ; \ 82 cmpl $0,x2apic_mode ; \ 83 je 1f ; \ 84 movl $(MSR_APIC_ISR0 + index),%ecx ; \ 85 rdmsr ; \ 86 jmp 2f ; \ 871: ; \ 88 movq lapic_map, %rdx ; /* pointer to local APIC */ \ 89 movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \ 902: ; \ 91 bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ 92 jz 3f ; \ 93 addl $(32 * index),%eax ; \ 94 movq %rsp, %rsi ; \ 95 movl %eax, %edi ; /* pass the IRQ */ \ 96 call lapic_handle_intr ; \ 973: ; \ 98 MEXITCOUNT ; \ 99 jmp doreti 100 101/* 102 * Handle "spurious INTerrupts". 103 * Notes: 104 * This is different than the "spurious INTerrupt" generated by an 105 * 8259 PIC for missing INTs. See the APIC documentation for details. 106 * This routine should NOT do an 'EOI' cycle. 107 */ 108 .text 109 SUPERALIGN_TEXT 110IDTVEC(spuriousint) 111 112 /* No EOI cycle used here */ 113 114 jmp doreti_iret 115 116 ISR_VEC(1, apic_isr1) 117 ISR_VEC(2, apic_isr2) 118 ISR_VEC(3, apic_isr3) 119 ISR_VEC(4, apic_isr4) 120 ISR_VEC(5, apic_isr5) 121 ISR_VEC(6, apic_isr6) 122 ISR_VEC(7, apic_isr7) 123 124/* 125 * Local APIC periodic timer handler. 126 */ 127 .text 128 SUPERALIGN_TEXT 129IDTVEC(timerint) 130 PUSH_FRAME 131 FAKE_MCOUNT(TF_RIP(%rsp)) 132 movq %rsp, %rdi 133 call lapic_handle_timer 134 MEXITCOUNT 135 jmp doreti 136 137/* 138 * Local APIC CMCI handler. 139 */ 140 .text 141 SUPERALIGN_TEXT 142IDTVEC(cmcint) 143 PUSH_FRAME 144 FAKE_MCOUNT(TF_RIP(%rsp)) 145 call lapic_handle_cmc 146 MEXITCOUNT 147 jmp doreti 148 149/* 150 * Local APIC error interrupt handler. 151 */ 152 .text 153 SUPERALIGN_TEXT 154IDTVEC(errorint) 155 PUSH_FRAME 156 FAKE_MCOUNT(TF_RIP(%rsp)) 157 call lapic_handle_error 158 MEXITCOUNT 159 jmp doreti 160 161#ifdef XENHVM 162/* 163 * Xen event channel upcall interrupt handler. 164 * Only used when the hypervisor supports direct vector callbacks. 165 */ 166 .text 167 SUPERALIGN_TEXT 168IDTVEC(xen_intr_upcall) 169 PUSH_FRAME 170 FAKE_MCOUNT(TF_RIP(%rsp)) 171 movq %rsp, %rdi 172 call xen_intr_handle_upcall 173 MEXITCOUNT 174 jmp doreti 175#endif 176 177#ifdef SMP 178/* 179 * Global address space TLB shootdown. 180 */ 181 .text 182 183 SUPERALIGN_TEXT 184invltlb_ret: 185 call as_lapic_eoi 186 POP_FRAME 187 jmp doreti_iret 188 189 SUPERALIGN_TEXT 190IDTVEC(invltlb) 191 PUSH_FRAME 192 193 call invltlb_handler 194 jmp invltlb_ret 195 196IDTVEC(invltlb_pcid) 197 PUSH_FRAME 198 199 call invltlb_pcid_handler 200 jmp invltlb_ret 201 202IDTVEC(invltlb_invpcid) 203 PUSH_FRAME 204 205 call invltlb_invpcid_handler 206 jmp invltlb_ret 207 208/* 209 * Single page TLB shootdown 210 */ 211 .text 212 213 SUPERALIGN_TEXT 214IDTVEC(invlpg) 215 PUSH_FRAME 216 217 call invlpg_handler 218 jmp invltlb_ret 219 220/* 221 * Page range TLB shootdown. 222 */ 223 .text 224 SUPERALIGN_TEXT 225IDTVEC(invlrng) 226 PUSH_FRAME 227 228 call invlrng_handler 229 jmp invltlb_ret 230 231/* 232 * Invalidate cache. 233 */ 234 .text 235 SUPERALIGN_TEXT 236IDTVEC(invlcache) 237 PUSH_FRAME 238 239 call invlcache_handler 240 jmp invltlb_ret 241 242/* 243 * Handler for IPIs sent via the per-cpu IPI bitmap. 244 */ 245 .text 246 SUPERALIGN_TEXT 247IDTVEC(ipi_intr_bitmap_handler) 248 PUSH_FRAME 249 250 call as_lapic_eoi 251 252 FAKE_MCOUNT(TF_RIP(%rsp)) 253 254 call ipi_bitmap_handler 255 MEXITCOUNT 256 jmp doreti 257 258/* 259 * Executed by a CPU when it receives an IPI_STOP from another CPU. 260 */ 261 .text 262 SUPERALIGN_TEXT 263IDTVEC(cpustop) 264 PUSH_FRAME 265 266 call as_lapic_eoi 267 268 call cpustop_handler 269 jmp doreti 270 271/* 272 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 273 */ 274 .text 275 SUPERALIGN_TEXT 276IDTVEC(cpususpend) 277 PUSH_FRAME 278 279 call cpususpend_handler 280 call as_lapic_eoi 281 jmp doreti 282 283/* 284 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 285 * 286 * - Calls the generic rendezvous action function. 287 */ 288 .text 289 SUPERALIGN_TEXT 290IDTVEC(rendezvous) 291 PUSH_FRAME 292#ifdef COUNT_IPIS 293 movl PCPU(CPUID), %eax 294 movq ipi_rendezvous_counts(,%rax,8), %rax 295 incq (%rax) 296#endif 297 call smp_rendezvous_action 298 call as_lapic_eoi 299 jmp doreti 300 301/* 302 * IPI handler whose purpose is to interrupt the CPU with minimum overhead. 303 * This is used by bhyve to force a host cpu executing in guest context to 304 * trap into the hypervisor. 305 * 306 * This handler is different from other IPI handlers in the following aspects: 307 * 308 * 1. It doesn't push a trapframe on the stack. 309 * 310 * This implies that a DDB backtrace involving 'justreturn' will skip the 311 * function that was interrupted by this handler. 312 * 313 * 2. It doesn't 'swapgs' when userspace is interrupted. 314 * 315 * The 'justreturn' handler does not access any pcpu data so it is not an 316 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI 317 * whose handler already doesn't trust GS.base when kernel code is interrupted. 318 */ 319 .text 320 SUPERALIGN_TEXT 321IDTVEC(justreturn) 322 pushq %rax 323 pushq %rcx 324 pushq %rdx 325 call as_lapic_eoi 326 popq %rdx 327 popq %rcx 328 popq %rax 329 jmp doreti_iret 330 331#endif /* SMP */ 332