1121986Sjhb/*- 2121986Sjhb * Copyright (c) 1989, 1990 William F. Jolitz. 3121986Sjhb * Copyright (c) 1990 The Regents of the University of California. 4121986Sjhb * All rights reserved. 5121986Sjhb * 6121986Sjhb * Redistribution and use in source and binary forms, with or without 7121986Sjhb * modification, are permitted provided that the following conditions 8121986Sjhb * are met: 9121986Sjhb * 1. Redistributions of source code must retain the above copyright 10121986Sjhb * notice, this list of conditions and the following disclaimer. 11121986Sjhb * 2. Redistributions in binary form must reproduce the above copyright 12121986Sjhb * notice, this list of conditions and the following disclaimer in the 13121986Sjhb * documentation and/or other materials provided with the distribution. 14121986Sjhb * 4. Neither the name of the University nor the names of its contributors 15121986Sjhb * may be used to endorse or promote products derived from this software 16121986Sjhb * without specific prior written permission. 17121986Sjhb * 18121986Sjhb * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19121986Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20121986Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21121986Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22121986Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23121986Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24121986Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25121986Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26121986Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27121986Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28121986Sjhb * SUCH DAMAGE. 29121986Sjhb * 3026168Sfsmp * from: vector.s, 386BSD 0.1 unknown origin 3150477Speter * $FreeBSD$ 3226168Sfsmp */ 3326168Sfsmp 34121986Sjhb/* 35121986Sjhb * Interrupt entry points for external interrupts triggered by I/O APICs 36121986Sjhb * as well as IPI handlers. 37121986Sjhb */ 3828043Sfsmp 39152701Sjhb#include "opt_smp.h" 40152701Sjhb 41121986Sjhb#include <machine/asmacros.h> 42214631Sjhb#include <x86/apicreg.h> 4326168Sfsmp 44121986Sjhb#include "assym.s" 4526168Sfsmp 4666716Sjhb/* 47121986Sjhb * I/O Interrupt Entry Point. Rather than having one entry point for 48121986Sjhb * each interrupt source, we use one entry point for each 32-bit word 49121986Sjhb * in the ISR. The handler determines the highest bit set in the ISR, 50121986Sjhb * translates that into a vector, and passes the vector to the 51121986Sjhb * lapic_handle_intr() function. 5228487Sfsmp */ 53121986Sjhb#define ISR_VEC(index, vec_name) \ 5493264Sdillon .text ; \ 5593264Sdillon SUPERALIGN_TEXT ; \ 5693264SdillonIDTVEC(vec_name) ; \ 57125405Sjhb PUSH_FRAME ; \ 58153135Sjhb SET_KERNEL_SREGS ; \ 59209483Skib cld ; \ 60129620Sbde FAKE_MCOUNT(TF_EIP(%esp)) ; \ 61121986Sjhb movl lapic, %edx ; /* pointer to local APIC */ \ 62121986Sjhb movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \ 63217360Sjhb bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ 64217360Sjhb jz 1f ; \ 65121986Sjhb addl $(32 * index),%eax ; \ 66165302Skmacy pushl %esp ; \ 67121986Sjhb pushl %eax ; /* pass the IRQ */ \ 68121986Sjhb call lapic_handle_intr ; \ 69165302Skmacy addl $8, %esp ; /* discard parameter */ \ 70217360Sjhb1: ; \ 7126168Sfsmp MEXITCOUNT ; \ 72217360Sjhb jmp doreti 7326168Sfsmp 7427251Sfsmp/* 7527352Sfsmp * Handle "spurious INTerrupts". 7627352Sfsmp * Notes: 7727352Sfsmp * This is different than the "spurious INTerrupt" generated by an 7827352Sfsmp * 8259 PIC for missing INTs. See the APIC documentation for details. 7927352Sfsmp * This routine should NOT do an 'EOI' cycle. 8027352Sfsmp */ 8127352Sfsmp .text 8227352Sfsmp SUPERALIGN_TEXT 83115907SjhbIDTVEC(spuriousint) 8427352Sfsmp 8527352Sfsmp /* No EOI cycle used here */ 8627352Sfsmp 8727352Sfsmp iret 8827352Sfsmp 89122620Sjhb ISR_VEC(1, apic_isr1) 90122620Sjhb ISR_VEC(2, apic_isr2) 91122620Sjhb ISR_VEC(3, apic_isr3) 92122620Sjhb ISR_VEC(4, apic_isr4) 93122620Sjhb ISR_VEC(5, apic_isr5) 94122690Sjhb ISR_VEC(6, apic_isr6) 95122690Sjhb ISR_VEC(7, apic_isr7) 96121986Sjhb 97141538Sjhb/* 98141538Sjhb * Local APIC periodic timer handler. 99141538Sjhb */ 100141538Sjhb .text 101141538Sjhb SUPERALIGN_TEXT 102141538SjhbIDTVEC(timerint) 103141538Sjhb PUSH_FRAME 104153135Sjhb SET_KERNEL_SREGS 105209483Skib cld 106141538Sjhb FAKE_MCOUNT(TF_EIP(%esp)) 107165302Skmacy pushl %esp 108141538Sjhb call lapic_handle_timer 109165302Skmacy add $4, %esp 110141538Sjhb MEXITCOUNT 111141538Sjhb jmp doreti 112141538Sjhb 113205851Sjhb/* 114208507Sjhb * Local APIC CMCI handler. 115208507Sjhb */ 116208507Sjhb .text 117208507Sjhb SUPERALIGN_TEXT 118208507SjhbIDTVEC(cmcint) 119208507Sjhb PUSH_FRAME 120208507Sjhb SET_KERNEL_SREGS 121209483Skib cld 122208507Sjhb FAKE_MCOUNT(TF_EIP(%esp)) 123208507Sjhb call lapic_handle_cmc 124208507Sjhb MEXITCOUNT 125208507Sjhb jmp doreti 126208507Sjhb 127208507Sjhb/* 128205851Sjhb * Local APIC error interrupt handler. 129205851Sjhb */ 130205851Sjhb .text 131205851Sjhb SUPERALIGN_TEXT 132205851SjhbIDTVEC(errorint) 133205851Sjhb PUSH_FRAME 134205851Sjhb SET_KERNEL_SREGS 135209483Skib cld 136205851Sjhb FAKE_MCOUNT(TF_EIP(%esp)) 137205851Sjhb call lapic_handle_error 138205851Sjhb MEXITCOUNT 139205851Sjhb jmp doreti 140205851Sjhb 141115907Sjhb#ifdef SMP 14227352Sfsmp/* 14399862Speter * Global address space TLB shootdown. 14427251Sfsmp */ 14526168Sfsmp .text 14626168Sfsmp SUPERALIGN_TEXT 147115907SjhbIDTVEC(invltlb) 14826168Sfsmp pushl %eax 14999862Speter pushl %ds 15099862Speter movl $KDSEL, %eax /* Kernel data selector */ 151128328Sjhb movl %eax, %ds 152125405Sjhb 153150697Sjhb#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) 15446129Sluoqi pushl %fs 15599862Speter movl $KPSEL, %eax /* Private space selector */ 156128328Sjhb movl %eax, %fs 15769971Sjake movl PCPU(CPUID), %eax 15846129Sluoqi popl %fs 159150697Sjhb#ifdef COUNT_XINVLTLB_HITS 16099862Speter incl xhits_gbl(,%eax,4) 161150697Sjhb#endif 162150697Sjhb#ifdef COUNT_IPIS 163150697Sjhb movl ipi_invltlb_counts(,%eax,4),%eax 164150697Sjhb incl (%eax) 165150697Sjhb#endif 166150697Sjhb#endif 16727251Sfsmp 16826168Sfsmp movl %cr3, %eax /* invalidate the TLB */ 16926168Sfsmp movl %eax, %cr3 17027251Sfsmp 171121986Sjhb movl lapic, %eax 172121986Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 17327251Sfsmp 17499862Speter lock 17599862Speter incl smp_tlb_wait 17699862Speter 17799862Speter popl %ds 17826168Sfsmp popl %eax 17926168Sfsmp iret 18026168Sfsmp 18127007Sfsmp/* 18299862Speter * Single page TLB shootdown 18399862Speter */ 18499862Speter .text 18599862Speter SUPERALIGN_TEXT 186115907SjhbIDTVEC(invlpg) 18799862Speter pushl %eax 18899862Speter pushl %ds 18999862Speter movl $KDSEL, %eax /* Kernel data selector */ 190128328Sjhb movl %eax, %ds 191125405Sjhb 192150697Sjhb#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) 19399862Speter pushl %fs 19499862Speter movl $KPSEL, %eax /* Private space selector */ 195128328Sjhb movl %eax, %fs 19699862Speter movl PCPU(CPUID), %eax 19799862Speter popl %fs 198150697Sjhb#ifdef COUNT_XINVLTLB_HITS 19999862Speter incl xhits_pg(,%eax,4) 200150697Sjhb#endif 201150697Sjhb#ifdef COUNT_IPIS 202150697Sjhb movl ipi_invlpg_counts(,%eax,4),%eax 203150697Sjhb incl (%eax) 204150697Sjhb#endif 205150697Sjhb#endif 20699862Speter 20799862Speter movl smp_tlb_addr1, %eax 20899862Speter invlpg (%eax) /* invalidate single page */ 20999862Speter 210121986Sjhb movl lapic, %eax 211121986Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 21299862Speter 21399862Speter lock 21499862Speter incl smp_tlb_wait 21599862Speter 21699862Speter popl %ds 21799862Speter popl %eax 21899862Speter iret 21999862Speter 22099862Speter/* 22199862Speter * Page range TLB shootdown. 22299862Speter */ 22399862Speter .text 22499862Speter SUPERALIGN_TEXT 225115907SjhbIDTVEC(invlrng) 22699862Speter pushl %eax 22799862Speter pushl %edx 22899862Speter pushl %ds 22999862Speter movl $KDSEL, %eax /* Kernel data selector */ 230128328Sjhb movl %eax, %ds 231125405Sjhb 232150697Sjhb#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) 23399862Speter pushl %fs 23499862Speter movl $KPSEL, %eax /* Private space selector */ 235128328Sjhb movl %eax, %fs 23699862Speter movl PCPU(CPUID), %eax 23799862Speter popl %fs 238150697Sjhb#ifdef COUNT_XINVLTLB_HITS 23999862Speter incl xhits_rng(,%eax,4) 240150697Sjhb#endif 241150697Sjhb#ifdef COUNT_IPIS 242150697Sjhb movl ipi_invlrng_counts(,%eax,4),%eax 243150697Sjhb incl (%eax) 244150697Sjhb#endif 245150697Sjhb#endif 24699862Speter 24799862Speter movl smp_tlb_addr1, %edx 24899862Speter movl smp_tlb_addr2, %eax 24999862Speter1: invlpg (%edx) /* invalidate single page */ 25099862Speter addl $PAGE_SIZE, %edx 251102329Speter cmpl %eax, %edx 25299862Speter jb 1b 25399862Speter 254121986Sjhb movl lapic, %eax 255121986Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 25699862Speter 25799862Speter lock 25899862Speter incl smp_tlb_wait 25999862Speter 26099862Speter popl %ds 26199862Speter popl %edx 26299862Speter popl %eax 26399862Speter iret 26499862Speter 26599862Speter/* 266158236Sjhb * Invalidate cache. 267158236Sjhb */ 268158236Sjhb .text 269158236Sjhb SUPERALIGN_TEXT 270158236SjhbIDTVEC(invlcache) 271158236Sjhb pushl %eax 272158236Sjhb pushl %ds 273158236Sjhb movl $KDSEL, %eax /* Kernel data selector */ 274158236Sjhb movl %eax, %ds 275158236Sjhb 276158236Sjhb#ifdef COUNT_IPIS 277158236Sjhb pushl %fs 278158236Sjhb movl $KPSEL, %eax /* Private space selector */ 279158236Sjhb movl %eax, %fs 280158236Sjhb movl PCPU(CPUID), %eax 281158236Sjhb popl %fs 282158236Sjhb movl ipi_invlcache_counts(,%eax,4),%eax 283158236Sjhb incl (%eax) 284158236Sjhb#endif 285158236Sjhb 286158236Sjhb wbinvd 287158236Sjhb 288158236Sjhb movl lapic, %eax 289158236Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 290158236Sjhb 291158236Sjhb lock 292158236Sjhb incl smp_tlb_wait 293158236Sjhb 294158236Sjhb popl %ds 295158236Sjhb popl %eax 296158236Sjhb iret 297158236Sjhb 298158236Sjhb/* 299153426Sjhb * Handler for IPIs sent via the per-cpu IPI bitmap. 30031638Sfsmp */ 301184113Skmacy#ifndef XEN 30231638Sfsmp .text 30331638Sfsmp SUPERALIGN_TEXT 304138528SupsIDTVEC(ipi_intr_bitmap_handler) 305125405Sjhb PUSH_FRAME 306153135Sjhb SET_KERNEL_SREGS 307209483Skib cld 30831638Sfsmp 309121986Sjhb movl lapic, %edx 310121986Sjhb movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ 311138528Sups 312129620Sbde FAKE_MCOUNT(TF_EIP(%esp)) 31393264Sdillon 314138528Sups call ipi_bitmap_handler 31576078Sjhb MEXITCOUNT 31676078Sjhb jmp doreti 317184113Skmacy#endif 31831638Sfsmp/* 319153141Sjhb * Executed by a CPU when it receives an IPI_STOP from another CPU. 32027007Sfsmp */ 32127007Sfsmp .text 32227007Sfsmp SUPERALIGN_TEXT 323115907SjhbIDTVEC(cpustop) 324153135Sjhb PUSH_FRAME 325153135Sjhb SET_KERNEL_SREGS 326209483Skib cld 32727007Sfsmp 328121986Sjhb movl lapic, %eax 329121986Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 33036135Stegge 331153141Sjhb call cpustop_handler 33227251Sfsmp 333153135Sjhb POP_FRAME 33427251Sfsmp iret 33527007Sfsmp 33648924Smsmith/* 337235796Siwasaki * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 338235796Siwasaki */ 339235796Siwasaki#ifndef XEN 340235796Siwasaki .text 341235796Siwasaki SUPERALIGN_TEXT 342235796SiwasakiIDTVEC(cpususpend) 343235796Siwasaki PUSH_FRAME 344235796Siwasaki SET_KERNEL_SREGS 345235796Siwasaki cld 346235796Siwasaki 347235796Siwasaki movl lapic, %eax 348235796Siwasaki movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 349235796Siwasaki 350235796Siwasaki call cpususpend_handler 351235796Siwasaki 352235796Siwasaki POP_FRAME 353235796Siwasaki jmp doreti_iret 354235796Siwasaki#endif 355235796Siwasaki 356235796Siwasaki/* 35748924Smsmith * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 35848924Smsmith * 35948924Smsmith * - Calls the generic rendezvous action function. 36048924Smsmith */ 36148924Smsmith .text 36248924Smsmith SUPERALIGN_TEXT 363115907SjhbIDTVEC(rendezvous) 364125405Sjhb PUSH_FRAME 365153135Sjhb SET_KERNEL_SREGS 366209483Skib cld 36748924Smsmith 368150697Sjhb#ifdef COUNT_IPIS 369150697Sjhb movl PCPU(CPUID), %eax 370150697Sjhb movl ipi_rendezvous_counts(,%eax,4), %eax 371150697Sjhb incl (%eax) 372150697Sjhb#endif 37373011Sjake call smp_rendezvous_action 37448924Smsmith 375121986Sjhb movl lapic, %eax 376121986Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 37748924Smsmith POP_FRAME 37848924Smsmith iret 37948924Smsmith 380112993Speter/* 381112993Speter * Clean up when we lose out on the lazy context switch optimization. 382112993Speter * ie: when we are about to release a PTD but a cpu is still borrowing it. 383112993Speter */ 384112993Speter SUPERALIGN_TEXT 385115907SjhbIDTVEC(lazypmap) 386125405Sjhb PUSH_FRAME 387153135Sjhb SET_KERNEL_SREGS 388209483Skib cld 389112993Speter 390112993Speter call pmap_lazyfix_action 391121986Sjhb 392153135Sjhb movl lapic, %eax 393121986Sjhb movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ 394112993Speter POP_FRAME 395112993Speter iret 396115907Sjhb#endif /* SMP */ 397