apic_vector.s revision 255040 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 *	from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: head/sys/i386/i386/apic_vector.s 255040 2013-08-29 19:52:18Z gibbs $
32 */
33
34/*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39#include "opt_smp.h"
40
41#include <machine/asmacros.h>
42#include <x86/apicreg.h>
43
44#include "assym.s"
45
46/*
47 * I/O Interrupt Entry Point.  Rather than having one entry point for
48 * each interrupt source, we use one entry point for each 32-bit word
49 * in the ISR.  The handler determines the highest bit set in the ISR,
50 * translates that into a vector, and passes the vector to the
51 * lapic_handle_intr() function.
52 */
53#define	ISR_VEC(index, vec_name)					\
54	.text ;								\
55	SUPERALIGN_TEXT ;						\
56IDTVEC(vec_name) ;							\
57	PUSH_FRAME ;							\
58	SET_KERNEL_SREGS ;						\
59	cld ;								\
60	FAKE_MCOUNT(TF_EIP(%esp)) ;					\
61	movl	lapic, %edx ;	/* pointer to local APIC */		\
62	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
63	bsrl	%eax, %eax ;	/* index of highest set bit in ISR */	\
64	jz	1f ;							\
65	addl	$(32 * index),%eax ;					\
66	pushl	%esp		;                                       \
67	pushl	%eax ;		/* pass the IRQ */			\
68	call	lapic_handle_intr ;					\
69	addl	$8, %esp ;	/* discard parameter */			\
701: ;									\
71	MEXITCOUNT ;							\
72	jmp	doreti
73
74/*
75 * Handle "spurious INTerrupts".
76 * Notes:
77 *  This is different than the "spurious INTerrupt" generated by an
78 *   8259 PIC for missing INTs.  See the APIC documentation for details.
79 *  This routine should NOT do an 'EOI' cycle.
80 */
81	.text
82	SUPERALIGN_TEXT
83IDTVEC(spuriousint)
84
85	/* No EOI cycle used here */
86
87	iret
88
89	ISR_VEC(1, apic_isr1)
90	ISR_VEC(2, apic_isr2)
91	ISR_VEC(3, apic_isr3)
92	ISR_VEC(4, apic_isr4)
93	ISR_VEC(5, apic_isr5)
94	ISR_VEC(6, apic_isr6)
95	ISR_VEC(7, apic_isr7)
96
97/*
98 * Local APIC periodic timer handler.
99 */
100	.text
101	SUPERALIGN_TEXT
102IDTVEC(timerint)
103	PUSH_FRAME
104	SET_KERNEL_SREGS
105	cld
106	FAKE_MCOUNT(TF_EIP(%esp))
107	pushl	%esp
108	call	lapic_handle_timer
109	add	$4, %esp
110	MEXITCOUNT
111	jmp	doreti
112
113/*
114 * Local APIC CMCI handler.
115 */
116	.text
117	SUPERALIGN_TEXT
118IDTVEC(cmcint)
119	PUSH_FRAME
120	SET_KERNEL_SREGS
121	cld
122	FAKE_MCOUNT(TF_EIP(%esp))
123	call	lapic_handle_cmc
124	MEXITCOUNT
125	jmp	doreti
126
127/*
128 * Local APIC error interrupt handler.
129 */
130	.text
131	SUPERALIGN_TEXT
132IDTVEC(errorint)
133	PUSH_FRAME
134	SET_KERNEL_SREGS
135	cld
136	FAKE_MCOUNT(TF_EIP(%esp))
137	call	lapic_handle_error
138	MEXITCOUNT
139	jmp	doreti
140
141#ifdef XENHVM
142/*
143 * Xen event channel upcall interrupt handler.
144 * Only used when the hypervisor supports direct vector callbacks.
145 */
146	.text
147	SUPERALIGN_TEXT
148IDTVEC(xen_intr_upcall)
149	PUSH_FRAME
150	SET_KERNEL_SREGS
151	cld
152	FAKE_MCOUNT(TF_EIP(%esp))
153	pushl	%esp
154	call	xen_intr_handle_upcall
155	add	$4, %esp
156	MEXITCOUNT
157	jmp	doreti
158#endif
159
160#ifdef SMP
161/*
162 * Global address space TLB shootdown.
163 */
164	.text
165	SUPERALIGN_TEXT
166IDTVEC(invltlb)
167	pushl	%eax
168	pushl	%ds
169	movl	$KDSEL, %eax		/* Kernel data selector */
170	movl	%eax, %ds
171
172#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
173	pushl	%fs
174	movl	$KPSEL, %eax		/* Private space selector */
175	movl	%eax, %fs
176	movl	PCPU(CPUID), %eax
177	popl	%fs
178#ifdef COUNT_XINVLTLB_HITS
179	incl	xhits_gbl(,%eax,4)
180#endif
181#ifdef COUNT_IPIS
182	movl	ipi_invltlb_counts(,%eax,4),%eax
183	incl	(%eax)
184#endif
185#endif
186
187	movl	%cr3, %eax		/* invalidate the TLB */
188	movl	%eax, %cr3
189
190	movl	lapic, %eax
191	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
192
193	lock
194	incl	smp_tlb_wait
195
196	popl	%ds
197	popl	%eax
198	iret
199
200/*
201 * Single page TLB shootdown
202 */
203	.text
204	SUPERALIGN_TEXT
205IDTVEC(invlpg)
206	pushl	%eax
207	pushl	%ds
208	movl	$KDSEL, %eax		/* Kernel data selector */
209	movl	%eax, %ds
210
211#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
212	pushl	%fs
213	movl	$KPSEL, %eax		/* Private space selector */
214	movl	%eax, %fs
215	movl	PCPU(CPUID), %eax
216	popl	%fs
217#ifdef COUNT_XINVLTLB_HITS
218	incl	xhits_pg(,%eax,4)
219#endif
220#ifdef COUNT_IPIS
221	movl	ipi_invlpg_counts(,%eax,4),%eax
222	incl	(%eax)
223#endif
224#endif
225
226	movl	smp_tlb_addr1, %eax
227	invlpg	(%eax)			/* invalidate single page */
228
229	movl	lapic, %eax
230	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
231
232	lock
233	incl	smp_tlb_wait
234
235	popl	%ds
236	popl	%eax
237	iret
238
239/*
240 * Page range TLB shootdown.
241 */
242	.text
243	SUPERALIGN_TEXT
244IDTVEC(invlrng)
245	pushl	%eax
246	pushl	%edx
247	pushl	%ds
248	movl	$KDSEL, %eax		/* Kernel data selector */
249	movl	%eax, %ds
250
251#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
252	pushl	%fs
253	movl	$KPSEL, %eax		/* Private space selector */
254	movl	%eax, %fs
255	movl	PCPU(CPUID), %eax
256	popl	%fs
257#ifdef COUNT_XINVLTLB_HITS
258	incl	xhits_rng(,%eax,4)
259#endif
260#ifdef COUNT_IPIS
261	movl	ipi_invlrng_counts(,%eax,4),%eax
262	incl	(%eax)
263#endif
264#endif
265
266	movl	smp_tlb_addr1, %edx
267	movl	smp_tlb_addr2, %eax
2681:	invlpg	(%edx)			/* invalidate single page */
269	addl	$PAGE_SIZE, %edx
270	cmpl	%eax, %edx
271	jb	1b
272
273	movl	lapic, %eax
274	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
275
276	lock
277	incl	smp_tlb_wait
278
279	popl	%ds
280	popl	%edx
281	popl	%eax
282	iret
283
284/*
285 * Invalidate cache.
286 */
287	.text
288	SUPERALIGN_TEXT
289IDTVEC(invlcache)
290	pushl	%eax
291	pushl	%ds
292	movl	$KDSEL, %eax		/* Kernel data selector */
293	movl	%eax, %ds
294
295#ifdef COUNT_IPIS
296	pushl	%fs
297	movl	$KPSEL, %eax		/* Private space selector */
298	movl	%eax, %fs
299	movl	PCPU(CPUID), %eax
300	popl	%fs
301	movl	ipi_invlcache_counts(,%eax,4),%eax
302	incl	(%eax)
303#endif
304
305	wbinvd
306
307	movl	lapic, %eax
308	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
309
310	lock
311	incl	smp_tlb_wait
312
313	popl	%ds
314	popl	%eax
315	iret
316
317/*
318 * Handler for IPIs sent via the per-cpu IPI bitmap.
319 */
320#ifndef XEN
321	.text
322	SUPERALIGN_TEXT
323IDTVEC(ipi_intr_bitmap_handler)
324	PUSH_FRAME
325	SET_KERNEL_SREGS
326	cld
327
328	movl	lapic, %edx
329	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
330
331	FAKE_MCOUNT(TF_EIP(%esp))
332
333	call	ipi_bitmap_handler
334	MEXITCOUNT
335	jmp	doreti
336#endif
337/*
338 * Executed by a CPU when it receives an IPI_STOP from another CPU.
339 */
340	.text
341	SUPERALIGN_TEXT
342IDTVEC(cpustop)
343	PUSH_FRAME
344	SET_KERNEL_SREGS
345	cld
346
347	movl	lapic, %eax
348	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
349
350	call	cpustop_handler
351
352	POP_FRAME
353	iret
354
355/*
356 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
357 */
358#ifndef XEN
359	.text
360	SUPERALIGN_TEXT
361IDTVEC(cpususpend)
362	PUSH_FRAME
363	SET_KERNEL_SREGS
364	cld
365
366	movl	lapic, %eax
367	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
368
369	call	cpususpend_handler
370
371	POP_FRAME
372	jmp	doreti_iret
373#endif
374
375/*
376 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
377 *
378 * - Calls the generic rendezvous action function.
379 */
380	.text
381	SUPERALIGN_TEXT
382IDTVEC(rendezvous)
383	PUSH_FRAME
384	SET_KERNEL_SREGS
385	cld
386
387#ifdef COUNT_IPIS
388	movl	PCPU(CPUID), %eax
389	movl	ipi_rendezvous_counts(,%eax,4), %eax
390	incl	(%eax)
391#endif
392	call	smp_rendezvous_action
393
394	movl	lapic, %eax
395	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
396	POP_FRAME
397	iret
398
399/*
400 * Clean up when we lose out on the lazy context switch optimization.
401 * ie: when we are about to release a PTD but a cpu is still borrowing it.
402 */
403	SUPERALIGN_TEXT
404IDTVEC(lazypmap)
405	PUSH_FRAME
406	SET_KERNEL_SREGS
407	cld
408
409	call	pmap_lazyfix_action
410
411	movl	lapic, %eax
412	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
413	POP_FRAME
414	iret
415#endif /* SMP */
416