apic_vector.s revision 129620 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 *	from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: head/sys/i386/i386/apic_vector.s 129620 2004-05-23 16:50:55Z bde $
32 */
33
34/*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39#include <machine/asmacros.h>
40#include <machine/apicreg.h>
41#include <machine/smptests.h>
42
43#include "assym.s"
44
45/*
46 * Macros to create and destroy a trap frame.
47 */
48#define PUSH_FRAME							\
49	pushl	$0 ;		/* dummy error code */			\
50	pushl	$0 ;		/* dummy trap type */			\
51	pushal ;		/* 8 ints */				\
52	pushl	%ds ;		/* save data and extra segments ... */	\
53	pushl	%es ;							\
54	pushl	%fs
55
56#define POP_FRAME							\
57	popl	%fs ;							\
58	popl	%es ;							\
59	popl	%ds ;							\
60	popal ;								\
61	addl	$4+4,%esp
62
63/*
64 * I/O Interrupt Entry Point.  Rather than having one entry point for
65 * each interrupt source, we use one entry point for each 32-bit word
66 * in the ISR.  The handler determines the highest bit set in the ISR,
67 * translates that into a vector, and passes the vector to the
68 * lapic_handle_intr() function.
69 */
70#define	ISR_VEC(index, vec_name)					\
71	.text ;								\
72	SUPERALIGN_TEXT ;						\
73IDTVEC(vec_name) ;							\
74	PUSH_FRAME ;							\
75	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
76	movl	%eax, %ds ;						\
77	movl	%eax, %es ;						\
78	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
79	movl	%eax, %fs ;						\
80	FAKE_MCOUNT(TF_EIP(%esp)) ;					\
81	movl	lapic, %edx ;	/* pointer to local APIC */		\
82	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
83	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
84	jz	2f ;							\
85	addl	$(32 * index),%eax ;					\
861: ;									\
87	pushl	%eax ;		/* pass the IRQ */			\
88	call	lapic_handle_intr ;					\
89	addl	$4, %esp ;	/* discard parameter */			\
90	MEXITCOUNT ;							\
91	jmp	doreti ;						\
922:	movl	$-1, %eax ;	/* send a vector of -1 */		\
93	jmp	1b
94
95/*
96 * Handle "spurious INTerrupts".
97 * Notes:
98 *  This is different than the "spurious INTerrupt" generated by an
99 *   8259 PIC for missing INTs.  See the APIC documentation for details.
100 *  This routine should NOT do an 'EOI' cycle.
101 */
102	.text
103	SUPERALIGN_TEXT
104IDTVEC(spuriousint)
105
106	/* No EOI cycle used here */
107
108	iret
109
110MCOUNT_LABEL(bintr2)
111	ISR_VEC(1, apic_isr1)
112	ISR_VEC(2, apic_isr2)
113	ISR_VEC(3, apic_isr3)
114	ISR_VEC(4, apic_isr4)
115	ISR_VEC(5, apic_isr5)
116	ISR_VEC(6, apic_isr6)
117	ISR_VEC(7, apic_isr7)
118MCOUNT_LABEL(eintr2)
119
120#ifdef SMP
121/*
122 * Global address space TLB shootdown.
123 */
124	.text
125	SUPERALIGN_TEXT
126IDTVEC(invltlb)
127	pushl	%eax
128	pushl	%ds
129	movl	$KDSEL, %eax		/* Kernel data selector */
130	movl	%eax, %ds
131
132#ifdef COUNT_XINVLTLB_HITS
133	pushl	%fs
134	movl	$KPSEL, %eax		/* Private space selector */
135	movl	%eax, %fs
136	movl	PCPU(CPUID), %eax
137	popl	%fs
138	incl	xhits_gbl(,%eax,4)
139#endif /* COUNT_XINVLTLB_HITS */
140
141	movl	%cr3, %eax		/* invalidate the TLB */
142	movl	%eax, %cr3
143
144	movl	lapic, %eax
145	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
146
147	lock
148	incl	smp_tlb_wait
149
150	popl	%ds
151	popl	%eax
152	iret
153
154/*
155 * Single page TLB shootdown
156 */
157	.text
158	SUPERALIGN_TEXT
159IDTVEC(invlpg)
160	pushl	%eax
161	pushl	%ds
162	movl	$KDSEL, %eax		/* Kernel data selector */
163	movl	%eax, %ds
164
165#ifdef COUNT_XINVLTLB_HITS
166	pushl	%fs
167	movl	$KPSEL, %eax		/* Private space selector */
168	movl	%eax, %fs
169	movl	PCPU(CPUID), %eax
170	popl	%fs
171	incl	xhits_pg(,%eax,4)
172#endif /* COUNT_XINVLTLB_HITS */
173
174	movl	smp_tlb_addr1, %eax
175	invlpg	(%eax)			/* invalidate single page */
176
177	movl	lapic, %eax
178	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
179
180	lock
181	incl	smp_tlb_wait
182
183	popl	%ds
184	popl	%eax
185	iret
186
187/*
188 * Page range TLB shootdown.
189 */
190	.text
191	SUPERALIGN_TEXT
192IDTVEC(invlrng)
193	pushl	%eax
194	pushl	%edx
195	pushl	%ds
196	movl	$KDSEL, %eax		/* Kernel data selector */
197	movl	%eax, %ds
198
199#ifdef COUNT_XINVLTLB_HITS
200	pushl	%fs
201	movl	$KPSEL, %eax		/* Private space selector */
202	movl	%eax, %fs
203	movl	PCPU(CPUID), %eax
204	popl	%fs
205	incl	xhits_rng(,%eax,4)
206#endif /* COUNT_XINVLTLB_HITS */
207
208	movl	smp_tlb_addr1, %edx
209	movl	smp_tlb_addr2, %eax
2101:	invlpg	(%edx)			/* invalidate single page */
211	addl	$PAGE_SIZE, %edx
212	cmpl	%eax, %edx
213	jb	1b
214
215	movl	lapic, %eax
216	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
217
218	lock
219	incl	smp_tlb_wait
220
221	popl	%ds
222	popl	%edx
223	popl	%eax
224	iret
225
226/*
227 * Forward hardclock to another CPU.  Pushes a clockframe and calls
228 * forwarded_hardclock().
229 */
230	.text
231	SUPERALIGN_TEXT
232IDTVEC(hardclock)
233	PUSH_FRAME
234	movl	$KDSEL, %eax	/* reload with kernel's data segment */
235	movl	%eax, %ds
236	movl	%eax, %es
237	movl	$KPSEL, %eax
238	movl	%eax, %fs
239
240	movl	lapic, %edx
241	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
242
243	pushl	$0		/* XXX convert trapframe to clockframe */
244	call	forwarded_hardclock
245	addl	$4, %esp	/* XXX convert clockframe to trapframe */
246	MEXITCOUNT
247	jmp	doreti
248
249/*
250 * Forward statclock to another CPU.  Pushes a clockframe and calls
251 * forwarded_statclock().
252 */
253	.text
254	SUPERALIGN_TEXT
255IDTVEC(statclock)
256	PUSH_FRAME
257	movl	$KDSEL, %eax	/* reload with kernel's data segment */
258	movl	%eax, %ds
259	movl	%eax, %es
260	movl	$KPSEL, %eax
261	movl	%eax, %fs
262
263	movl	lapic, %edx
264	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
265
266	FAKE_MCOUNT(TF_EIP(%esp))
267
268	pushl	$0		/* XXX convert trapframe to clockframe */
269	call	forwarded_statclock
270	addl	$4, %esp	/* XXX convert clockframe to trapframe */
271	MEXITCOUNT
272	jmp	doreti
273
274/*
275 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
276 *
277 * The other CPU has already executed aston() or need_resched() on our
278 * current process, so we simply need to ack the interrupt and return
279 * via doreti to run ast().
280 */
281
282	.text
283	SUPERALIGN_TEXT
284IDTVEC(cpuast)
285	PUSH_FRAME
286	movl	$KDSEL, %eax
287	movl	%eax, %ds		/* use KERNEL data segment */
288	movl	%eax, %es
289	movl	$KPSEL, %eax
290	movl	%eax, %fs
291
292	movl	lapic, %edx
293	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
294
295	FAKE_MCOUNT(TF_EIP(%esp))
296
297	MEXITCOUNT
298	jmp	doreti
299
300/*
301 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
302 *
303 *  - Signals its receipt.
304 *  - Waits for permission to restart.
305 *  - Signals its restart.
306 */
307	.text
308	SUPERALIGN_TEXT
309IDTVEC(cpustop)
310	pushl	%ebp
311	movl	%esp, %ebp
312	pushl	%eax
313	pushl	%ecx
314	pushl	%edx
315	pushl	%ds			/* save current data segment */
316	pushl	%es
317	pushl	%fs
318
319	movl	$KDSEL, %eax
320	movl	%eax, %ds		/* use KERNEL data segment */
321	movl	%eax, %es
322	movl	$KPSEL, %eax
323	movl	%eax, %fs
324
325	movl	lapic, %eax
326	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
327
328	movl	PCPU(CPUID), %eax
329	imull	$PCB_SIZE, %eax
330	leal	CNAME(stoppcbs)(%eax), %eax
331	pushl	%eax
332	call	CNAME(savectx)		/* Save process context */
333	addl	$4, %esp
334
335	movl	PCPU(CPUID), %eax
336
337	lock
338	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
3391:
340	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
341	jnc	1b
342
343	lock
344	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
345	lock
346	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
347
348	test	%eax, %eax
349	jnz	2f
350
351	movl	CNAME(cpustop_restartfunc), %eax
352	test	%eax, %eax
353	jz	2f
354	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
355
356	call	*%eax
3572:
358	popl	%fs
359	popl	%es
360	popl	%ds			/* restore previous data segment */
361	popl	%edx
362	popl	%ecx
363	popl	%eax
364	movl	%ebp, %esp
365	popl	%ebp
366	iret
367
368/*
369 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
370 *
371 * - Calls the generic rendezvous action function.
372 */
373	.text
374	SUPERALIGN_TEXT
375IDTVEC(rendezvous)
376	PUSH_FRAME
377	movl	$KDSEL, %eax
378	movl	%eax, %ds		/* use KERNEL data segment */
379	movl	%eax, %es
380	movl	$KPSEL, %eax
381	movl	%eax, %fs
382
383	call	smp_rendezvous_action
384
385	movl	lapic, %eax
386	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
387	POP_FRAME
388	iret
389
390/*
391 * Clean up when we lose out on the lazy context switch optimization.
392 * ie: when we are about to release a PTD but a cpu is still borrowing it.
393 */
394	SUPERALIGN_TEXT
395IDTVEC(lazypmap)
396	PUSH_FRAME
397	movl	$KDSEL, %eax
398	movl	%eax, %ds		/* use KERNEL data segment */
399	movl	%eax, %es
400	movl	$KPSEL, %eax
401	movl	%eax, %fs
402
403	call	pmap_lazyfix_action
404
405	movl	lapic, %eax
406	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
407	POP_FRAME
408	iret
409#endif /* SMP */
410