apic_vector.s revision 152529 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 *	from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: head/sys/i386/i386/apic_vector.s 152529 2005-11-16 20:55:57Z jhb $
32 */
33
34/*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39#include <machine/asmacros.h>
40#include <machine/apicreg.h>
41#include <machine/smptests.h>
42
43#include "assym.s"
44
45/*
46 * Macros to create and destroy a trap frame.
47 */
48#define PUSH_FRAME							\
49	pushl	$0 ;		/* dummy error code */			\
50	pushl	$0 ;		/* dummy trap type */			\
51	pushal ;		/* 8 ints */				\
52	pushl	%ds ;		/* save data and extra segments ... */	\
53	pushl	%es ;							\
54	pushl	%fs
55
56#define POP_FRAME							\
57	popl	%fs ;							\
58	popl	%es ;							\
59	popl	%ds ;							\
60	popal ;								\
61	addl	$4+4,%esp
62
63/*
64 * I/O Interrupt Entry Point.  Rather than having one entry point for
65 * each interrupt source, we use one entry point for each 32-bit word
66 * in the ISR.  The handler determines the highest bit set in the ISR,
67 * translates that into a vector, and passes the vector to the
68 * lapic_handle_intr() function.
69 */
70#define	ISR_VEC(index, vec_name)					\
71	.text ;								\
72	SUPERALIGN_TEXT ;						\
73IDTVEC(vec_name) ;							\
74	PUSH_FRAME ;							\
75	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
76	movl	%eax, %ds ;						\
77	movl	%eax, %es ;						\
78	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
79	movl	%eax, %fs ;						\
80	FAKE_MCOUNT(TF_EIP(%esp)) ;					\
81	movl	lapic, %edx ;	/* pointer to local APIC */		\
82	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
83	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
84	jz	2f ;							\
85	addl	$(32 * index),%eax ;					\
861: ;									\
87	pushl	%eax ;		/* pass the IRQ */			\
88	call	lapic_handle_intr ;					\
89	addl	$4, %esp ;	/* discard parameter */			\
90	MEXITCOUNT ;							\
91	jmp	doreti ;						\
922:	movl	$-1, %eax ;	/* send a vector of -1 */		\
93	jmp	1b
94
95/*
96 * Handle "spurious INTerrupts".
97 * Notes:
98 *  This is different than the "spurious INTerrupt" generated by an
99 *   8259 PIC for missing INTs.  See the APIC documentation for details.
100 *  This routine should NOT do an 'EOI' cycle.
101 */
102	.text
103	SUPERALIGN_TEXT
104IDTVEC(spuriousint)
105
106	/* No EOI cycle used here */
107
108	iret
109
110	ISR_VEC(1, apic_isr1)
111	ISR_VEC(2, apic_isr2)
112	ISR_VEC(3, apic_isr3)
113	ISR_VEC(4, apic_isr4)
114	ISR_VEC(5, apic_isr5)
115	ISR_VEC(6, apic_isr6)
116	ISR_VEC(7, apic_isr7)
117
118/*
119 * Local APIC periodic timer handler.
120 */
121	.text
122	SUPERALIGN_TEXT
123IDTVEC(timerint)
124	PUSH_FRAME
125	movl	$KDSEL, %eax	/* reload with kernel's data segment */
126	movl	%eax, %ds
127	movl	%eax, %es
128	movl	$KPSEL, %eax
129	movl	%eax, %fs
130
131	movl	lapic, %edx
132	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
133
134	FAKE_MCOUNT(TF_EIP(%esp))
135
136	pushl	$0		/* XXX convert trapframe to clockframe */
137	call	lapic_handle_timer
138	addl	$4, %esp	/* XXX convert clockframe to trapframe */
139	MEXITCOUNT
140	jmp	doreti
141
142#ifdef SMP
143/*
144 * Global address space TLB shootdown.
145 */
146	.text
147	SUPERALIGN_TEXT
148IDTVEC(invltlb)
149	pushl	%eax
150	pushl	%ds
151	movl	$KDSEL, %eax		/* Kernel data selector */
152	movl	%eax, %ds
153
154#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
155	pushl	%fs
156	movl	$KPSEL, %eax		/* Private space selector */
157	movl	%eax, %fs
158	movl	PCPU(CPUID), %eax
159	popl	%fs
160#ifdef COUNT_XINVLTLB_HITS
161	incl	xhits_gbl(,%eax,4)
162#endif
163#ifdef COUNT_IPIS
164	movl	ipi_invltlb_counts(,%eax,4),%eax
165	incl	(%eax)
166#endif
167#endif
168
169	movl	%cr3, %eax		/* invalidate the TLB */
170	movl	%eax, %cr3
171
172	movl	lapic, %eax
173	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
174
175	lock
176	incl	smp_tlb_wait
177
178	popl	%ds
179	popl	%eax
180	iret
181
182/*
183 * Single page TLB shootdown
184 */
185	.text
186	SUPERALIGN_TEXT
187IDTVEC(invlpg)
188	pushl	%eax
189	pushl	%ds
190	movl	$KDSEL, %eax		/* Kernel data selector */
191	movl	%eax, %ds
192
193#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
194	pushl	%fs
195	movl	$KPSEL, %eax		/* Private space selector */
196	movl	%eax, %fs
197	movl	PCPU(CPUID), %eax
198	popl	%fs
199#ifdef COUNT_XINVLTLB_HITS
200	incl	xhits_pg(,%eax,4)
201#endif
202#ifdef COUNT_IPIS
203	movl	ipi_invlpg_counts(,%eax,4),%eax
204	incl	(%eax)
205#endif
206#endif
207
208	movl	smp_tlb_addr1, %eax
209	invlpg	(%eax)			/* invalidate single page */
210
211	movl	lapic, %eax
212	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
213
214	lock
215	incl	smp_tlb_wait
216
217	popl	%ds
218	popl	%eax
219	iret
220
221/*
222 * Page range TLB shootdown.
223 */
224	.text
225	SUPERALIGN_TEXT
226IDTVEC(invlrng)
227	pushl	%eax
228	pushl	%edx
229	pushl	%ds
230	movl	$KDSEL, %eax		/* Kernel data selector */
231	movl	%eax, %ds
232
233#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
234	pushl	%fs
235	movl	$KPSEL, %eax		/* Private space selector */
236	movl	%eax, %fs
237	movl	PCPU(CPUID), %eax
238	popl	%fs
239#ifdef COUNT_XINVLTLB_HITS
240	incl	xhits_rng(,%eax,4)
241#endif
242#ifdef COUNT_IPIS
243	movl	ipi_invlrng_counts(,%eax,4),%eax
244	incl	(%eax)
245#endif
246#endif
247
248	movl	smp_tlb_addr1, %edx
249	movl	smp_tlb_addr2, %eax
2501:	invlpg	(%edx)			/* invalidate single page */
251	addl	$PAGE_SIZE, %edx
252	cmpl	%eax, %edx
253	jb	1b
254
255	movl	lapic, %eax
256	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
257
258	lock
259	incl	smp_tlb_wait
260
261	popl	%ds
262	popl	%edx
263	popl	%eax
264	iret
265
266/*
267 * Forward hardclock to another CPU.  Pushes a clockframe and calls
268 * forwarded_hardclock().
269 */
270	.text
271	SUPERALIGN_TEXT
272IDTVEC(ipi_intr_bitmap_handler)
273
274	PUSH_FRAME
275	movl	$KDSEL, %eax	/* reload with kernel's data segment */
276	movl	%eax, %ds
277	movl	%eax, %es
278	movl	$KPSEL, %eax
279	movl	%eax, %fs
280
281	movl	lapic, %edx
282	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
283
284	FAKE_MCOUNT(TF_EIP(%esp))
285
286	pushl	$0		/* XXX convert trapframe to clockframe */
287	call	ipi_bitmap_handler
288	addl	$4, %esp	/* XXX convert clockframe to trapframe */
289	MEXITCOUNT
290	jmp	doreti
291
292/*
293 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
294 *
295 *  - Signals its receipt.
296 *  - Waits for permission to restart.
297 *  - Signals its restart.
298 */
299	.text
300	SUPERALIGN_TEXT
301IDTVEC(cpustop)
302	pushl	%ebp
303	movl	%esp, %ebp
304	pushl	%eax
305	pushl	%ecx
306	pushl	%edx
307	pushl	%ds			/* save current data segment */
308	pushl	%es
309	pushl	%fs
310
311	movl	$KDSEL, %eax
312	movl	%eax, %ds		/* use KERNEL data segment */
313	movl	%eax, %es
314	movl	$KPSEL, %eax
315	movl	%eax, %fs
316
317	movl	lapic, %eax
318	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
319
320	movl	PCPU(CPUID), %eax
321	imull	$PCB_SIZE, %eax
322	leal	CNAME(stoppcbs)(%eax), %eax
323	pushl	%eax
324	call	CNAME(savectx)		/* Save process context */
325	addl	$4, %esp
326
327	movl	PCPU(CPUID), %eax
328
329	lock
330	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
3311:
332	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
333	jnc	1b
334
335	lock
336	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
337	lock
338	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
339
340	test	%eax, %eax
341	jnz	2f
342
343	movl	CNAME(cpustop_restartfunc), %eax
344	test	%eax, %eax
345	jz	2f
346	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
347
348	call	*%eax
3492:
350	popl	%fs
351	popl	%es
352	popl	%ds			/* restore previous data segment */
353	popl	%edx
354	popl	%ecx
355	popl	%eax
356	movl	%ebp, %esp
357	popl	%ebp
358	iret
359
360/*
361 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
362 *
363 * - Calls the generic rendezvous action function.
364 */
365	.text
366	SUPERALIGN_TEXT
367IDTVEC(rendezvous)
368	PUSH_FRAME
369	movl	$KDSEL, %eax
370	movl	%eax, %ds		/* use KERNEL data segment */
371	movl	%eax, %es
372	movl	$KPSEL, %eax
373	movl	%eax, %fs
374
375#ifdef COUNT_IPIS
376	movl	PCPU(CPUID), %eax
377	movl	ipi_rendezvous_counts(,%eax,4), %eax
378	incl	(%eax)
379#endif
380	call	smp_rendezvous_action
381
382	movl	lapic, %eax
383	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
384	POP_FRAME
385	iret
386
387/*
388 * Clean up when we lose out on the lazy context switch optimization.
389 * ie: when we are about to release a PTD but a cpu is still borrowing it.
390 */
391	SUPERALIGN_TEXT
392IDTVEC(lazypmap)
393	PUSH_FRAME
394	movl	$KDSEL, %eax
395	movl	%eax, %ds		/* use KERNEL data segment */
396	movl	%eax, %es
397	movl	$KPSEL, %eax
398	movl	%eax, %fs
399
400#ifdef COUNT_IPIS
401	movl	PCPU(CPUID), %eax
402	movl	ipi_lazypmap_counts(,%eax,4), %eax
403	incl	(%eax)
404#endif
405	call	pmap_lazyfix_action
406
407	movl	lapic, %eax
408	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
409	POP_FRAME
410	iret
411#endif /* SMP */
412