apic_vector.s revision 152701 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 *	from: vector.s, 386BSD 0.1 unknown origin
31 * $FreeBSD: head/sys/i386/i386/apic_vector.s 152701 2005-11-22 22:54:42Z jhb $
32 */
33
34/*
35 * Interrupt entry points for external interrupts triggered by I/O APICs
36 * as well as IPI handlers.
37 */
38
39#include "opt_smp.h"
40
41#include <machine/asmacros.h>
42#include <machine/apicreg.h>
43
44#include "assym.s"
45
46/*
47 * Macros to create and destroy a trap frame.
48 */
49#define PUSH_FRAME							\
50	pushl	$0 ;		/* dummy error code */			\
51	pushl	$0 ;		/* dummy trap type */			\
52	pushal ;		/* 8 ints */				\
53	pushl	%ds ;		/* save data and extra segments ... */	\
54	pushl	%es ;							\
55	pushl	%fs
56
57#define POP_FRAME							\
58	popl	%fs ;							\
59	popl	%es ;							\
60	popl	%ds ;							\
61	popal ;								\
62	addl	$4+4,%esp
63
64/*
65 * I/O Interrupt Entry Point.  Rather than having one entry point for
66 * each interrupt source, we use one entry point for each 32-bit word
67 * in the ISR.  The handler determines the highest bit set in the ISR,
68 * translates that into a vector, and passes the vector to the
69 * lapic_handle_intr() function.
70 */
71#define	ISR_VEC(index, vec_name)					\
72	.text ;								\
73	SUPERALIGN_TEXT ;						\
74IDTVEC(vec_name) ;							\
75	PUSH_FRAME ;							\
76	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
77	movl	%eax, %ds ;						\
78	movl	%eax, %es ;						\
79	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
80	movl	%eax, %fs ;						\
81	FAKE_MCOUNT(TF_EIP(%esp)) ;					\
82	movl	lapic, %edx ;	/* pointer to local APIC */		\
83	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
84	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
85	jz	2f ;							\
86	addl	$(32 * index),%eax ;					\
871: ;									\
88	pushl	%eax ;		/* pass the IRQ */			\
89	call	lapic_handle_intr ;					\
90	addl	$4, %esp ;	/* discard parameter */			\
91	MEXITCOUNT ;							\
92	jmp	doreti ;						\
932:	movl	$-1, %eax ;	/* send a vector of -1 */		\
94	jmp	1b
95
96/*
97 * Handle "spurious INTerrupts".
98 * Notes:
99 *  This is different than the "spurious INTerrupt" generated by an
100 *   8259 PIC for missing INTs.  See the APIC documentation for details.
101 *  This routine should NOT do an 'EOI' cycle.
102 */
103	.text
104	SUPERALIGN_TEXT
105IDTVEC(spuriousint)
106
107	/* No EOI cycle used here */
108
109	iret
110
111	ISR_VEC(1, apic_isr1)
112	ISR_VEC(2, apic_isr2)
113	ISR_VEC(3, apic_isr3)
114	ISR_VEC(4, apic_isr4)
115	ISR_VEC(5, apic_isr5)
116	ISR_VEC(6, apic_isr6)
117	ISR_VEC(7, apic_isr7)
118
119/*
120 * Local APIC periodic timer handler.
121 */
122	.text
123	SUPERALIGN_TEXT
124IDTVEC(timerint)
125	PUSH_FRAME
126	movl	$KDSEL, %eax	/* reload with kernel's data segment */
127	movl	%eax, %ds
128	movl	%eax, %es
129	movl	$KPSEL, %eax
130	movl	%eax, %fs
131
132	movl	lapic, %edx
133	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
134
135	FAKE_MCOUNT(TF_EIP(%esp))
136
137	pushl	$0		/* XXX convert trapframe to clockframe */
138	call	lapic_handle_timer
139	addl	$4, %esp	/* XXX convert clockframe to trapframe */
140	MEXITCOUNT
141	jmp	doreti
142
143#ifdef SMP
144/*
145 * Global address space TLB shootdown.
146 */
147	.text
148	SUPERALIGN_TEXT
149IDTVEC(invltlb)
150	pushl	%eax
151	pushl	%ds
152	movl	$KDSEL, %eax		/* Kernel data selector */
153	movl	%eax, %ds
154
155#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
156	pushl	%fs
157	movl	$KPSEL, %eax		/* Private space selector */
158	movl	%eax, %fs
159	movl	PCPU(CPUID), %eax
160	popl	%fs
161#ifdef COUNT_XINVLTLB_HITS
162	incl	xhits_gbl(,%eax,4)
163#endif
164#ifdef COUNT_IPIS
165	movl	ipi_invltlb_counts(,%eax,4),%eax
166	incl	(%eax)
167#endif
168#endif
169
170	movl	%cr3, %eax		/* invalidate the TLB */
171	movl	%eax, %cr3
172
173	movl	lapic, %eax
174	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
175
176	lock
177	incl	smp_tlb_wait
178
179	popl	%ds
180	popl	%eax
181	iret
182
183/*
184 * Single page TLB shootdown
185 */
186	.text
187	SUPERALIGN_TEXT
188IDTVEC(invlpg)
189	pushl	%eax
190	pushl	%ds
191	movl	$KDSEL, %eax		/* Kernel data selector */
192	movl	%eax, %ds
193
194#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
195	pushl	%fs
196	movl	$KPSEL, %eax		/* Private space selector */
197	movl	%eax, %fs
198	movl	PCPU(CPUID), %eax
199	popl	%fs
200#ifdef COUNT_XINVLTLB_HITS
201	incl	xhits_pg(,%eax,4)
202#endif
203#ifdef COUNT_IPIS
204	movl	ipi_invlpg_counts(,%eax,4),%eax
205	incl	(%eax)
206#endif
207#endif
208
209	movl	smp_tlb_addr1, %eax
210	invlpg	(%eax)			/* invalidate single page */
211
212	movl	lapic, %eax
213	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
214
215	lock
216	incl	smp_tlb_wait
217
218	popl	%ds
219	popl	%eax
220	iret
221
222/*
223 * Page range TLB shootdown.
224 */
225	.text
226	SUPERALIGN_TEXT
227IDTVEC(invlrng)
228	pushl	%eax
229	pushl	%edx
230	pushl	%ds
231	movl	$KDSEL, %eax		/* Kernel data selector */
232	movl	%eax, %ds
233
234#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
235	pushl	%fs
236	movl	$KPSEL, %eax		/* Private space selector */
237	movl	%eax, %fs
238	movl	PCPU(CPUID), %eax
239	popl	%fs
240#ifdef COUNT_XINVLTLB_HITS
241	incl	xhits_rng(,%eax,4)
242#endif
243#ifdef COUNT_IPIS
244	movl	ipi_invlrng_counts(,%eax,4),%eax
245	incl	(%eax)
246#endif
247#endif
248
249	movl	smp_tlb_addr1, %edx
250	movl	smp_tlb_addr2, %eax
2511:	invlpg	(%edx)			/* invalidate single page */
252	addl	$PAGE_SIZE, %edx
253	cmpl	%eax, %edx
254	jb	1b
255
256	movl	lapic, %eax
257	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
258
259	lock
260	incl	smp_tlb_wait
261
262	popl	%ds
263	popl	%edx
264	popl	%eax
265	iret
266
267/*
268 * Forward hardclock to another CPU.  Pushes a clockframe and calls
269 * forwarded_hardclock().
270 */
271	.text
272	SUPERALIGN_TEXT
273IDTVEC(ipi_intr_bitmap_handler)
274
275	PUSH_FRAME
276	movl	$KDSEL, %eax	/* reload with kernel's data segment */
277	movl	%eax, %ds
278	movl	%eax, %es
279	movl	$KPSEL, %eax
280	movl	%eax, %fs
281
282	movl	lapic, %edx
283	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
284
285	FAKE_MCOUNT(TF_EIP(%esp))
286
287	pushl	$0		/* XXX convert trapframe to clockframe */
288	call	ipi_bitmap_handler
289	addl	$4, %esp	/* XXX convert clockframe to trapframe */
290	MEXITCOUNT
291	jmp	doreti
292
293/*
294 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
295 *
296 *  - Signals its receipt.
297 *  - Waits for permission to restart.
298 *  - Signals its restart.
299 */
300	.text
301	SUPERALIGN_TEXT
302IDTVEC(cpustop)
303	pushl	%ebp
304	movl	%esp, %ebp
305	pushl	%eax
306	pushl	%ecx
307	pushl	%edx
308	pushl	%ds			/* save current data segment */
309	pushl	%es
310	pushl	%fs
311
312	movl	$KDSEL, %eax
313	movl	%eax, %ds		/* use KERNEL data segment */
314	movl	%eax, %es
315	movl	$KPSEL, %eax
316	movl	%eax, %fs
317
318	movl	lapic, %eax
319	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
320
321	movl	PCPU(CPUID), %eax
322	imull	$PCB_SIZE, %eax
323	leal	CNAME(stoppcbs)(%eax), %eax
324	pushl	%eax
325	call	CNAME(savectx)		/* Save process context */
326	addl	$4, %esp
327
328	movl	PCPU(CPUID), %eax
329
330	lock
331	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
3321:
333	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
334	jnc	1b
335
336	lock
337	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
338	lock
339	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
340
341	test	%eax, %eax
342	jnz	2f
343
344	movl	CNAME(cpustop_restartfunc), %eax
345	test	%eax, %eax
346	jz	2f
347	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
348
349	call	*%eax
3502:
351	popl	%fs
352	popl	%es
353	popl	%ds			/* restore previous data segment */
354	popl	%edx
355	popl	%ecx
356	popl	%eax
357	movl	%ebp, %esp
358	popl	%ebp
359	iret
360
361/*
362 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
363 *
364 * - Calls the generic rendezvous action function.
365 */
366	.text
367	SUPERALIGN_TEXT
368IDTVEC(rendezvous)
369	PUSH_FRAME
370	movl	$KDSEL, %eax
371	movl	%eax, %ds		/* use KERNEL data segment */
372	movl	%eax, %es
373	movl	$KPSEL, %eax
374	movl	%eax, %fs
375
376#ifdef COUNT_IPIS
377	movl	PCPU(CPUID), %eax
378	movl	ipi_rendezvous_counts(,%eax,4), %eax
379	incl	(%eax)
380#endif
381	call	smp_rendezvous_action
382
383	movl	lapic, %eax
384	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
385	POP_FRAME
386	iret
387
388/*
389 * Clean up when we lose out on the lazy context switch optimization.
390 * ie: when we are about to release a PTD but a cpu is still borrowing it.
391 */
392	SUPERALIGN_TEXT
393IDTVEC(lazypmap)
394	PUSH_FRAME
395	movl	$KDSEL, %eax
396	movl	%eax, %ds		/* use KERNEL data segment */
397	movl	%eax, %es
398	movl	$KPSEL, %eax
399	movl	%eax, %fs
400
401#ifdef COUNT_IPIS
402	movl	PCPU(CPUID), %eax
403	movl	ipi_lazypmap_counts(,%eax,4), %eax
404	incl	(%eax)
405#endif
406	call	pmap_lazyfix_action
407
408	movl	lapic, %eax
409	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
410	POP_FRAME
411	iret
412#endif /* SMP */
413