apic_vector.s revision 121986 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 *    must display the following acknowledgement:
16 *	This product includes software developed by the University of
17 *	California, Berkeley and its contributors.
18 * 4. Neither the name of the University nor the names of its contributors
19 *    may be used to endorse or promote products derived from this software
20 *    without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 *	from: vector.s, 386BSD 0.1 unknown origin
35 * $FreeBSD: head/sys/i386/i386/apic_vector.s 121986 2003-11-03 21:53:38Z jhb $
36 */
37
38/*
39 * Interrupt entry points for external interrupts triggered by I/O APICs
40 * as well as IPI handlers.
41 */
42
43#include <machine/asmacros.h>
44#include <machine/apicreg.h>
45#include <machine/smptests.h>
46
47#include "assym.s"
48
49/*
50 * Macros to create and destroy a trap frame.
51 */
52#define PUSH_FRAME							\
53	pushl	$0 ;		/* dummy error code */			\
54	pushl	$0 ;		/* dummy trap type */			\
55	pushal ;		/* 8 ints */				\
56	pushl	%ds ;		/* save data and extra segments ... */	\
57	pushl	%es ;							\
58	pushl	%fs
59
60#define POP_FRAME							\
61	popl	%fs ;							\
62	popl	%es ;							\
63	popl	%ds ;							\
64	popal ;								\
65	addl	$4+4,%esp
66
67/*
68 * I/O Interrupt Entry Point.  Rather than having one entry point for
69 * each interrupt source, we use one entry point for each 32-bit word
70 * in the ISR.  The handler determines the highest bit set in the ISR,
71 * translates that into a vector, and passes the vector to the
72 * lapic_handle_intr() function.
73 */
74#define	ISR_VEC(index, vec_name)					\
75	.text ;								\
76	SUPERALIGN_TEXT ;						\
77IDTVEC(vec_name) ;							\
78	PUSH_FRAME ;							\
79	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
80	mov	%ax, %ds ;						\
81	mov	%ax, %es ;						\
82	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
83	mov	%ax, %fs ;						\
84	movl	lapic, %edx ;	/* pointer to local APIC */		\
85	movl	PCPU(CURTHREAD), %ebx ;					\
86	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
87	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
88	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
89	jz	2f ;							\
90	addl	$(32 * index),%eax ;					\
911: ;									\
92	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX avoid double count */	\
93	pushl	%eax ;		/* pass the IRQ */			\
94	call	lapic_handle_intr ;					\
95	addl	$4, %esp ;	/* discard parameter */			\
96	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
97	MEXITCOUNT ;							\
98	jmp	doreti ;						\
992:	movl	$-1, %eax ;	/* send a vector of -1 */		\
100	jmp	1b
101
102/*
103 * Handle "spurious INTerrupts".
104 * Notes:
105 *  This is different than the "spurious INTerrupt" generated by an
106 *   8259 PIC for missing INTs.  See the APIC documentation for details.
107 *  This routine should NOT do an 'EOI' cycle.
108 */
109	.text
110	SUPERALIGN_TEXT
111IDTVEC(spuriousint)
112
113	/* No EOI cycle used here */
114
115	iret
116
117MCOUNT_LABEL(bintr2)
118	ISR_VEC(1,apic_isr1)
119	ISR_VEC(2,apic_isr2)
120	ISR_VEC(3,apic_isr3)
121	ISR_VEC(4,apic_isr4)
122	ISR_VEC(5,apic_isr5)
123MCOUNT_LABEL(eintr2)
124
125#ifdef SMP
126/*
127 * Global address space TLB shootdown.
128 */
129	.text
130	SUPERALIGN_TEXT
131IDTVEC(invltlb)
132	pushl	%eax
133	pushl	%ds
134	movl	$KDSEL, %eax		/* Kernel data selector */
135	mov	%ax, %ds
136
137#ifdef COUNT_XINVLTLB_HITS
138	pushl	%fs
139	movl	$KPSEL, %eax		/* Private space selector */
140	mov	%ax, %fs
141	movl	PCPU(CPUID), %eax
142	popl	%fs
143	incl	xhits_gbl(,%eax,4)
144#endif /* COUNT_XINVLTLB_HITS */
145
146	movl	%cr3, %eax		/* invalidate the TLB */
147	movl	%eax, %cr3
148
149	movl	lapic, %eax
150	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
151
152	lock
153	incl	smp_tlb_wait
154
155	popl	%ds
156	popl	%eax
157	iret
158
159/*
160 * Single page TLB shootdown
161 */
162	.text
163	SUPERALIGN_TEXT
164IDTVEC(invlpg)
165	pushl	%eax
166	pushl	%ds
167	movl	$KDSEL, %eax		/* Kernel data selector */
168	mov	%ax, %ds
169
170#ifdef COUNT_XINVLTLB_HITS
171	pushl	%fs
172	movl	$KPSEL, %eax		/* Private space selector */
173	mov	%ax, %fs
174	movl	PCPU(CPUID), %eax
175	popl	%fs
176	incl	xhits_pg(,%eax,4)
177#endif /* COUNT_XINVLTLB_HITS */
178
179	movl	smp_tlb_addr1, %eax
180	invlpg	(%eax)			/* invalidate single page */
181
182	movl	lapic, %eax
183	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
184
185	lock
186	incl	smp_tlb_wait
187
188	popl	%ds
189	popl	%eax
190	iret
191
192/*
193 * Page range TLB shootdown.
194 */
195	.text
196	SUPERALIGN_TEXT
197IDTVEC(invlrng)
198	pushl	%eax
199	pushl	%edx
200	pushl	%ds
201	movl	$KDSEL, %eax		/* Kernel data selector */
202	mov	%ax, %ds
203
204#ifdef COUNT_XINVLTLB_HITS
205	pushl	%fs
206	movl	$KPSEL, %eax		/* Private space selector */
207	mov	%ax, %fs
208	movl	PCPU(CPUID), %eax
209	popl	%fs
210	incl	xhits_rng(,%eax,4)
211#endif /* COUNT_XINVLTLB_HITS */
212
213	movl	smp_tlb_addr1, %edx
214	movl	smp_tlb_addr2, %eax
2151:	invlpg	(%edx)			/* invalidate single page */
216	addl	$PAGE_SIZE, %edx
217	cmpl	%eax, %edx
218	jb	1b
219
220	movl	lapic, %eax
221	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
222
223	lock
224	incl	smp_tlb_wait
225
226	popl	%ds
227	popl	%edx
228	popl	%eax
229	iret
230
231/*
232 * Forward hardclock to another CPU.  Pushes a clockframe and calls
233 * forwarded_hardclock().
234 */
235	.text
236	SUPERALIGN_TEXT
237IDTVEC(hardclock)
238	PUSH_FRAME
239	movl	$KDSEL, %eax	/* reload with kernel's data segment */
240	mov	%ax, %ds
241	mov	%ax, %es
242	movl	$KPSEL, %eax
243	mov	%ax, %fs
244
245	movl	lapic, %edx
246	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
247
248	movl	PCPU(CURTHREAD),%ebx
249	incl	TD_INTR_NESTING_LEVEL(%ebx)
250	pushl	$0		/* XXX convert trapframe to clockframe */
251	call	forwarded_hardclock
252	addl	$4, %esp	/* XXX convert clockframe to trapframe */
253	decl	TD_INTR_NESTING_LEVEL(%ebx)
254	MEXITCOUNT
255	jmp	doreti
256
257/*
258 * Forward statclock to another CPU.  Pushes a clockframe and calls
259 * forwarded_statclock().
260 */
261	.text
262	SUPERALIGN_TEXT
263IDTVEC(statclock)
264	PUSH_FRAME
265	movl	$KDSEL, %eax	/* reload with kernel's data segment */
266	mov	%ax, %ds
267	mov	%ax, %es
268	movl	$KPSEL, %eax
269	mov	%ax, %fs
270
271	movl	lapic, %edx
272	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
273
274	FAKE_MCOUNT(13*4(%esp))
275
276	movl	PCPU(CURTHREAD),%ebx
277	incl	TD_INTR_NESTING_LEVEL(%ebx)
278	pushl	$0		/* XXX convert trapframe to clockframe */
279	call	forwarded_statclock
280	addl	$4, %esp	/* XXX convert clockframe to trapframe */
281	decl	TD_INTR_NESTING_LEVEL(%ebx)
282	MEXITCOUNT
283	jmp	doreti
284
285/*
286 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
287 *
288 * The other CPU has already executed aston() or need_resched() on our
289 * current process, so we simply need to ack the interrupt and return
290 * via doreti to run ast().
291 */
292
293	.text
294	SUPERALIGN_TEXT
295IDTVEC(cpuast)
296	PUSH_FRAME
297	movl	$KDSEL, %eax
298	mov	%ax, %ds		/* use KERNEL data segment */
299	mov	%ax, %es
300	movl	$KPSEL, %eax
301	mov	%ax, %fs
302
303	movl	lapic, %edx
304	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
305
306	FAKE_MCOUNT(13*4(%esp))
307
308	MEXITCOUNT
309	jmp	doreti
310
311/*
312 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
313 *
314 *  - Signals its receipt.
315 *  - Waits for permission to restart.
316 *  - Signals its restart.
317 */
318	.text
319	SUPERALIGN_TEXT
320IDTVEC(cpustop)
321	pushl	%ebp
322	movl	%esp, %ebp
323	pushl	%eax
324	pushl	%ecx
325	pushl	%edx
326	pushl	%ds			/* save current data segment */
327	pushl	%es
328	pushl	%fs
329
330	movl	$KDSEL, %eax
331	mov	%ax, %ds		/* use KERNEL data segment */
332	mov	%ax, %es
333	movl	$KPSEL, %eax
334	mov	%ax, %fs
335
336	movl	lapic, %eax
337	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
338
339	movl	PCPU(CPUID), %eax
340	imull	$PCB_SIZE, %eax
341	leal	CNAME(stoppcbs)(%eax), %eax
342	pushl	%eax
343	call	CNAME(savectx)		/* Save process context */
344	addl	$4, %esp
345
346	movl	PCPU(CPUID), %eax
347
348	lock
349	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
3501:
351	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
352	jnc	1b
353
354	lock
355	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
356	lock
357	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
358
359	test	%eax, %eax
360	jnz	2f
361
362	movl	CNAME(cpustop_restartfunc), %eax
363	test	%eax, %eax
364	jz	2f
365	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
366
367	call	*%eax
3682:
369	popl	%fs
370	popl	%es
371	popl	%ds			/* restore previous data segment */
372	popl	%edx
373	popl	%ecx
374	popl	%eax
375	movl	%ebp, %esp
376	popl	%ebp
377	iret
378
379/*
380 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
381 *
382 * - Calls the generic rendezvous action function.
383 */
384	.text
385	SUPERALIGN_TEXT
386IDTVEC(rendezvous)
387	PUSH_FRAME
388	movl	$KDSEL, %eax
389	mov	%ax, %ds		/* use KERNEL data segment */
390	mov	%ax, %es
391	movl	$KPSEL, %eax
392	mov	%ax, %fs
393
394	call	smp_rendezvous_action
395
396	movl	lapic, %eax
397	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
398	POP_FRAME
399	iret
400
401/*
402 * Clean up when we lose out on the lazy context switch optimization.
403 * ie: when we are about to release a PTD but a cpu is still borrowing it.
404 */
405	SUPERALIGN_TEXT
406IDTVEC(lazypmap)
407	PUSH_FRAME
408	movl	$KDSEL, %eax
409	mov	%ax, %ds		/* use KERNEL data segment */
410	mov	%ax, %es
411	movl	$KPSEL, %eax
412	mov	%ax, %fs
413
414	call	pmap_lazyfix_action
415
416	movl	lapic, %eax
417	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
418	POP_FRAME
419	iret
420#endif /* SMP */
421