apic_vector.s revision 123106 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 *    must display the following acknowledgement:
16 *	This product includes software developed by the University of
17 *	California, Berkeley and its contributors.
18 * 4. Neither the name of the University nor the names of its contributors
19 *    may be used to endorse or promote products derived from this software
20 *    without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 *	from: vector.s, 386BSD 0.1 unknown origin
35 * $FreeBSD: head/sys/i386/i386/apic_vector.s 123106 2003-12-02 15:02:12Z bde $
36 */
37
38/*
39 * Interrupt entry points for external interrupts triggered by I/O APICs
40 * as well as IPI handlers.
41 */
42
43#include <machine/asmacros.h>
44#include <machine/apicreg.h>
45#include <machine/smptests.h>
46
47#include "assym.s"
48
49/*
50 * Macros to create and destroy a trap frame.
51 */
52#define PUSH_FRAME							\
53	pushl	$0 ;		/* dummy error code */			\
54	pushl	$0 ;		/* dummy trap type */			\
55	pushal ;		/* 8 ints */				\
56	pushl	%ds ;		/* save data and extra segments ... */	\
57	pushl	%es ;							\
58	pushl	%fs
59
60#define POP_FRAME							\
61	popl	%fs ;							\
62	popl	%es ;							\
63	popl	%ds ;							\
64	popal ;								\
65	addl	$4+4,%esp
66
67/*
68 * I/O Interrupt Entry Point.  Rather than having one entry point for
69 * each interrupt source, we use one entry point for each 32-bit word
70 * in the ISR.  The handler determines the highest bit set in the ISR,
71 * translates that into a vector, and passes the vector to the
72 * lapic_handle_intr() function.
73 */
74#define	ISR_VEC(index, vec_name)					\
75	.text ;								\
76	SUPERALIGN_TEXT ;						\
77IDTVEC(vec_name) ;							\
78	PUSH_FRAME ;							\
79	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
80	mov	%ax, %ds ;						\
81	mov	%ax, %es ;						\
82	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
83	mov	%ax, %fs ;						\
84	FAKE_MCOUNT(13*4(%esp)) ;					\
85	movl	lapic, %edx ;	/* pointer to local APIC */		\
86	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
87	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
88	jz	2f ;							\
89	addl	$(32 * index),%eax ;					\
901: ;									\
91	pushl	%eax ;		/* pass the IRQ */			\
92	call	lapic_handle_intr ;					\
93	addl	$4, %esp ;	/* discard parameter */			\
94	MEXITCOUNT ;							\
95	jmp	doreti ;						\
962:	movl	$-1, %eax ;	/* send a vector of -1 */		\
97	jmp	1b
98
99/*
100 * Handle "spurious INTerrupts".
101 * Notes:
102 *  This is different than the "spurious INTerrupt" generated by an
103 *   8259 PIC for missing INTs.  See the APIC documentation for details.
104 *  This routine should NOT do an 'EOI' cycle.
105 */
106	.text
107	SUPERALIGN_TEXT
108IDTVEC(spuriousint)
109
110	/* No EOI cycle used here */
111
112	iret
113
114MCOUNT_LABEL(bintr2)
115	ISR_VEC(1, apic_isr1)
116	ISR_VEC(2, apic_isr2)
117	ISR_VEC(3, apic_isr3)
118	ISR_VEC(4, apic_isr4)
119	ISR_VEC(5, apic_isr5)
120	ISR_VEC(6, apic_isr6)
121	ISR_VEC(7, apic_isr7)
122MCOUNT_LABEL(eintr2)
123
124#ifdef SMP
125/*
126 * Global address space TLB shootdown.
127 */
128	.text
129	SUPERALIGN_TEXT
130IDTVEC(invltlb)
131	pushl	%eax
132	pushl	%ds
133	movl	$KDSEL, %eax		/* Kernel data selector */
134	mov	%ax, %ds
135
136#ifdef COUNT_XINVLTLB_HITS
137	pushl	%fs
138	movl	$KPSEL, %eax		/* Private space selector */
139	mov	%ax, %fs
140	movl	PCPU(CPUID), %eax
141	popl	%fs
142	incl	xhits_gbl(,%eax,4)
143#endif /* COUNT_XINVLTLB_HITS */
144
145	movl	%cr3, %eax		/* invalidate the TLB */
146	movl	%eax, %cr3
147
148	movl	lapic, %eax
149	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
150
151	lock
152	incl	smp_tlb_wait
153
154	popl	%ds
155	popl	%eax
156	iret
157
158/*
159 * Single page TLB shootdown
160 */
161	.text
162	SUPERALIGN_TEXT
163IDTVEC(invlpg)
164	pushl	%eax
165	pushl	%ds
166	movl	$KDSEL, %eax		/* Kernel data selector */
167	mov	%ax, %ds
168
169#ifdef COUNT_XINVLTLB_HITS
170	pushl	%fs
171	movl	$KPSEL, %eax		/* Private space selector */
172	mov	%ax, %fs
173	movl	PCPU(CPUID), %eax
174	popl	%fs
175	incl	xhits_pg(,%eax,4)
176#endif /* COUNT_XINVLTLB_HITS */
177
178	movl	smp_tlb_addr1, %eax
179	invlpg	(%eax)			/* invalidate single page */
180
181	movl	lapic, %eax
182	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
183
184	lock
185	incl	smp_tlb_wait
186
187	popl	%ds
188	popl	%eax
189	iret
190
191/*
192 * Page range TLB shootdown.
193 */
194	.text
195	SUPERALIGN_TEXT
196IDTVEC(invlrng)
197	pushl	%eax
198	pushl	%edx
199	pushl	%ds
200	movl	$KDSEL, %eax		/* Kernel data selector */
201	mov	%ax, %ds
202
203#ifdef COUNT_XINVLTLB_HITS
204	pushl	%fs
205	movl	$KPSEL, %eax		/* Private space selector */
206	mov	%ax, %fs
207	movl	PCPU(CPUID), %eax
208	popl	%fs
209	incl	xhits_rng(,%eax,4)
210#endif /* COUNT_XINVLTLB_HITS */
211
212	movl	smp_tlb_addr1, %edx
213	movl	smp_tlb_addr2, %eax
2141:	invlpg	(%edx)			/* invalidate single page */
215	addl	$PAGE_SIZE, %edx
216	cmpl	%eax, %edx
217	jb	1b
218
219	movl	lapic, %eax
220	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
221
222	lock
223	incl	smp_tlb_wait
224
225	popl	%ds
226	popl	%edx
227	popl	%eax
228	iret
229
230/*
231 * Forward hardclock to another CPU.  Pushes a clockframe and calls
232 * forwarded_hardclock().
233 */
234	.text
235	SUPERALIGN_TEXT
236IDTVEC(hardclock)
237	PUSH_FRAME
238	movl	$KDSEL, %eax	/* reload with kernel's data segment */
239	mov	%ax, %ds
240	mov	%ax, %es
241	movl	$KPSEL, %eax
242	mov	%ax, %fs
243
244	movl	lapic, %edx
245	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
246
247	pushl	$0		/* XXX convert trapframe to clockframe */
248	call	forwarded_hardclock
249	addl	$4, %esp	/* XXX convert clockframe to trapframe */
250	MEXITCOUNT
251	jmp	doreti
252
253/*
254 * Forward statclock to another CPU.  Pushes a clockframe and calls
255 * forwarded_statclock().
256 */
257	.text
258	SUPERALIGN_TEXT
259IDTVEC(statclock)
260	PUSH_FRAME
261	movl	$KDSEL, %eax	/* reload with kernel's data segment */
262	mov	%ax, %ds
263	mov	%ax, %es
264	movl	$KPSEL, %eax
265	mov	%ax, %fs
266
267	movl	lapic, %edx
268	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
269
270	FAKE_MCOUNT(13*4(%esp))
271
272	pushl	$0		/* XXX convert trapframe to clockframe */
273	call	forwarded_statclock
274	addl	$4, %esp	/* XXX convert clockframe to trapframe */
275	MEXITCOUNT
276	jmp	doreti
277
278/*
279 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
280 *
281 * The other CPU has already executed aston() or need_resched() on our
282 * current process, so we simply need to ack the interrupt and return
283 * via doreti to run ast().
284 */
285
286	.text
287	SUPERALIGN_TEXT
288IDTVEC(cpuast)
289	PUSH_FRAME
290	movl	$KDSEL, %eax
291	mov	%ax, %ds		/* use KERNEL data segment */
292	mov	%ax, %es
293	movl	$KPSEL, %eax
294	mov	%ax, %fs
295
296	movl	lapic, %edx
297	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
298
299	FAKE_MCOUNT(13*4(%esp))
300
301	MEXITCOUNT
302	jmp	doreti
303
304/*
305 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
306 *
307 *  - Signals its receipt.
308 *  - Waits for permission to restart.
309 *  - Signals its restart.
310 */
311	.text
312	SUPERALIGN_TEXT
313IDTVEC(cpustop)
314	pushl	%ebp
315	movl	%esp, %ebp
316	pushl	%eax
317	pushl	%ecx
318	pushl	%edx
319	pushl	%ds			/* save current data segment */
320	pushl	%es
321	pushl	%fs
322
323	movl	$KDSEL, %eax
324	mov	%ax, %ds		/* use KERNEL data segment */
325	mov	%ax, %es
326	movl	$KPSEL, %eax
327	mov	%ax, %fs
328
329	movl	lapic, %eax
330	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
331
332	movl	PCPU(CPUID), %eax
333	imull	$PCB_SIZE, %eax
334	leal	CNAME(stoppcbs)(%eax), %eax
335	pushl	%eax
336	call	CNAME(savectx)		/* Save process context */
337	addl	$4, %esp
338
339	movl	PCPU(CPUID), %eax
340
341	lock
342	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
3431:
344	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
345	jnc	1b
346
347	lock
348	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
349	lock
350	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
351
352	test	%eax, %eax
353	jnz	2f
354
355	movl	CNAME(cpustop_restartfunc), %eax
356	test	%eax, %eax
357	jz	2f
358	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
359
360	call	*%eax
3612:
362	popl	%fs
363	popl	%es
364	popl	%ds			/* restore previous data segment */
365	popl	%edx
366	popl	%ecx
367	popl	%eax
368	movl	%ebp, %esp
369	popl	%ebp
370	iret
371
372/*
373 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
374 *
375 * - Calls the generic rendezvous action function.
376 */
377	.text
378	SUPERALIGN_TEXT
379IDTVEC(rendezvous)
380	PUSH_FRAME
381	movl	$KDSEL, %eax
382	mov	%ax, %ds		/* use KERNEL data segment */
383	mov	%ax, %es
384	movl	$KPSEL, %eax
385	mov	%ax, %fs
386
387	call	smp_rendezvous_action
388
389	movl	lapic, %eax
390	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
391	POP_FRAME
392	iret
393
394/*
395 * Clean up when we lose out on the lazy context switch optimization.
396 * ie: when we are about to release a PTD but a cpu is still borrowing it.
397 */
398	SUPERALIGN_TEXT
399IDTVEC(lazypmap)
400	PUSH_FRAME
401	movl	$KDSEL, %eax
402	mov	%ax, %ds		/* use KERNEL data segment */
403	mov	%ax, %es
404	movl	$KPSEL, %eax
405	mov	%ax, %fs
406
407	call	pmap_lazyfix_action
408
409	movl	lapic, %eax
410	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
411	POP_FRAME
412	iret
413#endif /* SMP */
414