apic_vector.s revision 125161 
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. All advertising materials mentioning features or use of this software
15 *    must display the following acknowledgement:
16 *	This product includes software developed by the University of
17 *	California, Berkeley and its contributors.
18 * 4. Neither the name of the University nor the names of its contributors
19 *    may be used to endorse or promote products derived from this software
20 *    without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 *	from: vector.s, 386BSD 0.1 unknown origin
35 * $FreeBSD: head/sys/i386/i386/apic_vector.s 125161 2004-01-28 20:44:08Z jhb $
36 */
37
38/*
39 * Interrupt entry points for external interrupts triggered by I/O APICs
40 * as well as IPI handlers.
41 */
42
43#include <machine/asmacros.h>
44#include <machine/apicreg.h>
45#include <machine/smptests.h>
46
47#include "assym.s"
48
49/*
50 * Macros to create and destroy a trap frame.
51 */
52#define PUSH_FRAME							\
53	pushl	$0 ;		/* dummy error code */			\
54	pushl	$0 ;		/* dummy trap type */			\
55	pushal ;		/* 8 ints */				\
56	pushl	%ds ;		/* save data and extra segments ... */	\
57	pushl	%es ;							\
58	pushl	%fs
59
60#define	PUSH_FRAME_AND_SET_SEGS						\
61	PUSH_FRAME ;							\
62	mov	%fs,%ax ;	/* get current per-cpu selector */	\
63	cmp	$KPSEL,%ax ;	/* are we already in the kernel? */	\
64	je	1f ;		/* skip expensive segment reloads */	\
65	mov	$KDSEL,%ax ;	/* load kernel ds, es and fs */		\
66	mov	%ax,%ds ;						\
67	mov	%ax,%es ;						\
68	mov	$KPSEL,%ax ;						\
69	mov	%ax,%fs ;						\
701:
71
72#define POP_FRAME							\
73	popl	%fs ;							\
74	popl	%es ;							\
75	popl	%ds ;							\
76	popal ;								\
77	addl	$4+4,%esp
78
79/*
80 * I/O Interrupt Entry Point.  Rather than having one entry point for
81 * each interrupt source, we use one entry point for each 32-bit word
82 * in the ISR.  The handler determines the highest bit set in the ISR,
83 * translates that into a vector, and passes the vector to the
84 * lapic_handle_intr() function.
85 */
86#define	ISR_VEC(index, vec_name)					\
87	.text ;								\
88	SUPERALIGN_TEXT ;						\
89IDTVEC(vec_name) ;							\
90	PUSH_FRAME_AND_SET_SEGS ;					\
91	FAKE_MCOUNT(13*4(%esp)) ;					\
92	movl	lapic, %edx ;	/* pointer to local APIC */		\
93	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
94	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
95	jz	2f ;							\
96	addl	$(32 * index),%eax ;					\
971: ;									\
98	pushl	%eax ;		/* pass the IRQ */			\
99	call	lapic_handle_intr ;					\
100	addl	$4, %esp ;	/* discard parameter */			\
101	MEXITCOUNT ;							\
102	jmp	doreti ;						\
1032:	movl	$-1, %eax ;	/* send a vector of -1 */		\
104	jmp	1b
105
106/*
107 * Handle "spurious INTerrupts".
108 * Notes:
109 *  This is different than the "spurious INTerrupt" generated by an
110 *   8259 PIC for missing INTs.  See the APIC documentation for details.
111 *  This routine should NOT do an 'EOI' cycle.
112 */
113	.text
114	SUPERALIGN_TEXT
115IDTVEC(spuriousint)
116
117	/* No EOI cycle used here */
118
119	iret
120
121MCOUNT_LABEL(bintr2)
122	ISR_VEC(1, apic_isr1)
123	ISR_VEC(2, apic_isr2)
124	ISR_VEC(3, apic_isr3)
125	ISR_VEC(4, apic_isr4)
126	ISR_VEC(5, apic_isr5)
127	ISR_VEC(6, apic_isr6)
128	ISR_VEC(7, apic_isr7)
129MCOUNT_LABEL(eintr2)
130
131#ifdef SMP
132/*
133 * Global address space TLB shootdown.
134 */
135	.text
136	SUPERALIGN_TEXT
137IDTVEC(invltlb)
138	pushl	%eax
139	pushl	%ds
140	mov	%ds,%ax			/* get current data selector */
141	cmp	$KDSEL,%ax		/* are we already in the kernel? */
142	je	1f			/* skip expensive segment reload */
143	movl	$KDSEL, %eax		/* Kernel data selector */
144	mov	%ax, %ds
1451:
146#ifdef COUNT_XINVLTLB_HITS
147	pushl	%fs
148	movl	$KPSEL, %eax		/* Private space selector */
149	mov	%ax, %fs
150	movl	PCPU(CPUID), %eax
151	popl	%fs
152	incl	xhits_gbl(,%eax,4)
153#endif /* COUNT_XINVLTLB_HITS */
154
155	movl	%cr3, %eax		/* invalidate the TLB */
156	movl	%eax, %cr3
157
158	movl	lapic, %eax
159	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
160
161	lock
162	incl	smp_tlb_wait
163
164	popl	%ds
165	popl	%eax
166	iret
167
168/*
169 * Single page TLB shootdown
170 */
171	.text
172	SUPERALIGN_TEXT
173IDTVEC(invlpg)
174	pushl	%eax
175	pushl	%ds
176	mov	%ds,%ax			/* get current data selector */
177	cmp	$KDSEL,%ax		/* are we already in the kernel? */
178	je	1f			/* skip expensive segment reload */
179	movl	$KDSEL, %eax		/* Kernel data selector */
180	mov	%ax, %ds
1811:
182#ifdef COUNT_XINVLTLB_HITS
183	pushl	%fs
184	movl	$KPSEL, %eax		/* Private space selector */
185	mov	%ax, %fs
186	movl	PCPU(CPUID), %eax
187	popl	%fs
188	incl	xhits_pg(,%eax,4)
189#endif /* COUNT_XINVLTLB_HITS */
190
191	movl	smp_tlb_addr1, %eax
192	invlpg	(%eax)			/* invalidate single page */
193
194	movl	lapic, %eax
195	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
196
197	lock
198	incl	smp_tlb_wait
199
200	popl	%ds
201	popl	%eax
202	iret
203
204/*
205 * Page range TLB shootdown.
206 */
207	.text
208	SUPERALIGN_TEXT
209IDTVEC(invlrng)
210	pushl	%eax
211	pushl	%edx
212	pushl	%ds
213	mov	%ds,%ax			/* get current data selector */
214	cmp	$KDSEL,%ax		/* are we already in the kernel? */
215	je	1f			/* skip expensive segment reload */
216	movl	$KDSEL, %eax		/* Kernel data selector */
217	mov	%ax, %ds
2181:
219#ifdef COUNT_XINVLTLB_HITS
220	pushl	%fs
221	movl	$KPSEL, %eax		/* Private space selector */
222	mov	%ax, %fs
223	movl	PCPU(CPUID), %eax
224	popl	%fs
225	incl	xhits_rng(,%eax,4)
226#endif /* COUNT_XINVLTLB_HITS */
227
228	movl	smp_tlb_addr1, %edx
229	movl	smp_tlb_addr2, %eax
2301:	invlpg	(%edx)			/* invalidate single page */
231	addl	$PAGE_SIZE, %edx
232	cmpl	%eax, %edx
233	jb	1b
234
235	movl	lapic, %eax
236	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
237
238	lock
239	incl	smp_tlb_wait
240
241	popl	%ds
242	popl	%edx
243	popl	%eax
244	iret
245
246/*
247 * Forward hardclock to another CPU.  Pushes a clockframe and calls
248 * forwarded_hardclock().
249 */
250	.text
251	SUPERALIGN_TEXT
252IDTVEC(hardclock)
253	PUSH_FRAME_AND_SET_SEGS
254
255	movl	lapic, %edx
256	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
257
258	pushl	$0		/* XXX convert trapframe to clockframe */
259	call	forwarded_hardclock
260	addl	$4, %esp	/* XXX convert clockframe to trapframe */
261	MEXITCOUNT
262	jmp	doreti
263
264/*
265 * Forward statclock to another CPU.  Pushes a clockframe and calls
266 * forwarded_statclock().
267 */
268	.text
269	SUPERALIGN_TEXT
270IDTVEC(statclock)
271	PUSH_FRAME_AND_SET_SEGS
272
273	movl	lapic, %edx
274	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
275
276	FAKE_MCOUNT(13*4(%esp))
277
278	pushl	$0		/* XXX convert trapframe to clockframe */
279	call	forwarded_statclock
280	addl	$4, %esp	/* XXX convert clockframe to trapframe */
281	MEXITCOUNT
282	jmp	doreti
283
284/*
285 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
286 *
287 * The other CPU has already executed aston() or need_resched() on our
288 * current process, so we simply need to ack the interrupt and return
289 * via doreti to run ast().
290 */
291
292	.text
293	SUPERALIGN_TEXT
294IDTVEC(cpuast)
295	PUSH_FRAME_AND_SET_SEGS
296
297	movl	lapic, %edx
298	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
299
300	FAKE_MCOUNT(13*4(%esp))
301
302	MEXITCOUNT
303	jmp	doreti
304
305/*
306 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
307 *
308 *  - Signals its receipt.
309 *  - Waits for permission to restart.
310 *  - Signals its restart.
311 */
312	.text
313	SUPERALIGN_TEXT
314IDTVEC(cpustop)
315	pushl	%ebp
316	movl	%esp, %ebp
317	pushl	%eax
318	pushl	%ecx
319	pushl	%edx
320	pushl	%ds			/* save current data segment */
321	pushl	%es
322	pushl	%fs
323
324	movl	$KDSEL, %eax
325	mov	%ax, %ds		/* use KERNEL data segment */
326	mov	%ax, %es
327	movl	$KPSEL, %eax
328	mov	%ax, %fs
329
330	movl	lapic, %eax
331	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
332
333	movl	PCPU(CPUID), %eax
334	imull	$PCB_SIZE, %eax
335	leal	CNAME(stoppcbs)(%eax), %eax
336	pushl	%eax
337	call	CNAME(savectx)		/* Save process context */
338	addl	$4, %esp
339
340	movl	PCPU(CPUID), %eax
341
342	lock
343	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
3441:
345	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
346	jnc	1b
347
348	lock
349	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
350	lock
351	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
352
353	test	%eax, %eax
354	jnz	2f
355
356	movl	CNAME(cpustop_restartfunc), %eax
357	test	%eax, %eax
358	jz	2f
359	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
360
361	call	*%eax
3622:
363	popl	%fs
364	popl	%es
365	popl	%ds			/* restore previous data segment */
366	popl	%edx
367	popl	%ecx
368	popl	%eax
369	movl	%ebp, %esp
370	popl	%ebp
371	iret
372
373/*
374 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
375 *
376 * - Calls the generic rendezvous action function.
377 */
378	.text
379	SUPERALIGN_TEXT
380IDTVEC(rendezvous)
381	PUSH_FRAME_AND_SET_SEGS
382
383	call	smp_rendezvous_action
384
385	movl	lapic, %eax
386	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
387	POP_FRAME
388	iret
389
390/*
391 * Clean up when we lose out on the lazy context switch optimization.
392 * ie: when we are about to release a PTD but a cpu is still borrowing it.
393 */
394	SUPERALIGN_TEXT
395IDTVEC(lazypmap)
396	PUSH_FRAME_AND_SET_SEGS
397
398	call	pmap_lazyfix_action
399
400	movl	lapic, %eax
401	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
402	POP_FRAME
403	iret
404#endif /* SMP */
405