apic_vector.s revision 91328 
1/*
2 *	from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: head/sys/i386/i386/apic_vector.s 91328 2002-02-26 20:33:41Z dillon $
4 */
5
6
7#include <machine/apic.h>
8#include <machine/smp.h>
9
10/* convert an absolute IRQ# into a bitmask */
11#define IRQ_BIT(irq_num)	(1 << (irq_num))
12
13/* make an index into the IO APIC from the IRQ# */
14#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
15
16/*
17 *
18 */
19#define PUSH_FRAME							\
20	pushl	$0 ;		/* dummy error code */			\
21	pushl	$0 ;		/* dummy trap type */			\
22	pushal ;							\
23	pushl	%ds ;		/* save data and extra segments ... */	\
24	pushl	%es ;							\
25	pushl	%fs
26
27#define POP_FRAME							\
28	popl	%fs ;							\
29	popl	%es ;							\
30	popl	%ds ;							\
31	popal ;								\
32	addl	$4+4,%esp
33
34/*
35 * Macros for interrupt entry, call to handler, and exit.
36 */
37
38#define	FAST_INTR(irq_num, vec_name)					\
39	.text ;								\
40	SUPERALIGN_TEXT ;						\
41IDTVEC(vec_name) ;							\
42	PUSH_FRAME ;							\
43	movl	$KDSEL,%eax ;						\
44	mov	%ax,%ds ;						\
45	mov	%ax,%es ;						\
46	movl	$KPSEL,%eax ;						\
47	mov	%ax,%fs ;						\
48	FAKE_MCOUNT(13*4(%esp)) ;					\
49	call	critical_enter ;					\
50	movl	PCPU(CURTHREAD),%ebx ;					\
51	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
52	pushl	intr_unit + (irq_num) * 4 ;				\
53	call	*intr_handler + (irq_num) * 4 ;	/* do the work ASAP */	\
54	addl	$4, %esp ;						\
55	movl	$0, lapic+LA_EOI ;					\
56	lock ; 								\
57	incl	cnt+V_INTR ;	/* book-keeping can wait */		\
58	movl	intr_countp + (irq_num) * 4, %eax ;			\
59	lock ; 								\
60	incl	(%eax) ;						\
61	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
62	call	critical_exit ;						\
63	MEXITCOUNT ;							\
64	jmp	doreti
65
66#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
67#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
68
69#define MASK_IRQ(irq_num)						\
70	ICU_LOCK ;				/* into critical reg */	\
71	testl	$IRQ_BIT(irq_num), apic_imen ;				\
72	jne	7f ;			/* masked, don't mask */	\
73	orl	$IRQ_BIT(irq_num), apic_imen ;	/* set the mask bit */	\
74	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
75	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
76	movl	%eax, (%ecx) ;			/* write the index */	\
77	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
78	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
79	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
807: ;						/* already masked */	\
81	ICU_UNLOCK
82/*
83 * Test to see whether we are handling an edge or level triggered INT.
84 *  Level-triggered INTs must still be masked as we don't clear the source,
85 *  and the EOI cycle would cause redundant INTs to occur.
86 */
87#define MASK_LEVEL_IRQ(irq_num)						\
88	testl	$IRQ_BIT(irq_num), apic_pin_trigger ;			\
89	jz	9f ;				/* edge, don't mask */	\
90	MASK_IRQ(irq_num) ;						\
919:
92
93
94#ifdef APIC_INTR_REORDER
95#define EOI_IRQ(irq_num)						\
96	movl	apic_isrbit_location + 8 * (irq_num), %eax ;		\
97	movl	(%eax), %eax ;						\
98	testl	apic_isrbit_location + 4 + 8 * (irq_num), %eax ;	\
99	jz	9f ;				/* not active */	\
100	movl	$0, lapic+LA_EOI ;					\
1019:
102
103#else
104#define EOI_IRQ(irq_num)						\
105	testl	$IRQ_BIT(irq_num), lapic+LA_ISR1;			\
106	jz	9f	;			/* not active */	\
107	movl	$0, lapic+LA_EOI;					\
1089:
109#endif
110
111
112/*
113 * Test to see if the source is currently masked, clear if so.
114 */
115#define UNMASK_IRQ(irq_num)					\
116	ICU_LOCK ;				/* into critical reg */	\
117	testl	$IRQ_BIT(irq_num), _apic_imen ;				\
118	je	7f ;			/* bit clear, not masked */	\
119	andl	$~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */	\
120	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
121	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
122	movl	%eax, (%ecx) ;			/* write the index */	\
123	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
124	andl	$~IOART_INTMASK, %eax ;		/* clear the mask */	\
125	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
1267: ;						/* already unmasked */	\
127	ICU_UNLOCK
128
129/*
130 * Slow, threaded interrupts.
131 *
132 * XXX Most of the parameters here are obsolete.  Fix this when we're
133 * done.
134 * XXX we really shouldn't return via doreti if we just schedule the
135 * interrupt handler and don't run anything.  We could just do an
136 * iret.  FIXME.
137 */
138#define	INTR(irq_num, vec_name, maybe_extra_ipending)			\
139	.text ;								\
140	SUPERALIGN_TEXT ;						\
141/* _XintrNN: entry point used by IDT/HWIs via _vec[]. */		\
142IDTVEC(vec_name) ;							\
143	PUSH_FRAME ;							\
144	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
145	mov	%ax, %ds ;						\
146	mov	%ax, %es ;						\
147	movl	$KPSEL, %eax ;						\
148	mov	%ax, %fs ;						\
149;									\
150	maybe_extra_ipending ;						\
151;									\
152	MASK_LEVEL_IRQ(irq_num) ;					\
153	EOI_IRQ(irq_num) ;						\
1540: ;									\
155	movl	PCPU(CURTHREAD),%ebx ;					\
156	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
157;	 								\
158	FAKE_MCOUNT(13*4(%esp)) ;		/* XXX avoid dbl cnt */ \
159	pushl	$irq_num;			/* pass the IRQ */	\
160	call	sched_ithd ;						\
161	addl	$4, %esp ;		/* discard the parameter */	\
162;									\
163	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
164	MEXITCOUNT ;							\
165	jmp	doreti
166
167/*
168 * Handle "spurious INTerrupts".
169 * Notes:
170 *  This is different than the "spurious INTerrupt" generated by an
171 *   8259 PIC for missing INTs.  See the APIC documentation for details.
172 *  This routine should NOT do an 'EOI' cycle.
173 */
174	.text
175	SUPERALIGN_TEXT
176	.globl Xspuriousint
177Xspuriousint:
178
179	/* No EOI cycle used here */
180
181	iret
182
183/*
184 * Global address space TLB shootdown.
185 */
186	.text
187	SUPERALIGN_TEXT
188	.globl	Xinvltlb
189Xinvltlb:
190	pushl	%eax
191	pushl	%ds
192	movl	$KDSEL, %eax		/* Kernel data selector */
193	mov	%ax, %ds
194
195#ifdef COUNT_XINVLTLB_HITS
196	pushl	%fs
197	movl	$KPSEL, %eax		/* Private space selector */
198	mov	%ax, %fs
199	movl	PCPU(CPUID), %eax
200	popl	%fs
201	incl	xhits_gbl(,%eax,4)
202#endif /* COUNT_XINVLTLB_HITS */
203
204	movl	%cr3, %eax		/* invalidate the TLB */
205	movl	%eax, %cr3
206
207	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
208
209	lock
210	incl	smp_tlb_wait
211
212	popl	%ds
213	popl	%eax
214	iret
215
216/*
217 * Single page TLB shootdown
218 */
219	.text
220	SUPERALIGN_TEXT
221	.globl	Xinvlpg
222Xinvlpg:
223	pushl	%eax
224	pushl	%ds
225	movl	$KDSEL, %eax		/* Kernel data selector */
226	mov	%ax, %ds
227
228#ifdef COUNT_XINVLTLB_HITS
229	pushl	%fs
230	movl	$KPSEL, %eax		/* Private space selector */
231	mov	%ax, %fs
232	movl	PCPU(CPUID), %eax
233	popl	%fs
234	ss
235	incl	xhits_pg(,%eax,4)
236#endif /* COUNT_XINVLTLB_HITS */
237
238	movl	smp_tlb_addr1, %eax
239	invlpg	(%eax)			/* invalidate single page */
240
241	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
242
243	lock
244	incl	smp_tlb_wait
245
246	popl	%ds
247	popl	%eax
248	iret
249
250/*
251 * Page range TLB shootdown.
252 */
253	.text
254	SUPERALIGN_TEXT
255	.globl	Xinvlrng
256Xinvlrng:
257	pushl	%eax
258	pushl	%edx
259	pushl	%ds
260	movl	$KDSEL, %eax		/* Kernel data selector */
261	mov	%ax, %ds
262
263#ifdef COUNT_XINVLTLB_HITS
264	pushl	%fs
265	movl	$KPSEL, %eax		/* Private space selector */
266	mov	%ax, %fs
267	movl	PCPU(CPUID), %eax
268	popl	%fs
269	incl	xhits_rng(,%eax,4)
270#endif /* COUNT_XINVLTLB_HITS */
271
272	movl	smp_tlb_addr1, %edx
273	movl	smp_tlb_addr2, %eax
2741:	invlpg	(%edx)			/* invalidate single page */
275	addl	$PAGE_SIZE, %edx
276	cmpl	%edx, %eax
277	jb	1b
278
279	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
280
281	lock
282	incl	smp_tlb_wait
283
284	popl	%ds
285	popl	%edx
286	popl	%eax
287	iret
288
289/*
290 * Forward hardclock to another CPU.  Pushes a trapframe and calls
291 * forwarded_hardclock().
292 */
293	.text
294	SUPERALIGN_TEXT
295	.globl Xhardclock
296Xhardclock:
297	PUSH_FRAME
298	movl	$KDSEL, %eax	/* reload with kernel's data segment */
299	mov	%ax, %ds
300	mov	%ax, %es
301	movl	$KPSEL, %eax
302	mov	%ax, %fs
303
304	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
305
306	movl	PCPU(CURTHREAD),%ebx
307	incl	TD_INTR_NESTING_LEVEL(%ebx)
308	call	forwarded_hardclock
309	decl	TD_INTR_NESTING_LEVEL(%ebx)
310	MEXITCOUNT
311	jmp	doreti
312
313/*
314 * Forward statclock to another CPU.  Pushes a trapframe and calls
315 * forwarded_statclock().
316 */
317	.text
318	SUPERALIGN_TEXT
319	.globl Xstatclock
320Xstatclock:
321	PUSH_FRAME
322	movl	$KDSEL, %eax	/* reload with kernel's data segment */
323	mov	%ax, %ds
324	mov	%ax, %es
325	movl	$KPSEL, %eax
326	mov	%ax, %fs
327
328	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
329
330	FAKE_MCOUNT(13*4(%esp))
331	movl	PCPU(CURTHREAD),%ebx
332	incl	TD_INTR_NESTING_LEVEL(%ebx)
333	call	forwarded_statclock
334	decl	TD_INTR_NESTING_LEVEL(%ebx)
335	MEXITCOUNT
336	jmp	doreti
337
338/*
339 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
340 *
341 * The other CPU has already executed aston() or need_resched() on our
342 * current process, so we simply need to ack the interrupt and return
343 * via doreti to run ast().
344 */
345
346	.text
347	SUPERALIGN_TEXT
348	.globl Xcpuast
349Xcpuast:
350	PUSH_FRAME
351	movl	$KDSEL, %eax
352	mov	%ax, %ds		/* use KERNEL data segment */
353	mov	%ax, %es
354	movl	$KPSEL, %eax
355	mov	%ax, %fs
356
357	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
358
359	FAKE_MCOUNT(13*4(%esp))
360
361	MEXITCOUNT
362	jmp	doreti
363
364/*
365 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
366 *
367 *  - Signals its receipt.
368 *  - Waits for permission to restart.
369 *  - Signals its restart.
370 */
371	.text
372	SUPERALIGN_TEXT
373	.globl Xcpustop
374Xcpustop:
375	pushl	%ebp
376	movl	%esp, %ebp
377	pushl	%eax
378	pushl	%ecx
379	pushl	%edx
380	pushl	%ds			/* save current data segment */
381	pushl	%fs
382
383	movl	$KDSEL, %eax
384	mov	%ax, %ds		/* use KERNEL data segment */
385	movl	$KPSEL, %eax
386	mov	%ax, %fs
387
388	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
389
390	movl	PCPU(CPUID), %eax
391	imull	$PCB_SIZE, %eax
392	leal	CNAME(stoppcbs)(%eax), %eax
393	pushl	%eax
394	call	CNAME(savectx)		/* Save process context */
395	addl	$4, %esp
396
397	movl	PCPU(CPUID), %eax
398
399	lock
400	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
4011:
402	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
403	jnc	1b
404
405	lock
406	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
407	lock
408	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
409
410	test	%eax, %eax
411	jnz	2f
412
413	movl	CNAME(cpustop_restartfunc), %eax
414	test	%eax, %eax
415	jz	2f
416	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
417
418	call	*%eax
4192:
420	popl	%fs
421	popl	%ds			/* restore previous data segment */
422	popl	%edx
423	popl	%ecx
424	popl	%eax
425	movl	%ebp, %esp
426	popl	%ebp
427	iret
428
429
430MCOUNT_LABEL(bintr)
431	FAST_INTR(0,fastintr0)
432	FAST_INTR(1,fastintr1)
433	FAST_INTR(2,fastintr2)
434	FAST_INTR(3,fastintr3)
435	FAST_INTR(4,fastintr4)
436	FAST_INTR(5,fastintr5)
437	FAST_INTR(6,fastintr6)
438	FAST_INTR(7,fastintr7)
439	FAST_INTR(8,fastintr8)
440	FAST_INTR(9,fastintr9)
441	FAST_INTR(10,fastintr10)
442	FAST_INTR(11,fastintr11)
443	FAST_INTR(12,fastintr12)
444	FAST_INTR(13,fastintr13)
445	FAST_INTR(14,fastintr14)
446	FAST_INTR(15,fastintr15)
447	FAST_INTR(16,fastintr16)
448	FAST_INTR(17,fastintr17)
449	FAST_INTR(18,fastintr18)
450	FAST_INTR(19,fastintr19)
451	FAST_INTR(20,fastintr20)
452	FAST_INTR(21,fastintr21)
453	FAST_INTR(22,fastintr22)
454	FAST_INTR(23,fastintr23)
455	FAST_INTR(24,fastintr24)
456	FAST_INTR(25,fastintr25)
457	FAST_INTR(26,fastintr26)
458	FAST_INTR(27,fastintr27)
459	FAST_INTR(28,fastintr28)
460	FAST_INTR(29,fastintr29)
461	FAST_INTR(30,fastintr30)
462	FAST_INTR(31,fastintr31)
463#define	CLKINTR_PENDING	movl $1,CNAME(clkintr_pending)
464/* Threaded interrupts */
465	INTR(0,intr0, CLKINTR_PENDING)
466	INTR(1,intr1,)
467	INTR(2,intr2,)
468	INTR(3,intr3,)
469	INTR(4,intr4,)
470	INTR(5,intr5,)
471	INTR(6,intr6,)
472	INTR(7,intr7,)
473	INTR(8,intr8,)
474	INTR(9,intr9,)
475	INTR(10,intr10,)
476	INTR(11,intr11,)
477	INTR(12,intr12,)
478	INTR(13,intr13,)
479	INTR(14,intr14,)
480	INTR(15,intr15,)
481	INTR(16,intr16,)
482	INTR(17,intr17,)
483	INTR(18,intr18,)
484	INTR(19,intr19,)
485	INTR(20,intr20,)
486	INTR(21,intr21,)
487	INTR(22,intr22,)
488	INTR(23,intr23,)
489	INTR(24,intr24,)
490	INTR(25,intr25,)
491	INTR(26,intr26,)
492	INTR(27,intr27,)
493	INTR(28,intr28,)
494	INTR(29,intr29,)
495	INTR(30,intr30,)
496	INTR(31,intr31,)
497MCOUNT_LABEL(eintr)
498
499/*
500 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
501 *
502 * - Calls the generic rendezvous action function.
503 */
504	.text
505	SUPERALIGN_TEXT
506	.globl	Xrendezvous
507Xrendezvous:
508	PUSH_FRAME
509	movl	$KDSEL, %eax
510	mov	%ax, %ds		/* use KERNEL data segment */
511	mov	%ax, %es
512	movl	$KPSEL, %eax
513	mov	%ax, %fs
514
515	call	smp_rendezvous_action
516
517	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
518	POP_FRAME
519	iret
520
521
522	.data
523
524	.globl	apic_pin_trigger
525apic_pin_trigger:
526	.long	0
527
528	.text
529