apic_vector.s revision 51114 
1/*
2 *	from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: head/sys/i386/i386/apic_vector.s 51114 1999-09-10 01:17:01Z msmith $
4 */
5
6
7#include <machine/apic.h>
8#include <machine/smp.h>
9
10#include "i386/isa/intr_machdep.h"
11
12
13#ifdef FAST_SIMPLELOCK
14
15#define GET_FAST_INTR_LOCK						\
16	pushl	$_fast_intr_lock ;		/* address of lock */	\
17	call	_s_lock ;			/* MP-safe */		\
18	addl	$4,%esp
19
20#define REL_FAST_INTR_LOCK						\
21	movl	$0, _fast_intr_lock
22
23#else /* FAST_SIMPLELOCK */
24
25#define GET_FAST_INTR_LOCK						\
26	call	_get_isrlock
27
28#define REL_FAST_INTR_LOCK						\
29	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
30	call	_MPrellock ;						\
31	add	$4, %esp
32
33#endif /* FAST_SIMPLELOCK */
34
35/* convert an absolute IRQ# into a bitmask */
36#define IRQ_BIT(irq_num)	(1 << (irq_num))
37
38/* make an index into the IO APIC from the IRQ# */
39#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
40
41
42/*
43 * Macros for interrupt interrupt entry, call to handler, and exit.
44 */
45
46#ifdef FAST_WITHOUTCPL
47
48/*
49 */
50#define	FAST_INTR(irq_num, vec_name)					\
51	.text ;								\
52	SUPERALIGN_TEXT ;						\
53IDTVEC(vec_name) ;							\
54	pushl	%eax ;		/* save only call-used registers */	\
55	pushl	%ecx ;							\
56	pushl	%edx ;							\
57	pushl	%ds ;							\
58	MAYBE_PUSHL_ES ;						\
59	pushl	%fs ;							\
60	movl	$KDSEL,%eax ;						\
61	movl	%ax,%ds ;						\
62	MAYBE_MOVW_AX_ES ;						\
63	movl	$KPSEL,%eax ;						\
64	movl	%ax,%fs ;						\
65	FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ;			\
66	pushl	_intr_unit + (irq_num) * 4 ;				\
67	GET_FAST_INTR_LOCK ;						\
68	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
69	REL_FAST_INTR_LOCK ;						\
70	addl	$4, %esp ;						\
71	movl	$0, lapic_eoi ;						\
72	lock ; 								\
73	incl	_cnt+V_INTR ;	/* book-keeping can wait */		\
74	movl	_intr_countp + (irq_num) * 4, %eax ;			\
75	lock ; 								\
76	incl	(%eax) ;						\
77	MEXITCOUNT ;							\
78	popl	%fs ;							\
79	MAYBE_POPL_ES ;							\
80	popl	%ds ;							\
81	popl	%edx ;							\
82	popl	%ecx ;							\
83	popl	%eax ;							\
84	iret
85
86#else /* FAST_WITHOUTCPL */
87
88#define	FAST_INTR(irq_num, vec_name)					\
89	.text ;								\
90	SUPERALIGN_TEXT ;						\
91IDTVEC(vec_name) ;							\
92	pushl	%eax ;		/* save only call-used registers */	\
93	pushl	%ecx ;							\
94	pushl	%edx ;							\
95	pushl	%ds ;							\
96	MAYBE_PUSHL_ES ;						\
97	pushl	%fs ;							\
98	movl	$KDSEL, %eax ;						\
99	movl	%ax, %ds ;						\
100	MAYBE_MOVW_AX_ES ;						\
101	movl	$KPSEL, %eax ;						\
102	movl	%ax, %fs ;						\
103	FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ;			\
104	GET_FAST_INTR_LOCK ;						\
105	pushl	_intr_unit + (irq_num) * 4 ;				\
106	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
107	addl	$4, %esp ;						\
108	movl	$0, lapic_eoi ;						\
109	lock ; 								\
110	incl	_cnt+V_INTR ;	/* book-keeping can wait */		\
111	movl	_intr_countp + (irq_num) * 4,%eax ;			\
112	lock ; 								\
113	incl	(%eax) ;						\
114	movl	_cpl, %eax ;	/* unmasking pending HWIs or SWIs? */	\
115	notl	%eax ;							\
116	andl	_ipending, %eax ;					\
117	jne	2f ; 		/* yes, maybe handle them */		\
1181: ;									\
119	MEXITCOUNT ;							\
120	REL_FAST_INTR_LOCK ;						\
121	popl	%fs ;							\
122	MAYBE_POPL_ES ;							\
123	popl	%ds ;							\
124	popl	%edx ;							\
125	popl	%ecx ;							\
126	popl	%eax ;							\
127	iret ;								\
128;									\
129	ALIGN_TEXT ;							\
1302: ;									\
131	cmpb	$3, _intr_nesting_level ;	/* enough stack? */	\
132	jae	1b ;		/* no, return */			\
133	movl	_cpl, %eax ;						\
134	/* XXX next line is probably unnecessary now. */		\
135	movl	$HWI_MASK|SWI_MASK, _cpl ;	/* limit nesting ... */	\
136	lock ; 								\
137	incb	_intr_nesting_level ;	/* ... really limit it ... */	\
138	sti ;			/* to do this as early as possible */	\
139	popl	%fs ;		/* discard most of thin frame ... */	\
140	MAYBE_POPL_ES ;		/* discard most of thin frame ... */	\
141	popl	%ecx ;		/* ... original %ds ... */		\
142	popl	%edx ;							\
143	xchgl	%eax, 4(%esp) ;	/* orig %eax; save cpl */		\
144	pushal ;		/* build fat frame (grrr) ... */	\
145	pushl	%ecx ;		/* ... actually %ds ... */		\
146	pushl	%es ;							\
147	pushl	%fs ;
148	movl	$KDSEL, %eax ;						\
149	movl	%ax, %es ;						\
150	movl	$KPSEL, %eax ;
151	movl	%ax, %fs ;
152	movl	(3+8+0)*4(%esp), %ecx ;	/* %ecx from thin frame ... */	\
153	movl	%ecx, (3+6)*4(%esp) ;	/* ... to fat frame ... */	\
154	movl	(3+8+1)*4(%esp), %eax ;	/* ... cpl from thin frame */	\
155	pushl	%eax ;							\
156	subl	$4, %esp ;	/* junk for unit number */		\
157	MEXITCOUNT ;							\
158	jmp	_doreti
159
160#endif /** FAST_WITHOUTCPL */
161
162
163/*
164 *
165 */
166#define PUSH_FRAME							\
167	pushl	$0 ;		/* dummy error code */			\
168	pushl	$0 ;		/* dummy trap type */			\
169	pushal ;							\
170	pushl	%ds ;		/* save data and extra segments ... */	\
171	pushl	%es ;							\
172	pushl	%fs
173
174#define POP_FRAME							\
175	popl	%fs ;							\
176	popl	%es ;							\
177	popl	%ds ;							\
178	popal ;								\
179	addl	$4+4,%esp
180
181#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
182#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
183
184#define MASK_IRQ(irq_num)						\
185	IMASK_LOCK ;				/* into critical reg */	\
186	testl	$IRQ_BIT(irq_num), _apic_imen ;				\
187	jne	7f ;			/* masked, don't mask */	\
188	orl	$IRQ_BIT(irq_num), _apic_imen ;	/* set the mask bit */	\
189	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
190	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
191	movl	%eax, (%ecx) ;			/* write the index */	\
192	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
193	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
194	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
1957: ;						/* already masked */	\
196	IMASK_UNLOCK
197/*
198 * Test to see whether we are handling an edge or level triggered INT.
199 *  Level-triggered INTs must still be masked as we don't clear the source,
200 *  and the EOI cycle would cause redundant INTs to occur.
201 */
202#define MASK_LEVEL_IRQ(irq_num)						\
203	testl	$IRQ_BIT(irq_num), _apic_pin_trigger ;			\
204	jz	9f ;				/* edge, don't mask */	\
205	MASK_IRQ(irq_num) ;						\
2069:
207
208
209#ifdef APIC_INTR_REORDER
210#define EOI_IRQ(irq_num)						\
211	movl	_apic_isrbit_location + 8 * (irq_num), %eax ;		\
212	movl	(%eax), %eax ;						\
213	testl	_apic_isrbit_location + 4 + 8 * (irq_num), %eax ;	\
214	jz	9f ;				/* not active */	\
215	movl	$0, lapic_eoi ;						\
216	APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;	\
2179:
218
219#else
220#define EOI_IRQ(irq_num)						\
221	testl	$IRQ_BIT(irq_num), lapic_isr1;				\
222	jz	9f	;			/* not active */	\
223	movl	$0, lapic_eoi;						\
224	APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ;	\
2259:
226#endif
227
228
229/*
230 * Test to see if the source is currntly masked, clear if so.
231 */
232#define UNMASK_IRQ(irq_num)					\
233	IMASK_LOCK ;				/* into critical reg */	\
234	testl	$IRQ_BIT(irq_num), _apic_imen ;				\
235	je	7f ;			/* bit clear, not masked */	\
236	andl	$~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */	\
237	movl	IOAPICADDR(irq_num),%ecx ;	/* ioapic addr */	\
238	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
239	movl	%eax,(%ecx) ;			/* write the index */	\
240	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
241	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
242	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
2437: ;									\
244	IMASK_UNLOCK
245
246#ifdef INTR_SIMPLELOCK
247#define ENLOCK
248#define DELOCK
249#define LATELOCK call	_get_isrlock
250#else
251#define ENLOCK \
252	ISR_TRYLOCK ;		/* XXX this is going away... */		\
253	testl	%eax, %eax ;			/* did we get it? */	\
254	jz	3f
255#define DELOCK	ISR_RELLOCK
256#define LATELOCK
257#endif
258
259#ifdef APIC_INTR_DIAGNOSTIC
260#ifdef APIC_INTR_DIAGNOSTIC_IRQ
261log_intr_event:
262	pushf
263	cli
264	pushl	$CNAME(apic_itrace_debuglock)
265	call	CNAME(s_lock_np)
266	addl	$4, %esp
267	movl	CNAME(apic_itrace_debugbuffer_idx), %ecx
268	andl	$32767, %ecx
269	movl	_cpuid, %eax
270	shll	$8,	%eax
271	orl	8(%esp), %eax
272	movw	%ax,	CNAME(apic_itrace_debugbuffer)(,%ecx,2)
273	incl	%ecx
274	andl	$32767, %ecx
275	movl	%ecx,	CNAME(apic_itrace_debugbuffer_idx)
276	pushl	$CNAME(apic_itrace_debuglock)
277	call	CNAME(s_unlock_np)
278	addl	$4, %esp
279	popf
280	ret
281
282
283#define APIC_ITRACE(name, irq_num, id)					\
284	lock ;					/* MP-safe */		\
285	incl	CNAME(name) + (irq_num) * 4 ;				\
286	pushl	%eax ;							\
287	pushl	%ecx ;							\
288	pushl	%edx ;							\
289	movl	$(irq_num), %eax ;					\
290	cmpl	$APIC_INTR_DIAGNOSTIC_IRQ, %eax ;			\
291	jne	7f ;							\
292	pushl	$id ;							\
293	call	log_intr_event ;					\
294	addl	$4, %esp ;						\
2957: ;									\
296	popl	%edx ;							\
297	popl	%ecx ;							\
298	popl	%eax
299#else
300#define APIC_ITRACE(name, irq_num, id)					\
301	lock ;					/* MP-safe */		\
302	incl	CNAME(name) + (irq_num) * 4
303#endif
304
305#define APIC_ITRACE_ENTER 1
306#define APIC_ITRACE_EOI 2
307#define APIC_ITRACE_TRYISRLOCK 3
308#define APIC_ITRACE_GOTISRLOCK 4
309#define APIC_ITRACE_ENTER2 5
310#define APIC_ITRACE_LEAVE 6
311#define APIC_ITRACE_UNMASK 7
312#define APIC_ITRACE_ACTIVE 8
313#define APIC_ITRACE_MASKED 9
314#define APIC_ITRACE_NOISRLOCK 10
315#define APIC_ITRACE_MASKED2 11
316#define APIC_ITRACE_SPLZ 12
317#define APIC_ITRACE_DORETI 13
318
319#else
320#define APIC_ITRACE(name, irq_num, id)
321#endif
322
323#ifdef CPL_AND_CML
324
325#define	INTR(irq_num, vec_name, maybe_extra_ipending)			\
326	.text ;								\
327	SUPERALIGN_TEXT ;						\
328/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */	\
329IDTVEC(vec_name) ;							\
330	PUSH_FRAME ;							\
331	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
332	movl	%ax, %ds ;						\
333	movl	%ax, %es ;						\
334	movl	$KPSEL, %eax ;						\
335	movl	%ax, %fs ;						\
336;									\
337	maybe_extra_ipending ;						\
338;									\
339	APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ;	\
340	lock ;					/* MP-safe */		\
341	btsl	$(irq_num), iactive ;		/* lazy masking */	\
342	jc	1f ;				/* already active */	\
343;									\
344	MASK_LEVEL_IRQ(irq_num) ;					\
345	EOI_IRQ(irq_num) ;						\
3460: ;									\
347	APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
348	ENLOCK ;							\
349;									\
350	APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
351	AVCPL_LOCK ;				/* MP-safe */		\
352	testl	$IRQ_BIT(irq_num), _cpl ;				\
353	jne	2f ;				/* this INT masked */	\
354	testl	$IRQ_BIT(irq_num), _cml ;				\
355	jne	2f ;				/* this INT masked */	\
356	orl	$IRQ_BIT(irq_num), _cil ;				\
357	AVCPL_UNLOCK ;							\
358;									\
359	incb	_intr_nesting_level ;					\
360;	 								\
361  /* entry point used by doreti_unpend for HWIs. */			\
362__CONCAT(Xresume,irq_num): ;						\
363	FAKE_MCOUNT(13*4(%esp)) ;		/* XXX avoid dbl cnt */ \
364	lock ;	incl	_cnt+V_INTR ;		/* tally interrupts */	\
365	movl	_intr_countp + (irq_num) * 4, %eax ;			\
366	lock ;	incl	(%eax) ;					\
367;									\
368	AVCPL_LOCK ;				/* MP-safe */		\
369	movl	_cml, %eax ;						\
370	pushl	%eax ;							\
371	orl	_intr_mask + (irq_num) * 4, %eax ;			\
372	movl	%eax, _cml ;						\
373	AVCPL_UNLOCK ;							\
374;									\
375	pushl	_intr_unit + (irq_num) * 4 ;				\
376	incl	_inside_intr ;						\
377	APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ;	\
378	sti ;								\
379	call	*_intr_handler + (irq_num) * 4 ;			\
380	cli ;								\
381	APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ;	\
382	decl	_inside_intr ;						\
383;									\
384	lock ;	andl $~IRQ_BIT(irq_num), iactive ;			\
385	lock ;	andl $~IRQ_BIT(irq_num), _cil ;				\
386	UNMASK_IRQ(irq_num) ;						\
387	APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ;	\
388	sti ;				/* doreti repeats cli/sti */	\
389	MEXITCOUNT ;							\
390	LATELOCK ;							\
391	jmp	_doreti ;						\
392;									\
393	ALIGN_TEXT ;							\
3941: ;						/* active */		\
395	APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ;	\
396	MASK_IRQ(irq_num) ;						\
397	EOI_IRQ(irq_num) ;						\
398	AVCPL_LOCK ;				/* MP-safe */		\
399	lock ;								\
400	orl	$IRQ_BIT(irq_num), _ipending ;				\
401	AVCPL_UNLOCK ;							\
402	lock ;								\
403	btsl	$(irq_num), iactive ;		/* still active */	\
404	jnc	0b ;				/* retry */		\
405	POP_FRAME ;							\
406	iret ;								\
407;									\
408	ALIGN_TEXT ;							\
4092: ;						/* masked by cpl|cml */	\
410	APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ;	\
411	lock ;								\
412	orl	$IRQ_BIT(irq_num), _ipending ;				\
413	AVCPL_UNLOCK ;							\
414	DELOCK ;		/* XXX this is going away... */		\
415	POP_FRAME ;							\
416	iret ;								\
417	ALIGN_TEXT ;							\
4183: ; 			/* other cpu has isr lock */			\
419	APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
420	AVCPL_LOCK ;				/* MP-safe */		\
421	lock ;								\
422	orl	$IRQ_BIT(irq_num), _ipending ;				\
423	testl	$IRQ_BIT(irq_num), _cpl ;				\
424	jne	4f ;				/* this INT masked */	\
425	testl	$IRQ_BIT(irq_num), _cml ;				\
426	jne	4f ;				/* this INT masked */	\
427	orl	$IRQ_BIT(irq_num), _cil ;				\
428	AVCPL_UNLOCK ;							\
429	call	forward_irq ;	/* forward irq to lock holder */	\
430	POP_FRAME ;	 			/* and return */	\
431	iret ;								\
432	ALIGN_TEXT ;							\
4334: ;	 					/* blocked */		\
434	APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
435	AVCPL_UNLOCK ;							\
436	POP_FRAME ;	 			/* and return */	\
437	iret
438
439#else /* CPL_AND_CML */
440
441
442#define	INTR(irq_num, vec_name, maybe_extra_ipending)			\
443	.text ;								\
444	SUPERALIGN_TEXT ;						\
445/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */	\
446IDTVEC(vec_name) ;							\
447	PUSH_FRAME ;							\
448	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
449	movl	%ax, %ds ;						\
450	movl	%ax, %es ;						\
451	movl	$KPSEL, %eax ;						\
452	movl	%ax, %fs ;						\
453;									\
454	maybe_extra_ipending ;						\
455;									\
456	APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ;	\
457	lock ;					/* MP-safe */		\
458	btsl	$(irq_num), iactive ;		/* lazy masking */	\
459	jc	1f ;				/* already active */	\
460;									\
461	MASK_LEVEL_IRQ(irq_num) ;					\
462	EOI_IRQ(irq_num) ;						\
4630: ;									\
464	APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
465	ISR_TRYLOCK ;		/* XXX this is going away... */		\
466	testl	%eax, %eax ;			/* did we get it? */	\
467	jz	3f ;				/* no */		\
468;									\
469	APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
470	AVCPL_LOCK ;				/* MP-safe */		\
471	testl	$IRQ_BIT(irq_num), _cpl ;				\
472	jne	2f ;				/* this INT masked */	\
473	AVCPL_UNLOCK ;							\
474;									\
475	incb	_intr_nesting_level ;					\
476;	 								\
477  /* entry point used by doreti_unpend for HWIs. */			\
478__CONCAT(Xresume,irq_num): ;						\
479	FAKE_MCOUNT(13*4(%esp)) ;		/* XXX avoid dbl cnt */ \
480	lock ;	incl	_cnt+V_INTR ;		/* tally interrupts */	\
481	movl	_intr_countp + (irq_num) * 4, %eax ;			\
482	lock ;	incl	(%eax) ;					\
483;									\
484	AVCPL_LOCK ;				/* MP-safe */		\
485	movl	_cpl, %eax ;						\
486	pushl	%eax ;							\
487	orl	_intr_mask + (irq_num) * 4, %eax ;			\
488	movl	%eax, _cpl ;						\
489	lock ;								\
490	andl	$~IRQ_BIT(irq_num), _ipending ;				\
491	AVCPL_UNLOCK ;							\
492;									\
493	pushl	_intr_unit + (irq_num) * 4 ;				\
494	APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ;	\
495	sti ;								\
496	call	*_intr_handler + (irq_num) * 4 ;			\
497	cli ;								\
498	APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ;	\
499;									\
500	lock ;	andl	$~IRQ_BIT(irq_num), iactive ;			\
501	UNMASK_IRQ(irq_num) ;						\
502	APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ;	\
503	sti ;				/* doreti repeats cli/sti */	\
504	MEXITCOUNT ;							\
505	jmp	_doreti ;						\
506;									\
507	ALIGN_TEXT ;							\
5081: ;						/* active  */		\
509	APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ;	\
510	MASK_IRQ(irq_num) ;						\
511	EOI_IRQ(irq_num) ;						\
512	AVCPL_LOCK ;				/* MP-safe */		\
513	lock ;								\
514	orl	$IRQ_BIT(irq_num), _ipending ;				\
515	AVCPL_UNLOCK ;							\
516	lock ;								\
517	btsl	$(irq_num), iactive ;		/* still active */	\
518	jnc	0b ;				/* retry */		\
519	POP_FRAME ;							\
520	iret ;		/* XXX:	 iactive bit might be 0 now */		\
521	ALIGN_TEXT ;							\
5222: ;				/* masked by cpl, leave iactive set */	\
523	APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ;	\
524	lock ;								\
525	orl	$IRQ_BIT(irq_num), _ipending ;				\
526	AVCPL_UNLOCK ;							\
527	ISR_RELLOCK ;		/* XXX this is going away... */		\
528	POP_FRAME ;							\
529	iret ;								\
530	ALIGN_TEXT ;							\
5313: ; 			/* other cpu has isr lock */			\
532	APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
533	AVCPL_LOCK ;				/* MP-safe */		\
534	lock ;								\
535	orl	$IRQ_BIT(irq_num), _ipending ;				\
536	testl	$IRQ_BIT(irq_num), _cpl ;				\
537	jne	4f ;				/* this INT masked */	\
538	AVCPL_UNLOCK ;							\
539	call	forward_irq ;	 /* forward irq to lock holder */	\
540	POP_FRAME ;	 			/* and return */	\
541	iret ;								\
542	ALIGN_TEXT ;							\
5434: ;	 					/* blocked */		\
544	APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
545	AVCPL_UNLOCK ;							\
546	POP_FRAME ;	 			/* and return */	\
547	iret
548
549#endif /* CPL_AND_CML */
550
551
552/*
553 * Handle "spurious INTerrupts".
554 * Notes:
555 *  This is different than the "spurious INTerrupt" generated by an
556 *   8259 PIC for missing INTs.  See the APIC documentation for details.
557 *  This routine should NOT do an 'EOI' cycle.
558 */
559	.text
560	SUPERALIGN_TEXT
561	.globl _Xspuriousint
562_Xspuriousint:
563
564	/* No EOI cycle used here */
565
566	iret
567
568
569/*
570 * Handle TLB shootdowns.
571 */
572	.text
573	SUPERALIGN_TEXT
574	.globl	_Xinvltlb
575_Xinvltlb:
576	pushl	%eax
577
578#ifdef COUNT_XINVLTLB_HITS
579	pushl	%fs
580	movl	$KPSEL, %eax
581	movl	%ax, %fs
582	movl	_cpuid, %eax
583	popl	%fs
584	ss
585	incl	_xhits(,%eax,4)
586#endif /* COUNT_XINVLTLB_HITS */
587
588	movl	%cr3, %eax		/* invalidate the TLB */
589	movl	%eax, %cr3
590
591	ss				/* stack segment, avoid %ds load */
592	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
593
594	popl	%eax
595	iret
596
597
598#ifdef BETTER_CLOCK
599
600/*
601 * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
602 *
603 *  - Stores current cpu state in checkstate_cpustate[cpuid]
604 *      0 == user, 1 == sys, 2 == intr
605 *  - Stores current process in checkstate_curproc[cpuid]
606 *
607 *  - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
608 *
609 * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
610 */
611
612	.text
613	SUPERALIGN_TEXT
614	.globl _Xcpucheckstate
615	.globl _checkstate_cpustate
616	.globl _checkstate_curproc
617	.globl _checkstate_pc
618_Xcpucheckstate:
619	pushl	%eax
620	pushl	%ebx
621	pushl	%ds			/* save current data segment */
622	pushl	%fs
623
624	movl	$KDSEL, %eax
625	movl	%ax, %ds		/* use KERNEL data segment */
626	movl	$KPSEL, %eax
627	movl	%ax, %fs
628
629	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
630
631	movl	$0, %ebx
632	movl	20(%esp), %eax
633	andl	$3, %eax
634	cmpl	$3, %eax
635	je	1f
636	testl	$PSL_VM, 24(%esp)
637	jne	1f
638	incl	%ebx			/* system or interrupt */
639#ifdef CPL_AND_CML
640	cmpl	$0, _inside_intr
641	je	1f
642	incl	%ebx			/* interrupt */
643#endif
6441:
645	movl	_cpuid, %eax
646	movl	%ebx, _checkstate_cpustate(,%eax,4)
647	movl	_curproc, %ebx
648	movl	%ebx, _checkstate_curproc(,%eax,4)
649	movl	16(%esp), %ebx
650	movl	%ebx, _checkstate_pc(,%eax,4)
651
652	lock				/* checkstate_probed_cpus |= (1<<id) */
653	btsl	%eax, _checkstate_probed_cpus
654
655	popl	%fs
656	popl	%ds			/* restore previous data segment */
657	popl	%ebx
658	popl	%eax
659	iret
660
661#endif /* BETTER_CLOCK */
662
663/*
664 * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
665 *
666 *  - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
667 *
668 *  - We need a better method of triggering asts on other cpus.
669 */
670
671	.text
672	SUPERALIGN_TEXT
673	.globl _Xcpuast
674_Xcpuast:
675	PUSH_FRAME
676	movl	$KDSEL, %eax
677	movl	%ax, %ds		/* use KERNEL data segment */
678	movl	%ax, %es
679	movl	$KPSEL, %eax
680	movl	%ax, %fs
681
682	movl	_cpuid, %eax
683	lock				/* checkstate_need_ast &= ~(1<<id) */
684	btrl	%eax, _checkstate_need_ast
685	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
686
687	lock
688	btsl	%eax, _checkstate_pending_ast
689	jc	1f
690
691	FAKE_MCOUNT(13*4(%esp))
692
693	/*
694	 * Giant locks do not come cheap.
695	 * A lot of cycles are going to be wasted here.
696	 */
697	call	_get_isrlock
698
699	AVCPL_LOCK
700#ifdef CPL_AND_CML
701	movl	_cml, %eax
702#else
703	movl	_cpl, %eax
704#endif
705	pushl	%eax
706	movl	$1, _astpending		/* XXX */
707	AVCPL_UNLOCK
708	lock
709	incb	_intr_nesting_level
710	sti
711
712	pushl	$0
713
714	movl	_cpuid, %eax
715	lock
716	btrl	%eax, _checkstate_pending_ast
717	lock
718	btrl	%eax, CNAME(resched_cpus)
719	jnc	2f
720	movl	$1, CNAME(want_resched)
721	lock
722	incl	CNAME(want_resched_cnt)
7232:
724	lock
725	incl	CNAME(cpuast_cnt)
726	MEXITCOUNT
727	jmp	_doreti
7281:
729	/* We are already in the process of delivering an ast for this CPU */
730	POP_FRAME
731	iret
732
733
734/*
735 *	 Executed by a CPU when it receives an XFORWARD_IRQ IPI.
736 */
737
738	.text
739	SUPERALIGN_TEXT
740	.globl _Xforward_irq
741_Xforward_irq:
742	PUSH_FRAME
743	movl	$KDSEL, %eax
744	movl	%ax, %ds		/* use KERNEL data segment */
745	movl	%ax, %es
746	movl	$KPSEL, %eax
747	movl	%ax, %fs
748
749	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
750
751	FAKE_MCOUNT(13*4(%esp))
752
753	ISR_TRYLOCK
754	testl	%eax,%eax		/* Did we get the lock ? */
755	jz  1f				/* No */
756
757	lock
758	incl	CNAME(forward_irq_hitcnt)
759	cmpb	$4, _intr_nesting_level
760	jae	2f
761
762	AVCPL_LOCK
763#ifdef CPL_AND_CML
764	movl	_cml, %eax
765#else
766	movl	_cpl, %eax
767#endif
768	pushl	%eax
769	AVCPL_UNLOCK
770	lock
771	incb	_intr_nesting_level
772	sti
773
774	pushl	$0
775
776	MEXITCOUNT
777	jmp	_doreti			/* Handle forwarded interrupt */
7781:
779	lock
780	incl	CNAME(forward_irq_misscnt)
781	call	forward_irq	/* Oops, we've lost the isr lock */
782	MEXITCOUNT
783	POP_FRAME
784	iret
7852:
786	lock
787	incl	CNAME(forward_irq_toodeepcnt)
7883:
789	ISR_RELLOCK
790	MEXITCOUNT
791	POP_FRAME
792	iret
793
794/*
795 *
796 */
797forward_irq:
798	MCOUNT
799	cmpl	$0,_invltlb_ok
800	jz	4f
801
802	cmpl	$0, CNAME(forward_irq_enabled)
803	jz	4f
804
805	movl	_mp_lock,%eax
806	cmpl	$FREE_LOCK,%eax
807	jne	1f
808	movl	$0, %eax		/* Pick CPU #0 if noone has lock */
8091:
810	shrl	$24,%eax
811	movl	_cpu_num_to_apic_id(,%eax,4),%ecx
812	shll	$24,%ecx
813	movl	lapic_icr_hi, %eax
814	andl	$~APIC_ID_MASK, %eax
815	orl	%ecx, %eax
816	movl	%eax, lapic_icr_hi
817
8182:
819	movl	lapic_icr_lo, %eax
820	andl	$APIC_DELSTAT_MASK,%eax
821	jnz	2b
822	movl	lapic_icr_lo, %eax
823	andl	$APIC_RESV2_MASK, %eax
824	orl	$(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax
825	movl	%eax, lapic_icr_lo
8263:
827	movl	lapic_icr_lo, %eax
828	andl	$APIC_DELSTAT_MASK,%eax
829	jnz	3b
8304:
831	ret
832
833/*
834 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
835 *
836 *  - Signals its receipt.
837 *  - Waits for permission to restart.
838 *  - Signals its restart.
839 */
840
841	.text
842	SUPERALIGN_TEXT
843	.globl _Xcpustop
844_Xcpustop:
845	pushl	%ebp
846	movl	%esp, %ebp
847	pushl	%eax
848	pushl	%ecx
849	pushl	%edx
850	pushl	%ds			/* save current data segment */
851	pushl	%fs
852
853	movl	$KDSEL, %eax
854	movl	%ax, %ds		/* use KERNEL data segment */
855	movl	$KPSEL, %eax
856	movl	%ax, %fs
857
858	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
859
860	movl	_cpuid, %eax
861	imull	$PCB_SIZE, %eax
862	leal	CNAME(stoppcbs)(%eax), %eax
863	pushl	%eax
864	call	CNAME(savectx)		/* Save process context */
865	addl	$4, %esp
866
867
868	movl	_cpuid, %eax
869
870	lock
871	btsl	%eax, _stopped_cpus	/* stopped_cpus |= (1<<id) */
8721:
873	btl	%eax, _started_cpus	/* while (!(started_cpus & (1<<id))) */
874	jnc	1b
875
876	lock
877	btrl	%eax, _started_cpus	/* started_cpus &= ~(1<<id) */
878	lock
879	btrl	%eax, _stopped_cpus	/* stopped_cpus &= ~(1<<id) */
880
881	test	%eax, %eax
882	jnz	2f
883
884	movl	CNAME(cpustop_restartfunc), %eax
885	test	%eax, %eax
886	jz	2f
887	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
888
889	call	%eax
8902:
891	popl	%fs
892	popl	%ds			/* restore previous data segment */
893	popl	%edx
894	popl	%ecx
895	popl	%eax
896	movl	%ebp, %esp
897	popl	%ebp
898	iret
899
900
901MCOUNT_LABEL(bintr)
902	FAST_INTR(0,fastintr0)
903	FAST_INTR(1,fastintr1)
904	FAST_INTR(2,fastintr2)
905	FAST_INTR(3,fastintr3)
906	FAST_INTR(4,fastintr4)
907	FAST_INTR(5,fastintr5)
908	FAST_INTR(6,fastintr6)
909	FAST_INTR(7,fastintr7)
910	FAST_INTR(8,fastintr8)
911	FAST_INTR(9,fastintr9)
912	FAST_INTR(10,fastintr10)
913	FAST_INTR(11,fastintr11)
914	FAST_INTR(12,fastintr12)
915	FAST_INTR(13,fastintr13)
916	FAST_INTR(14,fastintr14)
917	FAST_INTR(15,fastintr15)
918	FAST_INTR(16,fastintr16)
919	FAST_INTR(17,fastintr17)
920	FAST_INTR(18,fastintr18)
921	FAST_INTR(19,fastintr19)
922	FAST_INTR(20,fastintr20)
923	FAST_INTR(21,fastintr21)
924	FAST_INTR(22,fastintr22)
925	FAST_INTR(23,fastintr23)
926#define	CLKINTR_PENDING	movl $1,CNAME(clkintr_pending)
927	INTR(0,intr0, CLKINTR_PENDING)
928	INTR(1,intr1,)
929	INTR(2,intr2,)
930	INTR(3,intr3,)
931	INTR(4,intr4,)
932	INTR(5,intr5,)
933	INTR(6,intr6,)
934	INTR(7,intr7,)
935	INTR(8,intr8,)
936	INTR(9,intr9,)
937	INTR(10,intr10,)
938	INTR(11,intr11,)
939	INTR(12,intr12,)
940	INTR(13,intr13,)
941	INTR(14,intr14,)
942	INTR(15,intr15,)
943	INTR(16,intr16,)
944	INTR(17,intr17,)
945	INTR(18,intr18,)
946	INTR(19,intr19,)
947	INTR(20,intr20,)
948	INTR(21,intr21,)
949	INTR(22,intr22,)
950	INTR(23,intr23,)
951MCOUNT_LABEL(eintr)
952
953/*
954 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
955 *
956 * - Calls the generic rendezvous action function.
957 */
958	.text
959	SUPERALIGN_TEXT
960	.globl	_Xrendezvous
961_Xrendezvous:
962	PUSH_FRAME
963	movl	$KDSEL, %eax
964	movl	%ax, %ds		/* use KERNEL data segment */
965	movl	%ax, %es
966	movl	$KPSEL, %eax
967	movl	%ax, %fs
968
969	call	_smp_rendezvous_action
970
971	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
972	POP_FRAME
973	iret
974
975
976	.data
977/*
978 * Addresses of interrupt handlers.
979 *  XresumeNN: Resumption addresses for HWIs.
980 */
981	.globl _ihandlers
982_ihandlers:
983/*
984 * used by:
985 *  ipl.s:	doreti_unpend
986 */
987	.long	Xresume0,  Xresume1,  Xresume2,  Xresume3
988	.long	Xresume4,  Xresume5,  Xresume6,  Xresume7
989	.long	Xresume8,  Xresume9,  Xresume10, Xresume11
990	.long	Xresume12, Xresume13, Xresume14, Xresume15
991	.long	Xresume16, Xresume17, Xresume18, Xresume19
992	.long	Xresume20, Xresume21, Xresume22, Xresume23
993/*
994 * used by:
995 *  ipl.s:	doreti_unpend
996 *  apic_ipl.s:	splz_unpend
997 */
998	.long	_swi_null, swi_net, _swi_null, _swi_null
999	.long	_swi_vm, _swi_null, _softclock, _swi_null
1000
1001imasks:				/* masks for interrupt handlers */
1002	.space	NHWI*4		/* padding; HWI masks are elsewhere */
1003
1004	.long	SWI_TTY_MASK, SWI_NET_MASK, SWI_CAMNET_MASK, SWI_CAMBIO_MASK
1005	.long	SWI_VM_MASK, 0, SWI_CLOCK_MASK, 0
1006
1007/* active flag for lazy masking */
1008iactive:
1009	.long	0
1010
1011#ifdef COUNT_XINVLTLB_HITS
1012	.globl	_xhits
1013_xhits:
1014	.space	(NCPU * 4), 0
1015#endif /* COUNT_XINVLTLB_HITS */
1016
1017/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
1018	.globl _stopped_cpus, _started_cpus
1019_stopped_cpus:
1020	.long	0
1021_started_cpus:
1022	.long	0
1023
1024#ifdef BETTER_CLOCK
1025	.globl _checkstate_probed_cpus
1026_checkstate_probed_cpus:
1027	.long	0
1028#endif /* BETTER_CLOCK */
1029	.globl _checkstate_need_ast
1030_checkstate_need_ast:
1031	.long	0
1032_checkstate_pending_ast:
1033	.long	0
1034	.globl CNAME(forward_irq_misscnt)
1035	.globl CNAME(forward_irq_toodeepcnt)
1036	.globl CNAME(forward_irq_hitcnt)
1037	.globl CNAME(resched_cpus)
1038	.globl CNAME(want_resched_cnt)
1039	.globl CNAME(cpuast_cnt)
1040	.globl CNAME(cpustop_restartfunc)
1041CNAME(forward_irq_misscnt):
1042	.long 0
1043CNAME(forward_irq_hitcnt):
1044	.long 0
1045CNAME(forward_irq_toodeepcnt):
1046	.long 0
1047CNAME(resched_cpus):
1048	.long 0
1049CNAME(want_resched_cnt):
1050	.long 0
1051CNAME(cpuast_cnt):
1052	.long 0
1053CNAME(cpustop_restartfunc):
1054	.long 0
1055
1056
1057
1058	.globl	_apic_pin_trigger
1059_apic_pin_trigger:
1060	.long	0
1061
1062	.text
1063