1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * $FreeBSD$
31 */
32
33#include "opt_apic.h"
34#include "opt_npx.h"
35
36#include <machine/asmacros.h>
37#include <machine/psl.h>
38#include <machine/trap.h>
39
40#include "assym.s"
41
42#define	SEL_RPL_MASK	0x0002
43#define __HYPERVISOR_iret	23
44
45/* Offsets into shared_info_t. */
46
47#define evtchn_upcall_pending /* 0 */
48#define evtchn_upcall_mask       1
49
50#define	sizeof_vcpu_shift	6
51
52
53#ifdef SMP
54#define GET_VCPU_INFO(reg)	movl PCPU(CPUID),reg			; \
55				shl  $sizeof_vcpu_shift,reg		; \
56				addl HYPERVISOR_shared_info,reg
57#else
58#define GET_VCPU_INFO(reg)	movl HYPERVISOR_shared_info,reg
59#endif
60
61#define __DISABLE_INTERRUPTS(reg)	movb $1,evtchn_upcall_mask(reg)
62#define __ENABLE_INTERRUPTS(reg)	movb $0,evtchn_upcall_mask(reg)
63#define DISABLE_INTERRUPTS(reg)	GET_VCPU_INFO(reg)			; \
64				__DISABLE_INTERRUPTS(reg)
65#define ENABLE_INTERRUPTS(reg)	GET_VCPU_INFO(reg)			; \
66				__ENABLE_INTERRUPTS(reg)
67#define __TEST_PENDING(reg)	testb $0xFF,evtchn_upcall_pending(reg)
68
69#define POPA \
70        popl %edi; \
71        popl %esi; \
72        popl %ebp; \
73        popl %ebx; \
74        popl %ebx; \
75        popl %edx; \
76        popl %ecx; \
77        popl %eax;
78
79	.text
80
81/*****************************************************************************/
82/* Trap handling                                                             */
83/*****************************************************************************/
84/*
85 * Trap and fault vector routines.
86 *
87 * Most traps are 'trap gates', SDT_SYS386TGT.  A trap gate pushes state on
88 * the stack that mostly looks like an interrupt, but does not disable
89 * interrupts.  A few of the traps we are use are interrupt gates,
90 * SDT_SYS386IGT, which are nearly the same thing except interrupts are
91 * disabled on entry.
92 *
93 * The cpu will push a certain amount of state onto the kernel stack for
94 * the current process.  The amount of state depends on the type of trap
95 * and whether the trap crossed rings or not.  See i386/include/frame.h.
96 * At the very least the current EFLAGS (status register, which includes
97 * the interrupt disable state prior to the trap), the code segment register,
98 * and the return instruction pointer are pushed by the cpu.  The cpu
99 * will also push an 'error' code for certain traps.  We push a dummy
100 * error code for those traps where the cpu doesn't in order to maintain
101 * a consistent frame.  We also push a contrived 'trap number'.
102 *
103 * The cpu does not push the general registers, we must do that, and we
104 * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
105 * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
106 * must load them with appropriate values for supervisor mode operation.
107 */
108
109MCOUNT_LABEL(user)
110MCOUNT_LABEL(btrap)
111
112#define	TRAP(a)		pushl $(a) ; jmp alltraps
113
114IDTVEC(div)
115	pushl $0; TRAP(T_DIVIDE)
116IDTVEC(dbg)
117	pushl $0; TRAP(T_TRCTRAP)
118IDTVEC(nmi)
119	pushl $0; TRAP(T_NMI)
120IDTVEC(bpt)
121	pushl $0; TRAP(T_BPTFLT)
122IDTVEC(ofl)
123	pushl $0; TRAP(T_OFLOW)
124IDTVEC(bnd)
125	pushl $0; TRAP(T_BOUND)
126IDTVEC(ill)
127	pushl $0; TRAP(T_PRIVINFLT)
128IDTVEC(dna)
129	pushl $0; TRAP(T_DNA)
130IDTVEC(fpusegm)
131	pushl $0; TRAP(T_FPOPFLT)
132IDTVEC(tss)
133	TRAP(T_TSSFLT)
134IDTVEC(missing)
135	TRAP(T_SEGNPFLT)
136IDTVEC(stk)
137	TRAP(T_STKFLT)
138IDTVEC(prot)
139	TRAP(T_PROTFLT)
140IDTVEC(page)
141	TRAP(T_PAGEFLT)
142IDTVEC(mchk)
143	pushl $0; TRAP(T_MCHK)
144IDTVEC(rsvd)
145	pushl $0; TRAP(T_RESERVED)
146IDTVEC(fpu)
147	pushl $0; TRAP(T_ARITHTRAP)
148IDTVEC(align)
149	TRAP(T_ALIGNFLT)
150IDTVEC(xmm)
151	pushl $0; TRAP(T_XMMFLT)
152
153IDTVEC(hypervisor_callback)
154	pushl $0;
155	pushl $0;
156	pushal
157	pushl	%ds
158	pushl	%es
159	pushl	%fs
160upcall_with_regs_pushed:
161	SET_KERNEL_SREGS
162	FAKE_MCOUNT(TF_EIP(%esp))
163call_evtchn_upcall:
164	movl	TF_EIP(%esp),%eax
165	cmpl	$scrit,%eax
166	jb	10f
167	cmpl	$ecrit,%eax
168	jb	critical_region_fixup
169
17010:	pushl	%esp
171	call	evtchn_do_upcall
172	addl	$4,%esp
173
174	/*
175	 * Return via doreti to handle ASTs.
176	 */
177	MEXITCOUNT
178	jmp	doreti
179
180
181hypervisor_callback_pending:
182	DISABLE_INTERRUPTS(%esi)				/*	cli */
183	jmp	10b
184	/*
185	 * alltraps entry point.  Interrupts are enabled if this was a trap
186	 * gate (TGT), else disabled if this was an interrupt gate (IGT).
187	 * Note that int0x80_syscall is a trap gate.  Only page faults
188	 * use an interrupt gate.
189	 */
190	SUPERALIGN_TEXT
191	.globl	alltraps
192	.type	alltraps,@function
193alltraps:
194	pushal
195	pushl	%ds
196	pushl	%es
197	pushl	%fs
198
199alltraps_with_regs_pushed:
200	SET_KERNEL_SREGS
201	FAKE_MCOUNT(TF_EIP(%esp))
202
203calltrap:
204	push	%esp
205	call	trap
206	add	$4, %esp
207
208	/*
209	 * Return via doreti to handle ASTs.
210	 */
211	MEXITCOUNT
212	jmp	doreti
213
214/*
215 * SYSCALL CALL GATE (old entry point for a.out binaries)
216 *
217 * The intersegment call has been set up to specify one dummy parameter.
218 *
219 * This leaves a place to put eflags so that the call frame can be
220 * converted to a trap frame. Note that the eflags is (semi-)bogusly
221 * pushed into (what will be) tf_err and then copied later into the
222 * final spot. It has to be done this way because esp can't be just
223 * temporarily altered for the pushfl - an interrupt might come in
224 * and clobber the saved cs/eip.
225 */
226	SUPERALIGN_TEXT
227IDTVEC(lcall_syscall)
228	pushfl				/* save eflags */
229	popl	8(%esp)			/* shuffle into tf_eflags */
230	pushl	$7			/* sizeof "lcall 7,0" */
231	subl	$4,%esp			/* skip over tf_trapno */
232	pushal
233	pushl	%ds
234	pushl	%es
235	pushl	%fs
236	SET_KERNEL_SREGS
237	FAKE_MCOUNT(TF_EIP(%esp))
238	pushl	%esp
239	call	syscall
240	add	$4, %esp
241	MEXITCOUNT
242	jmp	doreti
243
244/*
245 * Call gate entry for FreeBSD ELF and Linux/NetBSD syscall (int 0x80)
246 *
247 * Even though the name says 'int0x80', this is actually a TGT (trap gate)
248 * rather then an IGT (interrupt gate).  Thus interrupts are enabled on
249 * entry just as they are for a normal syscall.
250 */
251	SUPERALIGN_TEXT
252IDTVEC(int0x80_syscall)
253	pushl	$2			/* sizeof "int 0x80" */
254	pushl	$0xBEEF			/* for debug */
255	pushal
256	pushl	%ds
257	pushl	%es
258	pushl	%fs
259	SET_KERNEL_SREGS
260	FAKE_MCOUNT(TF_EIP(%esp))
261	pushl	%esp
262	call	syscall
263	add	$4, %esp
264	MEXITCOUNT
265	jmp	doreti
266
267ENTRY(fork_trampoline)
268	pushl	%esp			/* trapframe pointer */
269	pushl	%ebx			/* arg1 */
270	pushl	%esi			/* function */
271	call	fork_exit
272	addl	$12,%esp
273	/* cut from syscall */
274
275	/*
276	 * Return via doreti to handle ASTs.
277	 */
278	MEXITCOUNT
279	jmp	doreti
280
281
282/*
283 * To efficiently implement classification of trap and interrupt handlers
284 * for profiling, there must be only trap handlers between the labels btrap
285 * and bintr, and only interrupt handlers between the labels bintr and
286 * eintr.  This is implemented (partly) by including files that contain
287 * some of the handlers.  Before including the files, set up a normal asm
288 * environment so that the included files doen't need to know that they are
289 * included.
290 */
291
292	.data
293	.p2align 4
294	.text
295	SUPERALIGN_TEXT
296MCOUNT_LABEL(bintr)
297
298#ifdef DEV_APIC
299	.data
300	.p2align 4
301	.text
302	SUPERALIGN_TEXT
303
304#include <i386/i386/apic_vector.s>
305#endif
306
307	.data
308	.p2align 4
309	.text
310	SUPERALIGN_TEXT
311#include <i386/i386/vm86bios.s>
312
313	.text
314MCOUNT_LABEL(eintr)
315
316/*
317 * void doreti(struct trapframe)
318 *
319 * Handle return from interrupts, traps and syscalls.
320 */
321	.text
322	SUPERALIGN_TEXT
323	.type	doreti,@function
324doreti:
325	FAKE_MCOUNT($bintr)		/* init "from" bintr -> doreti */
326doreti_next:
327#ifdef notyet
328	/*
329	 * Check if ASTs can be handled now.  PSL_VM must be checked first
330	 * since segment registers only have an RPL in non-VM86 mode.
331	 */
332	testl	$PSL_VM,TF_EFLAGS(%esp)	/* are we in vm86 mode? */
333	jz	doreti_notvm86
334	movl	PCPU(CURPCB),%ecx
335	testl	$PCB_VM86CALL,PCB_FLAGS(%ecx)	/* are we in a vm86 call? */
336	jz	doreti_ast		/* can handle ASTS now if not */
337  	jmp	doreti_exit
338
339doreti_notvm86:
340#endif
341	testb	$SEL_RPL_MASK,TF_CS(%esp) /* are we returning to user mode? */
342	jz	doreti_exit		/* can't handle ASTs now if not */
343
344doreti_ast:
345	/*
346	 * Check for ASTs atomically with returning.  Disabling CPU
347	 * interrupts provides sufficient locking even in the SMP case,
348	 * since we will be informed of any new ASTs by an IPI.
349	 */
350	DISABLE_INTERRUPTS(%esi)				/*	cli */
351	movl	PCPU(CURTHREAD),%eax
352	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%eax)
353	je	doreti_exit
354	ENABLE_INTERRUPTS(%esi)	/* sti */
355	pushl	%esp			/* pass a pointer to the trapframe */
356	call	ast
357	add	$4,%esp
358	jmp	doreti_ast
359
360	/*
361	 * doreti_exit:	pop registers, iret.
362	 *
363	 *	The segment register pop is a special case, since it may
364	 *	fault if (for example) a sigreturn specifies bad segment
365	 *	registers.  The fault is handled in trap.c.
366	 */
367doreti_exit:
368	ENABLE_INTERRUPTS(%esi) # reenable event callbacks (sti)
369
370	.globl	scrit
371scrit:
372	__TEST_PENDING(%esi)
373        jnz	hypervisor_callback_pending	/* More to go  */
374
375	MEXITCOUNT
376
377	.globl	doreti_popl_fs
378doreti_popl_fs:
379	popl	%fs
380	.globl	doreti_popl_es
381doreti_popl_es:
382	popl	%es
383	.globl	doreti_popl_ds
384doreti_popl_ds:
385	popl	%ds
386
387	/*
388	 * This is important: as nothing is atomic over here (we can get
389	 * interrupted any time), we use the critical_region_fixup() in
390	 * order to figure out where out stack is. Therefore, do NOT use
391	 * 'popal' here without fixing up the table!
392	 */
393	POPA
394	addl	$8,%esp
395	.globl	doreti_iret
396doreti_iret:
397	jmp	hypercall_page + (__HYPERVISOR_iret * 32)
398	.globl	ecrit
399ecrit:
400  	/*
401	 * doreti_iret_fault and friends.  Alternative return code for
402	 * the case where we get a fault in the doreti_exit code
403	 * above.  trap() (i386/i386/trap.c) catches this specific
404	 * case, sends the process a signal and continues in the
405	 * corresponding place in the code below.
406	 */
407	ALIGN_TEXT
408	.globl	doreti_iret_fault
409doreti_iret_fault:
410	subl	$8,%esp
411	pushal
412	pushl	%ds
413	.globl	doreti_popl_ds_fault
414doreti_popl_ds_fault:
415	pushl	%es
416	.globl	doreti_popl_es_fault
417doreti_popl_es_fault:
418	pushl	%fs
419	.globl	doreti_popl_fs_fault
420doreti_popl_fs_fault:
421	movl	$0,TF_ERR(%esp)	/* XXX should be the error code */
422	movl	$T_PROTFLT,TF_TRAPNO(%esp)
423	jmp	alltraps_with_regs_pushed
424
425	/*
426# [How we do the fixup]. We want to merge the current stack frame with the
427# just-interrupted frame. How we do this depends on where in the critical
428# region the interrupted handler was executing, and so how many saved
429# registers are in each frame. We do this quickly using the lookup table
430# 'critical_fixup_table'. For each byte offset in the critical region, it
431# provides the number of bytes which have already been popped from the
432# interrupted stack frame.
433*/
434
435.globl critical_region_fixup
436critical_region_fixup:
437	addl $critical_fixup_table-scrit,%eax
438	movzbl (%eax),%eax    # %eax contains num bytes popped
439        movl  %esp,%esi
440        add  %eax,%esi        # %esi points at end of src region
441        movl  %esp,%edi
442        add  $0x40,%edi       # %edi points at end of dst region
443        movl  %eax,%ecx
444        shr  $2,%ecx          # convert bytes to words
445        je   16f              # skip loop if nothing to copy
44615:     subl $4,%esi          # pre-decrementing copy loop
447        subl $4,%edi
448        movl (%esi),%eax
449        movl %eax,(%edi)
450        loop 15b
45116:     movl %edi,%esp        # final %edi is top of merged stack
452	jmp  hypervisor_callback_pending
453
454
455critical_fixup_table:
456.byte   0x0,0x0,0x0			#testb  $0x1,(%esi)
457.byte   0x0,0x0,0x0,0x0,0x0,0x0		#jne    ea
458.byte   0x0,0x0				#pop    %fs
459.byte   0x04				#pop    %es
460.byte   0x08				#pop    %ds
461.byte   0x0c				#pop    %edi
462.byte   0x10	                        #pop    %esi
463.byte   0x14	                        #pop    %ebp
464.byte   0x18	                        #pop    %ebx
465.byte   0x1c	                        #pop    %ebx
466.byte   0x20	                        #pop    %edx
467.byte   0x24	                        #pop    %ecx
468.byte   0x28	                        #pop    %eax
469.byte   0x2c,0x2c,0x2c                  #add    $0x8,%esp
470#if 0
471	.byte   0x34	                        #iret
472#endif
473.byte   0x34,0x34,0x34,0x34,0x34        #HYPERVISOR_iret
474
475
476/* # Hypervisor uses this for application faults while it executes.*/
477ENTRY(failsafe_callback)
478	pushal
479	call xen_failsafe_handler
480/*#	call install_safe_pf_handler */
481        movl 28(%esp),%ebx
4821:      movl %ebx,%ds
483        movl 32(%esp),%ebx
4842:      movl %ebx,%es
485        movl 36(%esp),%ebx
4863:      movl %ebx,%fs
487        movl 40(%esp),%ebx
4884:      movl %ebx,%gs
489/*#        call install_normal_pf_handler */
490	popal
491	addl $12,%esp
492	iret
493
494
495