1/*-
2 * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 *    notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 *    notice, this list of conditions and the following disclaimer in the
20 *    documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35#include <sys/cdefs.h>
36__FBSDID("$FreeBSD$");
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/lock.h>
42#include <sys/mutex.h>
43#include <sys/pioctl.h>
44#include <sys/bus.h>
45#include <sys/proc.h>
46#include <sys/ptrace.h>
47#include <sys/syscall.h>
48#include <sys/sysent.h>
49#ifdef KDB
50#include <sys/kdb.h>
51#endif
52
53#include <vm/vm.h>
54#include <vm/pmap.h>
55#include <vm/vm_kern.h>
56#include <vm/vm_map.h>
57#include <vm/vm_param.h>
58#include <vm/vm_extern.h>
59
60#ifdef FPE
61#include <machine/fpe.h>
62#endif
63#include <machine/frame.h>
64#include <machine/pcb.h>
65#include <machine/pcpu.h>
66
67#include <machine/resource.h>
68#include <machine/intr.h>
69
70#ifdef KDTRACE_HOOKS
71#include <sys/dtrace_bsd.h>
72#endif
73
74int (*dtrace_invop_jump_addr)(struct trapframe *);
75
76extern register_t fsu_intr_fault;
77
78/* Called from exception.S */
79void do_trap_supervisor(struct trapframe *);
80void do_trap_user(struct trapframe *);
81
82static __inline void
83call_trapsignal(struct thread *td, int sig, int code, void *addr)
84{
85	ksiginfo_t ksi;
86
87	ksiginfo_init_trap(&ksi);
88	ksi.ksi_signo = sig;
89	ksi.ksi_code = code;
90	ksi.ksi_addr = addr;
91	trapsignal(td, &ksi);
92}
93
94int
95cpu_fetch_syscall_args(struct thread *td)
96{
97	struct proc *p;
98	register_t *ap;
99	struct syscall_args *sa;
100	int nap;
101
102	nap = NARGREG;
103	p = td->td_proc;
104	sa = &td->td_sa;
105	ap = &td->td_frame->tf_a[0];
106
107	sa->code = td->td_frame->tf_t[0];
108
109	if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
110		sa->code = *ap++;
111		nap--;
112	}
113
114	if (p->p_sysent->sv_mask)
115		sa->code &= p->p_sysent->sv_mask;
116	if (sa->code >= p->p_sysent->sv_size)
117		sa->callp = &p->p_sysent->sv_table[0];
118	else
119		sa->callp = &p->p_sysent->sv_table[sa->code];
120
121	sa->narg = sa->callp->sy_narg;
122	memcpy(sa->args, ap, nap * sizeof(register_t));
123	if (sa->narg > nap)
124		panic("TODO: Could we have more then %d args?", NARGREG);
125
126	td->td_retval[0] = 0;
127	td->td_retval[1] = 0;
128
129	return (0);
130}
131
132#include "../../kern/subr_syscall.c"
133
134static void
135dump_regs(struct trapframe *frame)
136{
137	int n;
138	int i;
139
140	n = (sizeof(frame->tf_t) / sizeof(frame->tf_t[0]));
141	for (i = 0; i < n; i++)
142		printf("t[%d] == 0x%016lx\n", i, frame->tf_t[i]);
143
144	n = (sizeof(frame->tf_s) / sizeof(frame->tf_s[0]));
145	for (i = 0; i < n; i++)
146		printf("s[%d] == 0x%016lx\n", i, frame->tf_s[i]);
147
148	n = (sizeof(frame->tf_a) / sizeof(frame->tf_a[0]));
149	for (i = 0; i < n; i++)
150		printf("a[%d] == 0x%016lx\n", i, frame->tf_a[i]);
151
152	printf("sepc == 0x%016lx\n", frame->tf_sepc);
153	printf("sstatus == 0x%016lx\n", frame->tf_sstatus);
154}
155
156static void
157svc_handler(struct trapframe *frame)
158{
159	struct thread *td;
160
161	td = curthread;
162	td->td_frame = frame;
163
164	syscallenter(td);
165	syscallret(td);
166}
167
168static void
169data_abort(struct trapframe *frame, int usermode)
170{
171	struct vm_map *map;
172	uint64_t stval;
173	struct thread *td;
174	struct pcb *pcb;
175	vm_prot_t ftype;
176	vm_offset_t va;
177	struct proc *p;
178	int error, sig, ucode;
179
180#ifdef KDB
181	if (kdb_active) {
182		kdb_reenter();
183		return;
184	}
185#endif
186
187	td = curthread;
188	p = td->td_proc;
189	pcb = td->td_pcb;
190	stval = frame->tf_stval;
191
192	if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 ||
193	    WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
194	    "Kernel page fault") != 0)
195		goto fatal;
196
197	if (usermode) {
198		map = &td->td_proc->p_vmspace->vm_map;
199	} else {
200		/*
201		 * Enable interrupts for the duration of the page fault. For
202		 * user faults this was done already in do_trap_user().
203		 */
204		intr_enable();
205
206		if (stval >= VM_MAX_USER_ADDRESS) {
207			map = kernel_map;
208		} else {
209			if (pcb->pcb_onfault == 0)
210				goto fatal;
211			map = &td->td_proc->p_vmspace->vm_map;
212		}
213	}
214
215	va = trunc_page(stval);
216
217	if ((frame->tf_scause == EXCP_FAULT_STORE) ||
218	    (frame->tf_scause == EXCP_STORE_PAGE_FAULT)) {
219		ftype = VM_PROT_WRITE;
220	} else if (frame->tf_scause == EXCP_INST_PAGE_FAULT) {
221		ftype = VM_PROT_EXECUTE;
222	} else {
223		ftype = VM_PROT_READ;
224	}
225
226	if (pmap_fault_fixup(map->pmap, va, ftype))
227		goto done;
228
229	error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode);
230	if (error != KERN_SUCCESS) {
231		if (usermode) {
232			call_trapsignal(td, sig, ucode, (void *)stval);
233		} else {
234			if (pcb->pcb_onfault != 0) {
235				frame->tf_a[0] = error;
236				frame->tf_sepc = pcb->pcb_onfault;
237				return;
238			}
239			goto fatal;
240		}
241	}
242
243done:
244	if (usermode)
245		userret(td, frame);
246	return;
247
248fatal:
249	dump_regs(frame);
250	panic("Fatal page fault at %#lx: %#016lx", frame->tf_sepc, stval);
251}
252
253void
254do_trap_supervisor(struct trapframe *frame)
255{
256	uint64_t exception;
257
258	/* Ensure we came from supervisor mode, interrupts disabled */
259	KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) ==
260	    SSTATUS_SPP, ("Came from S mode with interrupts enabled"));
261
262	exception = (frame->tf_scause & EXCP_MASK);
263	if (frame->tf_scause & EXCP_INTR) {
264		/* Interrupt */
265		riscv_cpu_intr(frame);
266		return;
267	}
268
269#ifdef KDTRACE_HOOKS
270	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
271		return;
272#endif
273
274	CTR3(KTR_TRAP, "do_trap_supervisor: curthread: %p, sepc: %lx, frame: %p",
275	    curthread, frame->tf_sepc, frame);
276
277	switch(exception) {
278	case EXCP_FAULT_LOAD:
279	case EXCP_FAULT_STORE:
280	case EXCP_FAULT_FETCH:
281	case EXCP_STORE_PAGE_FAULT:
282	case EXCP_LOAD_PAGE_FAULT:
283		data_abort(frame, 0);
284		break;
285	case EXCP_BREAKPOINT:
286#ifdef KDTRACE_HOOKS
287		if (dtrace_invop_jump_addr != 0) {
288			dtrace_invop_jump_addr(frame);
289			break;
290		}
291#endif
292#ifdef KDB
293		kdb_trap(exception, 0, frame);
294#else
295		dump_regs(frame);
296		panic("No debugger in kernel.\n");
297#endif
298		break;
299	case EXCP_ILLEGAL_INSTRUCTION:
300		dump_regs(frame);
301		panic("Illegal instruction at 0x%016lx\n", frame->tf_sepc);
302		break;
303	default:
304		dump_regs(frame);
305		panic("Unknown kernel exception %x trap value %lx\n",
306		    exception, frame->tf_stval);
307	}
308}
309
310void
311do_trap_user(struct trapframe *frame)
312{
313	uint64_t exception;
314	struct thread *td;
315	struct pcb *pcb;
316
317	td = curthread;
318	td->td_frame = frame;
319	pcb = td->td_pcb;
320
321	/* Ensure we came from usermode, interrupts disabled */
322	KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0,
323	    ("Came from U mode with interrupts enabled"));
324
325	exception = (frame->tf_scause & EXCP_MASK);
326	if (frame->tf_scause & EXCP_INTR) {
327		/* Interrupt */
328		riscv_cpu_intr(frame);
329		return;
330	}
331	intr_enable();
332
333	CTR3(KTR_TRAP, "do_trap_user: curthread: %p, sepc: %lx, frame: %p",
334	    curthread, frame->tf_sepc, frame);
335
336	switch(exception) {
337	case EXCP_FAULT_LOAD:
338	case EXCP_FAULT_STORE:
339	case EXCP_FAULT_FETCH:
340	case EXCP_STORE_PAGE_FAULT:
341	case EXCP_LOAD_PAGE_FAULT:
342	case EXCP_INST_PAGE_FAULT:
343		data_abort(frame, 1);
344		break;
345	case EXCP_USER_ECALL:
346		frame->tf_sepc += 4;	/* Next instruction */
347		svc_handler(frame);
348		break;
349	case EXCP_ILLEGAL_INSTRUCTION:
350#ifdef FPE
351		if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) {
352			/*
353			 * May be a FPE trap. Enable FPE usage
354			 * for this thread and try again.
355			 */
356			fpe_state_clear();
357			frame->tf_sstatus &= ~SSTATUS_FS_MASK;
358			frame->tf_sstatus |= SSTATUS_FS_CLEAN;
359			pcb->pcb_fpflags |= PCB_FP_STARTED;
360			break;
361		}
362#endif
363		call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc);
364		userret(td, frame);
365		break;
366	case EXCP_BREAKPOINT:
367		call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc);
368		userret(td, frame);
369		break;
370	default:
371		dump_regs(frame);
372		panic("Unknown userland exception %x, trap value %lx\n",
373		    exception, frame->tf_stval);
374	}
375}
376