1/*-
2 * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 *    notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 *    notice, this list of conditions and the following disclaimer in the
20 *    documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35#include <sys/cdefs.h>
36#include "opt_ddb.h"
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/ktr.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>
44#include <sys/bus.h>
45#include <sys/proc.h>
46#include <sys/ptrace.h>
47#include <sys/syscall.h>
48#include <sys/sysent.h>
49#ifdef KDB
50#include <sys/kdb.h>
51#endif
52
53#include <vm/vm.h>
54#include <vm/pmap.h>
55#include <vm/vm_kern.h>
56#include <vm/vm_map.h>
57#include <vm/vm_param.h>
58#include <vm/vm_extern.h>
59
60#include <machine/fpe.h>
61#include <machine/frame.h>
62#include <machine/pcb.h>
63#include <machine/pcpu.h>
64
65#include <machine/resource.h>
66#include <machine/intr.h>
67
68#ifdef KDTRACE_HOOKS
69#include <sys/dtrace_bsd.h>
70#endif
71
72#ifdef DDB
73#include <ddb/ddb.h>
74#include <ddb/db_sym.h>
75#endif
76
77void intr_irq_handler(struct trapframe *tf);
78
79int (*dtrace_invop_jump_addr)(struct trapframe *);
80
81/* Called from exception.S */
82void do_trap_supervisor(struct trapframe *);
83void do_trap_user(struct trapframe *);
84
85static __inline void
86call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno)
87{
88	ksiginfo_t ksi;
89
90	ksiginfo_init_trap(&ksi);
91	ksi.ksi_signo = sig;
92	ksi.ksi_code = code;
93	ksi.ksi_addr = addr;
94	ksi.ksi_trapno = trapno;
95	trapsignal(td, &ksi);
96}
97
98int
99cpu_fetch_syscall_args(struct thread *td)
100{
101	struct proc *p;
102	syscallarg_t *ap, *dst_ap;
103	struct syscall_args *sa;
104
105	p = td->td_proc;
106	sa = &td->td_sa;
107	ap = &td->td_frame->tf_a[0];
108	dst_ap = &sa->args[0];
109
110	sa->code = td->td_frame->tf_t[0];
111	sa->original_code = sa->code;
112
113	if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) {
114		sa->code = *ap++;
115	} else {
116		*dst_ap++ = *ap++;
117	}
118
119	if (__predict_false(sa->code >= p->p_sysent->sv_size))
120		sa->callp = &nosys_sysent;
121	else
122		sa->callp = &p->p_sysent->sv_table[sa->code];
123
124	KASSERT(sa->callp->sy_narg <= nitems(sa->args),
125	    ("Syscall %d takes too many arguments", sa->code));
126
127	memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(*dst_ap));
128
129	td->td_retval[0] = 0;
130	td->td_retval[1] = 0;
131
132	return (0);
133}
134
135#include "../../kern/subr_syscall.c"
136
137static void
138print_with_symbol(const char *name, uint64_t value)
139{
140#ifdef DDB
141	c_db_sym_t sym;
142	db_expr_t sym_value;
143	db_expr_t offset;
144	const char *sym_name;
145#endif
146
147	printf("%7s: 0x%016lx", name, value);
148
149#ifdef DDB
150	if (value >= VM_MIN_KERNEL_ADDRESS) {
151		sym = db_search_symbol(value, DB_STGY_ANY, &offset);
152		if (sym != C_DB_SYM_NULL) {
153			db_symbol_values(sym, &sym_name, &sym_value);
154			if (offset != 0)
155				printf(" (%s + 0x%lx)", sym_name, offset);
156			else
157				printf(" (%s)", sym_name);
158		}
159	}
160#endif
161	printf("\n");
162}
163
164static void
165dump_regs(struct trapframe *frame)
166{
167	char name[6];
168	int i;
169
170	for (i = 0; i < nitems(frame->tf_t); i++) {
171		snprintf(name, sizeof(name), "t[%d]", i);
172		print_with_symbol(name, frame->tf_t[i]);
173	}
174
175	for (i = 0; i < nitems(frame->tf_s); i++) {
176		snprintf(name, sizeof(name), "s[%d]", i);
177		print_with_symbol(name, frame->tf_s[i]);
178	}
179
180	for (i = 0; i < nitems(frame->tf_a); i++) {
181		snprintf(name, sizeof(name), "a[%d]", i);
182		print_with_symbol(name, frame->tf_a[i]);
183	}
184
185	print_with_symbol("ra", frame->tf_ra);
186	print_with_symbol("sp", frame->tf_sp);
187	print_with_symbol("gp", frame->tf_gp);
188	print_with_symbol("tp", frame->tf_tp);
189	print_with_symbol("sepc", frame->tf_sepc);
190	printf("sstatus: 0x%016lx\n", frame->tf_sstatus);
191	printf("stval  : 0x%016lx\n", frame->tf_stval);
192}
193
194static void
195ecall_handler(void)
196{
197	struct thread *td;
198
199	td = curthread;
200
201	syscallenter(td);
202	syscallret(td);
203}
204
205static void
206page_fault_handler(struct trapframe *frame, int usermode)
207{
208	struct vm_map *map;
209	uint64_t stval;
210	struct thread *td;
211	struct pcb *pcb;
212	vm_prot_t ftype;
213	vm_offset_t va;
214	struct proc *p;
215	int error, sig, ucode;
216#ifdef KDB
217	bool handled;
218#endif
219
220#ifdef KDB
221	if (kdb_active) {
222		kdb_reenter();
223		return;
224	}
225#endif
226
227	td = curthread;
228	p = td->td_proc;
229	pcb = td->td_pcb;
230	stval = frame->tf_stval;
231
232	if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 ||
233	    WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
234	    "Kernel page fault") != 0)
235		goto fatal;
236
237	if (usermode) {
238		if (!VIRT_IS_VALID(stval)) {
239			call_trapsignal(td, SIGSEGV, SEGV_MAPERR, (void *)stval,
240			    frame->tf_scause & SCAUSE_CODE);
241			goto done;
242		}
243		map = &p->p_vmspace->vm_map;
244	} else {
245		/*
246		 * Enable interrupts for the duration of the page fault. For
247		 * user faults this was done already in do_trap_user().
248		 */
249		intr_enable();
250
251		if (stval >= VM_MIN_KERNEL_ADDRESS) {
252			map = kernel_map;
253		} else {
254			if (pcb->pcb_onfault == 0)
255				goto fatal;
256			map = &p->p_vmspace->vm_map;
257		}
258	}
259
260	va = trunc_page(stval);
261
262	if (frame->tf_scause == SCAUSE_STORE_PAGE_FAULT) {
263		ftype = VM_PROT_WRITE;
264	} else if (frame->tf_scause == SCAUSE_INST_PAGE_FAULT) {
265		ftype = VM_PROT_EXECUTE;
266	} else {
267		ftype = VM_PROT_READ;
268	}
269
270	if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype))
271		goto done;
272
273	error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode);
274	if (error != KERN_SUCCESS) {
275		if (usermode) {
276			call_trapsignal(td, sig, ucode, (void *)stval,
277			    frame->tf_scause & SCAUSE_CODE);
278		} else {
279			if (pcb->pcb_onfault != 0) {
280				frame->tf_a[0] = error;
281				frame->tf_sepc = pcb->pcb_onfault;
282				return;
283			}
284			goto fatal;
285		}
286	}
287
288done:
289	if (usermode)
290		userret(td, frame);
291	return;
292
293fatal:
294	dump_regs(frame);
295#ifdef KDB
296	if (debugger_on_trap) {
297		kdb_why = KDB_WHY_TRAP;
298		handled = kdb_trap(frame->tf_scause & SCAUSE_CODE, 0, frame);
299		kdb_why = KDB_WHY_UNSET;
300		if (handled)
301			return;
302	}
303#endif
304	panic("Fatal page fault at %#lx: %#lx", frame->tf_sepc, stval);
305}
306
307void
308do_trap_supervisor(struct trapframe *frame)
309{
310	uint64_t exception;
311
312	/* Ensure we came from supervisor mode, interrupts disabled */
313	KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) ==
314	    SSTATUS_SPP, ("Came from S mode with interrupts enabled"));
315
316	KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0,
317	    ("Came from S mode with SUM enabled"));
318
319	exception = frame->tf_scause & SCAUSE_CODE;
320	if ((frame->tf_scause & SCAUSE_INTR) != 0) {
321		/* Interrupt */
322		intr_irq_handler(frame);
323		return;
324	}
325
326#ifdef KDTRACE_HOOKS
327	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
328		return;
329#endif
330
331	CTR4(KTR_TRAP, "%s: exception=%lu, sepc=%#lx, stval=%#lx", __func__,
332	    exception, frame->tf_sepc, frame->tf_stval);
333
334	switch (exception) {
335	case SCAUSE_LOAD_ACCESS_FAULT:
336	case SCAUSE_STORE_ACCESS_FAULT:
337	case SCAUSE_INST_ACCESS_FAULT:
338		dump_regs(frame);
339		panic("Memory access exception at %#lx: %#lx",
340		    frame->tf_sepc, frame->tf_stval);
341		break;
342	case SCAUSE_LOAD_MISALIGNED:
343	case SCAUSE_STORE_MISALIGNED:
344	case SCAUSE_INST_MISALIGNED:
345		dump_regs(frame);
346		panic("Misaligned address exception at %#lx: %#lx",
347		    frame->tf_sepc, frame->tf_stval);
348		break;
349	case SCAUSE_STORE_PAGE_FAULT:
350	case SCAUSE_LOAD_PAGE_FAULT:
351	case SCAUSE_INST_PAGE_FAULT:
352		page_fault_handler(frame, 0);
353		break;
354	case SCAUSE_BREAKPOINT:
355#ifdef KDTRACE_HOOKS
356		if (dtrace_invop_jump_addr != NULL &&
357		    dtrace_invop_jump_addr(frame) == 0)
358				break;
359#endif
360#ifdef KDB
361		kdb_trap(exception, 0, frame);
362#else
363		dump_regs(frame);
364		panic("No debugger in kernel.");
365#endif
366		break;
367	case SCAUSE_ILLEGAL_INSTRUCTION:
368		dump_regs(frame);
369		panic("Illegal instruction 0x%0*lx at %#lx",
370		    (frame->tf_stval & 0x3) != 0x3 ? 4 : 8,
371		    frame->tf_stval, frame->tf_sepc);
372		break;
373	default:
374		dump_regs(frame);
375		panic("Unknown kernel exception %#lx trap value %#lx",
376		    exception, frame->tf_stval);
377	}
378}
379
380void
381do_trap_user(struct trapframe *frame)
382{
383	uint64_t exception;
384	struct thread *td;
385	struct pcb *pcb;
386
387	td = curthread;
388	pcb = td->td_pcb;
389
390	KASSERT(td->td_frame == frame,
391	    ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame));
392
393	/* Ensure we came from usermode, interrupts disabled */
394	KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0,
395	    ("Came from U mode with interrupts enabled"));
396
397	KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0,
398	    ("Came from U mode with SUM enabled"));
399
400	exception = frame->tf_scause & SCAUSE_CODE;
401	if ((frame->tf_scause & SCAUSE_INTR) != 0) {
402		/* Interrupt */
403		intr_irq_handler(frame);
404		return;
405	}
406	intr_enable();
407
408	CTR4(KTR_TRAP, "%s: exception=%lu, sepc=%#lx, stval=%#lx", __func__,
409	    exception, frame->tf_sepc, frame->tf_stval);
410
411	switch (exception) {
412	case SCAUSE_LOAD_ACCESS_FAULT:
413	case SCAUSE_STORE_ACCESS_FAULT:
414	case SCAUSE_INST_ACCESS_FAULT:
415		call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_sepc,
416		    exception);
417		userret(td, frame);
418		break;
419	case SCAUSE_LOAD_MISALIGNED:
420	case SCAUSE_STORE_MISALIGNED:
421	case SCAUSE_INST_MISALIGNED:
422		call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sepc,
423		    exception);
424		userret(td, frame);
425		break;
426	case SCAUSE_STORE_PAGE_FAULT:
427	case SCAUSE_LOAD_PAGE_FAULT:
428	case SCAUSE_INST_PAGE_FAULT:
429		page_fault_handler(frame, 1);
430		break;
431	case SCAUSE_ECALL_USER:
432		frame->tf_sepc += 4;	/* Next instruction */
433		ecall_handler();
434		break;
435	case SCAUSE_ILLEGAL_INSTRUCTION:
436		if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) {
437			/*
438			 * May be a FPE trap. Enable FPE usage
439			 * for this thread and try again.
440			 */
441			fpe_state_clear();
442			frame->tf_sstatus &= ~SSTATUS_FS_MASK;
443			frame->tf_sstatus |= SSTATUS_FS_CLEAN;
444			pcb->pcb_fpflags |= PCB_FP_STARTED;
445			break;
446		}
447		call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc,
448		    exception);
449		userret(td, frame);
450		break;
451	case SCAUSE_BREAKPOINT:
452		call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc,
453		    exception);
454		userret(td, frame);
455		break;
456	default:
457		dump_regs(frame);
458		panic("Unknown userland exception %#lx, trap value %#lx",
459		    exception, frame->tf_stval);
460	}
461}
462