trap.c revision 319206 
1/*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/trap.c 319206 2017-05-30 13:21:43Z andrew $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/lock.h>
35#include <sys/mutex.h>
36#include <sys/pioctl.h>
37#include <sys/proc.h>
38#include <sys/ptrace.h>
39#include <sys/syscall.h>
40#include <sys/sysent.h>
41#ifdef KDB
42#include <sys/kdb.h>
43#endif
44
45#include <vm/vm.h>
46#include <vm/pmap.h>
47#include <vm/vm_kern.h>
48#include <vm/vm_map.h>
49#include <vm/vm_param.h>
50#include <vm/vm_extern.h>
51
52#include <machine/frame.h>
53#include <machine/pcb.h>
54#include <machine/pcpu.h>
55
56#ifdef KDTRACE_HOOKS
57#include <sys/dtrace_bsd.h>
58#endif
59
60#ifdef VFP
61#include <machine/vfp.h>
62#endif
63
64#ifdef KDB
65#include <machine/db_machdep.h>
66#endif
67
68#ifdef DDB
69#include <ddb/db_output.h>
70#endif
71
72extern register_t fsu_intr_fault;
73
74/* Called from exception.S */
75void do_el1h_sync(struct thread *, struct trapframe *);
76void do_el0_sync(struct thread *, struct trapframe *);
77void do_el0_error(struct trapframe *);
78static void print_registers(struct trapframe *frame);
79
80int (*dtrace_invop_jump_addr)(struct trapframe *);
81
82static __inline void
83call_trapsignal(struct thread *td, int sig, int code, void *addr)
84{
85	ksiginfo_t ksi;
86
87	ksiginfo_init_trap(&ksi);
88	ksi.ksi_signo = sig;
89	ksi.ksi_code = code;
90	ksi.ksi_addr = addr;
91	trapsignal(td, &ksi);
92}
93
94int
95cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
96{
97	struct proc *p;
98	register_t *ap;
99	int nap;
100
101	nap = 8;
102	p = td->td_proc;
103	ap = td->td_frame->tf_x;
104
105	sa->code = td->td_frame->tf_x[8];
106
107	if (sa->code == SYS_syscall || sa->code == SYS___syscall) {
108		sa->code = *ap++;
109		nap--;
110	}
111
112	if (p->p_sysent->sv_mask)
113		sa->code &= p->p_sysent->sv_mask;
114	if (sa->code >= p->p_sysent->sv_size)
115		sa->callp = &p->p_sysent->sv_table[0];
116	else
117		sa->callp = &p->p_sysent->sv_table[sa->code];
118
119	sa->narg = sa->callp->sy_narg;
120	memcpy(sa->args, ap, nap * sizeof(register_t));
121	if (sa->narg > nap)
122		panic("ARM64TODO: Could we have more than 8 args?");
123
124	td->td_retval[0] = 0;
125	td->td_retval[1] = 0;
126
127	return (0);
128}
129
130#include "../../kern/subr_syscall.c"
131
132static void
133svc_handler(struct thread *td, struct trapframe *frame)
134{
135	struct syscall_args sa;
136	int error;
137
138	if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) {
139		error = syscallenter(td, &sa);
140		syscallret(td, error, &sa);
141	} else {
142		call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr);
143		userret(td, frame);
144	}
145}
146
147static void
148data_abort(struct thread *td, struct trapframe *frame, uint64_t esr,
149    uint64_t far, int lower)
150{
151	struct vm_map *map;
152	struct proc *p;
153	struct pcb *pcb;
154	vm_prot_t ftype;
155	vm_offset_t va;
156	int error, sig, ucode;
157
158	/*
159	 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
160	 * and Store-Exclusive instruction usage restrictions", state
161	 * of the exclusive monitors after data abort exception is unknown.
162	 */
163	clrex();
164
165#ifdef KDB
166	if (kdb_active) {
167		kdb_reenter();
168		return;
169	}
170#endif
171
172	pcb = td->td_pcb;
173
174	/*
175	 * Special case for fuswintr and suswintr. These can't sleep so
176	 * handle them early on in the trap handler.
177	 */
178	if (__predict_false(pcb->pcb_onfault == (vm_offset_t)&fsu_intr_fault)) {
179		frame->tf_elr = pcb->pcb_onfault;
180		return;
181	}
182
183	p = td->td_proc;
184	if (lower)
185		map = &p->p_vmspace->vm_map;
186	else {
187		/* The top bit tells us which range to use */
188		if ((far >> 63) == 1)
189			map = kernel_map;
190		else
191			map = &p->p_vmspace->vm_map;
192	}
193
194	if (pmap_fault(map->pmap, esr, far) == KERN_SUCCESS)
195		return;
196
197	KASSERT(td->td_md.md_spinlock_count == 0,
198	    ("data abort with spinlock held"));
199	if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
200	    WARN_GIANTOK, NULL, "Kernel page fault") != 0) {
201		print_registers(frame);
202		printf(" far: %16lx\n", far);
203		printf(" esr:         %.8lx\n", esr);
204		panic("data abort in critical section or under mutex");
205	}
206
207	va = trunc_page(far);
208	ftype = ((esr >> 6) & 1) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ;
209
210	/* Fault in the page. */
211	error = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
212	if (error != KERN_SUCCESS) {
213		if (lower) {
214			sig = SIGSEGV;
215			if (error == KERN_PROTECTION_FAILURE)
216				ucode = SEGV_ACCERR;
217			else
218				ucode = SEGV_MAPERR;
219			call_trapsignal(td, sig, ucode, (void *)far);
220		} else {
221			if (td->td_intr_nesting_level == 0 &&
222			    pcb->pcb_onfault != 0) {
223				frame->tf_x[0] = error;
224				frame->tf_elr = pcb->pcb_onfault;
225				return;
226			}
227
228			printf("Fatal data abort:\n");
229			print_registers(frame);
230			printf(" far: %16lx\n", far);
231			printf(" esr:         %.8lx\n", esr);
232
233#ifdef KDB
234			if (debugger_on_panic || kdb_active)
235				if (kdb_trap(ESR_ELx_EXCEPTION(esr), 0, frame))
236					return;
237#endif
238			panic("vm_fault failed: %lx", frame->tf_elr);
239		}
240	}
241
242	if (lower)
243		userret(td, frame);
244}
245
246static void
247print_registers(struct trapframe *frame)
248{
249	u_int reg;
250
251	for (reg = 0; reg < 31; reg++) {
252		printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg,
253		    frame->tf_x[reg]);
254	}
255	printf("  sp: %16lx\n", frame->tf_sp);
256	printf("  lr: %16lx\n", frame->tf_lr);
257	printf(" elr: %16lx\n", frame->tf_elr);
258	printf("spsr:         %8x\n", frame->tf_spsr);
259}
260
261void
262do_el1h_sync(struct thread *td, struct trapframe *frame)
263{
264	uint32_t exception;
265	uint64_t esr, far;
266
267	/* Read the esr register to get the exception details */
268	esr = frame->tf_esr;
269	exception = ESR_ELx_EXCEPTION(esr);
270
271#ifdef KDTRACE_HOOKS
272	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
273		return;
274#endif
275
276	CTR4(KTR_TRAP,
277	    "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td,
278	    esr, frame->tf_elr, frame);
279
280	switch(exception) {
281	case EXCP_FP_SIMD:
282	case EXCP_TRAP_FP:
283		print_registers(frame);
284		printf(" esr:         %.8lx\n", esr);
285		panic("VFP exception in the kernel");
286	case EXCP_INSN_ABORT:
287	case EXCP_DATA_ABORT:
288		far = READ_SPECIALREG(far_el1);
289		intr_enable();
290		data_abort(td, frame, esr, far, 0);
291		break;
292	case EXCP_BRK:
293#ifdef KDTRACE_HOOKS
294		if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \
295		    dtrace_invop_jump_addr != 0) {
296			dtrace_invop_jump_addr(frame);
297			break;
298		}
299#endif
300		/* FALLTHROUGH */
301	case EXCP_WATCHPT_EL1:
302	case EXCP_SOFTSTP_EL1:
303#ifdef KDB
304		kdb_trap(exception, 0, frame);
305#else
306		panic("No debugger in kernel.\n");
307#endif
308		break;
309	default:
310		print_registers(frame);
311		panic("Unknown kernel exception %x esr_el1 %lx\n", exception,
312		    esr);
313	}
314}
315
316/*
317 * The attempted execution of an instruction bit pattern that has no allocated
318 * instruction results in an exception with an unknown reason.
319 */
320static void
321el0_excp_unknown(struct trapframe *frame, uint64_t far)
322{
323	struct thread *td;
324
325	td = curthread;
326	call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far);
327	userret(td, frame);
328}
329
330void
331do_el0_sync(struct thread *td, struct trapframe *frame)
332{
333	uint32_t exception;
334	uint64_t esr, far;
335
336	/* Check we have a sane environment when entering from userland */
337	KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
338	    ("Invalid pcpu address from userland: %p (tpidr %lx)",
339	     get_pcpu(), READ_SPECIALREG(tpidr_el1)));
340
341	esr = frame->tf_esr;
342	exception = ESR_ELx_EXCEPTION(esr);
343	switch (exception) {
344	case EXCP_UNKNOWN:
345	case EXCP_INSN_ABORT_L:
346	case EXCP_DATA_ABORT_L:
347	case EXCP_DATA_ABORT:
348		far = READ_SPECIALREG(far_el1);
349	}
350	intr_enable();
351
352	CTR4(KTR_TRAP,
353	    "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr,
354	    frame->tf_elr, frame);
355
356	switch(exception) {
357	case EXCP_FP_SIMD:
358	case EXCP_TRAP_FP:
359#ifdef VFP
360		vfp_restore_state();
361#else
362		panic("VFP exception in userland");
363#endif
364		break;
365	case EXCP_SVC:
366		svc_handler(td, frame);
367		break;
368	case EXCP_INSN_ABORT_L:
369	case EXCP_DATA_ABORT_L:
370	case EXCP_DATA_ABORT:
371		data_abort(td, frame, esr, far, 1);
372		break;
373	case EXCP_UNKNOWN:
374		el0_excp_unknown(frame, far);
375		break;
376	case EXCP_SP_ALIGN:
377		call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp);
378		userret(td, frame);
379		break;
380	case EXCP_PC_ALIGN:
381		call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr);
382		userret(td, frame);
383		break;
384	case EXCP_BRK:
385		call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr);
386		userret(td, frame);
387		break;
388	case EXCP_MSR:
389		call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr);
390		userret(td, frame);
391		break;
392	case EXCP_SOFTSTP_EL0:
393		td->td_frame->tf_spsr &= ~PSR_SS;
394		td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
395		WRITE_SPECIALREG(MDSCR_EL1,
396		    READ_SPECIALREG(MDSCR_EL1) & ~DBG_MDSCR_SS);
397		call_trapsignal(td, SIGTRAP, TRAP_TRACE,
398		    (void *)frame->tf_elr);
399		userret(td, frame);
400		break;
401	default:
402		call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr);
403		userret(td, frame);
404		break;
405	}
406}
407
408void
409do_el0_error(struct trapframe *frame)
410{
411
412	panic("ARM64TODO: do_el0_error");
413}
414
415