trap.c revision 286072
1124861Sharti/*- 2124861Sharti * Copyright (c) 2014 Andrew Turner 3124861Sharti * All rights reserved. 4124861Sharti * 5124861Sharti * Redistribution and use in source and binary forms, with or without 6124861Sharti * modification, are permitted provided that the following conditions 7310901Sngie * are met: 8133211Sharti * 1. Redistributions of source code must retain the above copyright 9133211Sharti * notice, this list of conditions and the following disclaimer. 10133211Sharti * 2. Redistributions in binary form must reproduce the above copyright 11133211Sharti * notice, this list of conditions and the following disclaimer in the 12133211Sharti * documentation and/or other materials provided with the distribution. 13124861Sharti * 14124861Sharti * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15124861Sharti * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16310901Sngie * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17133211Sharti * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18133211Sharti * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19133211Sharti * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20133211Sharti * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21133211Sharti * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22133211Sharti * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23133211Sharti * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24133211Sharti * SUCH DAMAGE. 25133211Sharti * 26133211Sharti */ 27133211Sharti 28124861Sharti#include <sys/cdefs.h> 29142810Sharti__FBSDID("$FreeBSD: head/sys/arm64/arm64/trap.c 286072 2015-07-30 13:59:38Z zbb $"); 30124861Sharti 31124861Sharti#include <sys/param.h> 32124861Sharti#include <sys/systm.h> 33124861Sharti#include <sys/kernel.h> 34216294Ssyrinx#include <sys/lock.h> 35310732Sngie#include <sys/mutex.h> 36312057Sngie#include <sys/pioctl.h> 37124861Sharti#include <sys/proc.h> 38124861Sharti#include <sys/ptrace.h> 39310732Sngie#include <sys/syscall.h> 40310732Sngie#include <sys/sysent.h> 41142810Sharti#ifdef KDB 42124861Sharti#include <sys/kdb.h> 43310732Sngie#endif 44124861Sharti 45124861Sharti#include <vm/vm.h> 46124861Sharti#include <vm/pmap.h> 47124861Sharti#include <vm/vm_kern.h> 48124861Sharti#include <vm/vm_map.h> 49124861Sharti#include <vm/vm_param.h> 50124861Sharti#include <vm/vm_extern.h> 51124861Sharti 52124861Sharti#include <machine/frame.h> 53124861Sharti#include <machine/pcb.h> 54124861Sharti#include <machine/pcpu.h> 55124861Sharti#include <machine/vmparam.h> 56124861Sharti 57124861Sharti#ifdef KDTRACE_HOOKS 58124861Sharti#include <sys/dtrace_bsd.h> 59124861Sharti#endif 60124861Sharti 61124861Sharti#ifdef VFP 62312264Sngie#include <machine/vfp.h> 63124861Sharti#endif 64124861Sharti 65124861Sharti#ifdef KDB 66124861Sharti#include <machine/db_machdep.h> 67124861Sharti#endif 68124861Sharti 69124861Sharti#ifdef DDB 70124861Sharti#include <ddb/db_output.h> 71124861Sharti#endif 72312057Sngie 73312057Sngieextern register_t fsu_intr_fault; 74124861Sharti 75124861Sharti/* Called from exception.S */ 76124861Shartivoid do_el1h_sync(struct trapframe *); 77124861Shartivoid do_el0_sync(struct trapframe *); 78124861Shartivoid do_el0_error(struct trapframe *); 79124861Sharti 80124861Shartiint (*dtrace_invop_jump_addr)(struct trapframe *); 81124861Sharti 82124861Shartistatic __inline void 83124861Sharticall_trapsignal(struct thread *td, int sig, int code, void *addr) 84124861Sharti{ 85124861Sharti ksiginfo_t ksi; 86124861Sharti 87124861Sharti ksiginfo_init_trap(&ksi); 88124861Sharti ksi.ksi_signo = sig; 89124861Sharti ksi.ksi_code = code; 90124861Sharti ksi.ksi_addr = addr; 91124861Sharti trapsignal(td, &ksi); 92124861Sharti} 93124861Sharti 94128237Shartiint 95124861Sharticpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa) 96124861Sharti{ 97124861Sharti struct proc *p; 98124861Sharti register_t *ap; 99124861Sharti int nap; 100124861Sharti 101124861Sharti nap = 8; 102124861Sharti p = td->td_proc; 103124861Sharti ap = td->td_frame->tf_x; 104124861Sharti 105124861Sharti sa->code = td->td_frame->tf_x[8]; 106124861Sharti 107124861Sharti if (sa->code == SYS_syscall || sa->code == SYS___syscall) { 108124861Sharti sa->code = *ap++; 109124861Sharti nap--; 110124861Sharti } 111124861Sharti 112124861Sharti if (p->p_sysent->sv_mask) 113124861Sharti sa->code &= p->p_sysent->sv_mask; 114124861Sharti if (sa->code >= p->p_sysent->sv_size) 115124861Sharti sa->callp = &p->p_sysent->sv_table[0]; 116124861Sharti else 117124861Sharti sa->callp = &p->p_sysent->sv_table[sa->code]; 118124861Sharti 119124861Sharti sa->narg = sa->callp->sy_narg; 120124861Sharti memcpy(sa->args, ap, nap * sizeof(register_t)); 121128237Sharti if (sa->narg > nap) 122124861Sharti panic("TODO: Could we have more then 8 args?"); 123128237Sharti 124124861Sharti td->td_retval[0] = 0; 125124861Sharti td->td_retval[1] = 0; 126124861Sharti 127124861Sharti return (0); 128124861Sharti} 129124861Sharti 130124861Sharti#include "../../kern/subr_syscall.c" 131124861Sharti 132124861Shartistatic void 133124861Shartisvc_handler(struct trapframe *frame) 134124861Sharti{ 135124861Sharti struct syscall_args sa; 136124861Sharti struct thread *td; 137124861Sharti int error; 138124861Sharti 139124861Sharti td = curthread; 140124861Sharti td->td_frame = frame; 141124861Sharti 142124861Sharti error = syscallenter(td, &sa); 143124861Sharti syscallret(td, error, &sa); 144124861Sharti} 145124861Sharti 146124861Shartistatic void 147124861Shartidata_abort(struct trapframe *frame, uint64_t esr, int lower) 148124861Sharti{ 149311722Sngie struct vm_map *map; 150124861Sharti struct thread *td; 151128237Sharti struct proc *p; 152124861Sharti struct pcb *pcb; 153311722Sngie vm_prot_t ftype; 154124861Sharti vm_offset_t va; 155124861Sharti uint64_t far; 156124861Sharti int error, sig, ucode; 157124861Sharti 158124861Sharti td = curthread; 159124861Sharti pcb = td->td_pcb; 160124861Sharti 161124861Sharti /* 162124861Sharti * Special case for fuswintr and suswintr. These can't sleep so 163124861Sharti * handle them early on in the trap handler. 164124861Sharti */ 165124861Sharti if (__predict_false(pcb->pcb_onfault == (vm_offset_t)&fsu_intr_fault)) { 166124861Sharti frame->tf_elr = pcb->pcb_onfault; 167124861Sharti return; 168124861Sharti } 169124861Sharti 170124861Sharti far = READ_SPECIALREG(far_el1); 171124861Sharti p = td->td_proc; 172124861Sharti 173124861Sharti if (lower) 174124861Sharti map = &td->td_proc->p_vmspace->vm_map; 175124861Sharti else { 176124861Sharti /* The top bit tells us which range to use */ 177124861Sharti if ((far >> 63) == 1) 178124861Sharti map = kernel_map; 179124861Sharti else 180124861Sharti map = &td->td_proc->p_vmspace->vm_map; 181124861Sharti } 182124861Sharti 183124861Sharti va = trunc_page(far); 184124861Sharti ftype = ((esr >> 6) & 1) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 185124861Sharti 186124861Sharti if (map != kernel_map) { 187124861Sharti /* 188124861Sharti * Keep swapout from messing with us during this 189124861Sharti * critical time. 190124861Sharti */ 191124861Sharti PROC_LOCK(p); 192124861Sharti ++p->p_lock; 193124861Sharti PROC_UNLOCK(p); 194124861Sharti 195124861Sharti /* Fault in the user page: */ 196124861Sharti error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); 197124861Sharti 198124861Sharti PROC_LOCK(p); 199124861Sharti --p->p_lock; 200124861Sharti PROC_UNLOCK(p); 201124861Sharti } else { 202124861Sharti /* 203124861Sharti * Don't have to worry about process locking or stacks in the 204124861Sharti * kernel. 205124861Sharti */ 206124861Sharti error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); 207124861Sharti } 208124861Sharti 209124861Sharti if (error != KERN_SUCCESS) { 210124861Sharti if (lower) { 211124861Sharti sig = SIGSEGV; 212124861Sharti if (error == KERN_PROTECTION_FAILURE) 213124861Sharti ucode = SEGV_ACCERR; 214124861Sharti else 215124861Sharti ucode = SEGV_MAPERR; 216124861Sharti call_trapsignal(td, sig, ucode, (void *)far); 217124861Sharti } else { 218124861Sharti if (td->td_intr_nesting_level == 0 && 219124861Sharti pcb->pcb_onfault != 0) { 220124861Sharti frame->tf_x[0] = error; 221124861Sharti frame->tf_elr = pcb->pcb_onfault; 222124861Sharti return; 223124861Sharti } 224124861Sharti panic("vm_fault failed: %lx", frame->tf_elr); 225124861Sharti } 226124861Sharti } 227124861Sharti 228124861Sharti if (lower) 229124861Sharti userret(td, frame); 230124861Sharti} 231124861Sharti 232124861Shartivoid 233124861Shartido_el1h_sync(struct trapframe *frame) 234124861Sharti{ 235124861Sharti uint32_t exception; 236124861Sharti uint64_t esr; 237124861Sharti 238124861Sharti /* Read the esr register to get the exception details */ 239124861Sharti esr = READ_SPECIALREG(esr_el1); 240124861Sharti exception = ESR_ELx_EXCEPTION(esr); 241124861Sharti 242124861Sharti#ifdef KDTRACE_HOOKS 243124861Sharti if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 244124861Sharti return; 245124861Sharti#endif 246124861Sharti 247124861Sharti /* 248124861Sharti * Sanity check we are in an exception er can handle. The IL bit 249124861Sharti * is used to indicate the instruction length, except in a few 250124861Sharti * exceptions described in the ARMv8 ARM. 251124861Sharti * 252124861Sharti * It is unclear in some cases if the bit is implementation defined. 253124861Sharti * The Foundation Model and QEMU disagree on if the IL bit should 254124861Sharti * be set when we are in a data fault from the same EL and the ISV 255124861Sharti * bit (bit 24) is also set. 256124861Sharti */ 257124861Sharti KASSERT((esr & ESR_ELx_IL) == ESR_ELx_IL || 258124861Sharti (exception == EXCP_DATA_ABORT && ((esr & ISS_DATA_ISV) == 0)), 259124861Sharti ("Invalid instruction length in exception")); 260124861Sharti 261124861Sharti CTR4(KTR_TRAP, 262311722Sngie "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", 263124861Sharti curthread, esr, frame->tf_elr, frame); 264124861Sharti 265124861Sharti switch(exception) { 266124861Sharti case EXCP_FP_SIMD: 267124861Sharti case EXCP_TRAP_FP: 268124861Sharti panic("VFP exception in the kernel"); 269124861Sharti case EXCP_DATA_ABORT: 270124861Sharti data_abort(frame, esr, 0); 271124861Sharti break; 272124861Sharti case EXCP_BRK: 273124861Sharti#ifdef KDTRACE_HOOKS 274124861Sharti if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ 275124861Sharti dtrace_invop_jump_addr != 0) { 276124861Sharti dtrace_invop_jump_addr(frame); 277124861Sharti break; 278124861Sharti } 279124861Sharti#endif 280124861Sharti case EXCP_WATCHPT_EL1: 281124861Sharti case EXCP_SOFTSTP_EL1: 282124861Sharti#ifdef KDB 283124861Sharti kdb_trap(exception, 0, frame); 284124861Sharti#else 285124861Sharti panic("No debugger in kernel.\n"); 286124861Sharti#endif 287124861Sharti break; 288124861Sharti default: 289124861Sharti panic("Unknown kernel exception %x esr_el1 %lx\n", exception, 290124861Sharti esr); 291124861Sharti } 292124861Sharti} 293124861Sharti 294124861Shartivoid 295124861Shartido_el0_sync(struct trapframe *frame) 296124861Sharti{ 297124861Sharti uint32_t exception; 298124861Sharti uint64_t esr; 299124861Sharti 300124861Sharti /* Check we have a sane environment when entering from userland */ 301124861Sharti KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, 302124861Sharti ("Invalid pcpu address from userland: %p (tpidr %lx)", 303124861Sharti get_pcpu(), READ_SPECIALREG(tpidr_el1))); 304124861Sharti 305124861Sharti esr = READ_SPECIALREG(esr_el1); 306124861Sharti exception = ESR_ELx_EXCEPTION(esr); 307124861Sharti 308312059Sngie CTR4(KTR_TRAP, 309124861Sharti "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", 310312059Sngie curthread, esr, frame->tf_elr, frame); 311124861Sharti 312124861Sharti switch(exception) { 313124861Sharti case EXCP_FP_SIMD: 314124861Sharti case EXCP_TRAP_FP: 315124861Sharti#ifdef VFP 316124861Sharti vfp_restore_state(); 317124861Sharti#else 318124861Sharti panic("VFP exception in userland"); 319124861Sharti#endif 320124861Sharti break; 321124861Sharti case EXCP_SVC: 322124861Sharti /* 323124861Sharti * Ensure the svc_handler is being run with interrupts enabled. 324124861Sharti * They will be automatically restored when returning from 325124861Sharti * exception handler. 326124861Sharti */ 327124861Sharti intr_enable(); 328124861Sharti svc_handler(frame); 329124861Sharti break; 330124861Sharti case EXCP_INSN_ABORT_L: 331124861Sharti case EXCP_DATA_ABORT_L: 332124861Sharti data_abort(frame, esr, 1); 333124861Sharti break; 334124861Sharti default: 335124861Sharti panic("Unknown userland exception %x esr_el1 %lx\n", exception, 336124861Sharti esr); 337124861Sharti } 338124861Sharti} 339124861Sharti 340124861Shartivoid 341124861Shartido_el0_error(struct trapframe *frame) 342124861Sharti{ 343124861Sharti 344124861Sharti panic("do_el0_error"); 345240234Sglebius} 346124861Sharti 347124861Sharti