1132718Skan/*- 2132718Skan * Copyright (c) 2014 Andrew Turner 390075Sobrien * All rights reserved. 490075Sobrien * 590075Sobrien * Redistribution and use in source and binary forms, with or without 690075Sobrien * modification, are permitted provided that the following conditions 790075Sobrien * are met: 890075Sobrien * 1. Redistributions of source code must retain the above copyright 990075Sobrien * notice, this list of conditions and the following disclaimer. 1090075Sobrien * 2. Redistributions in binary form must reproduce the above copyright 1190075Sobrien * notice, this list of conditions and the following disclaimer in the 1290075Sobrien * documentation and/or other materials provided with the distribution. 1390075Sobrien * 1490075Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1590075Sobrien * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1690075Sobrien * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1790075Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1890075Sobrien * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1990075Sobrien * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2090075Sobrien * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2190075Sobrien * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2290075Sobrien * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2390075Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2490075Sobrien * SUCH DAMAGE. 2590075Sobrien * 2690075Sobrien */ 2790075Sobrien 2890075Sobrien#include <sys/cdefs.h> 2990075Sobrien__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/trap.c 344905 2019-03-08 00:20:37Z jhb $"); 3090075Sobrien 3190075Sobrien#include <sys/param.h> 3290075Sobrien#include <sys/systm.h> 3390075Sobrien#include <sys/kernel.h> 3490075Sobrien#include <sys/lock.h> 3590075Sobrien#include <sys/mutex.h> 3690075Sobrien#include <sys/pioctl.h> 3790075Sobrien#include <sys/proc.h> 3890075Sobrien#include <sys/ptrace.h> 3990075Sobrien#include <sys/syscall.h> 4090075Sobrien#include <sys/sysent.h> 4190075Sobrien#ifdef KDB 4290075Sobrien#include <sys/kdb.h> 4390075Sobrien#endif 4490075Sobrien 4590075Sobrien#include <vm/vm.h> 4690075Sobrien#include <vm/pmap.h> 4790075Sobrien#include <vm/vm_kern.h> 4890075Sobrien#include <vm/vm_map.h> 49117395Skan#include <vm/vm_param.h> 5090075Sobrien#include <vm/vm_extern.h> 5190075Sobrien 52117395Skan#include <machine/frame.h> 53132718Skan#include <machine/pcb.h> 5490075Sobrien#include <machine/pcpu.h> 5590075Sobrien 5690075Sobrien#ifdef KDTRACE_HOOKS 5790075Sobrien#include <sys/dtrace_bsd.h> 5890075Sobrien#endif 5990075Sobrien 6090075Sobrien#ifdef VFP 6190075Sobrien#include <machine/vfp.h> 6290075Sobrien#endif 6390075Sobrien 6490075Sobrien#ifdef KDB 6590075Sobrien#include <machine/db_machdep.h> 6690075Sobrien#endif 6790075Sobrien 6890075Sobrien#ifdef DDB 6990075Sobrien#include <ddb/db_output.h> 7090075Sobrien#endif 7190075Sobrien 7290075Sobrienextern register_t fsu_intr_fault; 7390075Sobrien 7490075Sobrien/* Called from exception.S */ 7590075Sobrienvoid do_el1h_sync(struct thread *, struct trapframe *); 7690075Sobrienvoid do_el0_sync(struct thread *, struct trapframe *); 7790075Sobrienvoid do_el0_error(struct trapframe *); 7890075Sobrienstatic void print_registers(struct trapframe *frame); 7990075Sobrien 8090075Sobrienint (*dtrace_invop_jump_addr)(struct trapframe *); 8190075Sobrien 8290075Sobrienstatic __inline void 8390075Sobriencall_trapsignal(struct thread *td, int sig, int code, void *addr) 8490075Sobrien{ 8590075Sobrien ksiginfo_t ksi; 8690075Sobrien 8790075Sobrien ksiginfo_init_trap(&ksi); 8890075Sobrien ksi.ksi_signo = sig; 8990075Sobrien ksi.ksi_code = code; 9090075Sobrien ksi.ksi_addr = addr; 9190075Sobrien trapsignal(td, &ksi); 9290075Sobrien} 9390075Sobrien 9490075Sobrienint 9590075Sobriencpu_fetch_syscall_args(struct thread *td) 9690075Sobrien{ 97132718Skan struct proc *p; 9890075Sobrien register_t *ap; 9990075Sobrien struct syscall_args *sa; 10090075Sobrien int nap; 10190075Sobrien 10290075Sobrien nap = 8; 10390075Sobrien p = td->td_proc; 10490075Sobrien ap = td->td_frame->tf_x; 10590075Sobrien sa = &td->td_sa; 10690075Sobrien 10790075Sobrien sa->code = td->td_frame->tf_x[8]; 108132718Skan 10990075Sobrien if (sa->code == SYS_syscall || sa->code == SYS___syscall) { 110132718Skan sa->code = *ap++; 11190075Sobrien nap--; 11290075Sobrien } 11390075Sobrien 11490075Sobrien if (p->p_sysent->sv_mask) 11590075Sobrien sa->code &= p->p_sysent->sv_mask; 11690075Sobrien if (sa->code >= p->p_sysent->sv_size) 11790075Sobrien sa->callp = &p->p_sysent->sv_table[0]; 11890075Sobrien else 11990075Sobrien sa->callp = &p->p_sysent->sv_table[sa->code]; 12090075Sobrien 12190075Sobrien sa->narg = sa->callp->sy_narg; 122132718Skan memcpy(sa->args, ap, nap * sizeof(register_t)); 12390075Sobrien if (sa->narg > nap) 124132718Skan panic("ARM64TODO: Could we have more than 8 args?"); 12590075Sobrien 12690075Sobrien td->td_retval[0] = 0; 12790075Sobrien td->td_retval[1] = 0; 12890075Sobrien 12990075Sobrien return (0); 13090075Sobrien} 13190075Sobrien 13290075Sobrien#include "../../kern/subr_syscall.c" 13390075Sobrien 13490075Sobrienstatic void 13590075Sobriensvc_handler(struct thread *td, struct trapframe *frame) 13690075Sobrien{ 137132718Skan int error; 13890075Sobrien 139132718Skan if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) { 14090075Sobrien error = syscallenter(td); 14190075Sobrien syscallret(td, error); 14290075Sobrien } else { 14390075Sobrien call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr); 144132718Skan userret(td, frame); 14590075Sobrien } 146132718Skan} 14790075Sobrien 14890075Sobrienstatic void 14990075Sobriendata_abort(struct thread *td, struct trapframe *frame, uint64_t esr, 15090075Sobrien uint64_t far, int lower) 15190075Sobrien{ 15290075Sobrien struct vm_map *map; 15390075Sobrien struct proc *p; 15490075Sobrien struct pcb *pcb; 15590075Sobrien vm_prot_t ftype; 15690075Sobrien vm_offset_t va; 15790075Sobrien int error, sig, ucode; 15890075Sobrien#ifdef KDB 159103445Skan bool handled; 16090075Sobrien#endif 16190075Sobrien 16290075Sobrien /* 163103445Skan * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive 164132718Skan * and Store-Exclusive instruction usage restrictions", state 16590075Sobrien * of the exclusive monitors after data abort exception is unknown. 166132718Skan */ 167117395Skan clrex(); 168117395Skan 169117395Skan#ifdef KDB 170132718Skan if (kdb_active) { 171132718Skan kdb_reenter(); 172132718Skan return; 173132718Skan } 174132718Skan#endif 175132718Skan 176117395Skan pcb = td->td_pcb; 177117395Skan 178117395Skan /* 179117395Skan * Special case for fuswintr and suswintr. These can't sleep so 180117395Skan * handle them early on in the trap handler. 181117395Skan */ 182132718Skan if (__predict_false(pcb->pcb_onfault == (vm_offset_t)&fsu_intr_fault)) { 183117395Skan frame->tf_elr = pcb->pcb_onfault; 184132718Skan return; 185132718Skan } 186132718Skan 187132718Skan p = td->td_proc; 188132718Skan if (lower) 189132718Skan map = &p->p_vmspace->vm_map; 190132718Skan else { 191132718Skan /* The top bit tells us which range to use */ 192132718Skan if ((far >> 63) == 1) 193132718Skan map = kernel_map; 194132718Skan else 195132718Skan map = &p->p_vmspace->vm_map; 196132718Skan } 197132718Skan 198132718Skan if (pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) 199132718Skan return; 200132718Skan 201132718Skan KASSERT(td->td_md.md_spinlock_count == 0, 202132718Skan ("data abort with spinlock held")); 203132718Skan if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | 20490075Sobrien WARN_GIANTOK, NULL, "Kernel page fault") != 0) { 20590075Sobrien print_registers(frame); 20690075Sobrien printf(" far: %16lx\n", far); 20790075Sobrien printf(" esr: %.8lx\n", esr); 20890075Sobrien panic("data abort in critical section or under mutex"); 209132718Skan } 21090075Sobrien 211132718Skan va = trunc_page(far); 21290075Sobrien ftype = ((esr >> 6) & 1) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 21390075Sobrien 21490075Sobrien /* Fault in the page. */ 215132718Skan error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); 21690075Sobrien if (error != KERN_SUCCESS) { 217132718Skan if (lower) { 21890075Sobrien sig = SIGSEGV; 21990075Sobrien if (error == KERN_PROTECTION_FAILURE) 22090075Sobrien ucode = SEGV_ACCERR; 22190075Sobrien else 22290075Sobrien ucode = SEGV_MAPERR; 22390075Sobrien call_trapsignal(td, sig, ucode, (void *)far); 22490075Sobrien } else { 225132718Skan if (td->td_intr_nesting_level == 0 && 22690075Sobrien pcb->pcb_onfault != 0) { 227132718Skan frame->tf_x[0] = error; 22890075Sobrien frame->tf_elr = pcb->pcb_onfault; 22990075Sobrien return; 23090075Sobrien } 23190075Sobrien 23290075Sobrien printf("Fatal data abort:\n"); 23390075Sobrien print_registers(frame); 23490075Sobrien printf(" far: %16lx\n", far); 23590075Sobrien printf(" esr: %.8lx\n", esr); 236132718Skan 23790075Sobrien#ifdef KDB 238132718Skan if (debugger_on_trap) { 23990075Sobrien kdb_why = KDB_WHY_TRAP; 24090075Sobrien handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0, 24190075Sobrien frame); 24290075Sobrien kdb_why = KDB_WHY_UNSET; 24390075Sobrien if (handled) 24490075Sobrien return; 245132718Skan } 24690075Sobrien#endif 247132718Skan panic("vm_fault failed: %lx", frame->tf_elr); 24890075Sobrien } 24990075Sobrien } 25090075Sobrien 25190075Sobrien if (lower) 25290075Sobrien userret(td, frame); 25390075Sobrien} 254132718Skan 25590075Sobrienstatic void 256132718Skanprint_registers(struct trapframe *frame) 257132718Skan{ 258132718Skan u_int reg; 259132718Skan 260132718Skan for (reg = 0; reg < 31; reg++) { 261132718Skan printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg, 262132718Skan frame->tf_x[reg]); 263132718Skan } 264132718Skan printf(" sp: %16lx\n", frame->tf_sp); 265132718Skan printf(" lr: %16lx\n", frame->tf_lr); 266132718Skan printf(" elr: %16lx\n", frame->tf_elr); 267132718Skan printf("spsr: %8x\n", frame->tf_spsr); 268132718Skan} 269132718Skan 27090075Sobrienvoid 27190075Sobriendo_el1h_sync(struct thread *td, struct trapframe *frame) 27290075Sobrien{ 27390075Sobrien uint32_t exception; 27490075Sobrien uint64_t esr, far; 275132718Skan 27690075Sobrien /* Read the esr register to get the exception details */ 277132718Skan esr = frame->tf_esr; 27890075Sobrien exception = ESR_ELx_EXCEPTION(esr); 27990075Sobrien 28090075Sobrien#ifdef KDTRACE_HOOKS 28190075Sobrien if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 28290075Sobrien return; 28390075Sobrien#endif 284132718Skan 28590075Sobrien CTR4(KTR_TRAP, 286132718Skan "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, 28790075Sobrien esr, frame->tf_elr, frame); 28890075Sobrien 28990075Sobrien switch(exception) { 29090075Sobrien case EXCP_FP_SIMD: 29190075Sobrien case EXCP_TRAP_FP: 29290075Sobrien print_registers(frame); 29390075Sobrien printf(" esr: %.8lx\n", esr); 294146895Skan panic("VFP exception in the kernel"); 295146895Skan case EXCP_INSN_ABORT: 296146895Skan case EXCP_DATA_ABORT: 297146895Skan far = READ_SPECIALREG(far_el1); 298146895Skan intr_enable(); 299146895Skan data_abort(td, frame, esr, far, 0); 300132718Skan break; 30190075Sobrien case EXCP_BRK: 302146895Skan#ifdef KDTRACE_HOOKS 303146895Skan if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ 304146895Skan dtrace_invop_jump_addr != 0) { 305146895Skan dtrace_invop_jump_addr(frame); 306146895Skan break; 307146895Skan } 308146895Skan#endif 309146895Skan /* FALLTHROUGH */ 310146895Skan case EXCP_WATCHPT_EL1: 311146895Skan case EXCP_SOFTSTP_EL1: 312146895Skan#ifdef KDB 313146895Skan kdb_trap(exception, 0, frame); 314146895Skan#else 315146895Skan panic("No debugger in kernel.\n"); 316132718Skan#endif 31790075Sobrien break; 31890075Sobrien default: 31990075Sobrien print_registers(frame); 32090075Sobrien panic("Unknown kernel exception %x esr_el1 %lx\n", exception, 32190075Sobrien esr); 32290075Sobrien } 323132718Skan} 32490075Sobrien 325132718Skan/* 32690075Sobrien * The attempted execution of an instruction bit pattern that has no allocated 32790075Sobrien * instruction results in an exception with an unknown reason. 32890075Sobrien */ 32990075Sobrienstatic void 33090075Sobrienel0_excp_unknown(struct trapframe *frame, uint64_t far) 331132718Skan{ 33290075Sobrien struct thread *td; 333132718Skan 33490075Sobrien td = curthread; 33590075Sobrien call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far); 33690075Sobrien userret(td, frame); 33790075Sobrien} 33890075Sobrien 33990075Sobrienvoid 34090075Sobriendo_el0_sync(struct thread *td, struct trapframe *frame) 341132718Skan{ 34290075Sobrien uint32_t exception; 343132718Skan uint64_t esr, far; 344132718Skan 345132718Skan /* Check we have a sane environment when entering from userland */ 346132718Skan KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, 347132718Skan ("Invalid pcpu address from userland: %p (tpidr %lx)", 348132718Skan get_pcpu(), READ_SPECIALREG(tpidr_el1))); 349132718Skan 350132718Skan esr = frame->tf_esr; 351132718Skan exception = ESR_ELx_EXCEPTION(esr); 352132718Skan switch (exception) { 353132718Skan case EXCP_UNKNOWN: 354132718Skan case EXCP_INSN_ABORT_L: 355132718Skan case EXCP_DATA_ABORT_L: 356132718Skan case EXCP_DATA_ABORT: 35790075Sobrien far = READ_SPECIALREG(far_el1); 35890075Sobrien } 35990075Sobrien intr_enable(); 36090075Sobrien 36190075Sobrien CTR4(KTR_TRAP, 36290075Sobrien "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, 36390075Sobrien frame->tf_elr, frame); 36490075Sobrien 36590075Sobrien switch(exception) { 36690075Sobrien case EXCP_FP_SIMD: 36790075Sobrien case EXCP_TRAP_FP: 36890075Sobrien#ifdef VFP 36990075Sobrien vfp_restore_state(); 37090075Sobrien#else 37190075Sobrien panic("VFP exception in userland"); 37290075Sobrien#endif 37390075Sobrien break; 37490075Sobrien case EXCP_SVC: 37590075Sobrien svc_handler(td, frame); 37690075Sobrien break; 37790075Sobrien case EXCP_INSN_ABORT_L: 37890075Sobrien case EXCP_DATA_ABORT_L: 379132718Skan case EXCP_DATA_ABORT: 38090075Sobrien data_abort(td, frame, esr, far, 1); 38190075Sobrien break; 38290075Sobrien case EXCP_UNKNOWN: 38390075Sobrien el0_excp_unknown(frame, far); 384132718Skan break; 38590075Sobrien case EXCP_SP_ALIGN: 38690075Sobrien call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp); 38790075Sobrien userret(td, frame); 38890075Sobrien break; 38990075Sobrien case EXCP_PC_ALIGN: 39090075Sobrien call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr); 391132718Skan userret(td, frame); 392132718Skan break; 39390075Sobrien case EXCP_BRK: 39490075Sobrien call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr); 39590075Sobrien userret(td, frame); 39690075Sobrien break; 39790075Sobrien case EXCP_MSR: 39890075Sobrien call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr); 39990075Sobrien userret(td, frame); 40090075Sobrien break; 40190075Sobrien case EXCP_SOFTSTP_EL0: 40290075Sobrien td->td_frame->tf_spsr &= ~PSR_SS; 40390075Sobrien td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; 40490075Sobrien WRITE_SPECIALREG(MDSCR_EL1, 40590075Sobrien READ_SPECIALREG(MDSCR_EL1) & ~DBG_MDSCR_SS); 406132718Skan call_trapsignal(td, SIGTRAP, TRAP_TRACE, 40790075Sobrien (void *)frame->tf_elr); 408132718Skan userret(td, frame); 40990075Sobrien break; 41090075Sobrien default: 41190075Sobrien call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr); 41290075Sobrien userret(td, frame); 41390075Sobrien break; 414132718Skan } 41590075Sobrien} 416132718Skan 41790075Sobrienvoid 41890075Sobriendo_el0_error(struct trapframe *frame) 41990075Sobrien{ 42090075Sobrien 42190075Sobrien panic("ARM64TODO: do_el0_error"); 422132718Skan} 42390075Sobrien 424132718Skan