1276638Sian/*- 2276638Sian * Copyright 2014 Olivier Houchard <cognet@FreeBSD.org> 3276638Sian * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com> 4276638Sian * Copyright 2014 Michal Meloun <meloun@miracle.cz> 5276638Sian * Copyright 2014 Andrew Turner <andrew@FreeBSD.org> 6276638Sian * All rights reserved. 7276638Sian * 8276638Sian * Redistribution and use in source and binary forms, with or without 9276638Sian * modification, are permitted provided that the following conditions 10276638Sian * are met: 11276638Sian * 1. Redistributions of source code must retain the above copyright 12276638Sian * notice, this list of conditions and the following disclaimer. 13276638Sian * 2. Redistributions in binary form must reproduce the above copyright 14276638Sian * notice, this list of conditions and the following disclaimer in the 15276638Sian * documentation and/or other materials provided with the distribution. 16276638Sian * 17276638Sian * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18276638Sian * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19276638Sian * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20276638Sian * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21276638Sian * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22276638Sian * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23276638Sian * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24276638Sian * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25276638Sian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26276638Sian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27276638Sian * SUCH DAMAGE. 28276638Sian */ 29276638Sian 30276638Sian#include "opt_ktrace.h" 31276638Sian 32276638Sian#include <sys/cdefs.h> 33276638Sian__FBSDID("$FreeBSD: stable/11/sys/arm/arm/trap-v6.c 344905 2019-03-08 00:20:37Z jhb $"); 34276638Sian 35276638Sian#include <sys/param.h> 36276638Sian#include <sys/bus.h> 37276638Sian#include <sys/systm.h> 38276638Sian#include <sys/proc.h> 39276638Sian#include <sys/kernel.h> 40276638Sian#include <sys/lock.h> 41276638Sian#include <sys/mutex.h> 42276638Sian#include <sys/signalvar.h> 43276638Sian#include <sys/ktr.h> 44331017Skevans#include <sys/vmmeter.h> 45276638Sian#ifdef KTRACE 46276638Sian#include <sys/uio.h> 47276638Sian#include <sys/ktrace.h> 48276638Sian#endif 49276638Sian 50276638Sian#include <vm/vm.h> 51276638Sian#include <vm/pmap.h> 52276638Sian#include <vm/vm_kern.h> 53276638Sian#include <vm/vm_map.h> 54276638Sian#include <vm/vm_extern.h> 55276638Sian#include <vm/vm_param.h> 56276638Sian 57276638Sian#include <machine/cpu.h> 58276638Sian#include <machine/frame.h> 59276638Sian#include <machine/machdep.h> 60276638Sian#include <machine/pcb.h> 61276638Sian 62276638Sian#ifdef KDB 63276638Sian#include <sys/kdb.h> 64276638Sian#include <machine/db_machdep.h> 65276638Sian#endif 66276638Sian 67291852Sandrew#ifdef KDTRACE_HOOKS 68291852Sandrew#include <sys/dtrace_bsd.h> 69291852Sandrew#endif 70291852Sandrew 71283947Sianextern char cachebailout[]; 72276638Sian 73277523Sgonzo#ifdef DEBUG 74277523Sgonzoint last_fault_code; /* For the benefit of pmap_fault_fixup() */ 75277523Sgonzo#endif 76277523Sgonzo 77276638Sianstruct ksig { 78276638Sian int sig; 79276638Sian u_long code; 80276638Sian vm_offset_t addr; 81276638Sian}; 82276638Sian 83276638Siantypedef int abort_func_t(struct trapframe *, u_int, u_int, u_int, u_int, 84276638Sian struct thread *, struct ksig *); 85276638Sian 86276638Sianstatic abort_func_t abort_fatal; 87276638Sianstatic abort_func_t abort_align; 88276638Sianstatic abort_func_t abort_icache; 89276638Sian 90276638Sianstruct abort { 91276638Sian abort_func_t *func; 92276638Sian const char *desc; 93276638Sian}; 94276638Sian 95276638Sian/* 96276638Sian * How are the aborts handled? 97276638Sian * 98276638Sian * Undefined Code: 99276638Sian * - Always fatal as we do not know what does it mean. 100276638Sian * Imprecise External Abort: 101276638Sian * - Always fatal, but can be handled somehow in the future. 102290369Sskra * Now, due to PCIe buggy hardware, ignored. 103276638Sian * Precise External Abort: 104276638Sian * - Always fatal, but who knows in the future??? 105276638Sian * Debug Event: 106276638Sian * - Special handling. 107276638Sian * External Translation Abort (L1 & L2) 108290369Sskra * - Always fatal as something is screwed up in page tables or hardware. 109276638Sian * Domain Fault (L1 & L2): 110276638Sian * - Always fatal as we do not play game with domains. 111276638Sian * Alignment Fault: 112290369Sskra * - Everything should be aligned in kernel with exception of user to kernel 113290369Sskra * and vice versa data copying, so if pcb_onfault is not set, it's fatal. 114276638Sian * We generate signal in case of abort from user mode. 115276638Sian * Instruction cache maintenance: 116276638Sian * - According to manual, this is translation fault during cache maintenance 117276638Sian * operation. So, it could be really complex in SMP case and fuzzy too 118276638Sian * for cache operations working on virtual addresses. For now, we will 119276638Sian * consider this abort as fatal. In fact, no cache maintenance on 120276638Sian * not mapped virtual addresses should be called. As cache maintenance 121276638Sian * operation (except DMB, DSB, and Flush Prefetch Buffer) are priviledged, 122276638Sian * the abort is fatal for user mode as well for now. (This is good place to 123276638Sian * note that cache maintenance on virtual address fill TLB.) 124276638Sian * Acces Bit (L1 & L2): 125276638Sian * - Fast hardware emulation for kernel and user mode. 126276638Sian * Translation Fault (L1 & L2): 127276638Sian * - Standard fault mechanism is held including vm_fault(). 128276638Sian * Permission Fault (L1 & L2): 129290369Sskra * - Fast hardware emulation of modify bits and in other cases, standard 130276638Sian * fault mechanism is held including vm_fault(). 131276638Sian */ 132276638Sian 133276638Sianstatic const struct abort aborts[] = { 134276638Sian {abort_fatal, "Undefined Code (0x000)"}, 135276638Sian {abort_align, "Alignment Fault"}, 136276638Sian {abort_fatal, "Debug Event"}, 137276638Sian {NULL, "Access Bit (L1)"}, 138283947Sian {NULL, "Instruction cache maintenance"}, 139276638Sian {NULL, "Translation Fault (L1)"}, 140276638Sian {NULL, "Access Bit (L2)"}, 141276638Sian {NULL, "Translation Fault (L2)"}, 142276638Sian 143276638Sian {abort_fatal, "External Abort"}, 144276638Sian {abort_fatal, "Domain Fault (L1)"}, 145276638Sian {abort_fatal, "Undefined Code (0x00A)"}, 146276638Sian {abort_fatal, "Domain Fault (L2)"}, 147276638Sian {abort_fatal, "External Translation Abort (L1)"}, 148276638Sian {NULL, "Permission Fault (L1)"}, 149276638Sian {abort_fatal, "External Translation Abort (L2)"}, 150276638Sian {NULL, "Permission Fault (L2)"}, 151276638Sian 152276638Sian {abort_fatal, "TLB Conflict Abort"}, 153276638Sian {abort_fatal, "Undefined Code (0x401)"}, 154276638Sian {abort_fatal, "Undefined Code (0x402)"}, 155276638Sian {abort_fatal, "Undefined Code (0x403)"}, 156276638Sian {abort_fatal, "Undefined Code (0x404)"}, 157276638Sian {abort_fatal, "Undefined Code (0x405)"}, 158276638Sian {abort_fatal, "Asynchronous External Abort"}, 159276638Sian {abort_fatal, "Undefined Code (0x407)"}, 160276638Sian 161276638Sian {abort_fatal, "Asynchronous Parity Error on Memory Access"}, 162276638Sian {abort_fatal, "Parity Error on Memory Access"}, 163276638Sian {abort_fatal, "Undefined Code (0x40A)"}, 164276638Sian {abort_fatal, "Undefined Code (0x40B)"}, 165276638Sian {abort_fatal, "Parity Error on Translation (L1)"}, 166276638Sian {abort_fatal, "Undefined Code (0x40D)"}, 167276638Sian {abort_fatal, "Parity Error on Translation (L2)"}, 168276638Sian {abort_fatal, "Undefined Code (0x40F)"} 169276638Sian}; 170276638Sian 171276638Sianstatic __inline void 172276638Siancall_trapsignal(struct thread *td, int sig, int code, vm_offset_t addr) 173276638Sian{ 174276638Sian ksiginfo_t ksi; 175276638Sian 176276638Sian CTR4(KTR_TRAP, "%s: addr: %#x, sig: %d, code: %d", 177276638Sian __func__, addr, sig, code); 178276638Sian 179276638Sian /* 180276638Sian * TODO: some info would be nice to know 181276638Sian * if we are serving data or prefetch abort. 182276638Sian */ 183276638Sian 184276638Sian ksiginfo_init_trap(&ksi); 185276638Sian ksi.ksi_signo = sig; 186276638Sian ksi.ksi_code = code; 187276638Sian ksi.ksi_addr = (void *)addr; 188276638Sian trapsignal(td, &ksi); 189276638Sian} 190276638Sian 191276638Sian/* 192276638Sian * abort_imprecise() handles the following abort: 193276638Sian * 194276638Sian * FAULT_EA_IMPREC - Imprecise External Abort 195276638Sian * 196276638Sian * The imprecise means that we don't know where the abort happened, 197276638Sian * thus FAR is undefined. The abort should not never fire, but hot 198290369Sskra * plugging or accidental hardware failure can be the cause of it. 199276638Sian * If the abort happens, it can even be on different (thread) context. 200276638Sian * Without any additional support, the abort is fatal, as we do not 201276638Sian * know what really happened. 202276638Sian * 203276638Sian * QQQ: Some additional functionality, like pcb_onfault but global, 204276638Sian * can be implemented. Imprecise handlers could be registered 205276638Sian * which tell us if the abort is caused by something they know 206276638Sian * about. They should return one of three codes like: 207276638Sian * FAULT_IS_MINE, 208276638Sian * FAULT_CAN_BE_MINE, 209276638Sian * FAULT_IS_NOT_MINE. 210276638Sian * The handlers should be called until some of them returns 211276638Sian * FAULT_IS_MINE value or all was called. If all handlers return 212276638Sian * FAULT_IS_NOT_MINE value, then the abort is fatal. 213276638Sian */ 214276638Sianstatic __inline void 215290541Sskraabort_imprecise(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode) 216276638Sian{ 217290369Sskra 218290369Sskra /* 219290369Sskra * XXX - We can got imprecise abort as result of access 220276638Sian * to not-present PCI/PCIe configuration space. 221276638Sian */ 222276638Sian#if 0 223276638Sian goto out; 224276638Sian#endif 225276638Sian abort_fatal(tf, FAULT_EA_IMPREC, fsr, 0, prefetch, curthread, NULL); 226276638Sian 227276638Sian /* 228276638Sian * Returning from this function means that we ignore 229276638Sian * the abort for good reason. Note that imprecise abort 230276638Sian * could fire any time even in user mode. 231276638Sian */ 232276638Sian 233276638Sian#if 0 234276638Sianout: 235276638Sian if (usermode) 236276638Sian userret(curthread, tf); 237276638Sian#endif 238276638Sian} 239276638Sian 240276638Sian/* 241276638Sian * abort_debug() handles the following abort: 242276638Sian * 243276638Sian * FAULT_DEBUG - Debug Event 244276638Sian * 245276638Sian */ 246276638Sianstatic __inline void 247290541Sskraabort_debug(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode, 248276638Sian u_int far) 249276638Sian{ 250290369Sskra 251276638Sian if (usermode) { 252276638Sian struct thread *td; 253276638Sian 254276638Sian td = curthread; 255276638Sian call_trapsignal(td, SIGTRAP, TRAP_BRKPT, far); 256276638Sian userret(td, tf); 257276638Sian } else { 258276638Sian#ifdef KDB 259294740Szbb kdb_trap((prefetch) ? T_BREAKPOINT : T_WATCHPOINT, 0, tf); 260276638Sian#else 261276638Sian printf("No debugger in kernel.\n"); 262276638Sian#endif 263276638Sian } 264276638Sian} 265276638Sian 266276638Sian/* 267276638Sian * Abort handler. 268276638Sian * 269276638Sian * FAR, FSR, and everything what can be lost after enabling 270276638Sian * interrupts must be grabbed before the interrupts will be 271276638Sian * enabled. Note that when interrupts will be enabled, we 272276638Sian * could even migrate to another CPU ... 273276638Sian * 274276638Sian * TODO: move quick cases to ASM 275276638Sian */ 276276638Sianvoid 277276638Sianabort_handler(struct trapframe *tf, int prefetch) 278276638Sian{ 279276638Sian struct thread *td; 280276638Sian vm_offset_t far, va; 281290541Sskra int idx, rv; 282276638Sian uint32_t fsr; 283276638Sian struct ksig ksig; 284276638Sian struct proc *p; 285276638Sian struct pcb *pcb; 286276638Sian struct vm_map *map; 287276638Sian struct vmspace *vm; 288276638Sian vm_prot_t ftype; 289290541Sskra bool usermode; 290331988Smmel int bp_harden; 291276638Sian#ifdef INVARIANTS 292276638Sian void *onfault; 293276638Sian#endif 294295255Sskra 295295255Sskra PCPU_INC(cnt.v_trap); 296276638Sian td = curthread; 297295255Sskra 298276638Sian fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get(); 299281647Sandrew#if __ARM_ARCH >= 7 300281647Sandrew far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get(); 301281647Sandrew#else 302276638Sian far = (prefetch) ? TRAPF_PC(tf) : cp15_dfar_get(); 303281647Sandrew#endif 304276638Sian 305276638Sian idx = FSR_TO_FAULT(fsr); 306276638Sian usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */ 307331988Smmel 308331988Smmel /* 309331988Smmel * Apply BP hardening by flushing the branch prediction cache 310331988Smmel * for prefaults on kernel addresses. 311331988Smmel */ 312331988Smmel if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS && 313331988Smmel (idx == FAULT_TRAN_L2 || idx == FAULT_PERM_L2))) { 314331988Smmel bp_harden = PCPU_GET(bp_harden_kind); 315331988Smmel if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL) 316331988Smmel _CP15_BPIALL(); 317331988Smmel else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU) 318331988Smmel _CP15_ICIALLU(); 319331988Smmel } 320331988Smmel 321276638Sian if (usermode) 322276638Sian td->td_frame = tf; 323276638Sian 324291094Sskra CTR6(KTR_TRAP, "%s: fsr %#x (idx %u) far %#x prefetch %u usermode %d", 325291094Sskra __func__, fsr, idx, far, prefetch, usermode); 326276638Sian 327276638Sian /* 328276638Sian * Firstly, handle aborts that are not directly related to mapping. 329276638Sian */ 330276638Sian if (__predict_false(idx == FAULT_EA_IMPREC)) { 331276638Sian abort_imprecise(tf, fsr, prefetch, usermode); 332276638Sian return; 333276638Sian } 334276638Sian 335276638Sian if (__predict_false(idx == FAULT_DEBUG)) { 336276638Sian abort_debug(tf, fsr, prefetch, usermode, far); 337276638Sian return; 338276638Sian } 339276638Sian 340290369Sskra /* 341290369Sskra * ARM has a set of unprivileged load and store instructions 342290369Sskra * (LDRT/LDRBT/STRT/STRBT ...) which are supposed to be used in other 343290369Sskra * than user mode and OS should recognize their aborts and behave 344290369Sskra * appropriately. However, there is no way how to do that reasonably 345290369Sskra * in general unless we restrict the handling somehow. 346290369Sskra * 347290369Sskra * For now, these instructions are used only in copyin()/copyout() 348290369Sskra * like functions where usermode buffers are checked in advance that 349290369Sskra * they are not from KVA space. Thus, no action is needed here. 350290369Sskra */ 351290369Sskra 352295255Sskra /* 353295255Sskra * (1) Handle access and R/W hardware emulation aborts. 354295255Sskra * (2) Check that abort is not on pmap essential address ranges. 355295255Sskra * There is no way how to fix it, so we don't even try. 356295255Sskra */ 357276638Sian rv = pmap_fault(PCPU_GET(curpmap), far, fsr, idx, usermode); 358294822Sskra if (rv == KERN_SUCCESS) 359276638Sian return; 360276638Sian#ifdef KDB 361276638Sian if (kdb_active) { 362276638Sian kdb_reenter(); 363276638Sian goto out; 364276638Sian } 365276638Sian#endif 366295255Sskra if (rv == KERN_INVALID_ADDRESS) 367295255Sskra goto nogo; 368295255Sskra 369276638Sian if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) { 370276638Sian /* 371276638Sian * Due to both processor errata and lazy TLB invalidation when 372276638Sian * access restrictions are removed from virtual pages, memory 373276638Sian * accesses that are allowed by the physical mapping layer may 374276638Sian * nonetheless cause one spurious page fault per virtual page. 375276638Sian * When the thread is executing a "no faulting" section that 376276638Sian * is bracketed by vm_fault_{disable,enable}_pagefaults(), 377276638Sian * every page fault is treated as a spurious page fault, 378276638Sian * unless it accesses the same virtual address as the most 379276638Sian * recent page fault within the same "no faulting" section. 380276638Sian */ 381276638Sian if (td->td_md.md_spurflt_addr != far || 382276638Sian (td->td_pflags & TDP_RESETSPUR) != 0) { 383276638Sian td->td_md.md_spurflt_addr = far; 384276638Sian td->td_pflags &= ~TDP_RESETSPUR; 385276638Sian 386276638Sian tlb_flush_local(far & ~PAGE_MASK); 387276638Sian return; 388276638Sian } 389276638Sian } else { 390276638Sian /* 391276638Sian * If we get a page fault while in a critical section, then 392276638Sian * it is most likely a fatal kernel page fault. The kernel 393276638Sian * is already going to panic trying to get a sleep lock to 394276638Sian * do the VM lookup, so just consider it a fatal trap so the 395276638Sian * kernel can print out a useful trap message and even get 396276638Sian * to the debugger. 397276638Sian * 398276638Sian * If we get a page fault while holding a non-sleepable 399276638Sian * lock, then it is most likely a fatal kernel page fault. 400276638Sian * If WITNESS is enabled, then it's going to whine about 401276638Sian * bogus LORs with various VM locks, so just skip to the 402276638Sian * fatal trap handling directly. 403276638Sian */ 404276638Sian if (td->td_critnest != 0 || 405276638Sian WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, 406276638Sian "Kernel page fault") != 0) { 407276638Sian abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig); 408276638Sian return; 409276638Sian } 410276638Sian } 411276638Sian 412276638Sian /* Re-enable interrupts if they were enabled previously. */ 413276638Sian if (td->td_md.md_spinlock_count == 0) { 414276638Sian if (__predict_true(tf->tf_spsr & PSR_I) == 0) 415276638Sian enable_interrupts(PSR_I); 416276638Sian if (__predict_true(tf->tf_spsr & PSR_F) == 0) 417276638Sian enable_interrupts(PSR_F); 418276638Sian } 419276638Sian 420276638Sian p = td->td_proc; 421276638Sian if (usermode) { 422276638Sian td->td_pticks = 0; 423284214Smjg if (td->td_cowgen != p->p_cowgen) 424284214Smjg thread_cow_update(td); 425276638Sian } 426276638Sian 427276638Sian /* Invoke the appropriate handler, if necessary. */ 428276638Sian if (__predict_false(aborts[idx].func != NULL)) { 429276638Sian if ((aborts[idx].func)(tf, idx, fsr, far, prefetch, td, &ksig)) 430276638Sian goto do_trapsignal; 431276638Sian goto out; 432276638Sian } 433276638Sian 434276638Sian /* 435295255Sskra * At this point, we're dealing with one of the following aborts: 436295255Sskra * 437295255Sskra * FAULT_ICACHE - I-cache maintenance 438295255Sskra * FAULT_TRAN_xx - Translation 439295255Sskra * FAULT_PERM_xx - Permission 440295255Sskra */ 441295255Sskra 442295255Sskra /* 443283947Sian * Don't pass faulting cache operation to vm_fault(). We don't want 444283947Sian * to handle all vm stuff at this moment. 445283947Sian */ 446283947Sian pcb = td->td_pcb; 447283947Sian if (__predict_false(pcb->pcb_onfault == cachebailout)) { 448283947Sian tf->tf_r0 = far; /* return failing address */ 449283947Sian tf->tf_pc = (register_t)pcb->pcb_onfault; 450283947Sian return; 451283947Sian } 452283947Sian 453290369Sskra /* Handle remaining I-cache aborts. */ 454283947Sian if (idx == FAULT_ICACHE) { 455283947Sian if (abort_icache(tf, idx, fsr, far, prefetch, td, &ksig)) 456283947Sian goto do_trapsignal; 457283947Sian goto out; 458283947Sian } 459283947Sian 460276638Sian va = trunc_page(far); 461276638Sian if (va >= KERNBASE) { 462276638Sian /* 463276638Sian * Don't allow user-mode faults in kernel address space. 464276638Sian */ 465276638Sian if (usermode) 466276638Sian goto nogo; 467276638Sian 468276638Sian map = kernel_map; 469276638Sian } else { 470276638Sian /* 471276638Sian * This is a fault on non-kernel virtual memory. If curproc 472276638Sian * is NULL or curproc->p_vmspace is NULL the fault is fatal. 473276638Sian */ 474276638Sian vm = (p != NULL) ? p->p_vmspace : NULL; 475276638Sian if (vm == NULL) 476276638Sian goto nogo; 477276638Sian 478276638Sian map = &vm->vm_map; 479276638Sian if (!usermode && (td->td_intr_nesting_level != 0 || 480276638Sian pcb->pcb_onfault == NULL)) { 481276638Sian abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig); 482276638Sian return; 483276638Sian } 484276638Sian } 485276638Sian 486276638Sian ftype = (fsr & FSR_WNR) ? VM_PROT_WRITE : VM_PROT_READ; 487276638Sian if (prefetch) 488276638Sian ftype |= VM_PROT_EXECUTE; 489276638Sian 490277523Sgonzo#ifdef DEBUG 491277523Sgonzo last_fault_code = fsr; 492277523Sgonzo#endif 493277523Sgonzo 494276638Sian#ifdef INVARIANTS 495276638Sian onfault = pcb->pcb_onfault; 496276638Sian pcb->pcb_onfault = NULL; 497276638Sian#endif 498276638Sian 499287625Skib /* Fault in the page. */ 500287625Skib rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL); 501276638Sian 502276638Sian#ifdef INVARIANTS 503276638Sian pcb->pcb_onfault = onfault; 504276638Sian#endif 505276638Sian 506276638Sian if (__predict_true(rv == KERN_SUCCESS)) 507276638Sian goto out; 508276638Siannogo: 509276638Sian if (!usermode) { 510276638Sian if (td->td_intr_nesting_level == 0 && 511276638Sian pcb->pcb_onfault != NULL) { 512276638Sian tf->tf_r0 = rv; 513276638Sian tf->tf_pc = (int)pcb->pcb_onfault; 514276638Sian return; 515276638Sian } 516276638Sian CTR2(KTR_TRAP, "%s: vm_fault() failed with %d", __func__, rv); 517276638Sian abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig); 518276638Sian return; 519276638Sian } 520276638Sian 521285389Sandrew ksig.sig = SIGSEGV; 522285389Sandrew ksig.code = (rv == KERN_PROTECTION_FAILURE) ? SEGV_ACCERR : SEGV_MAPERR; 523276638Sian ksig.addr = far; 524276638Sian 525276638Siando_trapsignal: 526276638Sian call_trapsignal(td, ksig.sig, ksig.code, ksig.addr); 527276638Sianout: 528276638Sian if (usermode) 529276638Sian userret(td, tf); 530276638Sian} 531276638Sian 532276638Sian/* 533276638Sian * abort_fatal() handles the following data aborts: 534290369Sskra * 535276638Sian * FAULT_DEBUG - Debug Event 536276638Sian * FAULT_ACCESS_xx - Acces Bit 537276638Sian * FAULT_EA_PREC - Precise External Abort 538276638Sian * FAULT_DOMAIN_xx - Domain Fault 539276638Sian * FAULT_EA_TRAN_xx - External Translation Abort 540276638Sian * FAULT_EA_IMPREC - Imprecise External Abort 541276638Sian * + all undefined codes for ABORT 542276638Sian * 543276638Sian * We should never see these on a properly functioning system. 544276638Sian * 545276638Sian * This function is also called by the other handlers if they 546276638Sian * detect a fatal problem. 547276638Sian * 548276638Sian * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort. 549276638Sian */ 550276638Sianstatic int 551290369Sskraabort_fatal(struct trapframe *tf, u_int idx, u_int fsr, u_int far, 552290369Sskra u_int prefetch, struct thread *td, struct ksig *ksig) 553276638Sian{ 554290541Sskra bool usermode; 555276638Sian const char *mode; 556276638Sian const char *rw_mode; 557276638Sian 558276638Sian usermode = TRAPF_USERMODE(tf); 559291852Sandrew#ifdef KDTRACE_HOOKS 560291852Sandrew if (!usermode) { 561291852Sandrew if (dtrace_trap_func != NULL && (*dtrace_trap_func)(tf, far)) 562291852Sandrew return (0); 563291852Sandrew } 564291852Sandrew#endif 565291852Sandrew 566276638Sian mode = usermode ? "user" : "kernel"; 567276638Sian rw_mode = fsr & FSR_WNR ? "write" : "read"; 568276638Sian disable_interrupts(PSR_I|PSR_F); 569276638Sian 570276638Sian if (td != NULL) { 571276638Sian printf("Fatal %s mode data abort: '%s' on %s\n", mode, 572276638Sian aborts[idx].desc, rw_mode); 573276638Sian printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr); 574276638Sian if (idx != FAULT_EA_IMPREC) 575276638Sian printf("%08x, ", far); 576276638Sian else 577276638Sian printf("Invalid, "); 578276638Sian printf("spsr=%08x\n", tf->tf_spsr); 579276638Sian } else { 580276638Sian printf("Fatal %s mode prefetch abort at 0x%08x\n", 581276638Sian mode, tf->tf_pc); 582276638Sian printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr); 583276638Sian } 584276638Sian 585276638Sian printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n", 586276638Sian tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3); 587276638Sian printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n", 588276638Sian tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7); 589276638Sian printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n", 590276638Sian tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11); 591276638Sian printf("r12=%08x, ", tf->tf_r12); 592276638Sian 593276638Sian if (usermode) 594276638Sian printf("usp=%08x, ulr=%08x", 595276638Sian tf->tf_usr_sp, tf->tf_usr_lr); 596276638Sian else 597276638Sian printf("ssp=%08x, slr=%08x", 598276638Sian tf->tf_svc_sp, tf->tf_svc_lr); 599276638Sian printf(", pc =%08x\n\n", tf->tf_pc); 600276638Sian 601276638Sian#ifdef KDB 602344905Sjhb if (debugger_on_trap) { 603335556Savg kdb_why = KDB_WHY_TRAP; 604276638Sian kdb_trap(fsr, 0, tf); 605335556Savg kdb_why = KDB_WHY_UNSET; 606335556Savg } 607276638Sian#endif 608276638Sian panic("Fatal abort"); 609276638Sian /*NOTREACHED*/ 610276638Sian} 611276638Sian 612276638Sian/* 613276638Sian * abort_align() handles the following data abort: 614276638Sian * 615276638Sian * FAULT_ALIGN - Alignment fault 616276638Sian * 617290369Sskra * Everything should be aligned in kernel with exception of user to kernel 618290369Sskra * and vice versa data copying, so if pcb_onfault is not set, it's fatal. 619290369Sskra * We generate signal in case of abort from user mode. 620276638Sian */ 621276638Sianstatic int 622290369Sskraabort_align(struct trapframe *tf, u_int idx, u_int fsr, u_int far, 623290369Sskra u_int prefetch, struct thread *td, struct ksig *ksig) 624276638Sian{ 625290541Sskra bool usermode; 626276638Sian 627276638Sian usermode = TRAPF_USERMODE(tf); 628276638Sian if (!usermode) { 629276638Sian if (td->td_intr_nesting_level == 0 && td != NULL && 630276638Sian td->td_pcb->pcb_onfault != NULL) { 631290472Sskra tf->tf_r0 = EFAULT; 632276638Sian tf->tf_pc = (int)td->td_pcb->pcb_onfault; 633276638Sian return (0); 634276638Sian } 635276638Sian abort_fatal(tf, idx, fsr, far, prefetch, td, ksig); 636276638Sian } 637276638Sian /* Deliver a bus error signal to the process */ 638290472Sskra ksig->code = BUS_ADRALN; 639276638Sian ksig->sig = SIGBUS; 640276638Sian ksig->addr = far; 641276638Sian return (1); 642276638Sian} 643276638Sian 644276638Sian/* 645276638Sian * abort_icache() handles the following data abort: 646276638Sian * 647276638Sian * FAULT_ICACHE - Instruction cache maintenance 648276638Sian * 649276638Sian * According to manual, FAULT_ICACHE is translation fault during cache 650276638Sian * maintenance operation. In fact, no cache maintenance operation on 651276638Sian * not mapped virtual addresses should be called. As cache maintenance 652276638Sian * operation (except DMB, DSB, and Flush Prefetch Buffer) are priviledged, 653276638Sian * the abort is concider as fatal for now. However, all the matter with 654276638Sian * cache maintenance operation on virtual addresses could be really complex 655276638Sian * and fuzzy in SMP case, so maybe in future standard fault mechanism 656276638Sian * should be held here including vm_fault() calling. 657276638Sian */ 658276638Sianstatic int 659290369Sskraabort_icache(struct trapframe *tf, u_int idx, u_int fsr, u_int far, 660290369Sskra u_int prefetch, struct thread *td, struct ksig *ksig) 661276638Sian{ 662290369Sskra 663276638Sian abort_fatal(tf, idx, fsr, far, prefetch, td, ksig); 664276638Sian return(0); 665276638Sian} 666