1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * linux/arch/arm/kernel/traps.c 4 * 5 * Copyright (C) 1995-2009 Russell King 6 * Fragments that appear the same as linux/arch/i386/kernel/traps.c (C) Linus Torvalds 7 * 8 * 'traps.c' handles hardware exceptions after we have saved some state in 9 * 'linux/arch/arm/lib/traps.S'. Mostly a debugging aid, but will probably 10 * kill the offending process. 11 */ 12#include <linux/signal.h> 13#include <linux/personality.h> 14#include <linux/kallsyms.h> 15#include <linux/spinlock.h> 16#include <linux/uaccess.h> 17#include <linux/hardirq.h> 18#include <linux/kdebug.h> 19#include <linux/kprobes.h> 20#include <linux/module.h> 21#include <linux/kexec.h> 22#include <linux/bug.h> 23#include <linux/delay.h> 24#include <linux/init.h> 25#include <linux/sched/signal.h> 26#include <linux/sched/debug.h> 27#include <linux/sched/task_stack.h> 28#include <linux/irq.h> 29 30#include <linux/atomic.h> 31#include <asm/cacheflush.h> 32#include <asm/exception.h> 33#include <asm/spectre.h> 34#include <asm/unistd.h> 35#include <asm/traps.h> 36#include <asm/ptrace.h> 37#include <asm/unwind.h> 38#include <asm/tls.h> 39#include <asm/stacktrace.h> 40#include <asm/system_misc.h> 41#include <asm/opcodes.h> 42 43 44static const char *handler[]= { 45 "prefetch abort", 46 "data abort", 47 "address exception", 48 "interrupt", 49 "undefined instruction", 50}; 51 52void *vectors_page; 53 54#ifdef CONFIG_DEBUG_USER 55unsigned int user_debug; 56 57static int __init user_debug_setup(char *str) 58{ 59 get_option(&str, &user_debug); 60 return 1; 61} 62__setup("user_debug=", user_debug_setup); 63#endif 64 65void dump_backtrace_entry(unsigned long where, unsigned long from, 66 unsigned long frame, const char *loglvl) 67{ 68 unsigned long end = frame + 4 + sizeof(struct pt_regs); 69 70 if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER) && 71 IS_ENABLED(CONFIG_CC_IS_GCC) && 72 end > ALIGN(frame, THREAD_SIZE)) { 73 /* 74 * If we are walking past the end of the stack, it may be due 75 * to the fact that we are on an IRQ or overflow stack. In this 76 * case, we can load the address of the other stack from the 77 * frame record. 78 */ 79 frame = ((unsigned long *)frame)[-2] - 4; 80 end = frame + 4 + sizeof(struct pt_regs); 81 } 82 83#ifndef CONFIG_KALLSYMS 84 printk("%sFunction entered at [<%08lx>] from [<%08lx>]\n", 85 loglvl, where, from); 86#elif defined CONFIG_BACKTRACE_VERBOSE 87 printk("%s[<%08lx>] (%ps) from [<%08lx>] (%pS)\n", 88 loglvl, where, (void *)where, from, (void *)from); 89#else 90 printk("%s %ps from %pS\n", loglvl, (void *)where, (void *)from); 91#endif 92 93 if (in_entry_text(from) && end <= ALIGN(frame, THREAD_SIZE)) 94 dump_mem(loglvl, "Exception stack", frame + 4, end); 95} 96 97void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl) 98{ 99 char str[80], *p; 100 unsigned int x; 101 int reg; 102 103 for (reg = 10, x = 0, p = str; reg >= 0; reg--) { 104 if (instruction & BIT(reg)) { 105 p += sprintf(p, " r%d:%08x", reg, *stack--); 106 if (++x == 6) { 107 x = 0; 108 p = str; 109 printk("%s%s\n", loglvl, str); 110 } 111 } 112 } 113 if (p != str) 114 printk("%s%s\n", loglvl, str); 115} 116 117#ifndef CONFIG_ARM_UNWIND 118/* 119 * Stack pointers should always be within the kernels view of 120 * physical memory. If it is not there, then we can't dump 121 * out any information relating to the stack. 122 */ 123static int verify_stack(unsigned long sp) 124{ 125 if (sp < PAGE_OFFSET || 126 (!IS_ENABLED(CONFIG_VMAP_STACK) && 127 sp > (unsigned long)high_memory && high_memory != NULL)) 128 return -EFAULT; 129 130 return 0; 131} 132#endif 133 134/* 135 * Dump out the contents of some memory nicely... 136 */ 137void dump_mem(const char *lvl, const char *str, unsigned long bottom, 138 unsigned long top) 139{ 140 unsigned long first; 141 int i; 142 143 printk("%s%s(0x%08lx to 0x%08lx)\n", lvl, str, bottom, top); 144 145 for (first = bottom & ~31; first < top; first += 32) { 146 unsigned long p; 147 char str[sizeof(" 12345678") * 8 + 1]; 148 149 memset(str, ' ', sizeof(str)); 150 str[sizeof(str) - 1] = '\0'; 151 152 for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { 153 if (p >= bottom && p < top) { 154 unsigned long val; 155 if (!get_kernel_nofault(val, (unsigned long *)p)) 156 sprintf(str + i * 9, " %08lx", val); 157 else 158 sprintf(str + i * 9, " ????????"); 159 } 160 } 161 printk("%s%04lx:%s\n", lvl, first & 0xffff, str); 162 } 163} 164 165static void dump_instr(const char *lvl, struct pt_regs *regs) 166{ 167 unsigned long addr = instruction_pointer(regs); 168 const int thumb = thumb_mode(regs); 169 const int width = thumb ? 4 : 8; 170 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; 171 int i; 172 173 /* 174 * Note that we now dump the code first, just in case the backtrace 175 * kills us. 176 */ 177 178 for (i = -4; i < 1 + !!thumb; i++) { 179 unsigned int val, bad; 180 181 if (thumb) { 182 u16 tmp; 183 184 if (user_mode(regs)) 185 bad = get_user(tmp, &((u16 __user *)addr)[i]); 186 else 187 bad = get_kernel_nofault(tmp, &((u16 *)addr)[i]); 188 189 val = __mem_to_opcode_thumb16(tmp); 190 } else { 191 if (user_mode(regs)) 192 bad = get_user(val, &((u32 __user *)addr)[i]); 193 else 194 bad = get_kernel_nofault(val, &((u32 *)addr)[i]); 195 196 val = __mem_to_opcode_arm(val); 197 } 198 199 if (!bad) 200 p += sprintf(p, i == 0 ? "(%0*x) " : "%0*x ", 201 width, val); 202 else { 203 p += sprintf(p, "bad PC value"); 204 break; 205 } 206 } 207 printk("%sCode: %s\n", lvl, str); 208} 209 210#ifdef CONFIG_ARM_UNWIND 211void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, 212 const char *loglvl) 213{ 214 unwind_backtrace(regs, tsk, loglvl); 215} 216#else 217void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, 218 const char *loglvl) 219{ 220 unsigned int fp, mode; 221 int ok = 1; 222 223 printk("%sCall trace: ", loglvl); 224 225 if (!tsk) 226 tsk = current; 227 228 if (regs) { 229 fp = frame_pointer(regs); 230 mode = processor_mode(regs); 231 } else if (tsk != current) { 232 fp = thread_saved_fp(tsk); 233 mode = 0x10; 234 } else { 235 asm("mov %0, fp" : "=r" (fp) : : "cc"); 236 mode = 0x10; 237 } 238 239 if (!fp) { 240 pr_cont("no frame pointer"); 241 ok = 0; 242 } else if (verify_stack(fp)) { 243 pr_cont("invalid frame pointer 0x%08x", fp); 244 ok = 0; 245 } else if (fp < (unsigned long)end_of_stack(tsk)) 246 pr_cont("frame pointer underflow"); 247 pr_cont("\n"); 248 249 if (ok) 250 c_backtrace(fp, mode, loglvl); 251} 252#endif 253 254void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) 255{ 256 dump_backtrace(NULL, tsk, loglvl); 257 barrier(); 258} 259 260#ifdef CONFIG_PREEMPT 261#define S_PREEMPT " PREEMPT" 262#elif defined(CONFIG_PREEMPT_RT) 263#define S_PREEMPT " PREEMPT_RT" 264#else 265#define S_PREEMPT "" 266#endif 267#ifdef CONFIG_SMP 268#define S_SMP " SMP" 269#else 270#define S_SMP "" 271#endif 272#ifdef CONFIG_THUMB2_KERNEL 273#define S_ISA " THUMB2" 274#else 275#define S_ISA " ARM" 276#endif 277 278static int __die(const char *str, int err, struct pt_regs *regs) 279{ 280 struct task_struct *tsk = current; 281 static int die_counter; 282 int ret; 283 284 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP S_ISA "\n", 285 str, err, ++die_counter); 286 287 /* trap and error numbers are mostly meaningless on ARM */ 288 ret = notify_die(DIE_OOPS, str, regs, err, tsk->thread.trap_no, SIGSEGV); 289 if (ret == NOTIFY_STOP) 290 return 1; 291 292 print_modules(); 293 __show_regs(regs); 294 __show_regs_alloc_free(regs); 295 pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n", 296 TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), end_of_stack(tsk)); 297 298 if (!user_mode(regs) || in_interrupt()) { 299 dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, 300 ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN) 301 + THREAD_SIZE); 302 dump_backtrace(regs, tsk, KERN_EMERG); 303 dump_instr(KERN_EMERG, regs); 304 } 305 306 return 0; 307} 308 309static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED; 310static int die_owner = -1; 311static unsigned int die_nest_count; 312 313static unsigned long oops_begin(void) 314{ 315 int cpu; 316 unsigned long flags; 317 318 oops_enter(); 319 320 /* racy, but better than risking deadlock. */ 321 raw_local_irq_save(flags); 322 cpu = smp_processor_id(); 323 if (!arch_spin_trylock(&die_lock)) { 324 if (cpu == die_owner) 325 /* nested oops. should stop eventually */; 326 else 327 arch_spin_lock(&die_lock); 328 } 329 die_nest_count++; 330 die_owner = cpu; 331 console_verbose(); 332 bust_spinlocks(1); 333 return flags; 334} 335 336static void oops_end(unsigned long flags, struct pt_regs *regs, int signr) 337{ 338 if (regs && kexec_should_crash(current)) 339 crash_kexec(regs); 340 341 bust_spinlocks(0); 342 die_owner = -1; 343 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); 344 die_nest_count--; 345 if (!die_nest_count) 346 /* Nest count reaches zero, release the lock. */ 347 arch_spin_unlock(&die_lock); 348 raw_local_irq_restore(flags); 349 oops_exit(); 350 351 if (in_interrupt()) 352 panic("Fatal exception in interrupt"); 353 if (panic_on_oops) 354 panic("Fatal exception"); 355 if (signr) 356 make_task_dead(signr); 357} 358 359/* 360 * This function is protected against re-entrancy. 361 */ 362void die(const char *str, struct pt_regs *regs, int err) 363{ 364 enum bug_trap_type bug_type = BUG_TRAP_TYPE_NONE; 365 unsigned long flags = oops_begin(); 366 int sig = SIGSEGV; 367 368 if (!user_mode(regs)) 369 bug_type = report_bug(regs->ARM_pc, regs); 370 if (bug_type != BUG_TRAP_TYPE_NONE) 371 str = "Oops - BUG"; 372 373 if (__die(str, err, regs)) 374 sig = 0; 375 376 oops_end(flags, regs, sig); 377} 378 379void arm_notify_die(const char *str, struct pt_regs *regs, 380 int signo, int si_code, void __user *addr, 381 unsigned long err, unsigned long trap) 382{ 383 if (user_mode(regs)) { 384 current->thread.error_code = err; 385 current->thread.trap_no = trap; 386 387 force_sig_fault(signo, si_code, addr); 388 } else { 389 die(str, regs, err); 390 } 391} 392 393#ifdef CONFIG_GENERIC_BUG 394 395int is_valid_bugaddr(unsigned long pc) 396{ 397#ifdef CONFIG_THUMB2_KERNEL 398 u16 bkpt; 399 u16 insn = __opcode_to_mem_thumb16(BUG_INSTR_VALUE); 400#else 401 u32 bkpt; 402 u32 insn = __opcode_to_mem_arm(BUG_INSTR_VALUE); 403#endif 404 405 if (get_kernel_nofault(bkpt, (void *)pc)) 406 return 0; 407 408 return bkpt == insn; 409} 410 411#endif 412 413static LIST_HEAD(undef_hook); 414static DEFINE_RAW_SPINLOCK(undef_lock); 415 416void register_undef_hook(struct undef_hook *hook) 417{ 418 unsigned long flags; 419 420 raw_spin_lock_irqsave(&undef_lock, flags); 421 list_add(&hook->node, &undef_hook); 422 raw_spin_unlock_irqrestore(&undef_lock, flags); 423} 424 425void unregister_undef_hook(struct undef_hook *hook) 426{ 427 unsigned long flags; 428 429 raw_spin_lock_irqsave(&undef_lock, flags); 430 list_del(&hook->node); 431 raw_spin_unlock_irqrestore(&undef_lock, flags); 432} 433 434static nokprobe_inline 435int call_undef_hook(struct pt_regs *regs, unsigned int instr) 436{ 437 struct undef_hook *hook; 438 unsigned long flags; 439 int (*fn)(struct pt_regs *regs, unsigned int instr) = NULL; 440 441 raw_spin_lock_irqsave(&undef_lock, flags); 442 list_for_each_entry(hook, &undef_hook, node) 443 if ((instr & hook->instr_mask) == hook->instr_val && 444 (regs->ARM_cpsr & hook->cpsr_mask) == hook->cpsr_val) 445 fn = hook->fn; 446 raw_spin_unlock_irqrestore(&undef_lock, flags); 447 448 return fn ? fn(regs, instr) : 1; 449} 450 451asmlinkage void do_undefinstr(struct pt_regs *regs) 452{ 453 unsigned int instr; 454 void __user *pc; 455 456 pc = (void __user *)instruction_pointer(regs); 457 458 if (processor_mode(regs) == SVC_MODE) { 459#ifdef CONFIG_THUMB2_KERNEL 460 if (thumb_mode(regs)) { 461 instr = __mem_to_opcode_thumb16(((u16 *)pc)[0]); 462 if (is_wide_instruction(instr)) { 463 u16 inst2; 464 inst2 = __mem_to_opcode_thumb16(((u16 *)pc)[1]); 465 instr = __opcode_thumb32_compose(instr, inst2); 466 } 467 } else 468#endif 469 instr = __mem_to_opcode_arm(*(u32 *) pc); 470 } else if (thumb_mode(regs)) { 471 if (get_user(instr, (u16 __user *)pc)) 472 goto die_sig; 473 instr = __mem_to_opcode_thumb16(instr); 474 if (is_wide_instruction(instr)) { 475 unsigned int instr2; 476 if (get_user(instr2, (u16 __user *)pc+1)) 477 goto die_sig; 478 instr2 = __mem_to_opcode_thumb16(instr2); 479 instr = __opcode_thumb32_compose(instr, instr2); 480 } 481 } else { 482 if (get_user(instr, (u32 __user *)pc)) 483 goto die_sig; 484 instr = __mem_to_opcode_arm(instr); 485 } 486 487 if (call_undef_hook(regs, instr) == 0) 488 return; 489 490die_sig: 491#ifdef CONFIG_DEBUG_USER 492 if (user_debug & UDBG_UNDEFINED) { 493 pr_info("%s (%d): undefined instruction: pc=%px\n", 494 current->comm, task_pid_nr(current), pc); 495 __show_regs(regs); 496 dump_instr(KERN_INFO, regs); 497 } 498#endif 499 arm_notify_die("Oops - undefined instruction", regs, 500 SIGILL, ILL_ILLOPC, pc, 0, 6); 501} 502NOKPROBE_SYMBOL(do_undefinstr) 503 504/* 505 * Handle FIQ similarly to NMI on x86 systems. 506 * 507 * The runtime environment for NMIs is extremely restrictive 508 * (NMIs can pre-empt critical sections meaning almost all locking is 509 * forbidden) meaning this default FIQ handling must only be used in 510 * circumstances where non-maskability improves robustness, such as 511 * watchdog or debug logic. 512 * 513 * This handler is not appropriate for general purpose use in drivers 514 * platform code and can be overrideen using set_fiq_handler. 515 */ 516asmlinkage void __exception_irq_entry handle_fiq_as_nmi(struct pt_regs *regs) 517{ 518 struct pt_regs *old_regs = set_irq_regs(regs); 519 520 nmi_enter(); 521 522 /* nop. FIQ handlers for special arch/arm features can be added here. */ 523 524 nmi_exit(); 525 526 set_irq_regs(old_regs); 527} 528 529/* 530 * bad_mode handles the impossible case in the vectors. If you see one of 531 * these, then it's extremely serious, and could mean you have buggy hardware. 532 * It never returns, and never tries to sync. We hope that we can at least 533 * dump out some state information... 534 */ 535asmlinkage void bad_mode(struct pt_regs *regs, int reason) 536{ 537 console_verbose(); 538 539 pr_crit("Bad mode in %s handler detected\n", handler[reason]); 540 541 die("Oops - bad mode", regs, 0); 542 local_irq_disable(); 543 panic("bad mode"); 544} 545 546static int bad_syscall(int n, struct pt_regs *regs) 547{ 548 if ((current->personality & PER_MASK) != PER_LINUX) { 549 send_sig(SIGSEGV, current, 1); 550 return regs->ARM_r0; 551 } 552 553#ifdef CONFIG_DEBUG_USER 554 if (user_debug & UDBG_SYSCALL) { 555 pr_err("[%d] %s: obsolete system call %08x.\n", 556 task_pid_nr(current), current->comm, n); 557 dump_instr(KERN_ERR, regs); 558 } 559#endif 560 561 arm_notify_die("Oops - bad syscall", regs, SIGILL, ILL_ILLTRP, 562 (void __user *)instruction_pointer(regs) - 563 (thumb_mode(regs) ? 2 : 4), 564 n, 0); 565 566 return regs->ARM_r0; 567} 568 569static inline int 570__do_cache_op(unsigned long start, unsigned long end) 571{ 572 int ret; 573 574 do { 575 unsigned long chunk = min(PAGE_SIZE, end - start); 576 577 if (fatal_signal_pending(current)) 578 return 0; 579 580 ret = flush_icache_user_range(start, start + chunk); 581 if (ret) 582 return ret; 583 584 cond_resched(); 585 start += chunk; 586 } while (start < end); 587 588 return 0; 589} 590 591static inline int 592do_cache_op(unsigned long start, unsigned long end, int flags) 593{ 594 if (end < start || flags) 595 return -EINVAL; 596 597 if (!access_ok((void __user *)start, end - start)) 598 return -EFAULT; 599 600 return __do_cache_op(start, end); 601} 602 603/* 604 * Handle all unrecognised system calls. 605 * 0x9f0000 - 0x9fffff are some more esoteric system calls 606 */ 607#define NR(x) ((__ARM_NR_##x) - __ARM_NR_BASE) 608asmlinkage int arm_syscall(int no, struct pt_regs *regs) 609{ 610 if ((no >> 16) != (__ARM_NR_BASE>> 16)) 611 return bad_syscall(no, regs); 612 613 switch (no & 0xffff) { 614 case 0: /* branch through 0 */ 615 arm_notify_die("branch through zero", regs, 616 SIGSEGV, SEGV_MAPERR, NULL, 0, 0); 617 return 0; 618 619 case NR(breakpoint): /* SWI BREAK_POINT */ 620 regs->ARM_pc -= thumb_mode(regs) ? 2 : 4; 621 ptrace_break(regs); 622 return regs->ARM_r0; 623 624 /* 625 * Flush a region from virtual address 'r0' to virtual address 'r1' 626 * _exclusive_. There is no alignment requirement on either address; 627 * user space does not need to know the hardware cache layout. 628 * 629 * r2 contains flags. It should ALWAYS be passed as ZERO until it 630 * is defined to be something else. For now we ignore it, but may 631 * the fires of hell burn in your belly if you break this rule. ;) 632 * 633 * (at a later date, we may want to allow this call to not flush 634 * various aspects of the cache. Passing '0' will guarantee that 635 * everything necessary gets flushed to maintain consistency in 636 * the specified region). 637 */ 638 case NR(cacheflush): 639 return do_cache_op(regs->ARM_r0, regs->ARM_r1, regs->ARM_r2); 640 641 case NR(usr26): 642 if (!(elf_hwcap & HWCAP_26BIT)) 643 break; 644 regs->ARM_cpsr &= ~MODE32_BIT; 645 return regs->ARM_r0; 646 647 case NR(usr32): 648 if (!(elf_hwcap & HWCAP_26BIT)) 649 break; 650 regs->ARM_cpsr |= MODE32_BIT; 651 return regs->ARM_r0; 652 653 case NR(set_tls): 654 set_tls(regs->ARM_r0); 655 return 0; 656 657 case NR(get_tls): 658 return current_thread_info()->tp_value[0]; 659 660 default: 661 /* Calls 9f00xx..9f07ff are defined to return -ENOSYS 662 if not implemented, rather than raising SIGILL. This 663 way the calling program can gracefully determine whether 664 a feature is supported. */ 665 if ((no & 0xffff) <= 0x7ff) 666 return -ENOSYS; 667 break; 668 } 669#ifdef CONFIG_DEBUG_USER 670 /* 671 * experience shows that these seem to indicate that 672 * something catastrophic has happened 673 */ 674 if (user_debug & UDBG_SYSCALL) { 675 pr_err("[%d] %s: arm syscall %d\n", 676 task_pid_nr(current), current->comm, no); 677 dump_instr(KERN_ERR, regs); 678 if (user_mode(regs)) { 679 __show_regs(regs); 680 c_backtrace(frame_pointer(regs), processor_mode(regs), KERN_ERR); 681 } 682 } 683#endif 684 arm_notify_die("Oops - bad syscall(2)", regs, SIGILL, ILL_ILLTRP, 685 (void __user *)instruction_pointer(regs) - 686 (thumb_mode(regs) ? 2 : 4), 687 no, 0); 688 return 0; 689} 690 691#ifdef CONFIG_TLS_REG_EMUL 692 693/* 694 * We might be running on an ARMv6+ processor which should have the TLS 695 * register but for some reason we can't use it, or maybe an SMP system 696 * using a pre-ARMv6 processor (there are apparently a few prototypes like 697 * that in existence) and therefore access to that register must be 698 * emulated. 699 */ 700 701static int get_tp_trap(struct pt_regs *regs, unsigned int instr) 702{ 703 int reg = (instr >> 12) & 15; 704 if (reg == 15) 705 return 1; 706 regs->uregs[reg] = current_thread_info()->tp_value[0]; 707 regs->ARM_pc += 4; 708 return 0; 709} 710 711static struct undef_hook arm_mrc_hook = { 712 .instr_mask = 0x0fff0fff, 713 .instr_val = 0x0e1d0f70, 714 .cpsr_mask = PSR_T_BIT, 715 .cpsr_val = 0, 716 .fn = get_tp_trap, 717}; 718 719static int __init arm_mrc_hook_init(void) 720{ 721 register_undef_hook(&arm_mrc_hook); 722 return 0; 723} 724 725late_initcall(arm_mrc_hook_init); 726 727#endif 728 729/* 730 * A data abort trap was taken, but we did not handle the instruction. 731 * Try to abort the user program, or panic if it was the kernel. 732 */ 733asmlinkage void 734baddataabort(int code, unsigned long instr, struct pt_regs *regs) 735{ 736 unsigned long addr = instruction_pointer(regs); 737 738#ifdef CONFIG_DEBUG_USER 739 if (user_debug & UDBG_BADABORT) { 740 pr_err("8<--- cut here ---\n"); 741 pr_err("[%d] %s: bad data abort: code %d instr 0x%08lx\n", 742 task_pid_nr(current), current->comm, code, instr); 743 dump_instr(KERN_ERR, regs); 744 show_pte(KERN_ERR, current->mm, addr); 745 } 746#endif 747 748 arm_notify_die("unknown data abort code", regs, 749 SIGILL, ILL_ILLOPC, (void __user *)addr, instr, 0); 750} 751 752void __readwrite_bug(const char *fn) 753{ 754 pr_err("%s called, but not implemented\n", fn); 755 BUG(); 756} 757EXPORT_SYMBOL(__readwrite_bug); 758 759#ifdef CONFIG_MMU 760void __pte_error(const char *file, int line, pte_t pte) 761{ 762 pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); 763} 764 765void __pmd_error(const char *file, int line, pmd_t pmd) 766{ 767 pr_err("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd)); 768} 769 770void __pgd_error(const char *file, int line, pgd_t pgd) 771{ 772 pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); 773} 774#endif 775 776asmlinkage void __div0(void) 777{ 778 pr_err("Division by zero in kernel.\n"); 779 dump_stack(); 780} 781EXPORT_SYMBOL(__div0); 782 783void abort(void) 784{ 785 BUG(); 786 787 /* if that doesn't kill us, halt */ 788 panic("Oops failed to kill thread"); 789} 790 791#ifdef CONFIG_KUSER_HELPERS 792static void __init kuser_init(void *vectors) 793{ 794 extern char __kuser_helper_start[], __kuser_helper_end[]; 795 int kuser_sz = __kuser_helper_end - __kuser_helper_start; 796 797 memcpy(vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz); 798 799 /* 800 * vectors + 0xfe0 = __kuser_get_tls 801 * vectors + 0xfe8 = hardware TLS instruction at 0xffff0fe8 802 */ 803 if (tls_emu || has_tls_reg) 804 memcpy(vectors + 0xfe0, vectors + 0xfe8, 4); 805} 806#else 807static inline void __init kuser_init(void *vectors) 808{ 809} 810#endif 811 812#ifndef CONFIG_CPU_V7M 813static void copy_from_lma(void *vma, void *lma_start, void *lma_end) 814{ 815 memcpy(vma, lma_start, lma_end - lma_start); 816} 817 818static void flush_vectors(void *vma, size_t offset, size_t size) 819{ 820 unsigned long start = (unsigned long)vma + offset; 821 unsigned long end = start + size; 822 823 flush_icache_range(start, end); 824} 825 826#ifdef CONFIG_HARDEN_BRANCH_HISTORY 827int spectre_bhb_update_vectors(unsigned int method) 828{ 829 extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[]; 830 extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[]; 831 void *vec_start, *vec_end; 832 833 if (system_state >= SYSTEM_FREEING_INITMEM) { 834 pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n", 835 smp_processor_id()); 836 return SPECTRE_VULNERABLE; 837 } 838 839 switch (method) { 840 case SPECTRE_V2_METHOD_LOOP8: 841 vec_start = __vectors_bhb_loop8_start; 842 vec_end = __vectors_bhb_loop8_end; 843 break; 844 845 case SPECTRE_V2_METHOD_BPIALL: 846 vec_start = __vectors_bhb_bpiall_start; 847 vec_end = __vectors_bhb_bpiall_end; 848 break; 849 850 default: 851 pr_err("CPU%u: unknown Spectre BHB state %d\n", 852 smp_processor_id(), method); 853 return SPECTRE_VULNERABLE; 854 } 855 856 copy_from_lma(vectors_page, vec_start, vec_end); 857 flush_vectors(vectors_page, 0, vec_end - vec_start); 858 859 return SPECTRE_MITIGATED; 860} 861#endif 862 863void __init early_trap_init(void *vectors_base) 864{ 865 extern char __stubs_start[], __stubs_end[]; 866 extern char __vectors_start[], __vectors_end[]; 867 unsigned i; 868 869 vectors_page = vectors_base; 870 871 /* 872 * Poison the vectors page with an undefined instruction. This 873 * instruction is chosen to be undefined for both ARM and Thumb 874 * ISAs. The Thumb version is an undefined instruction with a 875 * branch back to the undefined instruction. 876 */ 877 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) 878 ((u32 *)vectors_base)[i] = 0xe7fddef1; 879 880 /* 881 * Copy the vectors, stubs and kuser helpers (in entry-armv.S) 882 * into the vector page, mapped at 0xffff0000, and ensure these 883 * are visible to the instruction stream. 884 */ 885 copy_from_lma(vectors_base, __vectors_start, __vectors_end); 886 copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end); 887 888 kuser_init(vectors_base); 889 890 flush_vectors(vectors_base, 0, PAGE_SIZE * 2); 891} 892#else /* ifndef CONFIG_CPU_V7M */ 893void __init early_trap_init(void *vectors_base) 894{ 895 /* 896 * on V7-M there is no need to copy the vector table to a dedicated 897 * memory area. The address is configurable and so a table in the kernel 898 * image can be used. 899 */ 900} 901#endif 902 903#ifdef CONFIG_VMAP_STACK 904 905DECLARE_PER_CPU(u8 *, irq_stack_ptr); 906 907asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr); 908 909static int __init allocate_overflow_stacks(void) 910{ 911 u8 *stack; 912 int cpu; 913 914 for_each_possible_cpu(cpu) { 915 stack = (u8 *)__get_free_page(GFP_KERNEL); 916 if (WARN_ON(!stack)) 917 return -ENOMEM; 918 per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE]; 919 } 920 return 0; 921} 922early_initcall(allocate_overflow_stacks); 923 924asmlinkage void handle_bad_stack(struct pt_regs *regs) 925{ 926 unsigned long tsk_stk = (unsigned long)current->stack; 927#ifdef CONFIG_IRQSTACKS 928 unsigned long irq_stk = (unsigned long)raw_cpu_read(irq_stack_ptr); 929#endif 930 unsigned long ovf_stk = (unsigned long)raw_cpu_read(overflow_stack_ptr); 931 932 console_verbose(); 933 pr_emerg("Insufficient stack space to handle exception!"); 934 935 pr_emerg("Task stack: [0x%08lx..0x%08lx]\n", 936 tsk_stk, tsk_stk + THREAD_SIZE); 937#ifdef CONFIG_IRQSTACKS 938 pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n", 939 irq_stk - THREAD_SIZE, irq_stk); 940#endif 941 pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n", 942 ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk); 943 944 die("kernel stack overflow", regs, 0); 945} 946 947#ifndef CONFIG_ARM_LPAE 948/* 949 * Normally, we rely on the logic in do_translation_fault() to update stale PMD 950 * entries covering the vmalloc space in a task's page tables when it first 951 * accesses the region in question. Unfortunately, this is not sufficient when 952 * the task stack resides in the vmalloc region, as do_translation_fault() is a 953 * C function that needs a stack to run. 954 * 955 * So we need to ensure that these PMD entries are up to date *before* the MM 956 * switch. As we already have some logic in the MM switch path that takes care 957 * of this, let's trigger it by bumping the counter every time the core vmalloc 958 * code modifies a PMD entry in the vmalloc region. Use release semantics on 959 * the store so that other CPUs observing the counter's new value are 960 * guaranteed to see the updated page table entries as well. 961 */ 962void arch_sync_kernel_mappings(unsigned long start, unsigned long end) 963{ 964 if (start < VMALLOC_END && end > VMALLOC_START) 965 atomic_inc_return_release(&init_mm.context.vmalloc_seq); 966} 967#endif 968#endif 969