1// Copyright 2016 The Fuchsia Authors 2// Copyright (c) 2014 Travis Geiselbrecht 3// 4// Use of this source code is governed by a MIT-style 5// license that can be found in the LICENSE file or at 6// https://opensource.org/licenses/MIT 7 8#include <arch/arch_ops.h> 9#include <arch/arm64.h> 10#include <arch/arm64/exceptions.h> 11#include <arch/exception.h> 12#include <arch/user_copy.h> 13 14#include <bits.h> 15#include <debug.h> 16#include <inttypes.h> 17 18#include <kernel/interrupt.h> 19#include <kernel/thread.h> 20 21#include <platform.h> 22#include <stdio.h> 23#include <trace.h> 24#include <vm/fault.h> 25#include <vm/vm.h> 26 27#include <lib/counters.h> 28#include <lib/crashlog.h> 29 30#include <zircon/syscalls/exception.h> 31#include <zircon/types.h> 32 33#define LOCAL_TRACE 0 34 35#define DFSC_ALIGNMENT_FAULT 0b100001 36 37static void dump_iframe(const struct arm64_iframe_long* iframe) { 38 printf("iframe %p:\n", iframe); 39 printf("x0 %#18" PRIx64 " x1 %#18" PRIx64 " x2 %#18" PRIx64 " x3 %#18" PRIx64 "\n", iframe->r[0], iframe->r[1], iframe->r[2], iframe->r[3]); 40 printf("x4 %#18" PRIx64 " x5 %#18" PRIx64 " x6 %#18" PRIx64 " x7 %#18" PRIx64 "\n", iframe->r[4], iframe->r[5], iframe->r[6], iframe->r[7]); 41 printf("x8 %#18" PRIx64 " x9 %#18" PRIx64 " x10 %#18" PRIx64 " x11 %#18" PRIx64 "\n", iframe->r[8], iframe->r[9], iframe->r[10], iframe->r[11]); 42 printf("x12 %#18" PRIx64 " x13 %#18" PRIx64 " x14 %#18" PRIx64 " x15 %#18" PRIx64 "\n", iframe->r[12], iframe->r[13], iframe->r[14], iframe->r[15]); 43 printf("x16 %#18" PRIx64 " x17 %#18" PRIx64 " x18 %#18" PRIx64 " x19 %#18" PRIx64 "\n", iframe->r[16], iframe->r[17], iframe->r[18], iframe->r[19]); 44 printf("x20 %#18" PRIx64 " x21 %#18" PRIx64 " x22 %#18" PRIx64 " x23 %#18" PRIx64 "\n", iframe->r[20], iframe->r[21], iframe->r[22], iframe->r[23]); 45 printf("x24 %#18" PRIx64 " x25 %#18" PRIx64 " x26 %#18" PRIx64 " x27 %#18" PRIx64 "\n", iframe->r[24], iframe->r[25], iframe->r[26], iframe->r[27]); 46 printf("x28 %#18" PRIx64 " x29 %#18" PRIx64 " lr %#18" PRIx64 " usp %#18" PRIx64 "\n", iframe->r[28], iframe->r[29], iframe->lr, iframe->usp); 47 printf("elr %#18" PRIx64 "\n", iframe->elr); 48 printf("spsr %#18" PRIx64 "\n", iframe->spsr); 49} 50 51KCOUNTER(exceptions_brkpt, "kernel.exceptions.breakpoint"); 52KCOUNTER(exceptions_fpu, "kernel.exceptions.fpu"); 53KCOUNTER(exceptions_page, "kernel.exceptions.page_fault"); 54KCOUNTER(exceptions_irq, "kernel.exceptions.irq"); 55KCOUNTER(exceptions_unhandled, "kernel.exceptions.unhandled"); 56KCOUNTER(exceptions_user, "kernel.exceptions.user"); 57KCOUNTER(exceptions_unknown, "kernel.exceptions.unknown"); 58 59static zx_status_t try_dispatch_user_data_fault_exception( 60 zx_excp_type_t type, struct arm64_iframe_long* iframe, 61 uint32_t esr, uint64_t far) { 62 thread_t* thread = get_current_thread(); 63 arch_exception_context_t context = {}; 64 DEBUG_ASSERT(iframe != nullptr); 65 context.frame = iframe; 66 context.esr = esr; 67 context.far = far; 68 69 arch_enable_ints(); 70 DEBUG_ASSERT(thread->arch.suspended_general_regs == nullptr); 71 thread->arch.suspended_general_regs = iframe; 72 zx_status_t status = dispatch_user_exception(type, &context); 73 thread->arch.suspended_general_regs = nullptr; 74 arch_disable_ints(); 75 return status; 76} 77 78static zx_status_t try_dispatch_user_exception( 79 zx_excp_type_t type, struct arm64_iframe_long* iframe, uint32_t esr) { 80 return try_dispatch_user_data_fault_exception(type, iframe, esr, 0); 81} 82 83__NO_RETURN static void exception_die(struct arm64_iframe_long* iframe, uint32_t esr) { 84 platform_panic_start(); 85 86 uint32_t ec = BITS_SHIFT(esr, 31, 26); 87 uint32_t il = BIT(esr, 25); 88 uint32_t iss = BITS(esr, 24, 0); 89 90 /* fatal exception, die here */ 91 printf("ESR 0x%x: ec 0x%x, il 0x%x, iss 0x%x\n", esr, ec, il, iss); 92 dump_iframe(iframe); 93 crashlog.iframe = iframe; 94 95 platform_halt(HALT_ACTION_HALT, HALT_REASON_SW_PANIC); 96} 97 98static void arm64_unknown_handler(struct arm64_iframe_long* iframe, uint exception_flags, 99 uint32_t esr) { 100 /* this is for a lot of reasons, but most of them are undefined instructions */ 101 if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) { 102 /* trapped inside the kernel, this is bad */ 103 printf("unknown exception in kernel: PC at %#" PRIx64 "\n", iframe->elr); 104 exception_die(iframe, esr); 105 } 106 try_dispatch_user_exception(ZX_EXCP_UNDEFINED_INSTRUCTION, iframe, esr); 107} 108 109static void arm64_brk_handler(struct arm64_iframe_long* iframe, uint exception_flags, 110 uint32_t esr) { 111 if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) { 112 /* trapped inside the kernel, this is bad */ 113 printf("BRK in kernel: PC at %#" PRIx64 "\n", iframe->elr); 114 exception_die(iframe, esr); 115 } 116 try_dispatch_user_exception(ZX_EXCP_SW_BREAKPOINT, iframe, esr); 117} 118 119static void arm64_step_handler(struct arm64_iframe_long* iframe, uint exception_flags, 120 uint32_t esr) { 121 if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) { 122 /* trapped inside the kernel, this is bad */ 123 printf("software step in kernel: PC at %#" PRIx64 "\n", iframe->elr); 124 exception_die(iframe, esr); 125 } 126 try_dispatch_user_exception(ZX_EXCP_HW_BREAKPOINT, iframe, esr); 127} 128 129static void arm64_fpu_handler(struct arm64_iframe_long* iframe, uint exception_flags, 130 uint32_t esr) { 131 if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) { 132 /* we trapped a floating point instruction inside our own EL, this is bad */ 133 printf("invalid fpu use in kernel: PC at %#" PRIx64 "\n", 134 iframe->elr); 135 exception_die(iframe, esr); 136 } 137 arm64_fpu_exception(iframe, exception_flags); 138} 139 140static void arm64_instruction_abort_handler(struct arm64_iframe_long* iframe, uint exception_flags, 141 uint32_t esr) { 142 /* read the FAR register */ 143 uint64_t far = ARM64_READ_SYSREG(far_el1); 144 uint32_t ec = BITS_SHIFT(esr, 31, 26); 145 uint32_t iss = BITS(esr, 24, 0); 146 bool is_user = !BIT(ec, 0); 147 148 uint pf_flags = VMM_PF_FLAG_INSTRUCTION; 149 pf_flags |= is_user ? VMM_PF_FLAG_USER : 0; 150 /* Check if this was not permission fault */ 151 if ((iss & 0b111100) != 0b001100) { 152 pf_flags |= VMM_PF_FLAG_NOT_PRESENT; 153 } 154 155 LTRACEF("instruction abort: PC at %#" PRIx64 156 ", is_user %d, FAR %" PRIx64 ", esr 0x%x, iss 0x%x\n", 157 iframe->elr, is_user, far, esr, iss); 158 159 arch_enable_ints(); 160 kcounter_add(exceptions_page, 1); 161 CPU_STATS_INC(page_faults); 162 zx_status_t err = vmm_page_fault_handler(far, pf_flags); 163 arch_disable_ints(); 164 if (err >= 0) 165 return; 166 167 // If this is from user space, let the user exception handler 168 // get a shot at it. 169 if (is_user) { 170 kcounter_add(exceptions_user, 1); 171 if (try_dispatch_user_data_fault_exception(ZX_EXCP_FATAL_PAGE_FAULT, iframe, esr, far) == ZX_OK) 172 return; 173 } 174 175 printf("instruction abort: PC at %#" PRIx64 ", is_user %d, FAR %" PRIx64 "\n", 176 iframe->elr, is_user, far); 177 exception_die(iframe, esr); 178} 179 180static void arm64_data_abort_handler(struct arm64_iframe_long* iframe, uint exception_flags, 181 uint32_t esr) { 182 /* read the FAR register */ 183 uint64_t far = ARM64_READ_SYSREG(far_el1); 184 uint32_t ec = BITS_SHIFT(esr, 31, 26); 185 uint32_t iss = BITS(esr, 24, 0); 186 bool is_user = !BIT(ec, 0); 187 bool WnR = BIT(iss, 6); // Write not Read 188 bool CM = BIT(iss, 8); // cache maintenance op 189 190 uint pf_flags = 0; 191 // if it was marked Write but the cache maintenance bit was set, treat it as read 192 pf_flags |= (WnR && !CM) ? VMM_PF_FLAG_WRITE : 0; 193 pf_flags |= is_user ? VMM_PF_FLAG_USER : 0; 194 /* Check if this was not permission fault */ 195 if ((iss & 0b111100) != 0b001100) { 196 pf_flags |= VMM_PF_FLAG_NOT_PRESENT; 197 } 198 199 LTRACEF("data fault: PC at %#" PRIx64 200 ", is_user %d, FAR %#" PRIx64 ", esr 0x%x, iss 0x%x\n", 201 iframe->elr, is_user, far, esr, iss); 202 203 uint32_t dfsc = BITS(iss, 5, 0); 204 if (likely(dfsc != DFSC_ALIGNMENT_FAULT)) { 205 arch_enable_ints(); 206 kcounter_add(exceptions_page, 1); 207 zx_status_t err = vmm_page_fault_handler(far, pf_flags); 208 arch_disable_ints(); 209 if (err >= 0) { 210 return; 211 } 212 } 213 214 // Check if the current thread was expecting a data fault and 215 // we should return to its handler. 216 thread_t* thr = get_current_thread(); 217 if (thr->arch.data_fault_resume != NULL && is_user_address(far)) { 218 iframe->elr = (uintptr_t)thr->arch.data_fault_resume; 219 return; 220 } 221 222 // If this is from user space, let the user exception handler 223 // get a shot at it. 224 if (is_user) { 225 kcounter_add(exceptions_user, 1); 226 zx_excp_type_t excp_type = ZX_EXCP_FATAL_PAGE_FAULT; 227 if (unlikely(dfsc == DFSC_ALIGNMENT_FAULT)) { 228 excp_type = ZX_EXCP_UNALIGNED_ACCESS; 229 } 230 if (try_dispatch_user_data_fault_exception(excp_type, iframe, esr, far) == ZX_OK) 231 return; 232 } 233 234 /* decode the iss */ 235 if (BIT(iss, 24)) { /* ISV bit */ 236 printf("data fault: PC at %#" PRIx64 237 ", FAR %#" PRIx64 ", iss %#x (DFSC %#x)\n", 238 iframe->elr, far, iss, BITS(iss, 5, 0)); 239 } else { 240 printf("data fault: PC at %#" PRIx64 241 ", FAR %#" PRIx64 ", iss 0x%x\n", 242 iframe->elr, far, iss); 243 } 244 245 exception_die(iframe, esr); 246} 247 248static inline void arm64_restore_percpu_pointer() { 249 arm64_write_percpu_ptr(get_current_thread()->arch.current_percpu_ptr); 250} 251 252/* called from assembly */ 253extern "C" void arm64_sync_exception( 254 struct arm64_iframe_long* iframe, uint exception_flags, uint32_t esr) { 255 uint32_t ec = BITS_SHIFT(esr, 31, 26); 256 257 if (exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) { 258 // if we came from a lower level, restore the per cpu pointer 259 arm64_restore_percpu_pointer(); 260 } 261 262 switch (ec) { 263 case 0b000000: /* unknown reason */ 264 kcounter_add(exceptions_unknown, 1); 265 arm64_unknown_handler(iframe, exception_flags, esr); 266 break; 267 case 0b111000: /* BRK from arm32 */ 268 case 0b111100: /* BRK from arm64 */ 269 kcounter_add(exceptions_brkpt, 1); 270 arm64_brk_handler(iframe, exception_flags, esr); 271 break; 272 case 0b000111: /* floating point */ 273 kcounter_add(exceptions_fpu, 1); 274 arm64_fpu_handler(iframe, exception_flags, esr); 275 break; 276 case 0b010001: /* syscall from arm32 */ 277 case 0b010101: /* syscall from arm64 */ 278 printf("syscalls should be handled in assembly\n"); 279 exception_die(iframe, esr); 280 break; 281 case 0b100000: /* instruction abort from lower level */ 282 case 0b100001: /* instruction abort from same level */ 283 arm64_instruction_abort_handler(iframe, exception_flags, esr); 284 break; 285 case 0b100100: /* data abort from lower level */ 286 case 0b100101: /* data abort from same level */ 287 arm64_data_abort_handler(iframe, exception_flags, esr); 288 break; 289 case 0b110010: /* software step from lower level */ 290 case 0b110011: /* software step from same level */ 291 arm64_step_handler(iframe, exception_flags, esr); 292 break; 293 default: { 294 /* TODO: properly decode more of these */ 295 if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) { 296 /* trapped inside the kernel, this is bad */ 297 printf("unhandled exception in kernel: PC at %#" PRIx64 "\n", iframe->elr); 298 exception_die(iframe, esr); 299 } 300 /* let the user exception handler get a shot at it */ 301 kcounter_add(exceptions_unhandled, 1); 302 if (try_dispatch_user_exception(ZX_EXCP_GENERAL, iframe, esr) == ZX_OK) 303 break; 304 printf("unhandled synchronous exception\n"); 305 exception_die(iframe, esr); 306 } 307 } 308 309 /* if we came from user space, check to see if we have any signals to handle */ 310 if (unlikely(exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL)) { 311 /* in the case of receiving a kill signal, this function may not return, 312 * but the scheduler would have been invoked so it's fine. 313 */ 314 arm64_thread_process_pending_signals(iframe); 315 } 316 317 /* if we're returning to kernel space, make sure we restore the correct x18 */ 318 if ((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0) { 319 iframe->r[18] = (uint64_t)arm64_read_percpu_ptr(); 320 } 321} 322 323/* called from assembly */ 324extern "C" uint32_t arm64_irq(struct arm64_iframe_short* iframe, uint exception_flags) { 325 if (exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) { 326 // if we came from a lower level, restore the per cpu pointer 327 arm64_restore_percpu_pointer(); 328 } 329 330 LTRACEF("iframe %p, flags 0x%x\n", iframe, exception_flags); 331 332 int_handler_saved_state_t state; 333 int_handler_start(&state); 334 335 kcounter_add(exceptions_irq, 1); 336 platform_irq(iframe); 337 338 bool do_preempt = int_handler_finish(&state); 339 340 /* if we came from user space, check to see if we have any signals to handle */ 341 if (unlikely(exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL)) { 342 uint32_t exit_flags = 0; 343 if (thread_is_signaled(get_current_thread())) 344 exit_flags |= ARM64_IRQ_EXIT_THREAD_SIGNALED; 345 if (do_preempt) 346 exit_flags |= ARM64_IRQ_EXIT_RESCHEDULE; 347 return exit_flags; 348 } 349 350 /* preempt the thread if the interrupt has signaled it */ 351 if (do_preempt) 352 thread_preempt(); 353 354 /* if we're returning to kernel space, make sure we restore the correct x18 */ 355 if ((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0) { 356 iframe->r[18] = (uint64_t)arm64_read_percpu_ptr(); 357 } 358 359 return 0; 360} 361 362/* called from assembly */ 363extern "C" void arm64_finish_user_irq(uint32_t exit_flags, struct arm64_iframe_long* iframe) { 364 // we came from a lower level, so restore the per cpu pointer 365 arm64_restore_percpu_pointer(); 366 367 /* in the case of receiving a kill signal, this function may not return, 368 * but the scheduler would have been invoked so it's fine. 369 */ 370 if (unlikely(exit_flags & ARM64_IRQ_EXIT_THREAD_SIGNALED)) { 371 DEBUG_ASSERT(iframe != nullptr); 372 arm64_thread_process_pending_signals(iframe); 373 } 374 375 /* preempt the thread if the interrupt has signaled it */ 376 if (exit_flags & ARM64_IRQ_EXIT_RESCHEDULE) 377 thread_preempt(); 378} 379 380/* called from assembly */ 381extern "C" void arm64_invalid_exception(struct arm64_iframe_long* iframe, unsigned int which) { 382 // restore the percpu pointer (x18) unconditionally 383 arm64_restore_percpu_pointer(); 384 385 printf("invalid exception, which 0x%x\n", which); 386 dump_iframe(iframe); 387 388 platform_halt(HALT_ACTION_HALT, HALT_REASON_SW_PANIC); 389} 390 391/* called from assembly */ 392extern "C" void arm64_thread_process_pending_signals(struct arm64_iframe_long* iframe) { 393 thread_t* thread = get_current_thread(); 394 DEBUG_ASSERT(iframe != nullptr); 395 DEBUG_ASSERT(thread->arch.suspended_general_regs == nullptr); 396 397 thread->arch.suspended_general_regs = iframe; 398 thread_process_pending_signals(); 399 thread->arch.suspended_general_regs = nullptr; 400} 401 402void arch_dump_exception_context(const arch_exception_context_t* context) { 403 uint32_t ec = BITS_SHIFT(context->esr, 31, 26); 404 uint32_t iss = BITS(context->esr, 24, 0); 405 406 switch (ec) { 407 case 0b100000: /* instruction abort from lower level */ 408 case 0b100001: /* instruction abort from same level */ 409 printf("instruction abort: PC at %#" PRIx64 410 ", address %#" PRIx64 " IFSC %#x %s\n", 411 context->frame->elr, context->far, 412 BITS(context->esr, 5, 0), 413 BIT(ec, 0) ? "" : "user "); 414 415 break; 416 case 0b100100: /* data abort from lower level */ 417 case 0b100101: /* data abort from same level */ 418 printf("data abort: PC at %#" PRIx64 419 ", address %#" PRIx64 " %s%s\n", 420 context->frame->elr, context->far, 421 BIT(ec, 0) ? "" : "user ", 422 BIT(iss, 6) ? "write" : "read"); 423 } 424 425 dump_iframe(context->frame); 426 427 // try to dump the user stack 428 if (is_user_address(context->frame->usp)) { 429 uint8_t buf[256]; 430 if (arch_copy_from_user(buf, (void*)context->frame->usp, sizeof(buf)) == ZX_OK) { 431 printf("bottom of user stack at 0x%lx:\n", (vaddr_t)context->frame->usp); 432 hexdump_ex(buf, sizeof(buf), context->frame->usp); 433 } 434 } 435} 436 437void arch_fill_in_exception_context(const arch_exception_context_t* arch_context, zx_exception_report_t* report) { 438 zx_exception_context_t* zx_context = &report->context; 439 440 zx_context->arch.u.arm_64.esr = arch_context->esr; 441 442 // If there was a fatal page fault, fill in the address that caused the fault. 443 if (ZX_EXCP_FATAL_PAGE_FAULT == report->header.type) { 444 zx_context->arch.u.arm_64.far = arch_context->far; 445 } else { 446 zx_context->arch.u.arm_64.far = 0; 447 } 448} 449 450zx_status_t arch_dispatch_user_policy_exception(void) { 451 struct arm64_iframe_long frame = {}; 452 arch_exception_context_t context = {}; 453 context.frame = &frame; 454 return dispatch_user_exception(ZX_EXCP_POLICY_ERROR, &context); 455} 456