1// Copyright 2016 The Fuchsia Authors
2// Copyright (c) 2014 Travis Geiselbrecht
3//
4// Use of this source code is governed by a MIT-style
5// license that can be found in the LICENSE file or at
6// https://opensource.org/licenses/MIT
7
8#include <arch/arch_ops.h>
9#include <arch/arm64.h>
10#include <arch/arm64/exceptions.h>
11#include <arch/exception.h>
12#include <arch/user_copy.h>
13
14#include <bits.h>
15#include <debug.h>
16#include <inttypes.h>
17
18#include <kernel/interrupt.h>
19#include <kernel/thread.h>
20
21#include <platform.h>
22#include <stdio.h>
23#include <trace.h>
24#include <vm/fault.h>
25#include <vm/vm.h>
26
27#include <lib/counters.h>
28#include <lib/crashlog.h>
29
30#include <zircon/syscalls/exception.h>
31#include <zircon/types.h>
32
33#define LOCAL_TRACE 0
34
35#define DFSC_ALIGNMENT_FAULT 0b100001
36
37static void dump_iframe(const struct arm64_iframe_long* iframe) {
38    printf("iframe %p:\n", iframe);
39    printf("x0  %#18" PRIx64 " x1  %#18" PRIx64 " x2  %#18" PRIx64 " x3  %#18" PRIx64 "\n", iframe->r[0], iframe->r[1], iframe->r[2], iframe->r[3]);
40    printf("x4  %#18" PRIx64 " x5  %#18" PRIx64 " x6  %#18" PRIx64 " x7  %#18" PRIx64 "\n", iframe->r[4], iframe->r[5], iframe->r[6], iframe->r[7]);
41    printf("x8  %#18" PRIx64 " x9  %#18" PRIx64 " x10 %#18" PRIx64 " x11 %#18" PRIx64 "\n", iframe->r[8], iframe->r[9], iframe->r[10], iframe->r[11]);
42    printf("x12 %#18" PRIx64 " x13 %#18" PRIx64 " x14 %#18" PRIx64 " x15 %#18" PRIx64 "\n", iframe->r[12], iframe->r[13], iframe->r[14], iframe->r[15]);
43    printf("x16 %#18" PRIx64 " x17 %#18" PRIx64 " x18 %#18" PRIx64 " x19 %#18" PRIx64 "\n", iframe->r[16], iframe->r[17], iframe->r[18], iframe->r[19]);
44    printf("x20 %#18" PRIx64 " x21 %#18" PRIx64 " x22 %#18" PRIx64 " x23 %#18" PRIx64 "\n", iframe->r[20], iframe->r[21], iframe->r[22], iframe->r[23]);
45    printf("x24 %#18" PRIx64 " x25 %#18" PRIx64 " x26 %#18" PRIx64 " x27 %#18" PRIx64 "\n", iframe->r[24], iframe->r[25], iframe->r[26], iframe->r[27]);
46    printf("x28 %#18" PRIx64 " x29 %#18" PRIx64 " lr  %#18" PRIx64 " usp %#18" PRIx64 "\n", iframe->r[28], iframe->r[29], iframe->lr, iframe->usp);
47    printf("elr  %#18" PRIx64 "\n", iframe->elr);
48    printf("spsr %#18" PRIx64 "\n", iframe->spsr);
49}
50
51KCOUNTER(exceptions_brkpt, "kernel.exceptions.breakpoint");
52KCOUNTER(exceptions_fpu, "kernel.exceptions.fpu");
53KCOUNTER(exceptions_page, "kernel.exceptions.page_fault");
54KCOUNTER(exceptions_irq, "kernel.exceptions.irq");
55KCOUNTER(exceptions_unhandled, "kernel.exceptions.unhandled");
56KCOUNTER(exceptions_user, "kernel.exceptions.user");
57KCOUNTER(exceptions_unknown, "kernel.exceptions.unknown");
58
59static zx_status_t try_dispatch_user_data_fault_exception(
60    zx_excp_type_t type, struct arm64_iframe_long* iframe,
61    uint32_t esr, uint64_t far) {
62    thread_t* thread = get_current_thread();
63    arch_exception_context_t context = {};
64    DEBUG_ASSERT(iframe != nullptr);
65    context.frame = iframe;
66    context.esr = esr;
67    context.far = far;
68
69    arch_enable_ints();
70    DEBUG_ASSERT(thread->arch.suspended_general_regs == nullptr);
71    thread->arch.suspended_general_regs = iframe;
72    zx_status_t status = dispatch_user_exception(type, &context);
73    thread->arch.suspended_general_regs = nullptr;
74    arch_disable_ints();
75    return status;
76}
77
78static zx_status_t try_dispatch_user_exception(
79    zx_excp_type_t type, struct arm64_iframe_long* iframe, uint32_t esr) {
80    return try_dispatch_user_data_fault_exception(type, iframe, esr, 0);
81}
82
83__NO_RETURN static void exception_die(struct arm64_iframe_long* iframe, uint32_t esr) {
84    platform_panic_start();
85
86    uint32_t ec = BITS_SHIFT(esr, 31, 26);
87    uint32_t il = BIT(esr, 25);
88    uint32_t iss = BITS(esr, 24, 0);
89
90    /* fatal exception, die here */
91    printf("ESR 0x%x: ec 0x%x, il 0x%x, iss 0x%x\n", esr, ec, il, iss);
92    dump_iframe(iframe);
93    crashlog.iframe = iframe;
94
95    platform_halt(HALT_ACTION_HALT, HALT_REASON_SW_PANIC);
96}
97
98static void arm64_unknown_handler(struct arm64_iframe_long* iframe, uint exception_flags,
99                                  uint32_t esr) {
100    /* this is for a lot of reasons, but most of them are undefined instructions */
101    if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) {
102        /* trapped inside the kernel, this is bad */
103        printf("unknown exception in kernel: PC at %#" PRIx64 "\n", iframe->elr);
104        exception_die(iframe, esr);
105    }
106    try_dispatch_user_exception(ZX_EXCP_UNDEFINED_INSTRUCTION, iframe, esr);
107}
108
109static void arm64_brk_handler(struct arm64_iframe_long* iframe, uint exception_flags,
110                              uint32_t esr) {
111    if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) {
112        /* trapped inside the kernel, this is bad */
113        printf("BRK in kernel: PC at %#" PRIx64 "\n", iframe->elr);
114        exception_die(iframe, esr);
115    }
116    try_dispatch_user_exception(ZX_EXCP_SW_BREAKPOINT, iframe, esr);
117}
118
119static void arm64_step_handler(struct arm64_iframe_long* iframe, uint exception_flags,
120                               uint32_t esr) {
121    if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) {
122        /* trapped inside the kernel, this is bad */
123        printf("software step in kernel: PC at %#" PRIx64 "\n", iframe->elr);
124        exception_die(iframe, esr);
125    }
126    try_dispatch_user_exception(ZX_EXCP_HW_BREAKPOINT, iframe, esr);
127}
128
129static void arm64_fpu_handler(struct arm64_iframe_long* iframe, uint exception_flags,
130                              uint32_t esr) {
131    if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) {
132        /* we trapped a floating point instruction inside our own EL, this is bad */
133        printf("invalid fpu use in kernel: PC at %#" PRIx64 "\n",
134               iframe->elr);
135        exception_die(iframe, esr);
136    }
137    arm64_fpu_exception(iframe, exception_flags);
138}
139
140static void arm64_instruction_abort_handler(struct arm64_iframe_long* iframe, uint exception_flags,
141                                            uint32_t esr) {
142    /* read the FAR register */
143    uint64_t far = ARM64_READ_SYSREG(far_el1);
144    uint32_t ec = BITS_SHIFT(esr, 31, 26);
145    uint32_t iss = BITS(esr, 24, 0);
146    bool is_user = !BIT(ec, 0);
147
148    uint pf_flags = VMM_PF_FLAG_INSTRUCTION;
149    pf_flags |= is_user ? VMM_PF_FLAG_USER : 0;
150    /* Check if this was not permission fault */
151    if ((iss & 0b111100) != 0b001100) {
152        pf_flags |= VMM_PF_FLAG_NOT_PRESENT;
153    }
154
155    LTRACEF("instruction abort: PC at %#" PRIx64
156            ", is_user %d, FAR %" PRIx64 ", esr 0x%x, iss 0x%x\n",
157            iframe->elr, is_user, far, esr, iss);
158
159    arch_enable_ints();
160    kcounter_add(exceptions_page, 1);
161    CPU_STATS_INC(page_faults);
162    zx_status_t err = vmm_page_fault_handler(far, pf_flags);
163    arch_disable_ints();
164    if (err >= 0)
165        return;
166
167    // If this is from user space, let the user exception handler
168    // get a shot at it.
169    if (is_user) {
170        kcounter_add(exceptions_user, 1);
171        if (try_dispatch_user_data_fault_exception(ZX_EXCP_FATAL_PAGE_FAULT, iframe, esr, far) == ZX_OK)
172            return;
173    }
174
175    printf("instruction abort: PC at %#" PRIx64 ", is_user %d, FAR %" PRIx64 "\n",
176           iframe->elr, is_user, far);
177    exception_die(iframe, esr);
178}
179
180static void arm64_data_abort_handler(struct arm64_iframe_long* iframe, uint exception_flags,
181                                     uint32_t esr) {
182    /* read the FAR register */
183    uint64_t far = ARM64_READ_SYSREG(far_el1);
184    uint32_t ec = BITS_SHIFT(esr, 31, 26);
185    uint32_t iss = BITS(esr, 24, 0);
186    bool is_user = !BIT(ec, 0);
187    bool WnR = BIT(iss, 6); // Write not Read
188    bool CM = BIT(iss, 8);  // cache maintenance op
189
190    uint pf_flags = 0;
191    // if it was marked Write but the cache maintenance bit was set, treat it as read
192    pf_flags |= (WnR && !CM) ? VMM_PF_FLAG_WRITE : 0;
193    pf_flags |= is_user ? VMM_PF_FLAG_USER : 0;
194    /* Check if this was not permission fault */
195    if ((iss & 0b111100) != 0b001100) {
196        pf_flags |= VMM_PF_FLAG_NOT_PRESENT;
197    }
198
199    LTRACEF("data fault: PC at %#" PRIx64
200            ", is_user %d, FAR %#" PRIx64 ", esr 0x%x, iss 0x%x\n",
201            iframe->elr, is_user, far, esr, iss);
202
203    uint32_t dfsc = BITS(iss, 5, 0);
204    if (likely(dfsc != DFSC_ALIGNMENT_FAULT)) {
205        arch_enable_ints();
206        kcounter_add(exceptions_page, 1);
207        zx_status_t err = vmm_page_fault_handler(far, pf_flags);
208        arch_disable_ints();
209        if (err >= 0) {
210            return;
211        }
212    }
213
214    // Check if the current thread was expecting a data fault and
215    // we should return to its handler.
216    thread_t* thr = get_current_thread();
217    if (thr->arch.data_fault_resume != NULL && is_user_address(far)) {
218        iframe->elr = (uintptr_t)thr->arch.data_fault_resume;
219        return;
220    }
221
222    // If this is from user space, let the user exception handler
223    // get a shot at it.
224    if (is_user) {
225        kcounter_add(exceptions_user, 1);
226        zx_excp_type_t excp_type = ZX_EXCP_FATAL_PAGE_FAULT;
227        if (unlikely(dfsc == DFSC_ALIGNMENT_FAULT)) {
228            excp_type = ZX_EXCP_UNALIGNED_ACCESS;
229        }
230        if (try_dispatch_user_data_fault_exception(excp_type, iframe, esr, far) == ZX_OK)
231            return;
232    }
233
234    /* decode the iss */
235    if (BIT(iss, 24)) { /* ISV bit */
236        printf("data fault: PC at %#" PRIx64
237               ", FAR %#" PRIx64 ", iss %#x (DFSC %#x)\n",
238               iframe->elr, far, iss, BITS(iss, 5, 0));
239    } else {
240        printf("data fault: PC at %#" PRIx64
241               ", FAR %#" PRIx64 ", iss 0x%x\n",
242               iframe->elr, far, iss);
243    }
244
245    exception_die(iframe, esr);
246}
247
248static inline void arm64_restore_percpu_pointer() {
249    arm64_write_percpu_ptr(get_current_thread()->arch.current_percpu_ptr);
250}
251
252/* called from assembly */
253extern "C" void arm64_sync_exception(
254    struct arm64_iframe_long* iframe, uint exception_flags, uint32_t esr) {
255    uint32_t ec = BITS_SHIFT(esr, 31, 26);
256
257    if (exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) {
258        // if we came from a lower level, restore the per cpu pointer
259        arm64_restore_percpu_pointer();
260    }
261
262    switch (ec) {
263    case 0b000000: /* unknown reason */
264        kcounter_add(exceptions_unknown, 1);
265        arm64_unknown_handler(iframe, exception_flags, esr);
266        break;
267    case 0b111000: /* BRK from arm32 */
268    case 0b111100: /* BRK from arm64 */
269        kcounter_add(exceptions_brkpt, 1);
270        arm64_brk_handler(iframe, exception_flags, esr);
271        break;
272    case 0b000111: /* floating point */
273        kcounter_add(exceptions_fpu, 1);
274        arm64_fpu_handler(iframe, exception_flags, esr);
275        break;
276    case 0b010001: /* syscall from arm32 */
277    case 0b010101: /* syscall from arm64 */
278        printf("syscalls should be handled in assembly\n");
279        exception_die(iframe, esr);
280        break;
281    case 0b100000: /* instruction abort from lower level */
282    case 0b100001: /* instruction abort from same level */
283        arm64_instruction_abort_handler(iframe, exception_flags, esr);
284        break;
285    case 0b100100: /* data abort from lower level */
286    case 0b100101: /* data abort from same level */
287        arm64_data_abort_handler(iframe, exception_flags, esr);
288        break;
289    case 0b110010: /* software step from lower level */
290    case 0b110011: /* software step from same level */
291        arm64_step_handler(iframe, exception_flags, esr);
292        break;
293    default: {
294        /* TODO: properly decode more of these */
295        if (unlikely((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0)) {
296            /* trapped inside the kernel, this is bad */
297            printf("unhandled exception in kernel: PC at %#" PRIx64 "\n", iframe->elr);
298            exception_die(iframe, esr);
299        }
300        /* let the user exception handler get a shot at it */
301        kcounter_add(exceptions_unhandled, 1);
302        if (try_dispatch_user_exception(ZX_EXCP_GENERAL, iframe, esr) == ZX_OK)
303            break;
304        printf("unhandled synchronous exception\n");
305        exception_die(iframe, esr);
306    }
307    }
308
309    /* if we came from user space, check to see if we have any signals to handle */
310    if (unlikely(exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL)) {
311        /* in the case of receiving a kill signal, this function may not return,
312         * but the scheduler would have been invoked so it's fine.
313         */
314        arm64_thread_process_pending_signals(iframe);
315    }
316
317    /* if we're returning to kernel space, make sure we restore the correct x18 */
318    if ((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0) {
319        iframe->r[18] = (uint64_t)arm64_read_percpu_ptr();
320    }
321}
322
323/* called from assembly */
324extern "C" uint32_t arm64_irq(struct arm64_iframe_short* iframe, uint exception_flags) {
325    if (exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) {
326        // if we came from a lower level, restore the per cpu pointer
327        arm64_restore_percpu_pointer();
328    }
329
330    LTRACEF("iframe %p, flags 0x%x\n", iframe, exception_flags);
331
332    int_handler_saved_state_t state;
333    int_handler_start(&state);
334
335    kcounter_add(exceptions_irq, 1);
336    platform_irq(iframe);
337
338    bool do_preempt = int_handler_finish(&state);
339
340    /* if we came from user space, check to see if we have any signals to handle */
341    if (unlikely(exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL)) {
342        uint32_t exit_flags = 0;
343        if (thread_is_signaled(get_current_thread()))
344            exit_flags |= ARM64_IRQ_EXIT_THREAD_SIGNALED;
345        if (do_preempt)
346            exit_flags |= ARM64_IRQ_EXIT_RESCHEDULE;
347        return exit_flags;
348    }
349
350    /* preempt the thread if the interrupt has signaled it */
351    if (do_preempt)
352        thread_preempt();
353
354    /* if we're returning to kernel space, make sure we restore the correct x18 */
355    if ((exception_flags & ARM64_EXCEPTION_FLAG_LOWER_EL) == 0) {
356        iframe->r[18] = (uint64_t)arm64_read_percpu_ptr();
357    }
358
359    return 0;
360}
361
362/* called from assembly */
363extern "C" void arm64_finish_user_irq(uint32_t exit_flags, struct arm64_iframe_long* iframe) {
364    // we came from a lower level, so restore the per cpu pointer
365    arm64_restore_percpu_pointer();
366
367    /* in the case of receiving a kill signal, this function may not return,
368     * but the scheduler would have been invoked so it's fine.
369     */
370    if (unlikely(exit_flags & ARM64_IRQ_EXIT_THREAD_SIGNALED)) {
371        DEBUG_ASSERT(iframe != nullptr);
372        arm64_thread_process_pending_signals(iframe);
373    }
374
375    /* preempt the thread if the interrupt has signaled it */
376    if (exit_flags & ARM64_IRQ_EXIT_RESCHEDULE)
377        thread_preempt();
378}
379
380/* called from assembly */
381extern "C" void arm64_invalid_exception(struct arm64_iframe_long* iframe, unsigned int which) {
382    // restore the percpu pointer (x18) unconditionally
383    arm64_restore_percpu_pointer();
384
385    printf("invalid exception, which 0x%x\n", which);
386    dump_iframe(iframe);
387
388    platform_halt(HALT_ACTION_HALT, HALT_REASON_SW_PANIC);
389}
390
391/* called from assembly */
392extern "C" void arm64_thread_process_pending_signals(struct arm64_iframe_long* iframe) {
393    thread_t* thread = get_current_thread();
394    DEBUG_ASSERT(iframe != nullptr);
395    DEBUG_ASSERT(thread->arch.suspended_general_regs == nullptr);
396
397    thread->arch.suspended_general_regs = iframe;
398    thread_process_pending_signals();
399    thread->arch.suspended_general_regs = nullptr;
400}
401
402void arch_dump_exception_context(const arch_exception_context_t* context) {
403    uint32_t ec = BITS_SHIFT(context->esr, 31, 26);
404    uint32_t iss = BITS(context->esr, 24, 0);
405
406    switch (ec) {
407    case 0b100000: /* instruction abort from lower level */
408    case 0b100001: /* instruction abort from same level */
409        printf("instruction abort: PC at %#" PRIx64
410               ", address %#" PRIx64 " IFSC %#x %s\n",
411               context->frame->elr, context->far,
412               BITS(context->esr, 5, 0),
413               BIT(ec, 0) ? "" : "user ");
414
415        break;
416    case 0b100100: /* data abort from lower level */
417    case 0b100101: /* data abort from same level */
418        printf("data abort: PC at %#" PRIx64
419               ", address %#" PRIx64 " %s%s\n",
420               context->frame->elr, context->far,
421               BIT(ec, 0) ? "" : "user ",
422               BIT(iss, 6) ? "write" : "read");
423    }
424
425    dump_iframe(context->frame);
426
427    // try to dump the user stack
428    if (is_user_address(context->frame->usp)) {
429        uint8_t buf[256];
430        if (arch_copy_from_user(buf, (void*)context->frame->usp, sizeof(buf)) == ZX_OK) {
431            printf("bottom of user stack at 0x%lx:\n", (vaddr_t)context->frame->usp);
432            hexdump_ex(buf, sizeof(buf), context->frame->usp);
433        }
434    }
435}
436
437void arch_fill_in_exception_context(const arch_exception_context_t* arch_context, zx_exception_report_t* report) {
438    zx_exception_context_t* zx_context = &report->context;
439
440    zx_context->arch.u.arm_64.esr = arch_context->esr;
441
442    // If there was a fatal page fault, fill in the address that caused the fault.
443    if (ZX_EXCP_FATAL_PAGE_FAULT == report->header.type) {
444        zx_context->arch.u.arm_64.far = arch_context->far;
445    } else {
446        zx_context->arch.u.arm_64.far = 0;
447    }
448}
449
450zx_status_t arch_dispatch_user_policy_exception(void) {
451    struct arm64_iframe_long frame = {};
452    arch_exception_context_t context = {};
453    context.frame = &frame;
454    return dispatch_user_exception(ZX_EXCP_POLICY_ERROR, &context);
455}
456