// Copyright 2016 The Fuchsia Authors // Copyright (c) 2014 Travis Geiselbrecht // // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file or at // https://opensource.org/licenses/MIT #include #include #include #include #include #include .section .text.boot.vectab,"ax",@progbits .align 12 #define DW_REG_lr 30 #define DW_REG_sp 31 // The "current mode exception link register", which for our purposes is elr_el1. #define DW_REG_ELR_mode 33 #define lr x30 #define elr1 DW_REG_ELR_mode // offset where sp,elr,spsr,lr,mdscr goes in the iframe #define regsave_special_reg_offset (30 * 8) // offset where x20-x29 goes in the iframe #define regsave_high_reg_offset (20 * 8) .macro regsave_long // There are 5 regs @ regsave_special_reg_offset, plus one unused space = 6 to // maintain 16-byte padding. sub_from_sp (6*8) push_regs x28, x29 push_regs x26, x27 push_regs x24, x25 push_regs x22, x23 push_regs x20, x21 push_regs x18, x19 push_regs x16, x17 push_regs x14, x15 push_regs x12, x13 push_regs x10, x11 push_regs x8, x9 push_regs x6, x7 push_regs x4, x5 push_regs x2, x3 push_regs x0, x1 // Preserve x0-x7 for syscall arguments mrs x9, sp_el0 // x10 (containing elr_el1) is used in syscall handler mrs x10, elr_el1 mrs x11, spsr_el1 mrs x12, mdscr_el1 stp lr, x9, [sp, #regsave_special_reg_offset] .cfi_rel_offset lr, (regsave_special_reg_offset) .cfi_rel_offset sp, (regsave_special_reg_offset + 8) stp x10, x11, [sp, #regsave_special_reg_offset + 16] .cfi_rel_offset elr1, (regsave_special_reg_offset + 16) str x12, [sp, #regsave_special_reg_offset + 32] .endm .macro regsave_short // Align the same as the long version above. The 10 extra words correspond to // x20 through x29 which are not saved here. sub_from_sp ((6*8) + (10*8)) push_regs x18, x19 push_regs x16, x17 push_regs x14, x15 push_regs x12, x13 push_regs x10, x11 push_regs x8, x9 push_regs x6, x7 push_regs x4, x5 push_regs x2, x3 push_regs x0, x1 // Preserve x0-x7 to mirror regsave_long mrs x9, sp_el0 mrs x10, elr_el1 mrs x11, spsr_el1 stp lr, x9, [sp, #regsave_special_reg_offset] .cfi_rel_offset lr, (regsave_special_reg_offset) .cfi_rel_offset sp, (regsave_special_reg_offset + 8) stp x10, x11, [sp, #regsave_special_reg_offset + 16] .cfi_rel_offset elr1, (regsave_special_reg_offset + 16) .endm // convert a short iframe to a long one by patching in the additional 10 words to save .macro regsave_short_to_long stp x20, x21, [sp, #regsave_high_reg_offset] stp x22, x23, [sp, #regsave_high_reg_offset + 0x10] stp x24, x25, [sp, #regsave_high_reg_offset + 0x20] stp x26, x27, [sp, #regsave_high_reg_offset + 0x30] stp x28, x29, [sp, #regsave_high_reg_offset + 0x40] mrs x12, mdscr_el1 str x12, [sp, #regsave_special_reg_offset + 32] .endm // Once we pop the stack past the saved sp_el0, elr_el1 the userspace values // are inaccessible. .macro mark_lr_sp_inaccessible // TODO(dje): gdb tries to use some value for these even if "undefined", // as a workaround set their values to zero which will cause gdb to // terminate the backtrace. Need to revisit, file gdb bug if necessary. cfi_register_is_zero DW_REG_sp cfi_register_is_zero DW_REG_ELR_mode .endm .macro regrestore_long // Preserve x0-x1 for syscall returns (eventually x0-x7) ldp lr, x9, [sp, #regsave_special_reg_offset] .cfi_same_value lr ldp x10, x11, [sp, #regsave_special_reg_offset + 16] ldr x12, [sp, #regsave_special_reg_offset + 32] msr sp_el0, x9 msr elr_el1, x10 msr spsr_el1, x11 msr mdscr_el1, x12 pop_regs x0, x1 pop_regs x2, x3 pop_regs x4, x5 pop_regs x6, x7 pop_regs x8, x9 pop_regs x10, x11 pop_regs x12, x13 pop_regs x14, x15 pop_regs x16, x17 pop_regs x18, x19 pop_regs x20, x21 pop_regs x22, x23 pop_regs x24, x25 pop_regs x26, x27 pop_regs x28, x29 add_to_sp (6*8) mark_lr_sp_inaccessible .endm .macro regrestore_short // Preserve x0-x7 to mirror regrestore_long ldp lr, x9, [sp, #regsave_special_reg_offset] .cfi_same_value lr ldp x10, x11, [sp, #regsave_special_reg_offset + 16] msr sp_el0, x9 msr elr_el1, x10 msr spsr_el1, x11 pop_regs x0, x1 pop_regs x2, x3 pop_regs x4, x5 pop_regs x6, x7 pop_regs x8, x9 pop_regs x10, x11 pop_regs x12, x13 pop_regs x14, x15 pop_regs x16, x17 pop_regs x18, x19 add_to_sp ((6*8) + (10*8)) mark_lr_sp_inaccessible .endm .macro start_isr_cfi .cfi_startproc simple .cfi_signal_frame // The return address is in elr_el1, not lr. .cfi_return_column elr1 .cfi_def_cfa sp, 0 .endm .macro start_isr_func start_isr_cfi ALL_CFI_SAME_VALUE .cfi_undefined elr1 .endm .macro start_helper_cfi .cfi_startproc simple .cfi_signal_frame .cfi_def_cfa sp, (regsave_special_reg_offset + 4 * 8) .endm // The CFA offset of integer register |regno| (regno = 0-29). #define REG_CFA_OFFSET(regno) .cfi_offset x##regno, -((4 * 8) + ((30 - (regno)) * 8)) // Mark the locations of the registers based on the CFA so that the // location doesn't change as the regs are popped. .macro setup_short_helper_cfi REG_CFA_OFFSET(0) REG_CFA_OFFSET(1) REG_CFA_OFFSET(2) REG_CFA_OFFSET(3) REG_CFA_OFFSET(4) REG_CFA_OFFSET(5) REG_CFA_OFFSET(6) REG_CFA_OFFSET(7) REG_CFA_OFFSET(8) REG_CFA_OFFSET(9) REG_CFA_OFFSET(10) REG_CFA_OFFSET(11) REG_CFA_OFFSET(12) REG_CFA_OFFSET(13) REG_CFA_OFFSET(14) REG_CFA_OFFSET(15) REG_CFA_OFFSET(16) REG_CFA_OFFSET(17) REG_CFA_OFFSET(18) REG_CFA_OFFSET(19) .cfi_offset sp, -(3 * 8) .cfi_offset lr, -(4 * 8) .endm .macro setup_long_helper_cfi setup_short_helper_cfi REG_CFA_OFFSET(20) REG_CFA_OFFSET(21) REG_CFA_OFFSET(22) REG_CFA_OFFSET(23) REG_CFA_OFFSET(24) REG_CFA_OFFSET(25) REG_CFA_OFFSET(26) REG_CFA_OFFSET(27) REG_CFA_OFFSET(28) REG_CFA_OFFSET(29) .endm .macro start_short_helper start_helper_cfi setup_short_helper_cfi .endm .macro start_long_helper start_helper_cfi setup_long_helper_cfi .endm .macro invalid_exception, which start_isr_func regsave_long mov x1, #\which mov x0, sp bl arm64_invalid_exception b . .endm .macro irq_exception, exception_flags start_isr_func regsave_short msr daifclr, #1 /* reenable fiqs once elr and spsr have been saved */ mov x0, sp mov x1, \exception_flags bl arm64_irq cbnz x0, arm64_finish_user_irq_wrapper /* anything extra to do? */ msr daifset, #1 /* disable fiqs to protect elr and spsr restore */ b arm64_exc_shared_restore_short .endm .macro sync_exception, exception_flags, from_lower_el_64=0 start_isr_func regsave_long mrs x9, esr_el1 .if \from_lower_el_64 // If this is a syscall, x0-x7 contain args and x16 contains syscall num. // x10 contains elr_el1. lsr x11, x9, #26 // shift esr right 26 bits to get ec cmp x11, #0x15 // check for 64-bit syscall beq arm64_syscall_dispatcher // and jump to syscall handler .endif // Prepare the default sync_exception args mov x0, sp mov x1, \exception_flags mov w2, w9 bl arm64_sync_exception b arm64_exc_shared_restore_long .endm FUNCTION_LABEL(arm64_el1_exception_base) /* exceptions from current EL, using SP0 */ .org 0x000 LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_current_el_SP0) invalid_exception 0 END_FUNCTION(arm64_el1_sync_exc_current_el_SP0) .org 0x080 LOCAL_FUNCTION_LABEL(arm64_el1_irq_current_el_SP0) invalid_exception 1 END_FUNCTION(arm64_el1_irq_current_el_SP0) .org 0x100 LOCAL_FUNCTION_LABEL(arm64_el1_fiq_current_el_SP0) invalid_exception 2 END_FUNCTION(arm64_el1_fiq_current_el_SP0) .org 0x180 LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_current_el_SP0) invalid_exception 3 END_FUNCTION(arm64_el1_err_exc_current_el_SP0) /* exceptions from current EL, using SPx */ .org 0x200 LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_current_el_SPx) sync_exception #0 /* same EL, arm64 */ END_FUNCTION(arm64_el1_sync_exc_current_el_SPx) .org 0x280 LOCAL_FUNCTION_LABEL(arm64_el1_irq_current_el_SPx) irq_exception #0 /* same EL, arm64 */ END_FUNCTION(arm64_el1_irq_current_el_SPx) .org 0x300 LOCAL_FUNCTION_LABEL(arm64_el1_fiq_current_el_SPx) start_isr_func regsave_short mov x0, sp bl platform_fiq b arm64_exc_shared_restore_short END_FUNCTION(arm64_el1_fiq_current_el_SPx) .org 0x380 LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_current_el_SPx) invalid_exception 0x13 END_FUNCTION(arm64_el1_err_exc_current_el_SPx) /* exceptions from lower EL, running arm64 */ .org 0x400 LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_lower_el_64) sync_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL), 1 END_FUNCTION(arm64_el1_sync_exc_lower_el_64) .org 0x480 LOCAL_FUNCTION_LABEL(arm64_el1_irq_lower_el_64) irq_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL) END_FUNCTION(arm64_el1_irq_lower_el_64) .org 0x500 LOCAL_FUNCTION_LABEL(arm64_el1_fiq_lower_el_64) start_isr_func regsave_short mov x0, sp bl platform_fiq b arm64_exc_shared_restore_short END_FUNCTION(arm64_el1_fiq_lower_el_64) .org 0x580 LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_lower_el_64) invalid_exception 0x23 END_FUNCTION(arm64_el1_err_exc_lower_el_64) /* exceptions from lower EL, running arm32 */ .org 0x600 LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_lower_el_32) sync_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL|ARM64_EXCEPTION_FLAG_ARM32) END_FUNCTION(arm64_el1_sync_exc_lower_el_32) .org 0x680 LOCAL_FUNCTION_LABEL(arm64_el1_irq_lower_el_32) irq_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL|ARM64_EXCEPTION_FLAG_ARM32) END_FUNCTION(arm64_el1_irq_lower_el_32) .org 0x700 LOCAL_FUNCTION_LABEL(arm64_el1_fiq_lower_el_32) start_isr_func regsave_short mov x0, sp bl platform_fiq b arm64_exc_shared_restore_short END_FUNCTION(arm64_el1_fiq_lower_el_32) .org 0x780 LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_lower_el_32) invalid_exception 0x33 END_FUNCTION(arm64_el1_err_exc_lower_el_32) /* If an IRQ happened in userspace, and either the thread was signaled or needs to be rescheduled, then we end up here after arm64_irq returns. Suspending the thread requires constructing a long iframe in order to provide the values of all regs to any debugger that wishes to access them, but we can't do that until arm64_irq returns as we rely on the compiler to save/restore callee-saved regs. */ LOCAL_FUNCTION_LABEL(arm64_finish_user_irq_wrapper) start_short_helper /* if we're only here because of a need to reschedule then we don't need to construct a long iframe */ cmp x0, #ARM64_IRQ_EXIT_RESCHEDULE bne 1f mov x1, #0 /* don't need an iframe, just pass NULL */ bl arm64_finish_user_irq msr daifset, #1 /* disable fiqs to protect elr and spsr restore */ b arm64_exc_shared_restore_short 1: /* convert the short to a long frame */ regsave_short_to_long mov x1, sp bl arm64_finish_user_irq msr daifset, #1 /* disable fiqs to protect elr and spsr restore */ b arm64_exc_shared_restore_long END_FUNCTION(arm64_finish_user_irq_wrapper) LOCAL_FUNCTION_LABEL(arm64_exc_shared_restore_long) start_long_helper regrestore_long eret END_FUNCTION(arm64_exc_shared_restore_long) LOCAL_FUNCTION_LABEL(arm64_exc_shared_restore_short) start_short_helper regrestore_short eret END_FUNCTION(arm64_exc_shared_restore_short) // // Syscall args are in x0-x7 already. // pc is in x10 and needs to go in the next available register, // or the stack if the regs are full. // .macro pre_args, nargs .if \nargs == 8 push_regs x10, x11 // push twice to maintain alignment .else mov x\nargs, x10 .endif .endm .macro post_args, nargs .if \nargs == 8 pop_regs x10, x11 .endif b .Lpost_syscall .endm // // Expected state prior to arm64_syscall_dispatcher branch... // // x0-x7 - contains syscall arguments // x9 - contains esr_el1 // x10 - contains elr_el1 // x16 - contains syscall_num // sp - points to base of frame (frame->r[0]) // // Expected state prior to unknown_syscall and wrapper_syscall... // // x0-x7 - contains syscall arguments // x10 - contains userspace pc // LOCAL_FUNCTION_LABEL(arm64_syscall_dispatcher) start_isr_func // Restore per cpu pointer mrs x11, tpidr_el1 ldr x18, [x11, #CURRENT_PERCPU_PTR_OFFSET] // Verify syscall number and call the unknown handler if bad. cmp x16, #ZX_SYS_COUNT bhs .Lunknown_syscall // Jump to the right syscall wrapper. // call_wrapper_table is 4096 byte aligned so just use adrp adrp x12, call_wrapper_table ldr x12, [x12, x16, LSL#3] br x12 .Lunknown_syscall: mov x0, x16 pre_args 1 bl unknown_syscall post_args 1 .Lpost_syscall: // Upon return from syscall, x0 = status, x1 = thread signalled // Move the status to frame->r[0] for return to userspace. str x0, [sp] // Check for pending signals. If none, just return. cbz x1, arm64_exc_shared_restore_long mov x0, sp bl arm64_thread_process_pending_signals b arm64_exc_shared_restore_long END_FUNCTION(arm64_syscall_dispatcher) // Adds a label for making the syscall and adds it to the jump table. .macro syscall_dispatch nargs, syscall .pushsection .text.syscall-dispatch,"ax",%progbits LOCAL_FUNCTION(.Lcall_\syscall\()) pre_args \nargs bl wrapper_\syscall post_args \nargs END_FUNCTION(.Lcall_\syscall\()) .popsection .pushsection .rodata.syscall-table,"a",%progbits .quad .Lcall_\syscall .popsection .endm // Adds the label for the jump table. .macro start_syscall_dispatch .pushsection .rodata.syscall-table,"a",%progbits // align on 4096 byte boundary to save an instruction on table lookup .balign 4096 call_wrapper_table: .popsection .endm #include