1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8#include <arch/kernel/traps.h> 9#include <arch/object/vcpu.h> 10#include <arch/machine/registerset.h> 11#include <api/syscall.h> 12#include <machine/fpu.h> 13 14#include <sel4/benchmark_track_types.h> 15#include <benchmark/benchmark_track.h> 16#include <benchmark/benchmark_utilisation.h> 17#include <arch/machine.h> 18 19void VISIBLE NORETURN c_handle_undefined_instruction(void) 20{ 21 NODE_LOCK_SYS; 22 c_entry_hook(); 23 24#ifdef TRACK_KERNEL_ENTRIES 25 ksKernelEntry.path = Entry_UserLevelFault; 26 ksKernelEntry.word = getRegister(NODE_STATE(ksCurThread), NextIP); 27#endif 28 29#if defined(CONFIG_HAVE_FPU) && defined(CONFIG_ARCH_AARCH32) 30 /* We assume the first fault is a FP exception and enable FPU, if not already enabled */ 31 if (!isFpuEnable()) { 32 handleFPUFault(); 33 34 /* Restart the FP instruction that cause the fault */ 35 setNextPC(NODE_STATE(ksCurThread), getRestartPC(NODE_STATE(ksCurThread))); 36 } else { 37 handleUserLevelFault(0, 0); 38 } 39 40 restore_user_context(); 41 UNREACHABLE(); 42#endif 43 44 /* There's only one user-level fault on ARM, and the code is (0,0) */ 45#ifdef CONFIG_ARCH_AARCH32 46 handleUserLevelFault(0, 0); 47#else 48 handleUserLevelFault(getESR(), 0); 49#endif 50 restore_user_context(); 51 UNREACHABLE(); 52} 53 54#if defined(CONFIG_HAVE_FPU) && defined(CONFIG_ARCH_AARCH64) 55void VISIBLE NORETURN c_handle_enfp(void) 56{ 57 c_entry_hook(); 58 59 handleFPUFault(); 60 restore_user_context(); 61 UNREACHABLE(); 62} 63#endif /* CONFIG_HAVE_FPU */ 64 65static inline void NORETURN c_handle_vm_fault(vm_fault_type_t type) 66{ 67 NODE_LOCK_SYS; 68 c_entry_hook(); 69 70#ifdef TRACK_KERNEL_ENTRIES 71 ksKernelEntry.path = Entry_VMFault; 72 ksKernelEntry.word = getRegister(NODE_STATE(ksCurThread), NextIP); 73#endif 74 75 handleVMFaultEvent(type); 76 restore_user_context(); 77 UNREACHABLE(); 78} 79 80void VISIBLE NORETURN c_handle_data_fault(void) 81{ 82 c_handle_vm_fault(seL4_DataFault); 83} 84 85void VISIBLE NORETURN c_handle_instruction_fault(void) 86{ 87 c_handle_vm_fault(seL4_InstructionFault); 88} 89 90void VISIBLE NORETURN c_handle_interrupt(void) 91{ 92 NODE_LOCK_IRQ_IF(IRQT_TO_IRQ(getActiveIRQ()) != irq_remote_call_ipi); 93 c_entry_hook(); 94 95#ifdef TRACK_KERNEL_ENTRIES 96 ksKernelEntry.path = Entry_Interrupt; 97 ksKernelEntry.word = IRQT_TO_IRQ(getActiveIRQ()); 98#ifdef ENABLE_SMP_SUPPORT 99 ksKernelEntry.core = getCurrentCPUIndex(); 100#endif 101#endif 102 103 handleInterruptEntry(); 104 restore_user_context(); 105} 106 107void NORETURN slowpath(syscall_t syscall) 108{ 109 if (unlikely(syscall < SYSCALL_MIN || syscall > SYSCALL_MAX)) { 110#ifdef TRACK_KERNEL_ENTRIES 111 ksKernelEntry.path = Entry_UnknownSyscall; 112 /* ksKernelEntry.word word is already set to syscall */ 113#endif /* TRACK_KERNEL_ENTRIES */ 114 handleUnknownSyscall(syscall); 115 } else { 116#ifdef TRACK_KERNEL_ENTRIES 117 ksKernelEntry.is_fastpath = 0; 118#endif /* TRACK KERNEL ENTRIES */ 119 handleSyscall(syscall); 120 } 121 122 restore_user_context(); 123 UNREACHABLE(); 124} 125 126void VISIBLE c_handle_syscall(word_t cptr, word_t msgInfo, syscall_t syscall) 127{ 128 NODE_LOCK_SYS; 129 130 c_entry_hook(); 131#ifdef TRACK_KERNEL_ENTRIES 132 benchmark_debug_syscall_start(cptr, msgInfo, syscall); 133 ksKernelEntry.is_fastpath = 0; 134#endif /* DEBUG */ 135 136 slowpath(syscall); 137 UNREACHABLE(); 138} 139 140#ifdef CONFIG_FASTPATH 141ALIGN(L1_CACHE_LINE_SIZE) 142void VISIBLE c_handle_fastpath_call(word_t cptr, word_t msgInfo) 143{ 144 NODE_LOCK_SYS; 145 146 c_entry_hook(); 147#ifdef TRACK_KERNEL_ENTRIES 148 benchmark_debug_syscall_start(cptr, msgInfo, SysCall); 149 ksKernelEntry.is_fastpath = 1; 150#endif /* DEBUG */ 151 152 fastpath_call(cptr, msgInfo); 153 UNREACHABLE(); 154} 155 156ALIGN(L1_CACHE_LINE_SIZE) 157#ifdef CONFIG_KERNEL_MCS 158void VISIBLE c_handle_fastpath_reply_recv(word_t cptr, word_t msgInfo, word_t reply) 159#else 160void VISIBLE c_handle_fastpath_reply_recv(word_t cptr, word_t msgInfo) 161#endif 162{ 163 NODE_LOCK_SYS; 164 165 c_entry_hook(); 166#ifdef TRACK_KERNEL_ENTRIES 167 benchmark_debug_syscall_start(cptr, msgInfo, SysReplyRecv); 168 ksKernelEntry.is_fastpath = 1; 169#endif /* DEBUG */ 170 171#ifdef CONFIG_KERNEL_MCS 172 fastpath_reply_recv(cptr, msgInfo, reply); 173#else 174 fastpath_reply_recv(cptr, msgInfo); 175#endif 176 UNREACHABLE(); 177} 178 179#endif 180 181#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT 182VISIBLE NORETURN void c_handle_vcpu_fault(word_t hsr) 183{ 184 NODE_LOCK_SYS; 185 186 c_entry_hook(); 187 188#ifdef TRACK_KERNEL_ENTRIES 189 ksKernelEntry.path = Entry_VCPUFault; 190 ksKernelEntry.word = hsr; 191#endif 192 handleVCPUFault(hsr); 193 restore_user_context(); 194 UNREACHABLE(); 195} 196#endif /* CONFIG_ARM_HYPERVISOR_SUPPORT */ 197