1/*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include <config.h>
8#include <arch/kernel/traps.h>
9#include <arch/object/vcpu.h>
10#include <arch/machine/registerset.h>
11#include <api/syscall.h>
12#include <machine/fpu.h>
13
14#include <sel4/benchmark_track_types.h>
15#include <benchmark/benchmark_track.h>
16#include <benchmark/benchmark_utilisation.h>
17#include <arch/machine.h>
18
19void VISIBLE NORETURN c_handle_undefined_instruction(void)
20{
21    NODE_LOCK_SYS;
22    c_entry_hook();
23
24#ifdef TRACK_KERNEL_ENTRIES
25    ksKernelEntry.path = Entry_UserLevelFault;
26    ksKernelEntry.word = getRegister(NODE_STATE(ksCurThread), NextIP);
27#endif
28
29#if defined(CONFIG_HAVE_FPU) && defined(CONFIG_ARCH_AARCH32)
30    /* We assume the first fault is a FP exception and enable FPU, if not already enabled */
31    if (!isFpuEnable()) {
32        handleFPUFault();
33
34        /* Restart the FP instruction that cause the fault */
35        setNextPC(NODE_STATE(ksCurThread), getRestartPC(NODE_STATE(ksCurThread)));
36    } else {
37        handleUserLevelFault(0, 0);
38    }
39
40    restore_user_context();
41    UNREACHABLE();
42#endif
43
44    /* There's only one user-level fault on ARM, and the code is (0,0) */
45#ifdef CONFIG_ARCH_AARCH32
46    handleUserLevelFault(0, 0);
47#else
48    handleUserLevelFault(getESR(), 0);
49#endif
50    restore_user_context();
51    UNREACHABLE();
52}
53
54#if defined(CONFIG_HAVE_FPU) && defined(CONFIG_ARCH_AARCH64)
55void VISIBLE NORETURN c_handle_enfp(void)
56{
57    c_entry_hook();
58
59    handleFPUFault();
60    restore_user_context();
61    UNREACHABLE();
62}
63#endif /* CONFIG_HAVE_FPU */
64
65static inline void NORETURN c_handle_vm_fault(vm_fault_type_t type)
66{
67    NODE_LOCK_SYS;
68    c_entry_hook();
69
70#ifdef TRACK_KERNEL_ENTRIES
71    ksKernelEntry.path = Entry_VMFault;
72    ksKernelEntry.word = getRegister(NODE_STATE(ksCurThread), NextIP);
73#endif
74
75    handleVMFaultEvent(type);
76    restore_user_context();
77    UNREACHABLE();
78}
79
80void VISIBLE NORETURN c_handle_data_fault(void)
81{
82    c_handle_vm_fault(seL4_DataFault);
83}
84
85void VISIBLE NORETURN c_handle_instruction_fault(void)
86{
87    c_handle_vm_fault(seL4_InstructionFault);
88}
89
90void VISIBLE NORETURN c_handle_interrupt(void)
91{
92    NODE_LOCK_IRQ_IF(IRQT_TO_IRQ(getActiveIRQ()) != irq_remote_call_ipi);
93    c_entry_hook();
94
95#ifdef TRACK_KERNEL_ENTRIES
96    ksKernelEntry.path = Entry_Interrupt;
97    ksKernelEntry.word = IRQT_TO_IRQ(getActiveIRQ());
98#ifdef ENABLE_SMP_SUPPORT
99    ksKernelEntry.core = getCurrentCPUIndex();
100#endif
101#endif
102
103    handleInterruptEntry();
104    restore_user_context();
105}
106
107void NORETURN slowpath(syscall_t syscall)
108{
109    if (unlikely(syscall < SYSCALL_MIN || syscall > SYSCALL_MAX)) {
110#ifdef TRACK_KERNEL_ENTRIES
111        ksKernelEntry.path = Entry_UnknownSyscall;
112        /* ksKernelEntry.word word is already set to syscall */
113#endif /* TRACK_KERNEL_ENTRIES */
114        handleUnknownSyscall(syscall);
115    } else {
116#ifdef TRACK_KERNEL_ENTRIES
117        ksKernelEntry.is_fastpath = 0;
118#endif /* TRACK KERNEL ENTRIES */
119        handleSyscall(syscall);
120    }
121
122    restore_user_context();
123    UNREACHABLE();
124}
125
126void VISIBLE c_handle_syscall(word_t cptr, word_t msgInfo, syscall_t syscall)
127{
128    NODE_LOCK_SYS;
129
130    c_entry_hook();
131#ifdef TRACK_KERNEL_ENTRIES
132    benchmark_debug_syscall_start(cptr, msgInfo, syscall);
133    ksKernelEntry.is_fastpath = 0;
134#endif /* DEBUG */
135
136    slowpath(syscall);
137    UNREACHABLE();
138}
139
140#ifdef CONFIG_FASTPATH
141ALIGN(L1_CACHE_LINE_SIZE)
142void VISIBLE c_handle_fastpath_call(word_t cptr, word_t msgInfo)
143{
144    NODE_LOCK_SYS;
145
146    c_entry_hook();
147#ifdef TRACK_KERNEL_ENTRIES
148    benchmark_debug_syscall_start(cptr, msgInfo, SysCall);
149    ksKernelEntry.is_fastpath = 1;
150#endif /* DEBUG */
151
152    fastpath_call(cptr, msgInfo);
153    UNREACHABLE();
154}
155
156ALIGN(L1_CACHE_LINE_SIZE)
157#ifdef CONFIG_KERNEL_MCS
158void VISIBLE c_handle_fastpath_reply_recv(word_t cptr, word_t msgInfo, word_t reply)
159#else
160void VISIBLE c_handle_fastpath_reply_recv(word_t cptr, word_t msgInfo)
161#endif
162{
163    NODE_LOCK_SYS;
164
165    c_entry_hook();
166#ifdef TRACK_KERNEL_ENTRIES
167    benchmark_debug_syscall_start(cptr, msgInfo, SysReplyRecv);
168    ksKernelEntry.is_fastpath = 1;
169#endif /* DEBUG */
170
171#ifdef CONFIG_KERNEL_MCS
172    fastpath_reply_recv(cptr, msgInfo, reply);
173#else
174    fastpath_reply_recv(cptr, msgInfo);
175#endif
176    UNREACHABLE();
177}
178
179#endif
180
181#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
182VISIBLE NORETURN void c_handle_vcpu_fault(word_t hsr)
183{
184    NODE_LOCK_SYS;
185
186    c_entry_hook();
187
188#ifdef TRACK_KERNEL_ENTRIES
189    ksKernelEntry.path = Entry_VCPUFault;
190    ksKernelEntry.word = hsr;
191#endif
192    handleVCPUFault(hsr);
193    restore_user_context();
194    UNREACHABLE();
195}
196#endif /* CONFIG_ARM_HYPERVISOR_SUPPORT */
197