1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * The FRED specific kernel/user entry functions which are invoked from 4 * assembly code and dispatch to the associated handlers. 5 */ 6#include <linux/kernel.h> 7#include <linux/kdebug.h> 8#include <linux/nospec.h> 9 10#include <asm/desc.h> 11#include <asm/fred.h> 12#include <asm/idtentry.h> 13#include <asm/syscall.h> 14#include <asm/trapnr.h> 15#include <asm/traps.h> 16 17/* FRED EVENT_TYPE_OTHER vector numbers */ 18#define FRED_SYSCALL 1 19#define FRED_SYSENTER 2 20 21static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) 22{ 23 irqentry_state_t irq_state = irqentry_nmi_enter(regs); 24 25 instrumentation_begin(); 26 27 /* Panic on events from a high stack level */ 28 if (regs->fred_cs.sl > 0) { 29 pr_emerg("PANIC: invalid or fatal FRED event; event type %u " 30 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 31 regs->fred_ss.type, regs->fred_ss.vector, error_code, 32 fred_event_data(regs), regs->cs, regs->ip); 33 die("invalid or fatal FRED event", regs, error_code); 34 panic("invalid or fatal FRED event"); 35 } else { 36 unsigned long flags = oops_begin(); 37 int sig = SIGKILL; 38 39 pr_alert("BUG: invalid or fatal FRED event; event type %u " 40 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 41 regs->fred_ss.type, regs->fred_ss.vector, error_code, 42 fred_event_data(regs), regs->cs, regs->ip); 43 44 if (__die("Invalid or fatal FRED event", regs, error_code)) 45 sig = 0; 46 47 oops_end(flags, regs, sig); 48 } 49 50 instrumentation_end(); 51 irqentry_nmi_exit(regs, irq_state); 52} 53 54static noinstr void fred_intx(struct pt_regs *regs) 55{ 56 switch (regs->fred_ss.vector) { 57 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ 58 case X86_TRAP_BP: 59 return exc_int3(regs); 60 61 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ 62 case X86_TRAP_OF: 63 return exc_overflow(regs); 64 65#ifdef CONFIG_IA32_EMULATION 66 /* INT80 */ 67 case IA32_SYSCALL_VECTOR: 68 if (ia32_enabled()) 69 return fred_int80_emulation(regs); 70 fallthrough; 71#endif 72 73 default: 74 return exc_general_protection(regs, 0); 75 } 76} 77 78static __always_inline void fred_other(struct pt_regs *regs) 79{ 80 /* The compiler can fold these conditions into a single test */ 81 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.lm)) { 82 regs->orig_ax = regs->ax; 83 regs->ax = -ENOSYS; 84 do_syscall_64(regs, regs->orig_ax); 85 return; 86 } else if (ia32_enabled() && 87 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.lm)) { 88 regs->orig_ax = regs->ax; 89 regs->ax = -ENOSYS; 90 do_fast_syscall_32(regs); 91 return; 92 } else { 93 exc_invalid_op(regs); 94 return; 95 } 96} 97 98#define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function 99 100static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { 101 SYSVEC(ERROR_APIC_VECTOR, error_interrupt), 102 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), 103 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 104 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 105 106 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), 107 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), 108 SYSVEC(CALL_FUNCTION_VECTOR, call_function), 109 SYSVEC(REBOOT_VECTOR, reboot), 110 111 SYSVEC(THRESHOLD_APIC_VECTOR, threshold), 112 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), 113 SYSVEC(THERMAL_APIC_VECTOR, thermal), 114 115 SYSVEC(IRQ_WORK_VECTOR, irq_work), 116 117 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 118 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 119 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 120}; 121 122static bool fred_setup_done __initdata; 123 124void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) 125{ 126 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) 127 return; 128 129 if (WARN_ON_ONCE(fred_setup_done)) 130 return; 131 132 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) 133 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; 134} 135 136static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) 137{ 138 spurious_interrupt(regs, regs->fred_ss.vector); 139} 140 141void __init fred_complete_exception_setup(void) 142{ 143 unsigned int vector; 144 145 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) 146 set_bit(vector, system_vectors); 147 148 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { 149 if (sysvec_table[vector]) 150 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors); 151 else 152 sysvec_table[vector] = fred_handle_spurious_interrupt; 153 } 154 fred_setup_done = true; 155} 156 157static noinstr void fred_extint(struct pt_regs *regs) 158{ 159 unsigned int vector = regs->fred_ss.vector; 160 unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, 161 NR_SYSTEM_VECTORS); 162 163 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) 164 return; 165 166 if (likely(vector >= FIRST_SYSTEM_VECTOR)) { 167 irqentry_state_t state = irqentry_enter(regs); 168 169 instrumentation_begin(); 170 sysvec_table[index](regs); 171 instrumentation_end(); 172 irqentry_exit(regs, state); 173 } else { 174 common_interrupt(regs, vector); 175 } 176} 177 178static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) 179{ 180 /* Optimize for #PF. That's the only exception which matters performance wise */ 181 if (likely(regs->fred_ss.vector == X86_TRAP_PF)) 182 return exc_page_fault(regs, error_code); 183 184 switch (regs->fred_ss.vector) { 185 case X86_TRAP_DE: return exc_divide_error(regs); 186 case X86_TRAP_DB: return fred_exc_debug(regs); 187 case X86_TRAP_BR: return exc_bounds(regs); 188 case X86_TRAP_UD: return exc_invalid_op(regs); 189 case X86_TRAP_NM: return exc_device_not_available(regs); 190 case X86_TRAP_DF: return exc_double_fault(regs, error_code); 191 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); 192 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); 193 case X86_TRAP_SS: return exc_stack_segment(regs, error_code); 194 case X86_TRAP_GP: return exc_general_protection(regs, error_code); 195 case X86_TRAP_MF: return exc_coprocessor_error(regs); 196 case X86_TRAP_AC: return exc_alignment_check(regs, error_code); 197 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); 198 199#ifdef CONFIG_X86_MCE 200 case X86_TRAP_MC: return fred_exc_machine_check(regs); 201#endif 202#ifdef CONFIG_INTEL_TDX_GUEST 203 case X86_TRAP_VE: return exc_virtualization_exception(regs); 204#endif 205#ifdef CONFIG_X86_CET 206 case X86_TRAP_CP: return exc_control_protection(regs, error_code); 207#endif 208 default: return fred_bad_type(regs, error_code); 209 } 210 211} 212 213static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) 214{ 215 switch (regs->fred_ss.vector) { 216 case X86_TRAP_BP: return exc_int3(regs); 217 case X86_TRAP_OF: return exc_overflow(regs); 218 default: return fred_bad_type(regs, error_code); 219 } 220} 221 222__visible noinstr void fred_entry_from_user(struct pt_regs *regs) 223{ 224 unsigned long error_code = regs->orig_ax; 225 226 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 227 regs->orig_ax = -1; 228 229 switch (regs->fred_ss.type) { 230 case EVENT_TYPE_EXTINT: 231 return fred_extint(regs); 232 case EVENT_TYPE_NMI: 233 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 234 return fred_exc_nmi(regs); 235 break; 236 case EVENT_TYPE_HWEXC: 237 return fred_hwexc(regs, error_code); 238 case EVENT_TYPE_SWINT: 239 return fred_intx(regs); 240 case EVENT_TYPE_PRIV_SWEXC: 241 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 242 return fred_exc_debug(regs); 243 break; 244 case EVENT_TYPE_SWEXC: 245 return fred_swexc(regs, error_code); 246 case EVENT_TYPE_OTHER: 247 return fred_other(regs); 248 default: break; 249 } 250 251 return fred_bad_type(regs, error_code); 252} 253 254__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 255{ 256 unsigned long error_code = regs->orig_ax; 257 258 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 259 regs->orig_ax = -1; 260 261 switch (regs->fred_ss.type) { 262 case EVENT_TYPE_EXTINT: 263 return fred_extint(regs); 264 case EVENT_TYPE_NMI: 265 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 266 return fred_exc_nmi(regs); 267 break; 268 case EVENT_TYPE_HWEXC: 269 return fred_hwexc(regs, error_code); 270 case EVENT_TYPE_PRIV_SWEXC: 271 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 272 return fred_exc_debug(regs); 273 break; 274 case EVENT_TYPE_SWEXC: 275 return fred_swexc(regs, error_code); 276 default: break; 277 } 278 279 return fred_bad_type(regs, error_code); 280} 281 282#if IS_ENABLED(CONFIG_KVM_INTEL) 283__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) 284{ 285 switch (regs->fred_ss.type) { 286 case EVENT_TYPE_EXTINT: 287 return fred_extint(regs); 288 case EVENT_TYPE_NMI: 289 return fred_exc_nmi(regs); 290 default: 291 WARN_ON_ONCE(1); 292 } 293} 294#endif 295