1/* 2 * Copyright 2014, General Dynamics C4 Systems 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8#include <model/statedata.h> 9#include <kernel/stack.h> 10#include <machine/fpu.h> 11#include <arch/fastpath/fastpath.h> 12#include <arch/machine/debug.h> 13#include <benchmark/benchmark_track.h> 14#include <mode/stack.h> 15#include <arch/object/vcpu.h> 16#include <arch/kernel/traps.h> 17 18#include <api/syscall.h> 19#include <util.h> 20 21#ifdef CONFIG_VTX 22USED static void NORETURN vmlaunch_failed(void) 23{ 24 NODE_LOCK_SYS; 25 26 c_entry_hook(); 27 28 handleVmEntryFail(); 29 restore_user_context(); 30} 31 32static void NORETURN restore_vmx(void) 33{ 34 restoreVMCS(); 35#ifdef CONFIG_HARDWARE_DEBUG_API 36 /* Do not support breakpoints in VMs, so just disable all breakpoints */ 37 loadAllDisabledBreakpointState(ksCurThread); 38#endif 39#ifdef ENABLE_SMP_SUPPORT 40 NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->kernelSP = ((word_t)kernel_stack_alloc[getCurrentCPUIndex()]) + BIT( 41 CONFIG_KERNEL_STACK_BITS) - 4; 42#endif /* ENABLE_SMP_SUPPORT */ 43 if (NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->launched) { 44 /* attempt to do a vmresume */ 45 asm volatile( 46 // Set our stack pointer to the top of the tcb so we can efficiently pop 47 "movl %0, %%esp\n" 48 "popl %%eax\n" 49 "popl %%ebx\n" 50 "popl %%ecx\n" 51 "popl %%edx\n" 52 "popl %%esi\n" 53 "popl %%edi\n" 54 "popl %%ebp\n" 55 // Now do the vmresume 56 "vmresume\n" 57 // if we get here we failed 58#ifdef ENABLE_SMP_SUPPORT 59 "movl (%%esp), %%esp\n" 60#else 61 "leal kernel_stack_alloc + %c1, %%esp\n" 62#endif 63 "call vmlaunch_failed\n" 64 : 65 : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->gp_registers[VCPU_EAX]), 66 "i"(BIT(CONFIG_KERNEL_STACK_BITS) - sizeof(word_t)) 67 // Clobber memory so the compiler is forced to complete all stores 68 // before running this assembler 69 : "memory" 70 ); 71 } else { 72 /* attempt to do a vmlaunch */ 73 asm volatile( 74 // Set our stack pointer to the top of the tcb so we can efficiently pop 75 "movl %0, %%esp\n" 76 "popl %%eax\n" 77 "popl %%ebx\n" 78 "popl %%ecx\n" 79 "popl %%edx\n" 80 "popl %%esi\n" 81 "popl %%edi\n" 82 "popl %%ebp\n" 83 // Now do the vmresume 84 "vmlaunch\n" 85 // if we get here we failed 86#ifdef ENABLE_SMP_SUPPORT 87 "movl (%%esp), %%esp\n" 88#else 89 "leal kernel_stack_alloc + %c1, %%esp\n" 90#endif 91 "call vmlaunch_failed\n" 92 : 93 : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->gp_registers[VCPU_EAX]), 94 "i"(BIT(CONFIG_KERNEL_STACK_BITS) - sizeof(word_t)) 95 // Clobber memory so the compiler is forced to complete all stores 96 // before running this assembler 97 : "memory" 98 ); 99 } 100 UNREACHABLE(); 101} 102#endif 103 104void NORETURN VISIBLE restore_user_context(void); 105void NORETURN VISIBLE restore_user_context(void) 106{ 107 c_exit_hook(); 108 109 NODE_UNLOCK_IF_HELD; 110 111 /* we've now 'exited' the kernel. If we have a pending interrupt 112 * we should 'enter' it again */ 113 if (ARCH_NODE_STATE(x86KSPendingInterrupt) != int_invalid) { 114 /* put this in service */ 115 interrupt_t irq = servicePendingIRQ(); 116 /* reset our stack and jmp to the IRQ entry point */ 117 asm volatile( 118 "mov %[stack_top], %%esp\n" 119 "push %[syscall] \n" 120 "push %[irq]\n" 121 "call c_handle_interrupt" 122 : 123 : [stack_top] "r"(&(kernel_stack_alloc[CURRENT_CPU_INDEX()][BIT(CONFIG_KERNEL_STACK_BITS)])), 124 [syscall] "r"(0), /* syscall is unused for irq path */ 125 [irq] "r"(irq) 126 : "memory"); 127 UNREACHABLE(); 128 } 129 130#ifdef CONFIG_VTX 131 if (thread_state_ptr_get_tsType(&NODE_STATE(ksCurThread)->tcbState) == ThreadState_RunningVM) { 132 restore_vmx(); 133 } 134#endif 135 setKernelEntryStackPointer(NODE_STATE(ksCurThread)); 136 lazyFPURestore(NODE_STATE(ksCurThread)); 137 138#ifdef CONFIG_HARDWARE_DEBUG_API 139 restore_user_debug_context(NODE_STATE(ksCurThread)); 140#endif 141 142 if (config_set(CONFIG_KERNEL_X86_IBRS_BASIC)) { 143 x86_disable_ibrs(); 144 } 145 146 /* see if we entered via syscall */ 147 if (likely(NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[Error] == -1)) { 148 NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[FLAGS] &= ~FLAGS_IF; 149 asm volatile( 150 // Set our stack pointer to the top of the tcb so we can efficiently pop 151 "movl %0, %%esp\n" 152 // restore syscall number 153 "popl %%eax\n" 154 // cap/badge register 155 "popl %%ebx\n" 156 // skip ecx and edx, these will contain esp and NextIP due to sysenter/sysexit convention 157 "addl $8, %%esp\n" 158 // message info register 159 "popl %%esi\n" 160 // message register 161 "popl %%edi\n" 162 // message register 163 "popl %%ebp\n" 164 // skip FaultIP and Error (these are fake registers) 165 "addl $8, %%esp\n" 166 // restore NextIP 167 "popl %%edx\n" 168 // skip cs 169 "addl $4, %%esp\n" 170 "movl 4(%%esp), %%ecx\n" 171 "popfl\n" 172 "orl %[IFMASK], -4(%%esp)\n" 173 "sti\n" 174 "sysexit\n" 175 : 176 : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[EAX]), 177 [IFMASK]"i"(FLAGS_IF) 178 // Clobber memory so the compiler is forced to complete all stores 179 // before running this assembler 180 : "memory" 181 ); 182 } else { 183 asm volatile( 184 // Set our stack pointer to the top of the tcb so we can efficiently pop 185 "movl %0, %%esp\n" 186 "popl %%eax\n" 187 "popl %%ebx\n" 188 "popl %%ecx\n" 189 "popl %%edx\n" 190 "popl %%esi\n" 191 "popl %%edi\n" 192 "popl %%ebp\n" 193 // skip FaultIP and Error 194 "addl $8, %%esp\n" 195 "iret\n" 196 : 197 : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[EAX]) 198 // Clobber memory so the compiler is forced to complete all stores 199 // before running this assembler 200 : "memory" 201 ); 202 } 203 204 UNREACHABLE(); 205} 206