1/*
2 * Copyright 2014, General Dynamics C4 Systems
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include <config.h>
8#include <model/statedata.h>
9#include <kernel/stack.h>
10#include <machine/fpu.h>
11#include <arch/fastpath/fastpath.h>
12#include <arch/machine/debug.h>
13#include <benchmark/benchmark_track.h>
14#include <mode/stack.h>
15#include <arch/object/vcpu.h>
16#include <arch/kernel/traps.h>
17
18#include <api/syscall.h>
19#include <util.h>
20
21#ifdef CONFIG_VTX
22USED static void NORETURN vmlaunch_failed(void)
23{
24    NODE_LOCK_SYS;
25
26    c_entry_hook();
27
28    handleVmEntryFail();
29    restore_user_context();
30}
31
32static void NORETURN restore_vmx(void)
33{
34    restoreVMCS();
35#ifdef CONFIG_HARDWARE_DEBUG_API
36    /* Do not support breakpoints in VMs, so just disable all breakpoints */
37    loadAllDisabledBreakpointState(ksCurThread);
38#endif
39#ifdef ENABLE_SMP_SUPPORT
40    NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->kernelSP = ((word_t)kernel_stack_alloc[getCurrentCPUIndex()]) + BIT(
41                                                             CONFIG_KERNEL_STACK_BITS) - 4;
42#endif /* ENABLE_SMP_SUPPORT */
43    if (NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->launched) {
44        /* attempt to do a vmresume */
45        asm volatile(
46            // Set our stack pointer to the top of the tcb so we can efficiently pop
47            "movl %0, %%esp\n"
48            "popl %%eax\n"
49            "popl %%ebx\n"
50            "popl %%ecx\n"
51            "popl %%edx\n"
52            "popl %%esi\n"
53            "popl %%edi\n"
54            "popl %%ebp\n"
55            // Now do the vmresume
56            "vmresume\n"
57            // if we get here we failed
58#ifdef ENABLE_SMP_SUPPORT
59            "movl (%%esp), %%esp\n"
60#else
61            "leal kernel_stack_alloc + %c1, %%esp\n"
62#endif
63            "call vmlaunch_failed\n"
64            :
65            : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->gp_registers[VCPU_EAX]),
66            "i"(BIT(CONFIG_KERNEL_STACK_BITS) - sizeof(word_t))
67            // Clobber memory so the compiler is forced to complete all stores
68            // before running this assembler
69            : "memory"
70        );
71    } else {
72        /* attempt to do a vmlaunch */
73        asm volatile(
74            // Set our stack pointer to the top of the tcb so we can efficiently pop
75            "movl %0, %%esp\n"
76            "popl %%eax\n"
77            "popl %%ebx\n"
78            "popl %%ecx\n"
79            "popl %%edx\n"
80            "popl %%esi\n"
81            "popl %%edi\n"
82            "popl %%ebp\n"
83            // Now do the vmresume
84            "vmlaunch\n"
85            // if we get here we failed
86#ifdef ENABLE_SMP_SUPPORT
87            "movl (%%esp), %%esp\n"
88#else
89            "leal kernel_stack_alloc + %c1, %%esp\n"
90#endif
91            "call vmlaunch_failed\n"
92            :
93            : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbVCPU->gp_registers[VCPU_EAX]),
94            "i"(BIT(CONFIG_KERNEL_STACK_BITS) - sizeof(word_t))
95            // Clobber memory so the compiler is forced to complete all stores
96            // before running this assembler
97            : "memory"
98        );
99    }
100    UNREACHABLE();
101}
102#endif
103
104void NORETURN VISIBLE restore_user_context(void);
105void NORETURN VISIBLE restore_user_context(void)
106{
107    c_exit_hook();
108
109    NODE_UNLOCK_IF_HELD;
110
111    /* we've now 'exited' the kernel. If we have a pending interrupt
112     * we should 'enter' it again */
113    if (ARCH_NODE_STATE(x86KSPendingInterrupt) != int_invalid) {
114        /* put this in service */
115        interrupt_t irq = servicePendingIRQ();
116        /* reset our stack and jmp to the IRQ entry point */
117        asm volatile(
118            "mov %[stack_top], %%esp\n"
119            "push %[syscall] \n"
120            "push %[irq]\n"
121            "call c_handle_interrupt"
122            :
123            : [stack_top] "r"(&(kernel_stack_alloc[CURRENT_CPU_INDEX()][BIT(CONFIG_KERNEL_STACK_BITS)])),
124            [syscall] "r"(0), /* syscall is unused for irq path */
125            [irq] "r"(irq)
126            : "memory");
127        UNREACHABLE();
128    }
129
130#ifdef CONFIG_VTX
131    if (thread_state_ptr_get_tsType(&NODE_STATE(ksCurThread)->tcbState) == ThreadState_RunningVM) {
132        restore_vmx();
133    }
134#endif
135    setKernelEntryStackPointer(NODE_STATE(ksCurThread));
136    lazyFPURestore(NODE_STATE(ksCurThread));
137
138#ifdef CONFIG_HARDWARE_DEBUG_API
139    restore_user_debug_context(NODE_STATE(ksCurThread));
140#endif
141
142    if (config_set(CONFIG_KERNEL_X86_IBRS_BASIC)) {
143        x86_disable_ibrs();
144    }
145
146    /* see if we entered via syscall */
147    if (likely(NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[Error] == -1)) {
148        NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[FLAGS] &= ~FLAGS_IF;
149        asm volatile(
150            // Set our stack pointer to the top of the tcb so we can efficiently pop
151            "movl %0, %%esp\n"
152            // restore syscall number
153            "popl %%eax\n"
154            // cap/badge register
155            "popl %%ebx\n"
156            // skip ecx and edx, these will contain esp and NextIP due to sysenter/sysexit convention
157            "addl $8, %%esp\n"
158            // message info register
159            "popl %%esi\n"
160            // message register
161            "popl %%edi\n"
162            // message register
163            "popl %%ebp\n"
164            // skip FaultIP and Error (these are fake registers)
165            "addl $8, %%esp\n"
166            // restore NextIP
167            "popl %%edx\n"
168            // skip cs
169            "addl $4,  %%esp\n"
170            "movl 4(%%esp), %%ecx\n"
171            "popfl\n"
172            "orl %[IFMASK], -4(%%esp)\n"
173            "sti\n"
174            "sysexit\n"
175            :
176            : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[EAX]),
177            [IFMASK]"i"(FLAGS_IF)
178            // Clobber memory so the compiler is forced to complete all stores
179            // before running this assembler
180            : "memory"
181        );
182    } else {
183        asm volatile(
184            // Set our stack pointer to the top of the tcb so we can efficiently pop
185            "movl %0, %%esp\n"
186            "popl %%eax\n"
187            "popl %%ebx\n"
188            "popl %%ecx\n"
189            "popl %%edx\n"
190            "popl %%esi\n"
191            "popl %%edi\n"
192            "popl %%ebp\n"
193            // skip FaultIP and Error
194            "addl $8, %%esp\n"
195            "iret\n"
196            :
197            : "r"(&NODE_STATE(ksCurThread)->tcbArch.tcbContext.registers[EAX])
198            // Clobber memory so the compiler is forced to complete all stores
199            // before running this assembler
200            : "memory"
201        );
202    }
203
204    UNREACHABLE();
205}
206