1/*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#pragma once
8
9#include <util.h>
10#include <linker.h>
11#include <api/types.h>
12#include <api/syscall.h>
13#include <plat/machine/hardware.h>
14
15/* seL4 is always in the top of memory, so the high bits of pointers are always 1.
16   The autogenerated unpacking code doesn't know that, however, so will try to
17   conditionally sign extend (in 64-bit mode), which wastes cycles in the fast
18   path. Instead, we can do the unpacking ourselves and explicitly set the high
19   bits. */
20
21static inline tcb_t *endpoint_ptr_get_epQueue_tail_fp(endpoint_t *ep_ptr)
22{
23    uint64_t ret = ep_ptr->words[0] & 0xfffffffffffcull;
24    return unlikely(ret) ? TCB_PTR(ret | PPTR_BASE) : NULL;
25}
26
27static inline vspace_root_t *cap_vtable_cap_get_vspace_root_fp(cap_t vtable_cap)
28{
29    return PML4E_PTR(vtable_cap.words[1]);
30}
31
32static inline word_t cap_pml4_cap_get_capPML4MappedASID_fp(cap_t vtable_cap)
33{
34    return (uint32_t)vtable_cap.words[0];
35}
36
37static inline void FORCE_INLINE switchToThread_fp(tcb_t *thread, vspace_root_t *vroot, pde_t stored_hw_asid)
38{
39    word_t new_vroot = pptr_to_paddr(vroot);
40    /* the asid is the 12-bit PCID */
41    asid_t asid = (asid_t)(stored_hw_asid.words[0] & 0xfff);
42    cr3_t next_cr3 = makeCR3(new_vroot, asid);
43    if (likely(getCurrentUserCR3().words[0] != next_cr3.words[0])) {
44        SMP_COND_STATEMENT(tlb_bitmap_set(vroot, getCurrentCPUIndex());)
45        setCurrentUserCR3(next_cr3);
46    }
47
48#ifdef ENABLE_SMP_SUPPORT
49    asm volatile("movq %[value], %%gs:%c[offset]"
50                 :
51                 : [value] "r"(&thread->tcbArch.tcbContext.registers[Error + 1]),
52                 [offset] "i"(OFFSETOF(nodeInfo_t, currentThreadUserContext)));
53#endif /* ENABLE_SMP_SUPPORT */
54
55    if (config_set(CONFIG_KERNEL_X86_IBPB_ON_CONTEXT_SWITCH)) {
56        x86_ibpb();
57    }
58
59    if (config_set(CONFIG_KERNEL_X86_RSB_ON_CONTEXT_SWITCH)) {
60        x86_flush_rsb();
61    }
62
63#ifdef CONFIG_BENCHMARK_TRACK_UTILISATION
64    benchmark_utilisation_switch(NODE_STATE(ksCurThread), thread);
65#endif
66
67    NODE_STATE(ksCurThread) = thread;
68}
69
70#ifndef CONFIG_KERNEL_MCS
71static inline void thread_state_ptr_set_blockingIPCDiminish_np(thread_state_t *ts_ptr, word_t dim)
72{
73    ts_ptr->words[1] = (ts_ptr->words[1] & 1) | dim;
74}
75
76static inline void mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
77    mdb_node_t *node_ptr, word_t mdbNext,
78    word_t mdbRevocable, word_t mdbFirstBadged)
79{
80    node_ptr->words[1] = mdbNext | (mdbRevocable << 1) | mdbFirstBadged;
81}
82
83static inline void mdb_node_ptr_set_mdbPrev_np(mdb_node_t *node_ptr, word_t mdbPrev)
84{
85    node_ptr->words[0] = mdbPrev;
86}
87#endif
88
89static inline bool_t isValidVTableRoot_fp(cap_t vspace_root_cap)
90{
91    /* Check the cap is a pml4_cap, and that it is mapped. The fields are next
92       to each other, so they can be read and checked in parallel */
93    return (vspace_root_cap.words[0] >> (64 - 6)) == ((cap_pml4_cap << 1) | 0x1);
94}
95
96static inline void fastpath_copy_mrs(word_t length, tcb_t *src, tcb_t *dest)
97{
98    word_t i;
99    register_t reg;
100
101    /* assuming that length < n_msgRegisters */
102    for (i = 0; i < length; i ++) {
103        /* assuming that the message registers simply increment */
104        reg = msgRegisters[0] + i;
105        setRegister(dest, reg, getRegister(src, reg));
106    }
107}
108
109/* This is an accelerated check that msgLength, which appears
110   in the bottom of the msgInfo word, is <= 4 and that msgExtraCaps
111   which appears above it is zero. We are assuming that n_msgRegisters == 4
112   for this check to be useful. By masking out the bottom 3 bits, we are
113   really checking that n + 3 <= MASK(3), i.e. n + 3 <= 7 or n <= 4. */
114compile_assert(n_msgRegisters_eq_4, n_msgRegisters == 4)
115static inline int
116fastpath_mi_check(word_t msgInfo)
117{
118    return ((msgInfo & MASK(seL4_MsgLengthBits + seL4_MsgExtraCapBits))
119            + 3) & ~MASK(3);
120}
121
122static inline void NORETURN FORCE_INLINE fastpath_restore(word_t badge, word_t msgInfo, tcb_t *cur_thread)
123{
124    if (config_set(CONFIG_SYSENTER) && config_set(CONFIG_HARDWARE_DEBUG_API)
125        && ((getRegister(NODE_STATE(ksCurThread), FLAGS) & FLAGS_TF) != 0)) {
126        /* If single stepping using sysenter we need to do a return using iret to avoid
127         * a race condition in restoring the flags (which enables stepping and interrupts) and
128         * calling sysexit. This case is handled in restore_user_context so we just go there
129         */
130        restore_user_context();
131    }
132    NODE_UNLOCK;
133    c_exit_hook();
134    lazyFPURestore(cur_thread);
135
136    if (config_set(CONFIG_KERNEL_SKIM_WINDOW)) {
137        /* see restore_user_context for a full explanation of why we do this */
138        word_t *irqstack = x64KSIRQStack[CURRENT_CPU_INDEX()];
139        irqstack[0] = 0;
140        irqstack[1] = 0;
141        irqstack[2] = 0;
142        irqstack[3] = 0;
143        irqstack[4] = 0;
144        irqstack[5] = 0;
145    }
146
147#ifdef CONFIG_HARDWARE_DEBUG_API
148    restore_user_debug_context(cur_thread);
149#endif
150
151#ifdef ENABLE_SMP_SUPPORT
152#ifdef CONFIG_KERNEL_SKIM_WINDOW
153    word_t next_cr3 = MODE_NODE_STATE(x64KSCurrentUserCR3);
154#endif
155    swapgs();
156#endif /* ENABLE_SMP_SUPPORT */
157
158    if (config_set(CONFIG_KERNEL_X86_IBRS_BASIC)) {
159        x86_disable_ibrs();
160    }
161
162    if (config_set(CONFIG_SYSENTER)) {
163        cur_thread->tcbArch.tcbContext.registers[FLAGS] &= ~FLAGS_IF;
164
165#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW)
166        register word_t next_cr3_r11 asm("r11") = next_cr3;
167#endif
168        asm volatile(
169            "movq %%rcx, %%rsp\n"
170            "popq %%rax\n"
171            "popq %%rbx\n"
172            "popq %%rbp\n"
173            "popq %%r12\n"
174            "popq %%r13\n"
175            "popq %%r14\n"
176            // Skip RDX, we need to put NextIP into it
177            "addq $8, %%rsp\n"
178            "popq %%r10\n"
179            "popq %%r8\n"
180            "popq %%r9\n"
181            "popq %%r15\n"
182            // restore RFLAGS
183            "popfq\n"
184            // reset interrupt bit
185            "orq %[IF], -8(%%rsp)\n"
186            // Restore NextIP
187            "popq %%rdx\n"
188            // skip Error
189            "addq $8, %%rsp\n"
190            // restore RSP
191            "popq %%rcx\n"
192            // Skip FaultIP
193            "addq $8, %%rsp\n"
194#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW)
195            "popq %%rsp\n"
196            "movq %%r11, %%cr3\n"
197            "movq %%rsp, %%r11\n"
198#else
199            "popq %%r11\n"
200#ifdef CONFIG_KERNEL_SKIM_WINDOW
201            "movq (x64KSCurrentUserCR3), %%rsp\n"
202            "movq %%rsp, %%cr3\n"
203#endif /* CONFIG_KERNEL_SKIM_WINDOW */
204#endif /* defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) */
205            "sti\n"
206            "sysexitq\n"
207            :
208            : "c"(&cur_thread->tcbArch.tcbContext.registers[RAX]),
209            "D"(badge),
210            "S"(msgInfo),
211#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW)
212            "r"(next_cr3_r11),
213#endif
214            [IF] "i"(FLAGS_IF)
215            : "memory"
216        );
217    } else {
218        asm volatile(
219            // Set our stack pointer to the top of the tcb so we can efficiently pop
220            "movq %0, %%rsp\n"
221            "popq %%rax\n"
222            "popq %%rbx\n"
223            "popq %%rbp\n"
224            "popq %%r12\n"
225            "popq %%r13\n"
226            "popq %%r14\n"
227            "popq %%rdx\n"
228            "popq %%r10\n"
229            "popq %%r8\n"
230            "popq %%r9\n"
231            "popq %%r15\n"
232            //restore RFLAGS
233            "popq %%r11\n"
234            // Restore NextIP
235#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW)
236            "popq %%rsp\n"
237            "movq %%rcx, %%cr3\n"
238            "movq %%rsp, %%rcx\n"
239#else
240            "popq %%rcx\n"
241#ifdef CONFIG_KERNEL_SKIM_WINDOW
242            "movq (x64KSCurrentUserCR3), %%rsp\n"
243            "movq %%rsp, %%cr3\n"
244#endif /* CONFIG_KERNEL_SKIM_WINDOW */
245#endif /* defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) */
246            // clear RSP to not leak information to the user
247            "xor %%rsp, %%rsp\n"
248            // More register but we can ignore and are done restoring
249            // enable interrupt disabled by sysenter
250            "sysretq\n"
251            :
252            : "r"(&cur_thread->tcbArch.tcbContext.registers[RAX]),
253            "D"(badge),
254            "S"(msgInfo)
255#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW)
256            , "c"(next_cr3)
257#endif
258            : "memory"
259        );
260    }
261    UNREACHABLE();
262}
263
264