1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#pragma once 8 9#include <util.h> 10#include <linker.h> 11#include <api/types.h> 12#include <api/syscall.h> 13#include <plat/machine/hardware.h> 14 15/* seL4 is always in the top of memory, so the high bits of pointers are always 1. 16 The autogenerated unpacking code doesn't know that, however, so will try to 17 conditionally sign extend (in 64-bit mode), which wastes cycles in the fast 18 path. Instead, we can do the unpacking ourselves and explicitly set the high 19 bits. */ 20 21static inline tcb_t *endpoint_ptr_get_epQueue_tail_fp(endpoint_t *ep_ptr) 22{ 23 uint64_t ret = ep_ptr->words[0] & 0xfffffffffffcull; 24 return unlikely(ret) ? TCB_PTR(ret | PPTR_BASE) : NULL; 25} 26 27static inline vspace_root_t *cap_vtable_cap_get_vspace_root_fp(cap_t vtable_cap) 28{ 29 return PML4E_PTR(vtable_cap.words[1]); 30} 31 32static inline word_t cap_pml4_cap_get_capPML4MappedASID_fp(cap_t vtable_cap) 33{ 34 return (uint32_t)vtable_cap.words[0]; 35} 36 37static inline void FORCE_INLINE switchToThread_fp(tcb_t *thread, vspace_root_t *vroot, pde_t stored_hw_asid) 38{ 39 word_t new_vroot = pptr_to_paddr(vroot); 40 /* the asid is the 12-bit PCID */ 41 asid_t asid = (asid_t)(stored_hw_asid.words[0] & 0xfff); 42 cr3_t next_cr3 = makeCR3(new_vroot, asid); 43 if (likely(getCurrentUserCR3().words[0] != next_cr3.words[0])) { 44 SMP_COND_STATEMENT(tlb_bitmap_set(vroot, getCurrentCPUIndex());) 45 setCurrentUserCR3(next_cr3); 46 } 47 48#ifdef ENABLE_SMP_SUPPORT 49 asm volatile("movq %[value], %%gs:%c[offset]" 50 : 51 : [value] "r"(&thread->tcbArch.tcbContext.registers[Error + 1]), 52 [offset] "i"(OFFSETOF(nodeInfo_t, currentThreadUserContext))); 53#endif /* ENABLE_SMP_SUPPORT */ 54 55 if (config_set(CONFIG_KERNEL_X86_IBPB_ON_CONTEXT_SWITCH)) { 56 x86_ibpb(); 57 } 58 59 if (config_set(CONFIG_KERNEL_X86_RSB_ON_CONTEXT_SWITCH)) { 60 x86_flush_rsb(); 61 } 62 63#ifdef CONFIG_BENCHMARK_TRACK_UTILISATION 64 benchmark_utilisation_switch(NODE_STATE(ksCurThread), thread); 65#endif 66 67 NODE_STATE(ksCurThread) = thread; 68} 69 70#ifndef CONFIG_KERNEL_MCS 71static inline void thread_state_ptr_set_blockingIPCDiminish_np(thread_state_t *ts_ptr, word_t dim) 72{ 73 ts_ptr->words[1] = (ts_ptr->words[1] & 1) | dim; 74} 75 76static inline void mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( 77 mdb_node_t *node_ptr, word_t mdbNext, 78 word_t mdbRevocable, word_t mdbFirstBadged) 79{ 80 node_ptr->words[1] = mdbNext | (mdbRevocable << 1) | mdbFirstBadged; 81} 82 83static inline void mdb_node_ptr_set_mdbPrev_np(mdb_node_t *node_ptr, word_t mdbPrev) 84{ 85 node_ptr->words[0] = mdbPrev; 86} 87#endif 88 89static inline bool_t isValidVTableRoot_fp(cap_t vspace_root_cap) 90{ 91 /* Check the cap is a pml4_cap, and that it is mapped. The fields are next 92 to each other, so they can be read and checked in parallel */ 93 return (vspace_root_cap.words[0] >> (64 - 6)) == ((cap_pml4_cap << 1) | 0x1); 94} 95 96static inline void fastpath_copy_mrs(word_t length, tcb_t *src, tcb_t *dest) 97{ 98 word_t i; 99 register_t reg; 100 101 /* assuming that length < n_msgRegisters */ 102 for (i = 0; i < length; i ++) { 103 /* assuming that the message registers simply increment */ 104 reg = msgRegisters[0] + i; 105 setRegister(dest, reg, getRegister(src, reg)); 106 } 107} 108 109/* This is an accelerated check that msgLength, which appears 110 in the bottom of the msgInfo word, is <= 4 and that msgExtraCaps 111 which appears above it is zero. We are assuming that n_msgRegisters == 4 112 for this check to be useful. By masking out the bottom 3 bits, we are 113 really checking that n + 3 <= MASK(3), i.e. n + 3 <= 7 or n <= 4. */ 114compile_assert(n_msgRegisters_eq_4, n_msgRegisters == 4) 115static inline int 116fastpath_mi_check(word_t msgInfo) 117{ 118 return ((msgInfo & MASK(seL4_MsgLengthBits + seL4_MsgExtraCapBits)) 119 + 3) & ~MASK(3); 120} 121 122static inline void NORETURN FORCE_INLINE fastpath_restore(word_t badge, word_t msgInfo, tcb_t *cur_thread) 123{ 124 if (config_set(CONFIG_SYSENTER) && config_set(CONFIG_HARDWARE_DEBUG_API) 125 && ((getRegister(NODE_STATE(ksCurThread), FLAGS) & FLAGS_TF) != 0)) { 126 /* If single stepping using sysenter we need to do a return using iret to avoid 127 * a race condition in restoring the flags (which enables stepping and interrupts) and 128 * calling sysexit. This case is handled in restore_user_context so we just go there 129 */ 130 restore_user_context(); 131 } 132 NODE_UNLOCK; 133 c_exit_hook(); 134 lazyFPURestore(cur_thread); 135 136 if (config_set(CONFIG_KERNEL_SKIM_WINDOW)) { 137 /* see restore_user_context for a full explanation of why we do this */ 138 word_t *irqstack = x64KSIRQStack[CURRENT_CPU_INDEX()]; 139 irqstack[0] = 0; 140 irqstack[1] = 0; 141 irqstack[2] = 0; 142 irqstack[3] = 0; 143 irqstack[4] = 0; 144 irqstack[5] = 0; 145 } 146 147#ifdef CONFIG_HARDWARE_DEBUG_API 148 restore_user_debug_context(cur_thread); 149#endif 150 151#ifdef ENABLE_SMP_SUPPORT 152#ifdef CONFIG_KERNEL_SKIM_WINDOW 153 word_t next_cr3 = MODE_NODE_STATE(x64KSCurrentUserCR3); 154#endif 155 swapgs(); 156#endif /* ENABLE_SMP_SUPPORT */ 157 158 if (config_set(CONFIG_KERNEL_X86_IBRS_BASIC)) { 159 x86_disable_ibrs(); 160 } 161 162 if (config_set(CONFIG_SYSENTER)) { 163 cur_thread->tcbArch.tcbContext.registers[FLAGS] &= ~FLAGS_IF; 164 165#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) 166 register word_t next_cr3_r11 asm("r11") = next_cr3; 167#endif 168 asm volatile( 169 "movq %%rcx, %%rsp\n" 170 "popq %%rax\n" 171 "popq %%rbx\n" 172 "popq %%rbp\n" 173 "popq %%r12\n" 174 "popq %%r13\n" 175 "popq %%r14\n" 176 // Skip RDX, we need to put NextIP into it 177 "addq $8, %%rsp\n" 178 "popq %%r10\n" 179 "popq %%r8\n" 180 "popq %%r9\n" 181 "popq %%r15\n" 182 // restore RFLAGS 183 "popfq\n" 184 // reset interrupt bit 185 "orq %[IF], -8(%%rsp)\n" 186 // Restore NextIP 187 "popq %%rdx\n" 188 // skip Error 189 "addq $8, %%rsp\n" 190 // restore RSP 191 "popq %%rcx\n" 192 // Skip FaultIP 193 "addq $8, %%rsp\n" 194#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) 195 "popq %%rsp\n" 196 "movq %%r11, %%cr3\n" 197 "movq %%rsp, %%r11\n" 198#else 199 "popq %%r11\n" 200#ifdef CONFIG_KERNEL_SKIM_WINDOW 201 "movq (x64KSCurrentUserCR3), %%rsp\n" 202 "movq %%rsp, %%cr3\n" 203#endif /* CONFIG_KERNEL_SKIM_WINDOW */ 204#endif /* defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) */ 205 "sti\n" 206 "sysexitq\n" 207 : 208 : "c"(&cur_thread->tcbArch.tcbContext.registers[RAX]), 209 "D"(badge), 210 "S"(msgInfo), 211#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) 212 "r"(next_cr3_r11), 213#endif 214 [IF] "i"(FLAGS_IF) 215 : "memory" 216 ); 217 } else { 218 asm volatile( 219 // Set our stack pointer to the top of the tcb so we can efficiently pop 220 "movq %0, %%rsp\n" 221 "popq %%rax\n" 222 "popq %%rbx\n" 223 "popq %%rbp\n" 224 "popq %%r12\n" 225 "popq %%r13\n" 226 "popq %%r14\n" 227 "popq %%rdx\n" 228 "popq %%r10\n" 229 "popq %%r8\n" 230 "popq %%r9\n" 231 "popq %%r15\n" 232 //restore RFLAGS 233 "popq %%r11\n" 234 // Restore NextIP 235#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) 236 "popq %%rsp\n" 237 "movq %%rcx, %%cr3\n" 238 "movq %%rsp, %%rcx\n" 239#else 240 "popq %%rcx\n" 241#ifdef CONFIG_KERNEL_SKIM_WINDOW 242 "movq (x64KSCurrentUserCR3), %%rsp\n" 243 "movq %%rsp, %%cr3\n" 244#endif /* CONFIG_KERNEL_SKIM_WINDOW */ 245#endif /* defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) */ 246 // clear RSP to not leak information to the user 247 "xor %%rsp, %%rsp\n" 248 // More register but we can ignore and are done restoring 249 // enable interrupt disabled by sysenter 250 "sysretq\n" 251 : 252 : "r"(&cur_thread->tcbArch.tcbContext.registers[RAX]), 253 "D"(badge), 254 "S"(msgInfo) 255#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_KERNEL_SKIM_WINDOW) 256 , "c"(next_cr3) 257#endif 258 : "memory" 259 ); 260 } 261 UNREACHABLE(); 262} 263 264