1/* 2 * Copyright 2018, Data61 3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO) 4 * ABN 41 687 119 230. 5 * 6 * This software may be distributed and modified according to the terms of 7 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 8 * See "LICENSE_GPLv2.txt" for details. 9 * 10 * @TAG(DATA61_GPL) 11 */ 12 13/* 14 * 15 * Copyright 2016, 2017 Hesham Almatary, Data61/CSIRO <hesham.almatary@data61.csiro.au> 16 * Copyright 2015, 2016 Hesham Almatary <heshamelmatary@gmail.com> 17 */ 18 19#ifndef __ARCH_FASTPATH_H 20#define __ARCH_FASTPATH_H 21 22#include <config.h> 23#include <util.h> 24#include <linker.h> 25#include <api/types.h> 26#include <api/syscall.h> 27#include <api/types.h> 28#include <smp/lock.h> 29#include <arch/machine/hardware.h> 30 31void slowpath(syscall_t syscall) 32NORETURN; 33 34void fastpath_call(word_t cptr, word_t r_msgInfo) 35NORETURN; 36 37void fastpath_reply_recv(word_t cptr, word_t r_msgInfo) 38NORETURN; 39 40/* Use macros to not break verification */ 41#define endpoint_ptr_get_epQueue_tail_fp(ep_ptr) TCB_PTR(endpoint_ptr_get_epQueue_tail(ep_ptr)) 42#define cap_vtable_cap_get_vspace_root_fp(vtable_cap) PTE_PTR(cap_page_table_cap_get_capPTBasePtr(vtable_cap)) 43 44static inline void FORCE_INLINE 45switchToThread_fp(tcb_t *thread, pte_t *vroot, pte_t stored_hw_asid) 46{ 47 asid_t asid = (asid_t)(stored_hw_asid.words[0]); 48 49 setVSpaceRoot(addrFromPPtr(vroot), asid); 50 51 setRegister(thread, tp, thread->tcbIPCBuffer); 52 53 NODE_STATE(ksCurThread) = thread; 54} 55 56static inline void 57mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged( 58 mdb_node_t *node_ptr, word_t mdbNext, 59 word_t mdbRevocable, word_t mdbFirstBadged) 60{ 61 node_ptr->words[1] = mdbNext | (mdbRevocable << 1) | mdbFirstBadged; 62} 63 64static inline void 65mdb_node_ptr_set_mdbPrev_np(mdb_node_t *node_ptr, word_t mdbPrev) 66{ 67 node_ptr->words[0] = mdbPrev; 68} 69 70static inline bool_t 71isValidVTableRoot_fp(cap_t vspace_root_cap) 72{ 73 return cap_capType_equals(vspace_root_cap, cap_page_table_cap) && 74 cap_page_table_cap_get_capPTIsMapped(vspace_root_cap); 75} 76 77/* This is an accelerated check that msgLength, which appears 78 in the bottom of the msgInfo word, is <= 4 and that msgExtraCaps 79 which appears above it is zero. We are assuming that n_msgRegisters == 4 80 for this check to be useful. By masking out the bottom 3 bits, we are 81 really checking that n + 3 <= MASK(3), i.e. n + 3 <= 7 or n <= 4. */ 82compile_assert (n_msgRegisters_eq_4, n_msgRegisters == 4) 83static inline int 84fastpath_mi_check(word_t msgInfo) 85{ 86 return (msgInfo & MASK(seL4_MsgLengthBits + seL4_MsgExtraCapBits)) > 4; 87} 88 89static inline void 90fastpath_copy_mrs(word_t length, tcb_t *src, tcb_t *dest) 91{ 92 word_t i; 93 register_t reg; 94 95 /* assuming that length < n_msgRegisters */ 96 for (i = 0; i < length; i ++) { 97 /* assuming that the message registers simply increment */ 98 reg = msgRegisters[0] + i; 99 setRegister(dest, reg, getRegister(src, reg)); 100 } 101} 102 103static inline int 104fastpath_reply_cap_check(cap_t cap) 105{ 106 return cap_capType_equals(cap, cap_reply_cap); 107} 108 109/** DONT_TRANSLATE */ 110static inline void NORETURN 111fastpath_restore(word_t badge, word_t msgInfo, tcb_t *cur_thread) 112{ 113 NODE_UNLOCK_IF_HELD; 114 115 c_exit_hook(); 116 117 register word_t badge_reg asm("a0") = badge; 118 register word_t msgInfo_reg asm("a1") = msgInfo; 119 register word_t cur_thread_reg asm("t0") = TCB_REF(cur_thread); 120 121 asm volatile( 122 LOAD_S " ra, (0*%[REGSIZE])(t0) \n" 123 LOAD_S " sp, (1*%[REGSIZE])(t0) \n" 124 LOAD_S " gp, (2*%[REGSIZE])(t0) \n" 125 /* skip tp */ 126 /* skip x5/t0 */ 127 LOAD_S " t2, (6*%[REGSIZE])(t0) \n" 128 LOAD_S " s0, (7*%[REGSIZE])(t0) \n" 129 LOAD_S " s1, (8*%[REGSIZE])(t0) \n" 130 LOAD_S " a2, (11*%[REGSIZE])(t0) \n" 131 LOAD_S " a3, (12*%[REGSIZE])(t0) \n" 132 LOAD_S " a4, (13*%[REGSIZE])(t0) \n" 133 LOAD_S " a5, (14*%[REGSIZE])(t0) \n" 134 LOAD_S " a6, (15*%[REGSIZE])(t0) \n" 135 LOAD_S " a7, (16*%[REGSIZE])(t0) \n" 136 LOAD_S " s2, (17*%[REGSIZE])(t0) \n" 137 LOAD_S " s3, 18*%[REGSIZE](t0) \n" 138 LOAD_S " s4, 19*%[REGSIZE](t0) \n" 139 LOAD_S " s5, 20*%[REGSIZE](t0) \n" 140 LOAD_S " s6, 21*%[REGSIZE](t0) \n" 141 LOAD_S " s7, 22*%[REGSIZE](t0) \n" 142 LOAD_S " s8, 23*%[REGSIZE](t0) \n" 143 LOAD_S " s9, 24*%[REGSIZE](t0) \n" 144 LOAD_S " s10, 25*%[REGSIZE](t0)\n" 145 LOAD_S " s11, 26*%[REGSIZE](t0)\n" 146 LOAD_S " t3, 27*%[REGSIZE](t0) \n" 147 LOAD_S " t4, 28*%[REGSIZE](t0) \n" 148 LOAD_S " t5, 29*%[REGSIZE](t0) \n" 149 LOAD_S " t6, 30*%[REGSIZE](t0) \n" 150 /* Get next restored tp */ 151 LOAD_S " t1, 3*%[REGSIZE](t0) \n" 152 /* get restored tp */ 153 "add tp, t1, x0 \n" 154 /* get sepc */ 155 LOAD_S " t1, 34*%[REGSIZE](t0)\n" 156 "csrw sepc, t1 \n" 157 158 /* Write back sscratch with cur_thread_reg to get it back on the next trap entry */ 159 "csrw sscratch, t0\n" 160 161 LOAD_S " t1, 32*%[REGSIZE](t0) \n" 162 "csrw sstatus, t1\n" 163 164 LOAD_S " t1, (5*%[REGSIZE])(t0) \n" 165 LOAD_S " t0, (4*%[REGSIZE])(t0) \n" 166 "sret" 167 : /* no output */ 168 : "r" (cur_thread_reg), 169 [REGSIZE] "i" (sizeof(word_t)), 170 "r" (badge_reg), 171 "r" (msgInfo_reg) 172 : "memory" 173 ); 174 175 UNREACHABLE(); 176} 177#endif 178