1/*
2 * Copyright 2018, Data61
3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
4 * ABN 41 687 119 230.
5 *
6 * This software may be distributed and modified according to the terms of
7 * the GNU General Public License version 2. Note that NO WARRANTY is provided.
8 * See "LICENSE_GPLv2.txt" for details.
9 *
10 * @TAG(DATA61_GPL)
11 */
12
13/*
14 *
15 * Copyright 2016, 2017 Hesham Almatary, Data61/CSIRO <hesham.almatary@data61.csiro.au>
16 * Copyright 2015, 2016 Hesham Almatary <heshamelmatary@gmail.com>
17 */
18
19#ifndef __ARCH_FASTPATH_H
20#define __ARCH_FASTPATH_H
21
22#include <config.h>
23#include <util.h>
24#include <linker.h>
25#include <api/types.h>
26#include <api/syscall.h>
27#include <api/types.h>
28#include <smp/lock.h>
29#include <arch/machine/hardware.h>
30
31void slowpath(syscall_t syscall)
32NORETURN;
33
34void fastpath_call(word_t cptr, word_t r_msgInfo)
35NORETURN;
36
37void fastpath_reply_recv(word_t cptr, word_t r_msgInfo)
38NORETURN;
39
40/* Use macros to not break verification */
41#define endpoint_ptr_get_epQueue_tail_fp(ep_ptr) TCB_PTR(endpoint_ptr_get_epQueue_tail(ep_ptr))
42#define cap_vtable_cap_get_vspace_root_fp(vtable_cap) PTE_PTR(cap_page_table_cap_get_capPTBasePtr(vtable_cap))
43
44static inline void FORCE_INLINE
45switchToThread_fp(tcb_t *thread, pte_t *vroot, pte_t stored_hw_asid)
46{
47    asid_t asid = (asid_t)(stored_hw_asid.words[0]);
48
49    setVSpaceRoot(addrFromPPtr(vroot), asid);
50
51    setRegister(thread, tp, thread->tcbIPCBuffer);
52
53    NODE_STATE(ksCurThread) = thread;
54}
55
56static inline void
57mdb_node_ptr_mset_mdbNext_mdbRevocable_mdbFirstBadged(
58    mdb_node_t *node_ptr, word_t mdbNext,
59    word_t mdbRevocable, word_t mdbFirstBadged)
60{
61    node_ptr->words[1] = mdbNext | (mdbRevocable << 1) | mdbFirstBadged;
62}
63
64static inline void
65mdb_node_ptr_set_mdbPrev_np(mdb_node_t *node_ptr, word_t mdbPrev)
66{
67    node_ptr->words[0] = mdbPrev;
68}
69
70static inline bool_t
71isValidVTableRoot_fp(cap_t vspace_root_cap)
72{
73    return cap_capType_equals(vspace_root_cap, cap_page_table_cap) &&
74           cap_page_table_cap_get_capPTIsMapped(vspace_root_cap);
75}
76
77/* This is an accelerated check that msgLength, which appears
78   in the bottom of the msgInfo word, is <= 4 and that msgExtraCaps
79   which appears above it is zero. We are assuming that n_msgRegisters == 4
80   for this check to be useful. By masking out the bottom 3 bits, we are
81   really checking that n + 3 <= MASK(3), i.e. n + 3 <= 7 or n <= 4. */
82compile_assert (n_msgRegisters_eq_4, n_msgRegisters == 4)
83static inline int
84fastpath_mi_check(word_t msgInfo)
85{
86    return (msgInfo & MASK(seL4_MsgLengthBits + seL4_MsgExtraCapBits)) > 4;
87}
88
89static inline void
90fastpath_copy_mrs(word_t length, tcb_t *src, tcb_t *dest)
91{
92    word_t i;
93    register_t reg;
94
95    /* assuming that length < n_msgRegisters */
96    for (i = 0; i < length; i ++) {
97        /* assuming that the message registers simply increment */
98        reg = msgRegisters[0] + i;
99        setRegister(dest, reg, getRegister(src, reg));
100    }
101}
102
103static inline int
104fastpath_reply_cap_check(cap_t cap)
105{
106    return cap_capType_equals(cap, cap_reply_cap);
107}
108
109/** DONT_TRANSLATE */
110static inline void NORETURN
111fastpath_restore(word_t badge, word_t msgInfo, tcb_t *cur_thread)
112{
113    NODE_UNLOCK_IF_HELD;
114
115    c_exit_hook();
116
117    register word_t badge_reg asm("a0") = badge;
118    register word_t msgInfo_reg asm("a1") = msgInfo;
119    register word_t cur_thread_reg asm("t0") = TCB_REF(cur_thread);
120
121    asm volatile(
122        LOAD_S "  ra, (0*%[REGSIZE])(t0)  \n"
123        LOAD_S "  sp, (1*%[REGSIZE])(t0)  \n"
124        LOAD_S "  gp, (2*%[REGSIZE])(t0)  \n"
125        /* skip tp */
126        /* skip x5/t0 */
127        LOAD_S "  t2, (6*%[REGSIZE])(t0)  \n"
128        LOAD_S "  s0, (7*%[REGSIZE])(t0)  \n"
129        LOAD_S "  s1, (8*%[REGSIZE])(t0)  \n"
130        LOAD_S "  a2, (11*%[REGSIZE])(t0) \n"
131        LOAD_S "  a3, (12*%[REGSIZE])(t0) \n"
132        LOAD_S "  a4, (13*%[REGSIZE])(t0) \n"
133        LOAD_S "  a5, (14*%[REGSIZE])(t0) \n"
134        LOAD_S "  a6, (15*%[REGSIZE])(t0) \n"
135        LOAD_S "  a7, (16*%[REGSIZE])(t0) \n"
136        LOAD_S "  s2, (17*%[REGSIZE])(t0) \n"
137        LOAD_S "  s3, 18*%[REGSIZE](t0) \n"
138        LOAD_S "  s4, 19*%[REGSIZE](t0) \n"
139        LOAD_S "  s5, 20*%[REGSIZE](t0) \n"
140        LOAD_S "  s6, 21*%[REGSIZE](t0) \n"
141        LOAD_S "  s7, 22*%[REGSIZE](t0) \n"
142        LOAD_S "  s8, 23*%[REGSIZE](t0) \n"
143        LOAD_S "  s9, 24*%[REGSIZE](t0) \n"
144        LOAD_S "  s10, 25*%[REGSIZE](t0)\n"
145        LOAD_S "  s11, 26*%[REGSIZE](t0)\n"
146        LOAD_S "  t3, 27*%[REGSIZE](t0) \n"
147        LOAD_S "  t4, 28*%[REGSIZE](t0) \n"
148        LOAD_S "  t5, 29*%[REGSIZE](t0) \n"
149        LOAD_S "  t6, 30*%[REGSIZE](t0) \n"
150        /* Get next restored tp */
151        LOAD_S "  t1, 3*%[REGSIZE](t0)  \n"
152        /* get restored tp */
153        "add tp, t1, x0  \n"
154        /* get sepc */
155        LOAD_S "  t1, 34*%[REGSIZE](t0)\n"
156        "csrw sepc, t1  \n"
157
158        /* Write back sscratch with cur_thread_reg to get it back on the next trap entry */
159        "csrw sscratch, t0\n"
160
161        LOAD_S "  t1, 32*%[REGSIZE](t0) \n"
162        "csrw sstatus, t1\n"
163
164        LOAD_S "  t1, (5*%[REGSIZE])(t0) \n"
165        LOAD_S "  t0, (4*%[REGSIZE])(t0) \n"
166        "sret"
167        : /* no output */
168        : "r" (cur_thread_reg),
169        [REGSIZE] "i" (sizeof(word_t)),
170        "r" (badge_reg),
171        "r" (msgInfo_reg)
172        : "memory"
173    );
174
175    UNREACHABLE();
176}
177#endif
178