1// See LICENSE for license details. 2 3#include "mtrap.h" 4#include "atomic.h" 5#include "vm.h" 6#include "fp_emulation.h" 7#include "fdt.h" 8#include "uart.h" 9#include "uart16550.h" 10#include "uart16750.h" 11#include "finisher.h" 12#include "disabled_hart_mask.h" 13#include "htif.h" 14#include <string.h> 15#include <limits.h> 16 17pte_t* root_page_table; 18uintptr_t mem_size; 19volatile uint64_t* mtime; 20volatile uint32_t* plic_priorities; 21size_t plic_ndevs; 22void* kernel_start; 23void* kernel_end; 24 25static void mstatus_init() 26{ 27 // Enable FPU 28 if (supports_extension('D') || supports_extension('F')) 29 write_csr(mstatus, MSTATUS_FS); 30 31 // Enable user/supervisor use of perf counters 32 if (supports_extension('S')) 33 write_csr(scounteren, -1); 34 write_csr(mcounteren, -1); 35 36 // Enable software interrupts 37 write_csr(mie, MIP_MSIP); 38 39 // Disable paging 40 if (supports_extension('S')) 41 write_csr(sptbr, 0); 42} 43 44// send S-mode interrupts and most exceptions straight to S-mode 45static void delegate_traps() 46{ 47 if (!supports_extension('S')) 48 return; 49 50 uintptr_t interrupts = MIP_SSIP | MIP_STIP | MIP_SEIP; 51 uintptr_t exceptions = 52 (1U << CAUSE_MISALIGNED_FETCH) | 53 (1U << CAUSE_FETCH_PAGE_FAULT) | 54 (1U << CAUSE_BREAKPOINT) | 55 (1U << CAUSE_LOAD_PAGE_FAULT) | 56 (1U << CAUSE_STORE_PAGE_FAULT) | 57 (1U << CAUSE_USER_ECALL); 58 59 if (supports_extension('H')) { 60 exceptions |= (1 << CAUSE_HYPERVISOR_ECALL); 61 } 62 63 write_csr(mideleg, interrupts); 64 write_csr(medeleg, exceptions); 65 assert(read_csr(mideleg) == interrupts); 66 assert(read_csr(medeleg) == exceptions); 67} 68 69static void fp_init() 70{ 71 if (!supports_extension('D') && !supports_extension('F')) 72 return; 73 74 assert(read_csr(mstatus) & MSTATUS_FS); 75 76#ifdef __riscv_flen 77 for (int i = 0; i < 32; i++) 78 init_fp_reg(i); 79 write_csr(fcsr, 0); 80#else 81 uintptr_t fd_mask = (1 << ('F' - 'A')) | (1 << ('D' - 'A')); 82 clear_csr(misa, fd_mask); 83 assert(!(read_csr(misa) & fd_mask)); 84#endif 85} 86 87hls_t* hls_init(uintptr_t id) 88{ 89 hls_t* hls = OTHER_HLS(id); 90 memset(hls, 0, sizeof(*hls)); 91 return hls; 92} 93 94static void memory_init() 95{ 96 mem_size = mem_size / MEGAPAGE_SIZE * MEGAPAGE_SIZE; 97} 98 99static void hart_init() 100{ 101 mstatus_init(); 102 fp_init(); 103#ifndef BBL_BOOT_MACHINE 104 delegate_traps(); 105#endif /* BBL_BOOT_MACHINE */ 106 setup_pmp(); 107} 108 109static void plic_init() 110{ 111 for (size_t i = 1; i <= plic_ndevs; i++) 112 plic_priorities[i] = 1; 113} 114 115static void prci_test() 116{ 117 assert(!(read_csr(mip) & MIP_MSIP)); 118 *HLS()->ipi = 1; 119 assert(read_csr(mip) & MIP_MSIP); 120 *HLS()->ipi = 0; 121 122 assert(!(read_csr(mip) & MIP_MTIP)); 123 *HLS()->timecmp = 0; 124 assert(read_csr(mip) & MIP_MTIP); 125 *HLS()->timecmp = -1ULL; 126} 127 128static void hart_plic_init() 129{ 130 // clear pending interrupts 131 *HLS()->ipi = 0; 132 *HLS()->timecmp = -1ULL; 133 write_csr(mip, 0); 134 135 if (!plic_ndevs) 136 return; 137 138 size_t ie_words = (plic_ndevs + 8 * sizeof(uintptr_t) - 1) / 139 (8 * sizeof(uintptr_t)); 140 for (size_t i = 0; i < ie_words; i++) { 141 if (HLS()->plic_s_ie) { 142 // Supervisor not always present 143 HLS()->plic_s_ie[i] = ULONG_MAX; 144 } 145 } 146 *HLS()->plic_m_thresh = 1; 147 if (HLS()->plic_s_thresh) { 148 // Supervisor not always present 149 *HLS()->plic_s_thresh = 0; 150 } 151} 152 153static void wake_harts() 154{ 155 for (int hart = 0; hart < MAX_HARTS; ++hart) 156 if ((((~disabled_hart_mask & hart_mask) >> hart) & 1)) 157 *OTHER_HLS(hart)->ipi = 1; // wakeup the hart 158} 159 160void init_first_hart(uintptr_t hartid, uintptr_t dtb) 161{ 162 // Confirm console as early as possible 163 query_uart(dtb); 164 query_uart16550(dtb); 165 query_uart16750(dtb); 166 query_htif(dtb); 167 printm("bbl loader\r\n"); 168 169 hart_init(); 170 hls_init(0); // this might get called again from parse_config_string 171 172 // Find the power button early as well so die() works 173 query_finisher(dtb); 174 175 query_mem(dtb); 176 query_harts(dtb); 177 query_clint(dtb); 178 query_plic(dtb); 179 query_chosen(dtb); 180 181 wake_harts(); 182 183 plic_init(); 184 hart_plic_init(); 185 //prci_test(); 186 memory_init(); 187 boot_loader(dtb); 188} 189 190void init_other_hart(uintptr_t hartid, uintptr_t dtb) 191{ 192 hart_init(); 193 hart_plic_init(); 194 boot_other_hart(dtb); 195} 196 197void setup_pmp(void) 198{ 199 // Set up a PMP to permit access to all of memory. 200 // Ignore the illegal-instruction trap if PMPs aren't supported. 201 uintptr_t pmpc = PMP_NAPOT | PMP_R | PMP_W | PMP_X; 202 asm volatile ("la t0, 1f\n\t" 203 "csrrw t0, mtvec, t0\n\t" 204 "csrw pmpaddr0, %1\n\t" 205 "csrw pmpcfg0, %0\n\t" 206 ".align 2\n\t" 207 "1: csrw mtvec, t0" 208 : : "r" (pmpc), "r" (-1UL) : "t0"); 209} 210 211void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t arg0, uintptr_t arg1) 212{ 213 uintptr_t mstatus = read_csr(mstatus); 214 mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); 215 mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); 216 write_csr(mstatus, mstatus); 217 write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); 218#ifndef __riscv_flen 219 uintptr_t *p_fcsr = MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE; // the x0's save slot 220 *p_fcsr = 0; 221#endif 222 write_csr(mepc, fn); 223 224 register uintptr_t a0 asm ("a0") = arg0; 225 register uintptr_t a1 asm ("a1") = arg1; 226 asm volatile ("mret" : : "r" (a0), "r" (a1)); 227 __builtin_unreachable(); 228} 229 230void enter_machine_mode(void (*fn)(uintptr_t, uintptr_t), uintptr_t arg0, uintptr_t arg1) 231{ 232 uintptr_t mstatus = read_csr(mstatus); 233 mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); 234 write_csr(mstatus, mstatus); 235 write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); 236 237 /* Jump to the payload's entry point */ 238 fn(arg0, arg1); 239 240 __builtin_unreachable(); 241} 242