1/* 2 * Copyright 2017, Data61 3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO) 4 * ABN 41 687 119 230. 5 * 6 * This software may be distributed and modified according to the terms of 7 * the BSD 2-Clause license. Note that NO WARRANTY is provided. 8 * See "LICENSE_BSD2.txt" for details. 9 * 10 * @TAG(DATA61_BSD) 11 */ 12#include <autoconf.h> 13#include <rumprun/gen_config.h> 14#include <sel4/sel4.h> 15#include <sel4/helpers.h> 16#include <stdlib.h> 17#include <sel4utils/util.h> 18#include <allocman/vka.h> 19#include <allocman/bootstrap.h> 20#include <sel4platsupport/timer.h> 21#include <simple/simple_helpers.h> 22#include <sel4platsupport/io.h> 23#include <sel4utils/iommu_dma.h> 24#include <sel4utils/page_dma.h> 25#include <sel4utils/time_server/client.h> 26#include <sel4/benchmark_utilisation_types.h> 27#include <allocman/utspace/utspace.h> 28#include <sel4platsupport/arch/io.h> 29#include <platsupport/timer.h> 30#include <bmk-core/core.h> 31#include <bmk-core/pgalloc.h> 32#include <bmk-core/printf.h> 33#include <bmk-core/string.h> 34#include <bmk-core/sched.h> 35#include <bmk-core/mainthread.h> 36#include <sel4platsupport/platsupport.h> 37#include <bmk-core/types.h> 38#include <sel4/kernel.h> 39#include <sel4utils/stack.h> 40#include <rumprun-base/rumprun.h> 41#include <sys/mman.h> 42#include <sel4runtime.h> 43 44/* global static memory for init */ 45static sel4utils_alloc_data_t alloc_data; 46 47/* dimensions of virtual memory for the allocator to use */ 48#define ALLOCATOR_VIRTUAL_POOL_SIZE ((PAGE_SIZE_4K) * 4000) 49 50/* allocator static pool */ 51#define ALLOCATOR_STATIC_POOL_SIZE ((PAGE_SIZE_4K) * 20) 52static char allocator_mem_pool[ALLOCATOR_STATIC_POOL_SIZE]; 53 54/* Damn linker errors: This symbol is overrided by files_to_obj.sh it is included 55 here to prevent 'undefined reference to `_cpio_archive`' linker errors */ 56char _cpio_archive[1]; 57char _cpio_archive_end[1]; 58 59/* Environment global data */ 60struct env env = { 61 .spldepth = 0, 62 .mask_the_mask = 0, 63 .should_wakeup = 0 64}; 65 66extern vspace_t *muslc_this_vspace; 67extern reservation_t muslc_brk_reservation; 68extern void *muslc_brk_reservation_start; 69sel4utils_res_t muslc_brk_reservation_memory; 70 71int rumpns_plat_mprotect(void *addr, size_t len, int prot); 72int rumpns_plat_mprotect(void *addr, size_t len, int prot) 73{ 74 75 if (config_set(CONFIG_USE_LARGE_PAGES)) { 76 /* benchmarks use large pages, remapping them as small pages in order 77 * to mprotect them is not yet implemented */ 78 return 0; 79 } 80 81 /* check addr is aligned */ 82 uintptr_t uint_addr = (uintptr_t) addr; 83 if (uint_addr % BMK_PCPU_PAGE_SIZE != 0) { 84 return EINVAL; 85 } 86 87 /* align len to the nearest full page */ 88 len = ROUND_UP(len, BMK_PCPU_PAGE_SIZE); 89 90 /* check len isn't too big */ 91 if (uint_addr + len < uint_addr) { 92 return EINVAL; 93 } 94 95 reservation_t res = {0}; 96 uintptr_t *cookies = NULL; 97 seL4_CPtr *caps = NULL; 98 int error = 0; 99 100 int num_pages = len / BMK_PCPU_PAGE_SIZE; 101 caps = calloc(num_pages, sizeof(seL4_CPtr)); 102 if (caps == NULL) { 103 error = ENOMEM; 104 goto out; 105 } 106 107 cookies = calloc(num_pages, sizeof(uintptr_t)); 108 if (cookies == NULL) { 109 error = ENOMEM; 110 goto out; 111 } 112 113 /* get all the caps and check the mapping is valid */ 114 for (int i = 0; i < num_pages; i++) { 115 void *vaddr = (void *)(uint_addr + i * BMK_PCPU_PAGE_SIZE); 116 caps[i] = vspace_get_cap(&env.vspace, vaddr); 117 if (caps[i] == seL4_CapNull) { 118 error = ENOMEM; 119 goto out; 120 } 121 cookies[i] = vspace_get_cookie(&env.vspace, vaddr); 122 } 123 124 /* unmap them all */ 125 vspace_unmap_pages(&env.vspace, addr, num_pages, BMK_PCPU_PAGE_SHIFT, VSPACE_PRESERVE); 126 127 128 /* remap with new rights */ 129 seL4_CapRights_t rights = seL4_CapRights_new(false, false, prot & PROT_READ, prot & PROT_WRITE); 130 res = vspace_reserve_range_at(&env.vspace, addr, len, rights, true); 131 ZF_LOGF_IF(res.res == 0, "Failed to reserve range we just unmapped!"); 132 133 error = vspace_map_pages_at_vaddr(&env.vspace, caps, cookies, addr, num_pages, BMK_PCPU_PAGE_SHIFT, res); 134 if (error) { 135 error = ENOMEM; 136 } 137 138out: 139 if (res.res) { 140 vspace_free_reservation(&env.vspace, res); 141 } 142 143 if (cookies) { 144 free(cookies); 145 } 146 147 if (caps) { 148 free(caps); 149 } 150 151 return error; 152} 153 154static void init_allocator(env_t env) 155{ 156 UNUSED int error; 157 UNUSED reservation_t virtual_reservation; 158 159 /* initialise allocator */ 160 allocman_t *allocator = bootstrap_use_current_1level(simple_get_cnode(&env->simple), 161 simple_get_cnode_size_bits(&env->simple), 162 simple_last_valid_cap(&env->simple) + 1, 163 BIT(simple_get_cnode_size_bits(&env->simple)), 164 ALLOCATOR_STATIC_POOL_SIZE, 165 allocator_mem_pool); 166 ZF_LOGF_IF(allocator == NULL, "Failed to bootstrap allocator"); 167 allocman_make_vka(&env->vka, allocator); 168 169 int num_regions = custom_get_num_regions(&env->custom_simple); 170 pmem_region_t *regions = allocman_mspace_alloc(allocator, sizeof(pmem_region_t) * num_regions, &error); 171 ZF_LOGF_IF(error, "allocman_mspace_alloc failed to allocate regions"); 172 error = custom_get_region_list(&env->custom_simple, num_regions, regions); 173 ZF_LOGF_IF(num_regions != error, "calloc returned NULL"); 174 allocman_add_simple_untypeds_with_regions(allocator, &env->simple, num_regions, regions); 175 allocman_mspace_free(allocator, regions, sizeof(pmem_region_t) * num_regions); 176 177 error = custom_simple_vspace_bootstrap_frames(&env->custom_simple, &env->vspace, &alloc_data, &env->vka); 178 179 error = sel4utils_reserve_range_no_alloc(&env->vspace, &muslc_brk_reservation_memory, 1048576, seL4_AllRights, 1, 180 &muslc_brk_reservation_start); 181 ZF_LOGF_IF(error, "Failed to reserve_range"); 182 muslc_this_vspace = &env->vspace; 183 muslc_brk_reservation.res = &muslc_brk_reservation_memory; 184 185 /* switch the allocator to a virtual memory pool */ 186 void *vaddr; 187 virtual_reservation = vspace_reserve_range(&env->vspace, ALLOCATOR_VIRTUAL_POOL_SIZE, 188 seL4_AllRights, 1, &vaddr); 189 ZF_LOGF_IF(virtual_reservation.res == 0, "Failed to switch allocator to virtual memory pool"); 190 191 bootstrap_configure_virtual_pool(allocator, vaddr, ALLOCATOR_VIRTUAL_POOL_SIZE, 192 simple_get_pd(&env->simple)); 193 194} 195 196static void provide_vmem(env_t env) 197{ 198 void *osend; 199 200 bmk_core_init(BMK_THREAD_STACK_PAGE_ORDER); 201 202 vspace_new_pages_config_t config; 203 size_t rumprun_size = env->custom_simple.rumprun_memory_size; 204 int page_size_bits = BMK_PCPU_PAGE_SHIFT; 205 if (config_set(CONFIG_USE_LARGE_PAGES)) { 206 page_size_bits = sel4_page_size_bits_for_memory_region(rumprun_size); 207 } 208 209 env->rump_mapping_page_size_bits = page_size_bits; 210 env->rump_mapping_page_type = kobject_get_type(KOBJECT_FRAME, page_size_bits); 211 ZF_LOGW_IF(rumprun_size % BIT(page_size_bits) != 0, "Warning: Memory size is being truncated by: 0x%zx", 212 rumprun_size % BIT(page_size_bits)); 213 size_t rumprun_pages = rumprun_size / BIT(page_size_bits); 214 ZF_LOGI("num pages %zd with size: %d bits\n", rumprun_pages, page_size_bits); 215 if (default_vspace_new_pages_config(rumprun_pages, page_size_bits, &config)) { 216 ZF_LOGF("Failed to create config"); 217 } 218 if (vspace_new_pages_config_use_device_ut(true, &config)) { 219 ZF_LOGF("Failed to set device_ram"); 220 } 221 222 osend = vspace_new_pages_with_config(&env->vspace, &config, seL4_AllRights); 223 ZF_LOGF_IF(osend == NULL, "vspace returned null"); 224 225 ZF_LOGI("Starting paddr: %p\n", osend); 226 bmk_pgalloc_loadmem((uintptr_t) osend, (uintptr_t) osend + rumprun_size); 227 228 bmk_memsize = rumprun_size; 229} 230 231void rump_irq_handle(int intr, int soft_intr) 232{ 233 sync_bin_sem_wait(&env.spl_semaphore); 234 235 ZF_LOGF_IF(env.spldepth != 0, "spldepth should be 0. This thread should be blocked."); 236 if (env.should_wakeup != 0) { 237 seL4_Signal(env.custom_simple.timer_config.timer_ntfn); 238 } 239 240 env.mask_the_mask = 1; 241 242 isr(intr, soft_intr); 243 sync_bin_sem_post(&env.spl_semaphore); 244 env.mask_the_mask = 0; 245 246} 247 248static void wait_for_pci_interrupt(void *UNUSED _a, void *UNUSED _b, void *UNUSED _c) 249{ 250 env.mask_the_mask = 0; 251 while (1) { 252 seL4_Word sender_badge; 253 seL4_Wait(env.pci_notification.cptr, &sender_badge); 254 rump_irq_handle(sender_badge, 0); 255 } 256} 257 258static int stdio_handler(void *UNUSED _a) 259{ 260 if (env.custom_simple.get_char_handler) { 261 env.custom_simple.get_char_handler(); 262 } 263 return 0; 264} 265 266static void wait_for_stdio_interrupt(void *UNUSED _a, void *UNUSED _b, void *UNUSED _c) 267{ 268 int intr = bmk_isr_rumpkernel(stdio_handler, NULL, -1, SOFTWARE_EVENT); 269 while (1) { 270 seL4_Word sender_badge; 271 seL4_Wait(env.custom_simple.stdio_ep[0], &sender_badge); 272 rump_irq_handle(0, BIT(intr)); 273 } 274} 275 276void preinit_rumprun(custom_simple_t *custom_simple) 277{ 278 if (custom_simple != &env.custom_simple) { 279 env.custom_simple = *custom_simple; 280 } 281 if (&env.simple != env.custom_simple.simple) { 282 env.simple = *env.custom_simple.simple; 283 } 284 285 /* initialse cspace, vspace and untyped memory allocation */ 286 init_allocator(&env); 287 cons_init(); 288} 289 290int init_rumprun(custom_simple_t *custom_simple) 291{ 292 int res = sel4platsupport_new_io_ops(&env.vspace, &env.vka, &env.simple, &env.io_ops); 293 ZF_LOGF_IF(res != 0, "sel4platsupport_new_io_ops failed"); 294 295#ifdef CONFIG_ARCH_X86 296 res = sel4platsupport_get_io_port_ops(&env.io_ops.io_port_ops, &env.simple, &env.vka); 297 ZF_LOGF_IF(res != 0, "sel4platsupport_get_io_port_ops failed"); 298#endif 299 300 if (is_ltimer(custom_simple)) { 301 sel4utils_rpc_ltimer_init(&custom_simple->timer_config.ltimer.ltimer, env.io_ops, 302 custom_simple->rpc_ep, TIMER_LABEL); 303 } 304 305 res = vka_alloc_notification(&env.vka, &env.pci_notification); 306 ZF_LOGF_IF(res != 0, "Failed to allocate notification object"); 307 res = vka_alloc_notification(&env.vka, &env.spl_notification); 308 ZF_LOGF_IF(res != 0, "Failed to allocate notification object"); 309 sync_bin_sem_init(&env.spl_semaphore, env.spl_notification.cptr, 1); 310 311 sel4utils_thread_config_t thread_config = thread_config_default(&env.simple, 312 simple_get_cnode(&env.simple), seL4_NilData, seL4_CapNull, custom_get_priority(&env.custom_simple)); 313 314 res = sel4utils_configure_thread_config(&env.vka, &env.vspace, &env.vspace, thread_config, &env.pci_thread); 315 ZF_LOGF_IF(res != 0, "Configure thread failed"); 316 if (!custom_simple->camkes) { 317 res = sel4utils_configure_thread_config(&env.vka, &env.vspace, &env.vspace, thread_config, &env.stdio_thread); 318 ZF_LOGF_IF(res != 0, "Configure thread failed"); 319 } 320 321 322 seL4_CPtr auth = simple_get_tcb(&env.simple); 323 res = seL4_TCB_SetPriority(simple_get_tcb(&env.simple), auth, custom_get_priority(&env.custom_simple) - 1); 324 ZF_LOGF_IF(res != 0, "seL4_TCB_SetPriority thread failed"); 325 326 NAME_THREAD(env.pci_thread.tcb.cptr, "pci thread"); 327 res = sel4utils_start_thread(&env.pci_thread, wait_for_pci_interrupt, NULL, NULL, 328 1); 329 330 ZF_LOGF_IF(res != 0, "sel4utils_start_thread(wait_for_pci_interrupt) failed"); 331 332#ifdef CONFIG_IOMMU 333 seL4_CPtr io_space = simple_init_cap(&env.simple, seL4_CapIOSpace); 334 res = sel4utils_make_iommu_dma_alloc(&env.vka, &env.vspace, &env.io_ops.dma_manager, 1, &io_space); 335 ZF_LOGF_IF(res != 0, "sel4utils_make_iommu_dma_alloc failed"); 336#else 337 res = sel4utils_new_page_dma_alloc(&env.vka, &env.vspace, &env.io_ops.dma_manager); 338 ZF_LOGF_IF(res != 0, "sel4utils_new_page_dma_alloc failed"); 339#endif 340 341 res = arch_init_clocks(&env); 342 ZF_LOGF_IF(res != 0, "failed to init clocks"); 343 344 provide_vmem(&env); 345 intr_init(); 346 347 if (!custom_simple->camkes) { 348 res = sel4utils_start_thread(&env.stdio_thread, wait_for_stdio_interrupt, NULL, NULL, 1); 349 } 350 351 struct rumprun_boot_config rumprun_config = {(char *)custom_get_cmdline(&env.custom_simple), CONFIG_RUMPRUN_TMPFS_NUM_MiB}; 352 353 bmk_sched_startmain(bmk_mainthread, (void *) &rumprun_config); 354 355 return 0; 356} 357