1/*
2 * Copyright 2017, Data61
3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
4 * ABN 41 687 119 230.
5 *
6 * This software may be distributed and modified according to the terms of
7 * the BSD 2-Clause license. Note that NO WARRANTY is provided.
8 * See "LICENSE_BSD2.txt" for details.
9 *
10 * @TAG(DATA61_BSD)
11 */
12#include <autoconf.h>
13#include <rumprun/gen_config.h>
14#include <sel4/sel4.h>
15#include <sel4/helpers.h>
16#include <stdlib.h>
17#include <sel4utils/util.h>
18#include <allocman/vka.h>
19#include <allocman/bootstrap.h>
20#include <sel4platsupport/timer.h>
21#include <simple/simple_helpers.h>
22#include <sel4platsupport/io.h>
23#include <sel4utils/iommu_dma.h>
24#include <sel4utils/page_dma.h>
25#include <sel4utils/time_server/client.h>
26#include <sel4/benchmark_utilisation_types.h>
27#include <allocman/utspace/utspace.h>
28#include <sel4platsupport/arch/io.h>
29#include <platsupport/timer.h>
30#include <bmk-core/core.h>
31#include <bmk-core/pgalloc.h>
32#include <bmk-core/printf.h>
33#include <bmk-core/string.h>
34#include <bmk-core/sched.h>
35#include <bmk-core/mainthread.h>
36#include <sel4platsupport/platsupport.h>
37#include <bmk-core/types.h>
38#include <sel4/kernel.h>
39#include <sel4utils/stack.h>
40#include <rumprun-base/rumprun.h>
41#include <sys/mman.h>
42#include <sel4runtime.h>
43
44/* global static memory for init */
45static sel4utils_alloc_data_t alloc_data;
46
47/* dimensions of virtual memory for the allocator to use */
48#define ALLOCATOR_VIRTUAL_POOL_SIZE ((PAGE_SIZE_4K) * 4000)
49
50/* allocator static pool */
51#define ALLOCATOR_STATIC_POOL_SIZE ((PAGE_SIZE_4K) * 20)
52static char allocator_mem_pool[ALLOCATOR_STATIC_POOL_SIZE];
53
54/* Damn linker errors: This symbol is overrided by files_to_obj.sh it is included
55    here to prevent 'undefined reference to `_cpio_archive`' linker errors */
56char _cpio_archive[1];
57char _cpio_archive_end[1];
58
59/* Environment global data */
60struct env env = {
61    .spldepth = 0,
62    .mask_the_mask = 0,
63    .should_wakeup = 0
64};
65
66extern vspace_t *muslc_this_vspace;
67extern reservation_t muslc_brk_reservation;
68extern void *muslc_brk_reservation_start;
69sel4utils_res_t muslc_brk_reservation_memory;
70
71int rumpns_plat_mprotect(void *addr, size_t len, int prot);
72int rumpns_plat_mprotect(void *addr, size_t len, int prot)
73{
74
75    if (config_set(CONFIG_USE_LARGE_PAGES)) {
76        /* benchmarks use large pages, remapping them as small pages in order
77         * to mprotect them is not yet implemented */
78        return 0;
79    }
80
81    /* check addr is aligned */
82    uintptr_t uint_addr = (uintptr_t) addr;
83    if (uint_addr % BMK_PCPU_PAGE_SIZE != 0) {
84        return EINVAL;
85    }
86
87    /* align len to the nearest full page */
88    len = ROUND_UP(len, BMK_PCPU_PAGE_SIZE);
89
90    /* check len isn't too big */
91    if (uint_addr + len < uint_addr) {
92        return EINVAL;
93    }
94
95    reservation_t res = {0};
96    uintptr_t *cookies = NULL;
97    seL4_CPtr *caps = NULL;
98    int error = 0;
99
100    int num_pages = len / BMK_PCPU_PAGE_SIZE;
101    caps = calloc(num_pages, sizeof(seL4_CPtr));
102    if (caps == NULL) {
103        error = ENOMEM;
104        goto out;
105    }
106
107    cookies = calloc(num_pages, sizeof(uintptr_t));
108    if (cookies == NULL) {
109        error = ENOMEM;
110        goto out;
111    }
112
113    /* get all the caps and check the mapping is valid */
114    for (int i = 0; i < num_pages; i++) {
115        void *vaddr = (void *)(uint_addr + i * BMK_PCPU_PAGE_SIZE);
116        caps[i] = vspace_get_cap(&env.vspace, vaddr);
117        if (caps[i] == seL4_CapNull) {
118            error = ENOMEM;
119            goto out;
120        }
121        cookies[i] = vspace_get_cookie(&env.vspace, vaddr);
122    }
123
124    /* unmap them all */
125    vspace_unmap_pages(&env.vspace, addr, num_pages, BMK_PCPU_PAGE_SHIFT, VSPACE_PRESERVE);
126
127
128    /* remap with new rights */
129    seL4_CapRights_t rights = seL4_CapRights_new(false, false, prot & PROT_READ, prot & PROT_WRITE);
130    res = vspace_reserve_range_at(&env.vspace, addr, len, rights, true);
131    ZF_LOGF_IF(res.res == 0, "Failed to reserve range we just unmapped!");
132
133    error = vspace_map_pages_at_vaddr(&env.vspace, caps, cookies, addr, num_pages, BMK_PCPU_PAGE_SHIFT, res);
134    if (error) {
135        error = ENOMEM;
136    }
137
138out:
139    if (res.res) {
140        vspace_free_reservation(&env.vspace, res);
141    }
142
143    if (cookies) {
144        free(cookies);
145    }
146
147    if (caps) {
148        free(caps);
149    }
150
151    return error;
152}
153
154static void init_allocator(env_t env)
155{
156    UNUSED int error;
157    UNUSED reservation_t virtual_reservation;
158
159    /* initialise allocator */
160    allocman_t *allocator = bootstrap_use_current_1level(simple_get_cnode(&env->simple),
161                                                         simple_get_cnode_size_bits(&env->simple),
162                                                         simple_last_valid_cap(&env->simple) + 1,
163                                                         BIT(simple_get_cnode_size_bits(&env->simple)),
164                                                         ALLOCATOR_STATIC_POOL_SIZE,
165                                                         allocator_mem_pool);
166    ZF_LOGF_IF(allocator == NULL, "Failed to bootstrap allocator");
167    allocman_make_vka(&env->vka, allocator);
168
169    int num_regions = custom_get_num_regions(&env->custom_simple);
170    pmem_region_t *regions = allocman_mspace_alloc(allocator, sizeof(pmem_region_t) * num_regions, &error);
171    ZF_LOGF_IF(error, "allocman_mspace_alloc failed to allocate regions");
172    error = custom_get_region_list(&env->custom_simple, num_regions, regions);
173    ZF_LOGF_IF(num_regions != error, "calloc returned NULL");
174    allocman_add_simple_untypeds_with_regions(allocator, &env->simple, num_regions, regions);
175    allocman_mspace_free(allocator, regions, sizeof(pmem_region_t) * num_regions);
176
177    error = custom_simple_vspace_bootstrap_frames(&env->custom_simple, &env->vspace, &alloc_data, &env->vka);
178
179    error = sel4utils_reserve_range_no_alloc(&env->vspace, &muslc_brk_reservation_memory, 1048576, seL4_AllRights, 1,
180                                             &muslc_brk_reservation_start);
181    ZF_LOGF_IF(error, "Failed to reserve_range");
182    muslc_this_vspace = &env->vspace;
183    muslc_brk_reservation.res = &muslc_brk_reservation_memory;
184
185    /* switch the allocator to a virtual memory pool */
186    void *vaddr;
187    virtual_reservation = vspace_reserve_range(&env->vspace, ALLOCATOR_VIRTUAL_POOL_SIZE,
188                                               seL4_AllRights, 1, &vaddr);
189    ZF_LOGF_IF(virtual_reservation.res == 0, "Failed to switch allocator to virtual memory pool");
190
191    bootstrap_configure_virtual_pool(allocator, vaddr, ALLOCATOR_VIRTUAL_POOL_SIZE,
192                                     simple_get_pd(&env->simple));
193
194}
195
196static void provide_vmem(env_t env)
197{
198    void *osend;
199
200    bmk_core_init(BMK_THREAD_STACK_PAGE_ORDER);
201
202    vspace_new_pages_config_t config;
203    size_t rumprun_size = env->custom_simple.rumprun_memory_size;
204    int page_size_bits = BMK_PCPU_PAGE_SHIFT;
205    if (config_set(CONFIG_USE_LARGE_PAGES)) {
206        page_size_bits = sel4_page_size_bits_for_memory_region(rumprun_size);
207    }
208
209    env->rump_mapping_page_size_bits = page_size_bits;
210    env->rump_mapping_page_type = kobject_get_type(KOBJECT_FRAME, page_size_bits);
211    ZF_LOGW_IF(rumprun_size % BIT(page_size_bits) != 0, "Warning: Memory size is being truncated by: 0x%zx",
212               rumprun_size % BIT(page_size_bits));
213    size_t rumprun_pages = rumprun_size / BIT(page_size_bits);
214    ZF_LOGI("num pages %zd with size: %d bits\n", rumprun_pages, page_size_bits);
215    if (default_vspace_new_pages_config(rumprun_pages, page_size_bits, &config)) {
216        ZF_LOGF("Failed to create config");
217    }
218    if (vspace_new_pages_config_use_device_ut(true, &config)) {
219        ZF_LOGF("Failed to set device_ram");
220    }
221
222    osend = vspace_new_pages_with_config(&env->vspace, &config, seL4_AllRights);
223    ZF_LOGF_IF(osend == NULL, "vspace returned null");
224
225    ZF_LOGI("Starting paddr: %p\n", osend);
226    bmk_pgalloc_loadmem((uintptr_t) osend, (uintptr_t) osend + rumprun_size);
227
228    bmk_memsize = rumprun_size;
229}
230
231void rump_irq_handle(int intr, int soft_intr)
232{
233    sync_bin_sem_wait(&env.spl_semaphore);
234
235    ZF_LOGF_IF(env.spldepth != 0, "spldepth should be 0.  This thread should be blocked.");
236    if (env.should_wakeup != 0) {
237        seL4_Signal(env.custom_simple.timer_config.timer_ntfn);
238    }
239
240    env.mask_the_mask = 1;
241
242    isr(intr, soft_intr);
243    sync_bin_sem_post(&env.spl_semaphore);
244    env.mask_the_mask = 0;
245
246}
247
248static void wait_for_pci_interrupt(void *UNUSED _a, void *UNUSED _b, void *UNUSED _c)
249{
250    env.mask_the_mask = 0;
251    while (1) {
252        seL4_Word sender_badge;
253        seL4_Wait(env.pci_notification.cptr, &sender_badge);
254        rump_irq_handle(sender_badge, 0);
255    }
256}
257
258static int stdio_handler(void *UNUSED _a)
259{
260    if (env.custom_simple.get_char_handler) {
261        env.custom_simple.get_char_handler();
262    }
263    return 0;
264}
265
266static void wait_for_stdio_interrupt(void *UNUSED _a, void *UNUSED _b, void *UNUSED _c)
267{
268    int intr = bmk_isr_rumpkernel(stdio_handler, NULL, -1, SOFTWARE_EVENT);
269    while (1) {
270        seL4_Word sender_badge;
271        seL4_Wait(env.custom_simple.stdio_ep[0], &sender_badge);
272        rump_irq_handle(0, BIT(intr));
273    }
274}
275
276void preinit_rumprun(custom_simple_t *custom_simple)
277{
278    if (custom_simple != &env.custom_simple) {
279        env.custom_simple = *custom_simple;
280    }
281    if (&env.simple != env.custom_simple.simple) {
282        env.simple = *env.custom_simple.simple;
283    }
284
285    /* initialse cspace, vspace and untyped memory allocation */
286    init_allocator(&env);
287    cons_init();
288}
289
290int init_rumprun(custom_simple_t *custom_simple)
291{
292    int res = sel4platsupport_new_io_ops(&env.vspace, &env.vka, &env.simple, &env.io_ops);
293    ZF_LOGF_IF(res != 0, "sel4platsupport_new_io_ops failed");
294
295#ifdef CONFIG_ARCH_X86
296    res = sel4platsupport_get_io_port_ops(&env.io_ops.io_port_ops, &env.simple, &env.vka);
297    ZF_LOGF_IF(res != 0, "sel4platsupport_get_io_port_ops failed");
298#endif
299
300    if (is_ltimer(custom_simple)) {
301        sel4utils_rpc_ltimer_init(&custom_simple->timer_config.ltimer.ltimer, env.io_ops,
302                                  custom_simple->rpc_ep, TIMER_LABEL);
303    }
304
305    res = vka_alloc_notification(&env.vka, &env.pci_notification);
306    ZF_LOGF_IF(res != 0, "Failed to allocate notification object");
307    res = vka_alloc_notification(&env.vka, &env.spl_notification);
308    ZF_LOGF_IF(res != 0, "Failed to allocate notification object");
309    sync_bin_sem_init(&env.spl_semaphore, env.spl_notification.cptr, 1);
310
311    sel4utils_thread_config_t thread_config = thread_config_default(&env.simple,
312                                                                    simple_get_cnode(&env.simple), seL4_NilData, seL4_CapNull, custom_get_priority(&env.custom_simple));
313
314    res = sel4utils_configure_thread_config(&env.vka, &env.vspace, &env.vspace, thread_config, &env.pci_thread);
315    ZF_LOGF_IF(res != 0, "Configure thread failed");
316    if (!custom_simple->camkes) {
317        res = sel4utils_configure_thread_config(&env.vka, &env.vspace, &env.vspace, thread_config, &env.stdio_thread);
318        ZF_LOGF_IF(res != 0, "Configure thread failed");
319    }
320
321
322    seL4_CPtr auth = simple_get_tcb(&env.simple);
323    res = seL4_TCB_SetPriority(simple_get_tcb(&env.simple), auth, custom_get_priority(&env.custom_simple) - 1);
324    ZF_LOGF_IF(res != 0, "seL4_TCB_SetPriority thread failed");
325
326    NAME_THREAD(env.pci_thread.tcb.cptr, "pci thread");
327    res = sel4utils_start_thread(&env.pci_thread, wait_for_pci_interrupt, NULL, NULL,
328                                 1);
329
330    ZF_LOGF_IF(res != 0, "sel4utils_start_thread(wait_for_pci_interrupt) failed");
331
332#ifdef CONFIG_IOMMU
333    seL4_CPtr io_space = simple_init_cap(&env.simple, seL4_CapIOSpace);
334    res = sel4utils_make_iommu_dma_alloc(&env.vka, &env.vspace, &env.io_ops.dma_manager, 1, &io_space);
335    ZF_LOGF_IF(res != 0, "sel4utils_make_iommu_dma_alloc failed");
336#else
337    res = sel4utils_new_page_dma_alloc(&env.vka, &env.vspace, &env.io_ops.dma_manager);
338    ZF_LOGF_IF(res != 0, "sel4utils_new_page_dma_alloc failed");
339#endif
340
341    res = arch_init_clocks(&env);
342    ZF_LOGF_IF(res != 0, "failed to init clocks");
343
344    provide_vmem(&env);
345    intr_init();
346
347    if (!custom_simple->camkes) {
348        res = sel4utils_start_thread(&env.stdio_thread, wait_for_stdio_interrupt, NULL, NULL, 1);
349    }
350
351    struct rumprun_boot_config rumprun_config = {(char *)custom_get_cmdline(&env.custom_simple), CONFIG_RUMPRUN_TMPFS_NUM_MiB};
352
353    bmk_sched_startmain(bmk_mainthread, (void *) &rumprun_config);
354
355    return 0;
356}
357