1/*
2 * Copyright 2014, General Dynamics C4 Systems
3 *
4 * This software may be distributed and modified according to the terms of
5 * the GNU General Public License version 2. Note that NO WARRANTY is provided.
6 * See "LICENSE_GPLv2.txt" for details.
7 *
8 * @TAG(GD_GPL)
9 */
10
11#include <config.h>
12#include <assert.h>
13#include <kernel/boot.h>
14#include <machine/io.h>
15#include <model/statedata.h>
16#include <object/interrupt.h>
17#include <arch/machine.h>
18#include <arch/kernel/boot.h>
19#include <arch/kernel/vspace.h>
20#include <arch/benchmark.h>
21#include <arch/user_access.h>
22#include <arch/object/iospace.h>
23#include <linker.h>
24#include <plat/machine/hardware.h>
25#include <machine.h>
26#include <plat/machine/timer.h>
27#include <arch/machine/timer.h>
28#include <arch/machine/fpu.h>
29#include <arch/machine/tlb.h>
30
31/* pointer to the end of boot code/data in kernel image */
32/* need a fake array to get the pointer from the linker script */
33extern char ki_boot_end[1];
34/* pointer to end of kernel image */
35extern char ki_end[1];
36
37#ifdef ENABLE_SMP_SUPPORT
38/* sync variable to prevent other nodes from booting
39 * until kernel data structures initialized */
40BOOT_DATA static volatile int node_boot_lock = 0;
41#endif /* ENABLE_SMP_SUPPORT */
42
43/**
44 * Split mem_reg about reserved_reg. If memory exists in the lower
45 * segment, insert it. If memory exists in the upper segment, return it.
46 */
47BOOT_CODE static region_t
48insert_region_excluded(region_t mem_reg, region_t reserved_reg)
49{
50    region_t residual_reg = mem_reg;
51    bool_t result UNUSED;
52
53    if (reserved_reg.start < mem_reg.start) {
54        /* Reserved region is below the provided mem_reg. */
55        mem_reg.end = 0;
56        mem_reg.start = 0;
57        /* Fit the residual around the reserved region */
58        if (reserved_reg.end > residual_reg.start) {
59            residual_reg.start = reserved_reg.end;
60        }
61    } else if (mem_reg.end > reserved_reg.start) {
62        /* Split mem_reg around reserved_reg */
63        mem_reg.end = reserved_reg.start;
64        residual_reg.start = reserved_reg.end;
65    } else {
66        /* reserved_reg is completely above mem_reg */
67        residual_reg.start = 0;
68        residual_reg.end = 0;
69    }
70    /* Add the lower region if it exists */
71    if (mem_reg.start < mem_reg.end) {
72        result = insert_region(mem_reg);
73        assert(result);
74    }
75    /* Validate the upper region */
76    if (residual_reg.start > residual_reg.end) {
77        residual_reg.start = residual_reg.end;
78    }
79
80    return residual_reg;
81}
82
83BOOT_CODE static region_t
84get_reserved_region(int i, pptr_t res_reg_end)
85{
86    region_t res_reg = mode_reserved_region[i];
87
88    /* Force ordering and exclusivity of reserved regions. */
89    assert(res_reg.start < res_reg.end);
90    assert(res_reg_end <= res_reg.start);
91    return res_reg;
92}
93
94BOOT_CODE static int
95get_num_reserved_region(void)
96{
97    return sizeof(mode_reserved_region) / sizeof(region_t);
98}
99
100BOOT_CODE static void
101init_freemem(region_t ui_reg)
102{
103    word_t i;
104    bool_t result UNUSED;
105    region_t cur_reg;
106    region_t res_reg[] = {
107        {
108            .start = kernelBase,
109            .end   = (pptr_t)ki_end
110        },
111        {
112            .start = ui_reg.start,
113            .end = ui_reg.end
114        },
115    };
116
117    for (i = 0; i < MAX_NUM_FREEMEM_REG; i++) {
118        ndks_boot.freemem[i] = REG_EMPTY;
119    }
120
121    /* Force ordering and exclusivity of reserved regions. */
122    assert(res_reg[0].start < res_reg[0].end);
123    assert(res_reg[1].start < res_reg[1].end);
124    assert(res_reg[0].end  <= res_reg[1].start);
125    for (i = 0; i < get_num_avail_p_regs(); i++) {
126        cur_reg = paddr_to_pptr_reg(get_avail_p_reg(i));
127        /* Adjust region if it exceeds the kernel window
128         * Note that we compare physical address in case of overflow.
129         */
130        if (pptr_to_paddr((void*)cur_reg.end) > PADDR_TOP) {
131            cur_reg.end = PPTR_TOP;
132        }
133        if (pptr_to_paddr((void*)cur_reg.start) > PADDR_TOP) {
134            cur_reg.start = PPTR_TOP;
135        }
136
137        cur_reg = insert_region_excluded(cur_reg, res_reg[0]);
138        cur_reg = insert_region_excluded(cur_reg, res_reg[1]);
139
140        /* Check any reserved mode specific reagion */
141        region_t mode_res_reg = res_reg[1];
142        for (int m = 0; m < get_num_reserved_region(); m++) {
143            mode_res_reg = get_reserved_region(i, mode_res_reg.end);
144            cur_reg = insert_region_excluded(cur_reg, mode_res_reg);
145        }
146
147        if (cur_reg.start != cur_reg.end) {
148            result = insert_region(cur_reg);
149            assert(result);
150        }
151    }
152}
153
154BOOT_CODE static void
155init_irqs(cap_t root_cnode_cap)
156{
157    irq_t i;
158
159    for (i = 0; i <= maxIRQ; i++) {
160        setIRQState(IRQInactive, i);
161    }
162    setIRQState(IRQTimer, KERNEL_TIMER_IRQ);
163#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
164    setIRQState(IRQReserved, INTERRUPT_VGIC_MAINTENANCE);
165#endif
166#ifdef CONFIG_ARM_SMMU
167    setIRQState(IRQReserved, INTERRUPT_SMMU);
168#endif
169
170#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT
171#ifdef KERNEL_PMU_IRQ
172    setIRQState(IRQReserved, KERNEL_PMU_IRQ);
173#if (defined CONFIG_PLAT_TX1 && defined ENABLE_SMP_SUPPORT)
174//SELFOUR-1252
175#error "This platform doesn't support tracking CPU utilisation on multicore"
176#endif /* CONFIG_PLAT_TX1 && ENABLE_SMP_SUPPORT */
177#else
178#error "This platform doesn't support tracking CPU utilisation feature"
179#endif /* KERNEL_TIMER_IRQ */
180#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */
181
182#ifdef ENABLE_SMP_SUPPORT
183    setIRQState(IRQIPI, irq_remote_call_ipi);
184    setIRQState(IRQIPI, irq_reschedule_ipi);
185#endif /* ENABLE_SMP_SUPPORT */
186
187    /* provide the IRQ control cap */
188    write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIRQControl), cap_irq_control_cap_new());
189}
190
191BOOT_CODE static bool_t
192create_untypeds(cap_t root_cnode_cap, region_t boot_mem_reuse_reg)
193{
194    seL4_SlotPos   slot_pos_before;
195    seL4_SlotPos   slot_pos_after;
196    region_t       dev_reg;
197    word_t         i;
198
199    slot_pos_before = ndks_boot.slot_pos_cur;
200    create_kernel_untypeds(root_cnode_cap, boot_mem_reuse_reg, slot_pos_before);
201    for (i = 0; i < get_num_dev_p_regs(); i++) {
202        dev_reg = paddr_to_pptr_reg(get_dev_p_reg(i));
203        if (!create_untypeds_for_region(root_cnode_cap, true,
204                                        dev_reg, slot_pos_before)) {
205            return false;
206        }
207    }
208
209    slot_pos_after = ndks_boot.slot_pos_cur;
210    ndks_boot.bi_frame->untyped = (seL4_SlotRegion) {
211        slot_pos_before, slot_pos_after
212    };
213    return true;
214
215}
216
217/** This and only this function initialises the CPU.
218 *
219 * It does NOT initialise any kernel state.
220 * @return For the verification build, this currently returns true always.
221 */
222BOOT_CODE static bool_t
223init_cpu(void)
224{
225    bool_t haveHWFPU;
226
227#ifdef CONFIG_ARCH_AARCH64
228    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
229        if (!checkTCR_EL2()) {
230            return false;
231        }
232    }
233#endif
234
235    activate_global_pd();
236    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
237        vcpu_boot_init();
238    }
239
240#ifdef CONFIG_HARDWARE_DEBUG_API
241    if (!Arch_initHardwareBreakpoints()) {
242        printf("Kernel built with CONFIG_HARDWARE_DEBUG_API, but this board doesn't "
243               "reliably support it.\n");
244        return false;
245    }
246#endif
247
248    /* Setup kernel stack pointer.
249     * On ARM SMP, the array index here is the CPU ID
250     */
251#ifndef CONFIG_ARCH_ARM_V6
252    word_t stack_top = ((word_t) kernel_stack_alloc[SMP_TERNARY(getCurrentCPUIndex(), 0)]) + BIT(CONFIG_KERNEL_STACK_BITS);
253#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_ARCH_AARCH64)
254    /* the least 12 bits are used to store logical core ID */
255    stack_top |= getCurrentCPUIndex();
256#endif
257    setKernelStack(stack_top);
258#endif /* CONFIG_ARCH_ARM_V6 */
259
260#ifdef CONFIG_ARCH_AARCH64
261    /* initialise CPU's exception vector table */
262    setVtable((pptr_t)arm_vector_table);
263#endif /* CONFIG_ARCH_AARCH64 */
264
265    haveHWFPU = fpsimd_HWCapTest();
266
267    /* Disable FPU to avoid channels where a platform has an FPU but doesn't make use of it */
268    if (haveHWFPU) {
269        disableFpu();
270    }
271
272#ifdef CONFIG_HAVE_FPU
273    if (haveHWFPU) {
274        if (!fpsimd_init()) {
275            return false;
276        }
277    } else {
278        printf("Platform claims to have FP hardware, but does not!");
279        return false;
280    }
281#endif /* CONFIG_HAVE_FPU */
282
283    cpu_initLocalIRQController();
284
285#ifdef CONFIG_ENABLE_BENCHMARKS
286    armv_init_ccnt();
287#endif /* CONFIG_ENABLE_BENCHMARKS */
288
289    /* Export selected CPU features for access by PL0 */
290    armv_init_user_access();
291
292    initTimer();
293
294    return true;
295}
296
297/* This and only this function initialises the platform. It does NOT initialise any kernel state. */
298
299BOOT_CODE static void
300init_plat(void)
301{
302    initIRQController();
303    initL2Cache();
304}
305
306#ifdef ENABLE_SMP_SUPPORT
307BOOT_CODE static bool_t
308try_init_kernel_secondary_core(void)
309{
310    /* need to first wait until some kernel init has been done */
311    while (!node_boot_lock);
312
313    /* Perform cpu init */
314    init_cpu();
315
316    /* Enable per-CPU timer interrupts */
317    maskInterrupt(false, KERNEL_TIMER_IRQ);
318
319    NODE_LOCK_SYS;
320
321    ksNumCPUs++;
322
323    init_core_state(SchedulerAction_ResumeCurrentThread);
324
325    return true;
326}
327
328BOOT_CODE static void
329release_secondary_cpus(void)
330{
331
332    /* release the cpus at the same time */
333    node_boot_lock = 1;
334
335#ifndef CONFIG_ARCH_AARCH64
336    /* At this point in time the other CPUs do *not* have the seL4 global pd set.
337     * However, they still have a PD from the elfloader (which is mapping mmemory
338     * as strongly ordered uncached, as a result we need to explicitly clean
339     * the cache for it to see the update of node_boot_lock
340     *
341     * For ARMv8, the elfloader sets the page table entries as inner shareable
342     * (so is the attribute of the seL4 global PD) when SMP is enabled, and
343     * turns on the cache. Thus, we do not need to clean and invaliate the cache.
344     */
345    cleanInvalidateL1Caches();
346    plat_cleanInvalidateCache();
347#endif
348
349    /* Wait until all the secondary cores are done initialising */
350    while (ksNumCPUs != CONFIG_MAX_NUM_NODES) {
351        /* perform a memory release+acquire to get new values of ksNumCPUs */
352        __atomic_signal_fence(__ATOMIC_ACQ_REL);
353    }
354}
355#endif /* ENABLE_SMP_SUPPORT */
356
357/* Main kernel initialisation function. */
358
359static BOOT_CODE bool_t
360try_init_kernel(
361    paddr_t ui_p_reg_start,
362    paddr_t ui_p_reg_end,
363    sword_t pv_offset,
364    vptr_t  v_entry
365)
366{
367    cap_t root_cnode_cap;
368    cap_t it_ap_cap;
369    cap_t it_pd_cap;
370    cap_t ipcbuf_cap;
371    region_t ui_reg = paddr_to_pptr_reg((p_region_t) {
372        ui_p_reg_start, ui_p_reg_end
373    });
374    pptr_t bi_frame_pptr;
375    vptr_t bi_frame_vptr;
376    vptr_t ipcbuf_vptr;
377    create_frames_of_region_ret_t create_frames_ret;
378
379    /* convert from physical addresses to userland vptrs */
380    v_region_t ui_v_reg;
381    v_region_t it_v_reg;
382    ui_v_reg.start = ui_p_reg_start - pv_offset;
383    ui_v_reg.end   = ui_p_reg_end   - pv_offset;
384
385    ipcbuf_vptr = ui_v_reg.end;
386    bi_frame_vptr = ipcbuf_vptr + BIT(PAGE_BITS);
387
388    /* The region of the initial thread is the user image + ipcbuf and boot info */
389    it_v_reg.start = ui_v_reg.start;
390    it_v_reg.end = bi_frame_vptr + BIT(PAGE_BITS);
391
392    if (it_v_reg.end > kernelBase) {
393        printf("Userland image virtual end address too high\n");
394        return false;
395    }
396
397    /* setup virtual memory for the kernel */
398    map_kernel_window();
399
400    /* initialise the CPU */
401    if (!init_cpu()) {
402        return false;
403    }
404
405    /* debug output via serial port is only available from here */
406    printf("Bootstrapping kernel\n");
407
408    /* initialise the platform */
409    init_plat();
410
411    /* make the free memory available to alloc_region() */
412    init_freemem(ui_reg);
413
414    /* create the root cnode */
415    root_cnode_cap = create_root_cnode();
416    if (cap_get_capType(root_cnode_cap) == cap_null_cap) {
417        return false;
418    }
419
420    /* create the cap for managing thread domains */
421    create_domain_cap(root_cnode_cap);
422
423    /* create the IRQ CNode */
424    if (!create_irq_cnode()) {
425        return false;
426    }
427
428    /* initialise the IRQ states and provide the IRQ control cap */
429    init_irqs(root_cnode_cap);
430
431    /* create the bootinfo frame */
432    bi_frame_pptr = allocate_bi_frame(0, CONFIG_MAX_NUM_NODES, ipcbuf_vptr);
433    if (!bi_frame_pptr) {
434        return false;
435    }
436
437    if (config_set(CONFIG_ARM_SMMU)) {
438        ndks_boot.bi_frame->ioSpaceCaps = create_iospace_caps(root_cnode_cap);
439        if (ndks_boot.bi_frame->ioSpaceCaps.start == 0 &&
440                ndks_boot.bi_frame->ioSpaceCaps.end == 0) {
441            return false;
442        }
443    } else {
444        ndks_boot.bi_frame->ioSpaceCaps = S_REG_EMPTY;
445    }
446
447    /* Construct an initial address space with enough virtual addresses
448     * to cover the user image + ipc buffer and bootinfo frames */
449    it_pd_cap = create_it_address_space(root_cnode_cap, it_v_reg);
450    if (cap_get_capType(it_pd_cap) == cap_null_cap) {
451        return false;
452    }
453
454    /* Create and map bootinfo frame cap */
455    create_bi_frame_cap(
456        root_cnode_cap,
457        it_pd_cap,
458        bi_frame_pptr,
459        bi_frame_vptr
460    );
461
462    /* create the initial thread's IPC buffer */
463    ipcbuf_cap = create_ipcbuf_frame(root_cnode_cap, it_pd_cap, ipcbuf_vptr);
464    if (cap_get_capType(ipcbuf_cap) == cap_null_cap) {
465        return false;
466    }
467
468    /* create all userland image frames */
469    create_frames_ret =
470        create_frames_of_region(
471            root_cnode_cap,
472            it_pd_cap,
473            ui_reg,
474            true,
475            pv_offset
476        );
477    if (!create_frames_ret.success) {
478        return false;
479    }
480    ndks_boot.bi_frame->userImageFrames = create_frames_ret.region;
481
482    /* create/initialise the initial thread's ASID pool */
483    it_ap_cap = create_it_asid_pool(root_cnode_cap);
484    if (cap_get_capType(it_ap_cap) == cap_null_cap) {
485        return false;
486    }
487    write_it_asid_pool(it_ap_cap, it_pd_cap);
488
489    /* create the idle thread */
490    if (!create_idle_thread()) {
491        return false;
492    }
493
494    /* Before creating the initial thread (which also switches to it)
495     * we clean the cache so that any page table information written
496     * as a result of calling create_frames_of_region will be correctly
497     * read by the hardware page table walker */
498    cleanInvalidateL1Caches();
499
500    /* create the initial thread */
501    tcb_t *initial = create_initial_thread(
502                         root_cnode_cap,
503                         it_pd_cap,
504                         v_entry,
505                         bi_frame_vptr,
506                         ipcbuf_vptr,
507                         ipcbuf_cap
508                     );
509
510    if (initial == NULL) {
511        return false;
512    }
513
514    init_core_state(initial);
515
516    /* create all of the untypeds. Both devices and kernel window memory */
517    if (!create_untypeds(
518                root_cnode_cap,
519    (region_t) {
520    kernelBase, (pptr_t)ki_boot_end
521    } /* reusable boot code/data */
522            )) {
523        return false;
524    }
525
526    /* no shared-frame caps (ARM has no multikernel support) */
527    ndks_boot.bi_frame->sharedFrames = S_REG_EMPTY;
528
529    /* finalise the bootinfo frame */
530    bi_finalise();
531
532    /* make everything written by the kernel visible to userland. Cleaning to PoC is not
533     * strictly neccessary, but performance is not critical here so clean and invalidate
534     * everything to PoC */
535    cleanInvalidateL1Caches();
536    invalidateLocalTLB();
537    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
538        invalidateHypTLB();
539    }
540
541
542    ksNumCPUs = 1;
543
544    /* initialize BKL before booting up other cores */
545    SMP_COND_STATEMENT(clh_lock_init());
546    SMP_COND_STATEMENT(release_secondary_cpus());
547
548    /* grab BKL before leaving the kernel */
549    NODE_LOCK_SYS;
550
551    printf("Booting all finished, dropped to user space\n");
552
553    /* kernel successfully initialized */
554    return true;
555}
556
557BOOT_CODE VISIBLE void
558init_kernel(
559    paddr_t ui_p_reg_start,
560    paddr_t ui_p_reg_end,
561    sword_t pv_offset,
562    vptr_t  v_entry
563)
564{
565    bool_t result;
566
567#ifdef ENABLE_SMP_SUPPORT
568    /* we assume there exists a cpu with id 0 and will use it for bootstrapping */
569    if (getCurrentCPUIndex() == 0) {
570        result = try_init_kernel(ui_p_reg_start,
571                                 ui_p_reg_end,
572                                 pv_offset,
573                                 v_entry);
574    } else {
575        result = try_init_kernel_secondary_core();
576    }
577
578#else
579    result = try_init_kernel(ui_p_reg_start,
580                             ui_p_reg_end,
581                             pv_offset,
582                             v_entry);
583
584#endif /* ENABLE_SMP_SUPPORT */
585
586    if (!result) {
587        fail ("Kernel init failed for some reason :(");
588    }
589
590    schedule();
591    activateThread();
592}
593
594