1/*
2 * Copyright 2014, General Dynamics C4 Systems
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include <config.h>
8#include <assert.h>
9#include <kernel/boot.h>
10#include <machine/io.h>
11#include <model/statedata.h>
12#include <object/interrupt.h>
13#include <arch/machine.h>
14#include <arch/kernel/boot.h>
15#include <arch/kernel/vspace.h>
16#include <arch/benchmark.h>
17#include <arch/user_access.h>
18#include <arch/object/iospace.h>
19#include <linker.h>
20#include <plat/machine/hardware.h>
21#include <machine.h>
22#include <arch/machine/timer.h>
23#include <arch/machine/fpu.h>
24#include <arch/machine/tlb.h>
25
26#ifdef CONFIG_ARM_SMMU
27#include <drivers/smmu/smmuv2.h>
28#endif
29/* pointer to the end of boot code/data in kernel image */
30/* need a fake array to get the pointer from the linker script */
31extern char ki_boot_end[1];
32/* pointer to end of kernel image */
33extern char ki_end[1];
34
35#ifdef ENABLE_SMP_SUPPORT
36/* sync variable to prevent other nodes from booting
37 * until kernel data structures initialized */
38BOOT_DATA static volatile int node_boot_lock = 0;
39#endif /* ENABLE_SMP_SUPPORT */
40
41#define ARCH_RESERVED 3 // kernel + user image + dtb
42#define MAX_RESERVED (ARCH_RESERVED + MODE_RESERVED)
43BOOT_DATA static region_t reserved[MAX_RESERVED];
44
45BOOT_CODE static void arch_init_freemem(p_region_t ui_p_reg, p_region_t dtb_p_reg, v_region_t it_v_reg,
46                                        word_t extra_bi_size_bits)
47{
48    reserved[0].start = KERNEL_ELF_BASE;
49    reserved[0].end = (pptr_t)ki_end;
50
51    int index = 1;
52    if (dtb_p_reg.start) {
53        /* the dtb region could be empty */
54        reserved[index].start = (pptr_t) paddr_to_pptr(dtb_p_reg.start);
55        reserved[index].end = (pptr_t) paddr_to_pptr(dtb_p_reg.end);
56        index++;
57    }
58
59    if (MODE_RESERVED > 1) {
60        printf("MODE_RESERVED > 1 unsupported!\n");
61        halt();
62    }
63
64    if (ui_p_reg.start < PADDR_TOP) {
65        region_t ui_reg = paddr_to_pptr_reg(ui_p_reg);
66        if (MODE_RESERVED == 1) {
67            if (ui_reg.end > mode_reserved_region[0].start) {
68                reserved[index] = mode_reserved_region[0];
69                index++;
70                reserved[index].start = ui_reg.start;
71                reserved[index].end = ui_reg.end;
72            } else {
73                reserved[index].start = ui_reg.start;
74                reserved[index].end = ui_reg.end;
75                index++;
76                reserved[index] = mode_reserved_region[0];
77            }
78            index++;
79        } else {
80            reserved[index].start = ui_reg.start;
81            reserved[index].end = ui_reg.end;
82            index++;
83        }
84    } else if (MODE_RESERVED == 1) {
85        reserved[index] = mode_reserved_region[0];
86        index++;
87    }
88
89    init_freemem(get_num_avail_p_regs(), get_avail_p_regs(), index, reserved, it_v_reg, extra_bi_size_bits);
90}
91
92
93BOOT_CODE static void init_irqs(cap_t root_cnode_cap)
94{
95    unsigned i;
96
97    for (i = 0; i <= maxIRQ ; i++) {
98        setIRQState(IRQInactive, CORE_IRQ_TO_IRQT(0, i));
99    }
100    setIRQState(IRQTimer, CORE_IRQ_TO_IRQT(0, KERNEL_TIMER_IRQ));
101#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
102    setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, INTERRUPT_VGIC_MAINTENANCE));
103    setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, INTERRUPT_VTIMER_EVENT));
104#endif
105#ifdef CONFIG_TK1_SMMU
106    setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, INTERRUPT_SMMU));
107#endif
108
109#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT
110#ifdef KERNEL_PMU_IRQ
111    setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, KERNEL_PMU_IRQ));
112#if (defined CONFIG_PLAT_TX1 && defined ENABLE_SMP_SUPPORT)
113//SELFOUR-1252
114#error "This platform doesn't support tracking CPU utilisation on multicore"
115#endif /* CONFIG_PLAT_TX1 && ENABLE_SMP_SUPPORT */
116#else
117#error "This platform doesn't support tracking CPU utilisation feature"
118#endif /* KERNEL_TIMER_IRQ */
119#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */
120
121#ifdef ENABLE_SMP_SUPPORT
122    setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_remote_call_ipi));
123    setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_reschedule_ipi));
124#endif
125
126    /* provide the IRQ control cap */
127    write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIRQControl), cap_irq_control_cap_new());
128}
129
130#ifdef CONFIG_ARM_SMMU
131BOOT_CODE static void init_smmu(cap_t root_cnode_cap)
132{
133    plat_smmu_init();
134    /*provide the SID and CB control cap*/
135    write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapSMMUSIDControl), cap_sid_control_cap_new());
136    write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapSMMUCBControl), cap_cb_control_cap_new());
137}
138
139#endif
140
141BOOT_CODE static bool_t create_untypeds(cap_t root_cnode_cap, region_t boot_mem_reuse_reg)
142{
143    seL4_SlotPos   slot_pos_before;
144    seL4_SlotPos   slot_pos_after;
145
146    slot_pos_before = ndks_boot.slot_pos_cur;
147    create_device_untypeds(root_cnode_cap, slot_pos_before);
148    create_kernel_untypeds(root_cnode_cap, boot_mem_reuse_reg, slot_pos_before);
149
150    slot_pos_after = ndks_boot.slot_pos_cur;
151    ndks_boot.bi_frame->untyped = (seL4_SlotRegion) {
152        slot_pos_before, slot_pos_after
153    };
154    return true;
155
156}
157
158/** This and only this function initialises the CPU.
159 *
160 * It does NOT initialise any kernel state.
161 * @return For the verification build, this currently returns true always.
162 */
163BOOT_CODE static bool_t init_cpu(void)
164{
165    bool_t haveHWFPU;
166
167#ifdef CONFIG_ARCH_AARCH64
168    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
169        if (!checkTCR_EL2()) {
170            return false;
171        }
172    }
173#endif
174
175    activate_global_pd();
176    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
177        vcpu_boot_init();
178    }
179
180#ifdef CONFIG_HARDWARE_DEBUG_API
181    if (!Arch_initHardwareBreakpoints()) {
182        printf("Kernel built with CONFIG_HARDWARE_DEBUG_API, but this board doesn't "
183               "reliably support it.\n");
184        return false;
185    }
186#endif
187
188    /* Setup kernel stack pointer.
189     * On ARM SMP, the array index here is the CPU ID
190     */
191#ifndef CONFIG_ARCH_ARM_V6
192    word_t stack_top = ((word_t) kernel_stack_alloc[SMP_TERNARY(getCurrentCPUIndex(), 0)]) + BIT(CONFIG_KERNEL_STACK_BITS);
193#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_ARCH_AARCH64)
194    /* the least 12 bits are used to store logical core ID */
195    stack_top |= getCurrentCPUIndex();
196#endif
197    setKernelStack(stack_top);
198#endif /* CONFIG_ARCH_ARM_V6 */
199
200#ifdef CONFIG_ARCH_AARCH64
201    /* initialise CPU's exception vector table */
202    setVtable((pptr_t)arm_vector_table);
203#endif /* CONFIG_ARCH_AARCH64 */
204
205    haveHWFPU = fpsimd_HWCapTest();
206
207    /* Disable FPU to avoid channels where a platform has an FPU but doesn't make use of it */
208    if (haveHWFPU) {
209        disableFpu();
210    }
211
212#ifdef CONFIG_HAVE_FPU
213    if (haveHWFPU) {
214        if (!fpsimd_init()) {
215            return false;
216        }
217    } else {
218        printf("Platform claims to have FP hardware, but does not!");
219        return false;
220    }
221#endif /* CONFIG_HAVE_FPU */
222
223    cpu_initLocalIRQController();
224
225#ifdef CONFIG_ENABLE_BENCHMARKS
226    arm_init_ccnt();
227#endif /* CONFIG_ENABLE_BENCHMARKS */
228
229    /* Export selected CPU features for access by PL0 */
230    armv_init_user_access();
231
232    initTimer();
233
234    return true;
235}
236
237/* This and only this function initialises the platform. It does NOT initialise any kernel state. */
238
239BOOT_CODE static void init_plat(void)
240{
241    initIRQController();
242    initL2Cache();
243#ifdef CONFIG_ARM_SMMU
244    plat_smmu_init();
245#endif
246}
247
248#ifdef ENABLE_SMP_SUPPORT
249BOOT_CODE static bool_t try_init_kernel_secondary_core(void)
250{
251    unsigned i;
252
253    /* need to first wait until some kernel init has been done */
254    while (!node_boot_lock);
255
256    /* Perform cpu init */
257    init_cpu();
258
259    for (i = 0; i < NUM_PPI; i++) {
260        maskInterrupt(true, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), i));
261    }
262    setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_remote_call_ipi));
263    setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_reschedule_ipi));
264    /* Enable per-CPU timer interrupts */
265    setIRQState(IRQTimer, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), KERNEL_TIMER_IRQ));
266#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
267    setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), INTERRUPT_VGIC_MAINTENANCE));
268    setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), INTERRUPT_VTIMER_EVENT));
269#endif /* CONFIG_ARM_HYPERVISOR_SUPPORT */
270    NODE_LOCK_SYS;
271
272    ksNumCPUs++;
273
274    init_core_state(SchedulerAction_ResumeCurrentThread);
275
276    return true;
277}
278
279BOOT_CODE static void release_secondary_cpus(void)
280{
281
282    /* release the cpus at the same time */
283    node_boot_lock = 1;
284
285#ifndef CONFIG_ARCH_AARCH64
286    /* At this point in time the other CPUs do *not* have the seL4 global pd set.
287     * However, they still have a PD from the elfloader (which is mapping mmemory
288     * as strongly ordered uncached, as a result we need to explicitly clean
289     * the cache for it to see the update of node_boot_lock
290     *
291     * For ARMv8, the elfloader sets the page table entries as inner shareable
292     * (so is the attribute of the seL4 global PD) when SMP is enabled, and
293     * turns on the cache. Thus, we do not need to clean and invaliate the cache.
294     */
295    cleanInvalidateL1Caches();
296    plat_cleanInvalidateL2Cache();
297#endif
298
299    /* Wait until all the secondary cores are done initialising */
300    while (ksNumCPUs != CONFIG_MAX_NUM_NODES) {
301        /* perform a memory release+acquire to get new values of ksNumCPUs */
302        __atomic_signal_fence(__ATOMIC_ACQ_REL);
303    }
304}
305#endif /* ENABLE_SMP_SUPPORT */
306
307/* Main kernel initialisation function. */
308
309static BOOT_CODE bool_t try_init_kernel(
310    paddr_t ui_p_reg_start,
311    paddr_t ui_p_reg_end,
312    sword_t pv_offset,
313    vptr_t  v_entry,
314    paddr_t dtb_addr_start,
315    paddr_t dtb_addr_end
316)
317{
318    cap_t root_cnode_cap;
319    cap_t it_ap_cap;
320    cap_t it_pd_cap;
321    cap_t ipcbuf_cap;
322    p_region_t ui_p_reg = (p_region_t) {
323        ui_p_reg_start, ui_p_reg_end
324    };
325    region_t ui_reg = paddr_to_pptr_reg(ui_p_reg);
326    region_t dtb_reg;
327    word_t extra_bi_size;
328    pptr_t extra_bi_offset = 0;
329    vptr_t extra_bi_frame_vptr;
330    vptr_t bi_frame_vptr;
331    vptr_t ipcbuf_vptr;
332    create_frames_of_region_ret_t create_frames_ret;
333    create_frames_of_region_ret_t extra_bi_ret;
334
335    /* convert from physical addresses to userland vptrs */
336    v_region_t ui_v_reg;
337    v_region_t it_v_reg;
338    ui_v_reg.start = ui_p_reg_start - pv_offset;
339    ui_v_reg.end   = ui_p_reg_end   - pv_offset;
340
341    ipcbuf_vptr = ui_v_reg.end;
342    bi_frame_vptr = ipcbuf_vptr + BIT(PAGE_BITS);
343    extra_bi_frame_vptr = bi_frame_vptr + BIT(PAGE_BITS);
344
345    /* If no DTB was provided, skip allocating extra bootinfo */
346    p_region_t dtb_p_reg = {
347        dtb_addr_start, ROUND_UP(dtb_addr_end, PAGE_BITS)
348    };
349    if (dtb_addr_start == 0) {
350        extra_bi_size = 0;
351        dtb_reg = (region_t) {
352            0, 0
353        };
354    } else {
355        dtb_reg = paddr_to_pptr_reg(dtb_p_reg);
356        extra_bi_size = sizeof(seL4_BootInfoHeader) + (dtb_reg.end - dtb_reg.start);
357    }
358    word_t extra_bi_size_bits = calculate_extra_bi_size_bits(extra_bi_size);
359
360    /* The region of the initial thread is the user image + ipcbuf and boot info */
361    it_v_reg.start = ui_v_reg.start;
362    it_v_reg.end = extra_bi_frame_vptr + BIT(extra_bi_size_bits);
363
364    if (it_v_reg.end >= USER_TOP) {
365        printf("Userland image virtual end address too high\n");
366        return false;
367    }
368
369    /* setup virtual memory for the kernel */
370    map_kernel_window();
371
372    /* initialise the CPU */
373    if (!init_cpu()) {
374        return false;
375    }
376
377    /* debug output via serial port is only available from here */
378    printf("Bootstrapping kernel\n");
379
380    /* initialise the platform */
381    init_plat();
382
383    arch_init_freemem(ui_p_reg, dtb_p_reg, it_v_reg, extra_bi_size_bits);
384
385    /* create the root cnode */
386    root_cnode_cap = create_root_cnode();
387    if (cap_get_capType(root_cnode_cap) == cap_null_cap) {
388        return false;
389    }
390
391    /* create the cap for managing thread domains */
392    create_domain_cap(root_cnode_cap);
393
394    /* initialise the IRQ states and provide the IRQ control cap */
395    init_irqs(root_cnode_cap);
396
397#ifdef CONFIG_ARM_SMMU
398    /* initialise the SMMU and provide the SMMU control caps*/
399    init_smmu(root_cnode_cap);
400#endif
401    populate_bi_frame(0, CONFIG_MAX_NUM_NODES, ipcbuf_vptr, extra_bi_size);
402
403    /* put DTB in the bootinfo block, if present. */
404    seL4_BootInfoHeader header;
405    if (dtb_reg.start) {
406        header.id = SEL4_BOOTINFO_HEADER_FDT;
407        header.len = sizeof(header) + dtb_reg.end - dtb_reg.start;
408        *(seL4_BootInfoHeader *)(rootserver.extra_bi + extra_bi_offset) = header;
409        extra_bi_offset += sizeof(header);
410        memcpy((void *)(rootserver.extra_bi + extra_bi_offset), (void *)dtb_reg.start,
411               dtb_reg.end - dtb_reg.start);
412        extra_bi_offset += (dtb_reg.end - dtb_reg.start);
413    }
414
415    if (extra_bi_size > extra_bi_offset) {
416        /* provde a chunk for any leftover padding in the extended boot info */
417        header.id = SEL4_BOOTINFO_HEADER_PADDING;
418        header.len = (extra_bi_size - extra_bi_offset);
419        *(seL4_BootInfoHeader *)(rootserver.extra_bi + extra_bi_offset) = header;
420    }
421
422    if (config_set(CONFIG_TK1_SMMU)) {
423        ndks_boot.bi_frame->ioSpaceCaps = create_iospace_caps(root_cnode_cap);
424        if (ndks_boot.bi_frame->ioSpaceCaps.start == 0 &&
425            ndks_boot.bi_frame->ioSpaceCaps.end == 0) {
426            return false;
427        }
428    } else {
429        ndks_boot.bi_frame->ioSpaceCaps = S_REG_EMPTY;
430    }
431
432    /* Construct an initial address space with enough virtual addresses
433     * to cover the user image + ipc buffer and bootinfo frames */
434    it_pd_cap = create_it_address_space(root_cnode_cap, it_v_reg);
435    if (cap_get_capType(it_pd_cap) == cap_null_cap) {
436        return false;
437    }
438
439    /* Create and map bootinfo frame cap */
440    create_bi_frame_cap(
441        root_cnode_cap,
442        it_pd_cap,
443        bi_frame_vptr
444    );
445
446    /* create and map extra bootinfo region */
447    if (extra_bi_size > 0) {
448        region_t extra_bi_region = {
449            .start = rootserver.extra_bi,
450            .end = rootserver.extra_bi + extra_bi_size
451        };
452        extra_bi_ret =
453            create_frames_of_region(
454                root_cnode_cap,
455                it_pd_cap,
456                extra_bi_region,
457                true,
458                pptr_to_paddr((void *)extra_bi_region.start) - extra_bi_frame_vptr
459            );
460        if (!extra_bi_ret.success) {
461            return false;
462        }
463        ndks_boot.bi_frame->extraBIPages = extra_bi_ret.region;
464    }
465
466#ifdef CONFIG_KERNEL_MCS
467    init_sched_control(root_cnode_cap, CONFIG_MAX_NUM_NODES);
468#endif
469
470    /* create the initial thread's IPC buffer */
471    ipcbuf_cap = create_ipcbuf_frame_cap(root_cnode_cap, it_pd_cap, ipcbuf_vptr);
472    if (cap_get_capType(ipcbuf_cap) == cap_null_cap) {
473        return false;
474    }
475
476    /* create all userland image frames */
477    create_frames_ret =
478        create_frames_of_region(
479            root_cnode_cap,
480            it_pd_cap,
481            ui_reg,
482            true,
483            pv_offset
484        );
485    if (!create_frames_ret.success) {
486        return false;
487    }
488    ndks_boot.bi_frame->userImageFrames = create_frames_ret.region;
489
490    /* create/initialise the initial thread's ASID pool */
491    it_ap_cap = create_it_asid_pool(root_cnode_cap);
492    if (cap_get_capType(it_ap_cap) == cap_null_cap) {
493        return false;
494    }
495    write_it_asid_pool(it_ap_cap, it_pd_cap);
496
497#ifdef CONFIG_KERNEL_MCS
498    NODE_STATE(ksCurTime) = getCurrentTime();
499#endif
500
501    /* create the idle thread */
502    if (!create_idle_thread()) {
503        return false;
504    }
505
506    /* Before creating the initial thread (which also switches to it)
507     * we clean the cache so that any page table information written
508     * as a result of calling create_frames_of_region will be correctly
509     * read by the hardware page table walker */
510    cleanInvalidateL1Caches();
511
512    /* create the initial thread */
513    tcb_t *initial = create_initial_thread(
514                         root_cnode_cap,
515                         it_pd_cap,
516                         v_entry,
517                         bi_frame_vptr,
518                         ipcbuf_vptr,
519                         ipcbuf_cap
520                     );
521
522    if (initial == NULL) {
523        return false;
524    }
525
526    init_core_state(initial);
527
528    /* create all of the untypeds. Both devices and kernel window memory */
529    if (!create_untypeds(
530            root_cnode_cap,
531    (region_t) {
532    KERNEL_ELF_BASE, (pptr_t)ki_boot_end
533    } /* reusable boot code/data */
534        )) {
535        return false;
536    }
537
538    /* no shared-frame caps (ARM has no multikernel support) */
539    ndks_boot.bi_frame->sharedFrames = S_REG_EMPTY;
540
541    /* finalise the bootinfo frame */
542    bi_finalise();
543
544    /* make everything written by the kernel visible to userland. Cleaning to PoC is not
545     * strictly neccessary, but performance is not critical here so clean and invalidate
546     * everything to PoC */
547    cleanInvalidateL1Caches();
548    invalidateLocalTLB();
549    if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) {
550        invalidateHypTLB();
551    }
552
553
554    ksNumCPUs = 1;
555
556    /* initialize BKL before booting up other cores */
557    SMP_COND_STATEMENT(clh_lock_init());
558    SMP_COND_STATEMENT(release_secondary_cpus());
559
560    /* grab BKL before leaving the kernel */
561    NODE_LOCK_SYS;
562
563    printf("Booting all finished, dropped to user space\n");
564
565    /* kernel successfully initialized */
566    return true;
567}
568
569BOOT_CODE VISIBLE void init_kernel(
570    paddr_t ui_p_reg_start,
571    paddr_t ui_p_reg_end,
572    sword_t pv_offset,
573    vptr_t  v_entry,
574    paddr_t dtb_addr_p,
575    uint32_t dtb_size
576)
577{
578    bool_t result;
579    paddr_t dtb_end_p = 0;
580
581    if (dtb_addr_p) {
582        dtb_end_p = dtb_addr_p + dtb_size;
583    }
584
585#ifdef ENABLE_SMP_SUPPORT
586    /* we assume there exists a cpu with id 0 and will use it for bootstrapping */
587    if (getCurrentCPUIndex() == 0) {
588        result = try_init_kernel(ui_p_reg_start,
589                                 ui_p_reg_end,
590                                 pv_offset,
591                                 v_entry,
592                                 dtb_addr_p, dtb_end_p);
593    } else {
594        result = try_init_kernel_secondary_core();
595    }
596
597#else
598    result = try_init_kernel(ui_p_reg_start,
599                             ui_p_reg_end,
600                             pv_offset,
601                             v_entry,
602                             dtb_addr_p, dtb_end_p);
603
604#endif /* ENABLE_SMP_SUPPORT */
605
606    if (!result) {
607        fail("Kernel init failed for some reason :(");
608    }
609
610#ifdef CONFIG_KERNEL_MCS
611    NODE_STATE(ksCurTime) = getCurrentTime();
612    NODE_STATE(ksConsumed) = 0;
613#endif
614    schedule();
615    activateThread();
616}
617
618