1/*
2 * Copyright (c) 2016, 2017, ETH Zurich.
3 * Copyright (c) 2016, Hewlett Packard Enterprise Development LP.
4 * All rights reserved.
5 *
6 * This file is distributed under the terms in the attached LICENSE file.
7 * If you do not find this file, copies can be found by writing to:
8 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group.
9 */
10
11/* CPU driver VM initialisation.
12
13   This is the entry point on booting the first core, and needs to deal with
14   the state left by UEFI.  The CPU is mostly configured, in particular
15   translation is enabled, and all RAM is mapped 1-1.  We'll also be in either
16   EL3 or EL2.  We need to map the EL1 kernel window (TTBR1), drop to EL1, and
17   jump to the next routine, which has already been relocated for us.
18 */
19
20#include <stdio.h>
21#include <stdbool.h>
22
23#include <barrelfish_kpi/types.h>
24#include <init.h>
25#include <offsets.h>
26#include <sysreg.h>
27#include <dev/armv8_dev.h>
28
29#include <multiboot2.h>
30#include <barrelfish_kpi/arm_core_data.h>
31
32
33void eret(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3);
34
35void boot_bsp_init(uint32_t magic, lpaddr_t pointer, lpaddr_t stack)
36    __attribute__((noreturn));
37void boot_app_init(lpaddr_t context)
38    __attribute__((noreturn));
39
40/* low level debugging facilities */
41#if DEBUG
42#ifdef THUNDERX
43#include <dev/pl011_uart_dev.h>
44
45#define CN88XX_MAP_UART0_OFFSET 0x87E024000000UL
46
47static pl011_uart_t uart;
48
49static void debug_uart_initialize(void) {
50    pl011_uart_initialize(&uart, (mackerel_addr_t) CN88XX_MAP_UART0_OFFSET);
51}
52
53static void debug_serial_putc(char c)
54{
55    while(pl011_uart_FR_txff_rdf(&uart) == 1) ;
56    pl011_uart_DR_rawwr(&uart, c);
57}
58#elif defined(XGENE)
59#include <dev/apm88xxxx/apm88xxxx_pc16550_dev.h>
60
61#define CN88XX_MAP_UART0_OFFSET 0x87E024000000UL
62
63apm88xxxx_pc16550_t uart;
64
65static void debug_uart_initialize(void) {
66    apm88xxxx_pc16550_initialize(&uart, (mackerel_addr_t)0x1C020000);
67}
68
69static void debug_serial_putc(char c)
70{
71    // Wait until FIFO can hold more characters
72    while(!apm88xxxx_pc16550_LSR_thre_rdf(&uart));
73    // Write character
74    apm88xxxx_pc16550_THR_thr_wrf(&uart, c);
75}
76
77#endif
78
79static void debug_serial_putchar(char c) {
80    if (c == '\n') {
81        debug_serial_putc('\r');
82    }
83    debug_serial_putc(c);
84}
85
86
87static void debug_print_string(char *str)
88{
89    while (str && *str) {
90        debug_serial_putchar(*str);
91        str++;
92    }
93}
94#else
95#define debug_print_string(x)
96#define debug_uart_initialize()
97#endif
98
99
100void (*cpu_driver_entry)(uint32_t magic, lpaddr_t pointer, lpaddr_t stack);
101
102static void configure_tcr(void) {
103    armv8_TCR_EL1_t tcr_el1 = armv8_TCR_EL1_rd(NULL);
104    // disable top byte ignored, EL1
105    tcr_el1 = armv8_TCR_EL1_TBI1_insert(tcr_el1, 0);
106    // disable top byte ignored, EL0
107    tcr_el1 = armv8_TCR_EL1_TBI0_insert(tcr_el1, 0);
108    // 48b IPA
109    tcr_el1 = armv8_TCR_EL1_IPS_insert(tcr_el1, 5);
110    // 4kB granule
111    tcr_el1 = armv8_TCR_EL1_TG1_insert(tcr_el1, armv8_KB_4);
112    // Walks inner shareable
113    tcr_el1 = armv8_TCR_EL1_SH1_insert(tcr_el1, armv8_inner_shareable);
114    // Walks outer WB WA
115    tcr_el1 = armv8_TCR_EL1_ORGN1_insert(tcr_el1, armv8_WbRaWa_cache);
116    // Walks inner WB WA
117    tcr_el1 = armv8_TCR_EL1_IRGN1_insert(tcr_el1, armv8_WbRaWa_cache);
118    // enable EL1 translation
119    tcr_el1 = armv8_TCR_EL1_EPD1_insert(tcr_el1, 0);
120    // 48b kernel VA
121    tcr_el1 = armv8_TCR_EL1_T1SZ_insert(tcr_el1, 16);
122    // 4kB granule
123    tcr_el1 = armv8_TCR_EL1_TG0_insert(tcr_el1, armv8_KB_4);
124    // Walks inner shareable
125    tcr_el1 = armv8_TCR_EL1_SH0_insert(tcr_el1, armv8_inner_shareable);
126    // Walks outer WB WA
127    tcr_el1 = armv8_TCR_EL1_ORGN0_insert(tcr_el1, armv8_WbRaWa_cache);
128    // Walks inner WB WA
129    tcr_el1 = armv8_TCR_EL1_IRGN0_insert(tcr_el1, armv8_WbRaWa_cache);
130    // enable EL0 translation
131    tcr_el1 = armv8_TCR_EL1_EPD0_insert(tcr_el1, 0);
132    // 48b user VA
133    tcr_el1 = armv8_TCR_EL1_T0SZ_insert(tcr_el1, 16);
134    armv8_TCR_EL1_wr(NULL, tcr_el1);
135}
136
137
138#define    DAIF_FIQ_BIT   (1 << 0)
139#define    DAIF_IRQ_BIT   (1 << 1)
140
141
142static void armv8_disable_interrupts(void)
143{
144    __asm volatile("msr DAIFSet, #3\n");
145}
146
147static void armv8_set_tcr(uint8_t el)
148{
149    switch(el) {
150        case 3:
151            //sysreg_write_ttbr0_el2(addr);
152        case 2:
153        {
154            armv8_TCR_EL2_t reg = 0;
155            reg = armv8_TCR_EL2_PS_insert(reg, 5);
156            reg = armv8_TCR_EL2_T0SZ_insert(reg, (64 - 48));
157            armv8_TCR_EL2_wr(NULL, reg);
158            break;
159        }
160        case 1:
161        {
162            armv8_TCR_EL1_t reg = 0;
163          // TODO: figure out what to set  reg = armv8_TCR_EL1_PS_insert(reg, 5);
164            reg = armv8_TCR_EL1_T0SZ_insert(reg, (64 - 48));
165            armv8_TCR_EL1_wr(NULL, reg);
166            break;
167        }
168        default:
169            assert("should not happen");
170            return;
171    }
172}
173
174static void armv8_set_ttbr0(uint8_t el, lpaddr_t addr)
175{
176    switch(el) {
177        case 3:
178            //sysreg_write_ttbr0_el2(addr);
179        case 2:
180            armv8_TTBR0_EL2_wr(NULL, addr);
181            armv8_TTBR0_EL1_wr(NULL, addr);
182            break;
183        case 1:
184            armv8_TTBR0_EL1_wr(NULL, addr);
185            break;
186        default:
187            assert("should not happen");
188            return;
189    }
190    __asm volatile("isb");
191}
192
193static void armv8_enable_mmu(uint8_t el)
194{
195    switch(el) {
196        case 3:
197            armv8_SCTLR_EL3_M_wrf(NULL, 0x1);
198            __asm volatile("tlbi    alle3\n  isb");
199            break;
200        case 2:
201            armv8_SCTLR_EL2_M_wrf(NULL, 0x1);
202            __asm volatile("tlbi    alle2\n  isb");
203            break;
204        case 1:
205            armv8_SCTLR_EL1_M_wrf(NULL, 0x1);
206            __asm volatile("tlbi    vmalle1\n  isb");
207            break;
208        default:
209            assert("should not happen");
210            return;
211    }
212    __asm volatile("dsb sy\n isb");
213}
214
215static void armv8_invalidate_tlb(uint8_t el)
216{
217    switch(el) {
218        case 3:
219            armv8_SCTLR_EL3_M_wrf(NULL, 0x1);
220            __asm volatile("tlbi    alle3");
221            break;
222        case 2:
223            armv8_SCTLR_EL2_M_wrf(NULL, 0x1);
224            __asm volatile("tlbi    alle2");
225            break;
226        case 1:
227            armv8_SCTLR_EL1_M_wrf(NULL, 0x1);
228            __asm volatile("tlbi    vmalle1");
229            break;
230        default:
231            assert("should not happen");
232            return;
233    }
234    __asm volatile("dsb sy\n isb");
235}
236
237static void armv8_invalidate_icache(void)
238{
239    __asm volatile(
240      "ic      iallu \n"
241      "dsb     sy \n"
242      "isb \n"
243      );
244}
245
246static void armv8_instruction_synchronization_barrier(void)
247{
248    __asm volatile("isb");
249}
250
251static void configure_spsr(uint8_t el) {
252    armv8_SPSR_EL2_t spsr = 0;
253    /* mask the exceptions */
254    spsr = armv8_SPSR_EL2_D_insert(spsr, 1);
255    spsr = armv8_SPSR_EL2_A_insert(spsr, 1);
256    spsr = armv8_SPSR_EL2_I_insert(spsr, 1);
257    spsr = armv8_SPSR_EL2_F_insert(spsr, 1);
258
259    /* set el1 and use the SP_ELx stack */
260    spsr = armv8_SPSR_EL2_M_lo_insert(spsr, (1<<2) | 1);
261
262    switch(el) {
263    case 3:
264        armv8_SPSR_EL3_wr(NULL, spsr);
265        return;
266    case 2:
267        armv8_SPSR_EL2_wr(NULL, spsr);
268        break;
269    case 1:
270        armv8_SPSR_EL1_wr(NULL, spsr);
271        return;
272    default:
273        return;
274    }
275}
276
277static void configure_ttbr1(uint8_t el)
278{
279    lpaddr_t ttbr1_el1;
280    switch(el) {
281    case 3:
282        ttbr1_el1= armv8_TTBR0_EL3_rawrd(NULL);
283        break;
284    case 2:
285        ttbr1_el1= armv8_TTBR0_EL2_rawrd(NULL);
286        break;
287    case 1:
288        ttbr1_el1= armv8_TTBR0_EL1_rawrd(NULL);
289        break;
290    default:
291        return;
292    }
293    armv8_TTBR1_EL1_rawwr(NULL, ttbr1_el1);
294}
295
296static void configure_mair(void)
297{
298    /* Set memory type 0, for kernel use. */
299    // attr0 = Normal Memory, Inner Write-back non transient
300    // attr1 = Device-nGnRnE memory
301    armv8_MAIR_EL1_wr(NULL, 0x00ff);
302}
303
304static void configure_sctlr(void)
305/* Enable EL0/1 translation. */
306{
307
308    armv8_SCTLR_EL1_t val = 0;
309
310    /* Traps EL0 execution of cache maintenance instructions to EL1 */
311    val = armv8_SCTLR_EL1_UCI_insert(val, 0x1);
312
313    /* write permissions implies execute never */
314    //val = armv8_SCTLR_EL1_WXN_insert(val, 0x1);
315
316    /* don't trap WFI/WFE instructions to EL1 */
317    val = armv8_SCTLR_EL1_nTWE_insert(val, 0x1);
318    val = armv8_SCTLR_EL1_nTWI_insert(val, 0x1);
319
320    /* disable Traps EL0 accesses to the CTR_EL0 to EL1*/
321    val = armv8_SCTLR_EL1_UCT_insert(val, 0x1);
322
323    /* Allow EL0 to do DC ZVA */
324    val = armv8_SCTLR_EL1_DZE_insert(val, 0x1);
325
326    /* enable instruction cache */
327    val = armv8_SCTLR_EL1_I_insert(val, 0x1);
328
329    /*
330     * EL0 execution of MRS , MSR(register) , or MSR(immediate) instructions
331     * that access the DAIF is not trapped to EL1.
332     */
333    //val = armv8_SCTLR_EL1_UMA_insert(val, 0x1);
334
335    /*
336     * Enables accesses to the DMB, DSB, and ISB System
337     * instructions in the (coproc== 1111 ) encoding space from EL0
338     */
339    val = armv8_SCTLR_EL1_CP15BEN_insert(val, 0x1);
340
341    /* Enable SP alignment checks */
342    val = armv8_SCTLR_EL1_SA0_insert(val, 0x1);
343    val = armv8_SCTLR_EL1_SA_insert(val, 0x1);
344
345    /* enable data cachable */
346    val = armv8_SCTLR_EL1_C_insert(val, 0x1);
347
348    /* enable alignment checks */
349    val = armv8_SCTLR_EL1_A_insert(val, 0x1);
350
351    /* enable mmu */
352    val = armv8_SCTLR_EL1_M_insert(val, 0x1);
353
354    armv8_SCTLR_EL1_wr(NULL, val);
355}
356
357static void configure_el3_traps(void)
358{
359
360    /* If we've started in EL3, that most likely means we're in the
361     * simulator.  We don't use it at all, so just disable all traps to
362     * EL3, and drop to non-secure EL2 (if it exists). */
363
364    armv8_SCR_EL3_t val = 0;
365
366    /* Don't trap secure timer access. */
367    val = armv8_SCR_EL3_ST_insert(val, 0x1);
368
369    /* Next EL is AArch64. */
370    val = armv8_SCR_EL3_RW_insert(val, 0x1);
371
372    /* HVC is enabled. */
373    val = armv8_SCR_EL3_HCE_insert(val, 0x1);
374
375    /* SMC is disabled. */
376    val = armv8_SCR_EL3_SMD_insert(val, 0x1);
377
378    /* External aborts don't trap to EL3. */
379    val = armv8_SCR_EL3_EA_insert(val, 0x1);
380
381    /* FIQs don't trap to EL3. */
382    val = armv8_SCR_EL3_FIQ_insert(val, 0x1);
383
384    /* IRQs don't trap to EL3. */
385    val = armv8_SCR_EL3_IRQ_insert(val, 0x1);
386
387    /* EL0 and EL1 are non-secure. */
388    val = armv8_SCR_EL3_NS_insert(val, 0x1);
389
390    armv8_SCR_EL3_wr(NULL, val);
391
392    /* We don't need to set SCTLR_EL3, as we're not using it. */
393
394    armv8_MDCR_EL3_t mdcr = 0;
395    /* Allow event counting in secure state. */
396    armv8_MDCR_EL3_SPME_insert(mdcr, 0x1);
397    armv8_MDCR_EL3_wr(NULL, mdcr);
398}
399
400static void configure_el2_traps(void)
401{
402    /* check if EL2 is implemented */
403    if (armv8_ID_AA64PFR0_EL1_EL2_rdf(NULL) == armv8_ID_EL_NOT_IMPLEMENTED) {
404        return;
405    }
406
407    /* configure EL2 traps & mmu */
408
409    armv8_HCR_EL2_t val = 0;
410
411    /* For the Non-secure EL1&0 translation regime, for permitted accesses to a
412     * memory location that use a common definition of the Shareability and
413     * Cacheability of the location, there might be a loss of coherency if the
414     * Inner Cacheability attribute for those accesses differs from the Outer
415     * Cacheability attribute.*/
416    val = armv8_HCR_EL2_MIOCNCE_insert(val, 1);
417
418    /* Set the mode to be AARCH64 */
419    val = armv8_HCR_EL2_RW_insert(val, 1);
420
421    /* HVC instructions are UNDEFINED at EL2 and Non-secure EL1. Any resulting
422     * exception is taken to the Exception level at which the HVC instruction
423     * is executed.
424     *
425     * XXX: this will disable Hypervisor calls entirely, revisit for ARRAKIS
426     */
427    val = armv8_HCR_EL2_HCD_insert(val, 1);
428
429    armv8_HCR_EL2_wr(NULL, val);
430
431
432    /* disable traps to EL2 for timer accesses */
433
434    armv8_CNTHCTL_EL2_t cnthctl;
435    cnthctl = armv8_CNTHCTL_EL2_rd(NULL);
436    cnthctl = armv8_CNTHCTL_EL2_EL1PCEN_insert(cnthctl, 0x1);
437    cnthctl = armv8_CNTHCTL_EL2_EL1PCTEN_insert(cnthctl, 0x1);
438    armv8_CNTHCTL_EL2_wr(NULL, cnthctl);
439}
440
441static void configure_el1_traps(void)
442{
443    /* disable traps for FP/SIMD access  */
444    armv8_CPACR_EL1_FPEN_wrf(NULL, armv8_fpen_trap_none);
445}
446
447static void drop_to_el2(uint32_t magic, lpaddr_t pointer, lpaddr_t stack)
448{
449    /* write the stack pointer for EL1 */
450    armv8_SP_EL1_wr(NULL, stack + KERNEL_OFFSET);
451
452    /* Set the jump target */
453    armv8_ELR_EL3_wr(NULL, (uint64_t)cpu_driver_entry);
454
455    /* call exception return */
456    eret(magic, pointer + KERNEL_OFFSET, stack + KERNEL_OFFSET, 0);
457}
458
459static void drop_to_el1(uint32_t magic, lpaddr_t pointer, lpaddr_t stack)
460{
461    /* write the stack pointer for EL1 */
462    armv8_SP_EL1_wr(NULL, stack + KERNEL_OFFSET);
463
464    /* Set the jump target */
465    armv8_ELR_EL2_wr(NULL, (uint64_t)cpu_driver_entry);
466
467    /* call exception return */
468    eret(magic, pointer + KERNEL_OFFSET, stack + KERNEL_OFFSET, 0);
469}
470
471static void jump_to_cpudriver(uint32_t magic, lpaddr_t pointer, lpaddr_t stack)
472{
473    // We are in EL1, so call arch_init directly.
474
475    // we may need to re set the stack pointer
476    uint64_t sp = sysreg_read_sp();
477    if (sp < KERNEL_OFFSET) {
478        sysreg_write_sp(sp + KERNEL_OFFSET);
479    }
480    cpu_driver_entry(magic, pointer + KERNEL_OFFSET, stack + KERNEL_OFFSET);
481
482}
483
484
485/* On entry:
486
487   Execution is starting in LOW addresses
488   Pointers to stack and multiboot are LOW addresses
489   Single core running (not guaranteed to be core 0)
490   CPU is in highest implemented exception level
491   MMU enabled, 4k translation granule, 1:1 mapping of all RAM
492   Little-endian mode
493   Core caches (L1&L2) and TLB enabled
494   Non-architectural caches disabled (e.g. L3)
495   Interrupts enabled
496   Generic timer initialized and enabled
497   >= 128KiB stack
498   ACPI tables available
499   Register x0 contains the multiboot magic value
500   Register x1 contains a pointer to multiboot image
501   Register x1 contains a pointer to top entry in the kernel stack
502 */
503static void boot_generic_init(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) {
504
505    uint8_t el = armv8_CurrentEL_EL_rdf(NULL);
506
507    /* Configure the EL1 translation regime. */
508    configure_tcr();
509
510    /* Copy the current TTBR for EL1. */
511    configure_ttbr1(el);
512
513    /* configure memory attributes */
514    configure_mair();
515
516    /* Enable EL0/1 translation. */
517    configure_sctlr();
518
519    /* configure spsr */
520    configure_spsr(el);
521
522    /* configure EL 1 traps*/
523    configure_el1_traps();
524
525    switch(el) {
526    case 3:
527        configure_el3_traps();
528        configure_el2_traps();
529        drop_to_el2(magic, pointer, stack);
530        break;
531    case 2:
532        configure_el2_traps();
533        drop_to_el1(magic, pointer, stack);
534        break;
535    case 1:
536        jump_to_cpudriver(magic, pointer, stack);
537        break;
538    default:
539        break;
540    }
541}
542
543/**
544 * @brief initializes an application core
545 *
546 * @param state pointer to the armv8_core_data structure
547 *
548 * This function is intended to bring the core to the same state as if it
549 * has been booted by the UEFI boot loader.
550 */
551void boot_app_init(lpaddr_t state)
552{
553
554    debug_uart_initialize();
555    debug_print_string("APP BOOTING\n");
556
557    struct armv8_core_data *cd = (struct armv8_core_data *)state;
558
559    cpu_driver_entry = (void *)cd->cpu_driver_entry;
560
561    uint8_t current_el = armv8_CurrentEL_EL_rdf(NULL);
562
563    if (current_el == 2) {
564        uint64_t zero = 0;
565        __asm volatile("MSR CPTR_EL2, %[zero]" : : [zero] "r" (zero));
566    }
567
568    // /* disable interrupts */
569    armv8_disable_interrupts();
570
571    /* set the ttbr0/1 */
572    armv8_set_ttbr0(current_el, cd->page_table_root);
573
574    /* set the TCR */
575    armv8_set_tcr(current_el);
576
577    /* enable MMU */
578    armv8_enable_mmu(current_el);
579
580    /* invalidate TLB */
581    armv8_invalidate_tlb(current_el);
582
583    /* invalidate icache */
584    armv8_invalidate_icache();
585    armv8_instruction_synchronization_barrier();
586
587    boot_generic_init(cd->boot_magic, state, cd->cpu_driver_stack);
588
589    while(1) {
590        __asm volatile("wfi \n");
591    }
592}
593
594/* On entry:
595
596   Execution is starting in LOW addresses
597   Pointers to stack and multiboot are LOW addresses
598   Single core running (not guaranteed to be core 0)
599   CPU is in highest implemented exception level
600   MMU enabled, 4k translation granule, 1:1 mapping of all RAM
601   Little-endian mode
602   Core caches (L1&L2) and TLB enabled
603   Non-architectural caches disabled (e.g. L3)
604   Interrupts enabled
605   Generic timer initialized and enabled
606   >= 128KiB stack
607   ACPI tables available
608   Register x0 contains the multiboot magic value
609   Register x1 contains a pointer to multiboot image
610   Register x1 contains a pointer to top entry in the kernel stack
611 */
612void
613boot_bsp_init(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) {
614
615    debug_uart_initialize();
616    debug_print_string("BSP BOOTING\n");
617
618    /* Boot magic must be set */
619    if (magic != MULTIBOOT2_BOOTLOADER_MAGIC) {
620        goto stop;
621    }
622
623    /*
624     * get the first entry in the multiboot structure, this holds a pointer
625     * to the CPU driver entry point
626     */
627    struct multiboot_header *mbhdr = (void *)pointer;
628
629    struct multiboot_header_tag *mb;
630    mb = (struct multiboot_header_tag *)(mbhdr + 1);
631
632    if (mb->type == MULTIBOOT_TAG_TYPE_EFI64) {
633        cpu_driver_entry = (void *)((struct multiboot_tag_efi64 *)mb)->pointer;
634    }
635
636    /* disable interrupts */
637    armv8_disable_interrupts();
638
639    boot_generic_init(magic, pointer, stack);
640
641    stop:
642    while(1) {
643        __asm volatile("wfi \n");
644    }
645}
646