1/* 2 * Copyright (c) 2016, 2017, ETH Zurich. 3 * Copyright (c) 2016, Hewlett Packard Enterprise Development LP. 4 * All rights reserved. 5 * 6 * This file is distributed under the terms in the attached LICENSE file. 7 * If you do not find this file, copies can be found by writing to: 8 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group. 9 */ 10 11/* CPU driver VM initialisation. 12 13 This is the entry point on booting the first core, and needs to deal with 14 the state left by UEFI. The CPU is mostly configured, in particular 15 translation is enabled, and all RAM is mapped 1-1. We'll also be in either 16 EL3 or EL2. We need to map the EL1 kernel window (TTBR1), drop to EL1, and 17 jump to the next routine, which has already been relocated for us. 18 */ 19 20#include <stdio.h> 21#include <stdbool.h> 22 23#include <barrelfish_kpi/types.h> 24#include <init.h> 25#include <offsets.h> 26#include <sysreg.h> 27#include <dev/armv8_dev.h> 28 29#include <multiboot2.h> 30#include <barrelfish_kpi/arm_core_data.h> 31 32 33void eret(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3); 34 35void boot_bsp_init(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) 36 __attribute__((noreturn)); 37void boot_app_init(lpaddr_t context) 38 __attribute__((noreturn)); 39 40/* low level debugging facilities */ 41#if DEBUG 42#ifdef THUNDERX 43#include <dev/pl011_uart_dev.h> 44 45#define CN88XX_MAP_UART0_OFFSET 0x87E024000000UL 46 47static pl011_uart_t uart; 48 49static void debug_uart_initialize(void) { 50 pl011_uart_initialize(&uart, (mackerel_addr_t) CN88XX_MAP_UART0_OFFSET); 51} 52 53static void debug_serial_putc(char c) 54{ 55 while(pl011_uart_FR_txff_rdf(&uart) == 1) ; 56 pl011_uart_DR_rawwr(&uart, c); 57} 58#elif defined(XGENE) 59#include <dev/apm88xxxx/apm88xxxx_pc16550_dev.h> 60 61#define CN88XX_MAP_UART0_OFFSET 0x87E024000000UL 62 63apm88xxxx_pc16550_t uart; 64 65static void debug_uart_initialize(void) { 66 apm88xxxx_pc16550_initialize(&uart, (mackerel_addr_t)0x1C020000); 67} 68 69static void debug_serial_putc(char c) 70{ 71 // Wait until FIFO can hold more characters 72 while(!apm88xxxx_pc16550_LSR_thre_rdf(&uart)); 73 // Write character 74 apm88xxxx_pc16550_THR_thr_wrf(&uart, c); 75} 76 77#endif 78 79static void debug_serial_putchar(char c) { 80 if (c == '\n') { 81 debug_serial_putc('\r'); 82 } 83 debug_serial_putc(c); 84} 85 86 87static void debug_print_string(char *str) 88{ 89 while (str && *str) { 90 debug_serial_putchar(*str); 91 str++; 92 } 93} 94#else 95#define debug_print_string(x) 96#define debug_uart_initialize() 97#endif 98 99 100void (*cpu_driver_entry)(uint32_t magic, lpaddr_t pointer, lpaddr_t stack); 101 102static void configure_tcr(void) { 103 armv8_TCR_EL1_t tcr_el1 = armv8_TCR_EL1_rd(NULL); 104 // disable top byte ignored, EL1 105 tcr_el1 = armv8_TCR_EL1_TBI1_insert(tcr_el1, 0); 106 // disable top byte ignored, EL0 107 tcr_el1 = armv8_TCR_EL1_TBI0_insert(tcr_el1, 0); 108 // 48b IPA 109 tcr_el1 = armv8_TCR_EL1_IPS_insert(tcr_el1, 5); 110 // 4kB granule 111 tcr_el1 = armv8_TCR_EL1_TG1_insert(tcr_el1, armv8_KB_4); 112 // Walks inner shareable 113 tcr_el1 = armv8_TCR_EL1_SH1_insert(tcr_el1, armv8_inner_shareable); 114 // Walks outer WB WA 115 tcr_el1 = armv8_TCR_EL1_ORGN1_insert(tcr_el1, armv8_WbRaWa_cache); 116 // Walks inner WB WA 117 tcr_el1 = armv8_TCR_EL1_IRGN1_insert(tcr_el1, armv8_WbRaWa_cache); 118 // enable EL1 translation 119 tcr_el1 = armv8_TCR_EL1_EPD1_insert(tcr_el1, 0); 120 // 48b kernel VA 121 tcr_el1 = armv8_TCR_EL1_T1SZ_insert(tcr_el1, 16); 122 // 4kB granule 123 tcr_el1 = armv8_TCR_EL1_TG0_insert(tcr_el1, armv8_KB_4); 124 // Walks inner shareable 125 tcr_el1 = armv8_TCR_EL1_SH0_insert(tcr_el1, armv8_inner_shareable); 126 // Walks outer WB WA 127 tcr_el1 = armv8_TCR_EL1_ORGN0_insert(tcr_el1, armv8_WbRaWa_cache); 128 // Walks inner WB WA 129 tcr_el1 = armv8_TCR_EL1_IRGN0_insert(tcr_el1, armv8_WbRaWa_cache); 130 // enable EL0 translation 131 tcr_el1 = armv8_TCR_EL1_EPD0_insert(tcr_el1, 0); 132 // 48b user VA 133 tcr_el1 = armv8_TCR_EL1_T0SZ_insert(tcr_el1, 16); 134 armv8_TCR_EL1_wr(NULL, tcr_el1); 135} 136 137 138#define DAIF_FIQ_BIT (1 << 0) 139#define DAIF_IRQ_BIT (1 << 1) 140 141 142static void armv8_disable_interrupts(void) 143{ 144 __asm volatile("msr DAIFSet, #3\n"); 145} 146 147static void armv8_set_tcr(uint8_t el) 148{ 149 switch(el) { 150 case 3: 151 //sysreg_write_ttbr0_el2(addr); 152 case 2: 153 { 154 armv8_TCR_EL2_t reg = 0; 155 reg = armv8_TCR_EL2_PS_insert(reg, 5); 156 reg = armv8_TCR_EL2_T0SZ_insert(reg, (64 - 48)); 157 armv8_TCR_EL2_wr(NULL, reg); 158 break; 159 } 160 case 1: 161 { 162 armv8_TCR_EL1_t reg = 0; 163 // TODO: figure out what to set reg = armv8_TCR_EL1_PS_insert(reg, 5); 164 reg = armv8_TCR_EL1_T0SZ_insert(reg, (64 - 48)); 165 armv8_TCR_EL1_wr(NULL, reg); 166 break; 167 } 168 default: 169 assert("should not happen"); 170 return; 171 } 172} 173 174static void armv8_set_ttbr0(uint8_t el, lpaddr_t addr) 175{ 176 switch(el) { 177 case 3: 178 //sysreg_write_ttbr0_el2(addr); 179 case 2: 180 armv8_TTBR0_EL2_wr(NULL, addr); 181 armv8_TTBR0_EL1_wr(NULL, addr); 182 break; 183 case 1: 184 armv8_TTBR0_EL1_wr(NULL, addr); 185 break; 186 default: 187 assert("should not happen"); 188 return; 189 } 190 __asm volatile("isb"); 191} 192 193static void armv8_enable_mmu(uint8_t el) 194{ 195 switch(el) { 196 case 3: 197 armv8_SCTLR_EL3_M_wrf(NULL, 0x1); 198 __asm volatile("tlbi alle3\n isb"); 199 break; 200 case 2: 201 armv8_SCTLR_EL2_M_wrf(NULL, 0x1); 202 __asm volatile("tlbi alle2\n isb"); 203 break; 204 case 1: 205 armv8_SCTLR_EL1_M_wrf(NULL, 0x1); 206 __asm volatile("tlbi vmalle1\n isb"); 207 break; 208 default: 209 assert("should not happen"); 210 return; 211 } 212 __asm volatile("dsb sy\n isb"); 213} 214 215static void armv8_invalidate_tlb(uint8_t el) 216{ 217 switch(el) { 218 case 3: 219 armv8_SCTLR_EL3_M_wrf(NULL, 0x1); 220 __asm volatile("tlbi alle3"); 221 break; 222 case 2: 223 armv8_SCTLR_EL2_M_wrf(NULL, 0x1); 224 __asm volatile("tlbi alle2"); 225 break; 226 case 1: 227 armv8_SCTLR_EL1_M_wrf(NULL, 0x1); 228 __asm volatile("tlbi vmalle1"); 229 break; 230 default: 231 assert("should not happen"); 232 return; 233 } 234 __asm volatile("dsb sy\n isb"); 235} 236 237static void armv8_invalidate_icache(void) 238{ 239 __asm volatile( 240 "ic iallu \n" 241 "dsb sy \n" 242 "isb \n" 243 ); 244} 245 246static void armv8_instruction_synchronization_barrier(void) 247{ 248 __asm volatile("isb"); 249} 250 251static void configure_spsr(uint8_t el) { 252 armv8_SPSR_EL2_t spsr = 0; 253 /* mask the exceptions */ 254 spsr = armv8_SPSR_EL2_D_insert(spsr, 1); 255 spsr = armv8_SPSR_EL2_A_insert(spsr, 1); 256 spsr = armv8_SPSR_EL2_I_insert(spsr, 1); 257 spsr = armv8_SPSR_EL2_F_insert(spsr, 1); 258 259 /* set el1 and use the SP_ELx stack */ 260 spsr = armv8_SPSR_EL2_M_lo_insert(spsr, (1<<2) | 1); 261 262 switch(el) { 263 case 3: 264 armv8_SPSR_EL3_wr(NULL, spsr); 265 return; 266 case 2: 267 armv8_SPSR_EL2_wr(NULL, spsr); 268 break; 269 case 1: 270 armv8_SPSR_EL1_wr(NULL, spsr); 271 return; 272 default: 273 return; 274 } 275} 276 277static void configure_ttbr1(uint8_t el) 278{ 279 lpaddr_t ttbr1_el1; 280 switch(el) { 281 case 3: 282 ttbr1_el1= armv8_TTBR0_EL3_rawrd(NULL); 283 break; 284 case 2: 285 ttbr1_el1= armv8_TTBR0_EL2_rawrd(NULL); 286 break; 287 case 1: 288 ttbr1_el1= armv8_TTBR0_EL1_rawrd(NULL); 289 break; 290 default: 291 return; 292 } 293 armv8_TTBR1_EL1_rawwr(NULL, ttbr1_el1); 294} 295 296static void configure_mair(void) 297{ 298 /* Set memory type 0, for kernel use. */ 299 // attr0 = Normal Memory, Inner Write-back non transient 300 // attr1 = Device-nGnRnE memory 301 armv8_MAIR_EL1_wr(NULL, 0x00ff); 302} 303 304static void configure_sctlr(void) 305/* Enable EL0/1 translation. */ 306{ 307 308 armv8_SCTLR_EL1_t val = 0; 309 310 /* Traps EL0 execution of cache maintenance instructions to EL1 */ 311 val = armv8_SCTLR_EL1_UCI_insert(val, 0x1); 312 313 /* write permissions implies execute never */ 314 //val = armv8_SCTLR_EL1_WXN_insert(val, 0x1); 315 316 /* don't trap WFI/WFE instructions to EL1 */ 317 val = armv8_SCTLR_EL1_nTWE_insert(val, 0x1); 318 val = armv8_SCTLR_EL1_nTWI_insert(val, 0x1); 319 320 /* disable Traps EL0 accesses to the CTR_EL0 to EL1*/ 321 val = armv8_SCTLR_EL1_UCT_insert(val, 0x1); 322 323 /* Allow EL0 to do DC ZVA */ 324 val = armv8_SCTLR_EL1_DZE_insert(val, 0x1); 325 326 /* enable instruction cache */ 327 val = armv8_SCTLR_EL1_I_insert(val, 0x1); 328 329 /* 330 * EL0 execution of MRS , MSR(register) , or MSR(immediate) instructions 331 * that access the DAIF is not trapped to EL1. 332 */ 333 //val = armv8_SCTLR_EL1_UMA_insert(val, 0x1); 334 335 /* 336 * Enables accesses to the DMB, DSB, and ISB System 337 * instructions in the (coproc== 1111 ) encoding space from EL0 338 */ 339 val = armv8_SCTLR_EL1_CP15BEN_insert(val, 0x1); 340 341 /* Enable SP alignment checks */ 342 val = armv8_SCTLR_EL1_SA0_insert(val, 0x1); 343 val = armv8_SCTLR_EL1_SA_insert(val, 0x1); 344 345 /* enable data cachable */ 346 val = armv8_SCTLR_EL1_C_insert(val, 0x1); 347 348 /* enable alignment checks */ 349 val = armv8_SCTLR_EL1_A_insert(val, 0x1); 350 351 /* enable mmu */ 352 val = armv8_SCTLR_EL1_M_insert(val, 0x1); 353 354 armv8_SCTLR_EL1_wr(NULL, val); 355} 356 357static void configure_el3_traps(void) 358{ 359 360 /* If we've started in EL3, that most likely means we're in the 361 * simulator. We don't use it at all, so just disable all traps to 362 * EL3, and drop to non-secure EL2 (if it exists). */ 363 364 armv8_SCR_EL3_t val = 0; 365 366 /* Don't trap secure timer access. */ 367 val = armv8_SCR_EL3_ST_insert(val, 0x1); 368 369 /* Next EL is AArch64. */ 370 val = armv8_SCR_EL3_RW_insert(val, 0x1); 371 372 /* HVC is enabled. */ 373 val = armv8_SCR_EL3_HCE_insert(val, 0x1); 374 375 /* SMC is disabled. */ 376 val = armv8_SCR_EL3_SMD_insert(val, 0x1); 377 378 /* External aborts don't trap to EL3. */ 379 val = armv8_SCR_EL3_EA_insert(val, 0x1); 380 381 /* FIQs don't trap to EL3. */ 382 val = armv8_SCR_EL3_FIQ_insert(val, 0x1); 383 384 /* IRQs don't trap to EL3. */ 385 val = armv8_SCR_EL3_IRQ_insert(val, 0x1); 386 387 /* EL0 and EL1 are non-secure. */ 388 val = armv8_SCR_EL3_NS_insert(val, 0x1); 389 390 armv8_SCR_EL3_wr(NULL, val); 391 392 /* We don't need to set SCTLR_EL3, as we're not using it. */ 393 394 armv8_MDCR_EL3_t mdcr = 0; 395 /* Allow event counting in secure state. */ 396 armv8_MDCR_EL3_SPME_insert(mdcr, 0x1); 397 armv8_MDCR_EL3_wr(NULL, mdcr); 398} 399 400static void configure_el2_traps(void) 401{ 402 /* check if EL2 is implemented */ 403 if (armv8_ID_AA64PFR0_EL1_EL2_rdf(NULL) == armv8_ID_EL_NOT_IMPLEMENTED) { 404 return; 405 } 406 407 /* configure EL2 traps & mmu */ 408 409 armv8_HCR_EL2_t val = 0; 410 411 /* For the Non-secure EL1&0 translation regime, for permitted accesses to a 412 * memory location that use a common definition of the Shareability and 413 * Cacheability of the location, there might be a loss of coherency if the 414 * Inner Cacheability attribute for those accesses differs from the Outer 415 * Cacheability attribute.*/ 416 val = armv8_HCR_EL2_MIOCNCE_insert(val, 1); 417 418 /* Set the mode to be AARCH64 */ 419 val = armv8_HCR_EL2_RW_insert(val, 1); 420 421 /* HVC instructions are UNDEFINED at EL2 and Non-secure EL1. Any resulting 422 * exception is taken to the Exception level at which the HVC instruction 423 * is executed. 424 * 425 * XXX: this will disable Hypervisor calls entirely, revisit for ARRAKIS 426 */ 427 val = armv8_HCR_EL2_HCD_insert(val, 1); 428 429 armv8_HCR_EL2_wr(NULL, val); 430 431 432 /* disable traps to EL2 for timer accesses */ 433 434 armv8_CNTHCTL_EL2_t cnthctl; 435 cnthctl = armv8_CNTHCTL_EL2_rd(NULL); 436 cnthctl = armv8_CNTHCTL_EL2_EL1PCEN_insert(cnthctl, 0x1); 437 cnthctl = armv8_CNTHCTL_EL2_EL1PCTEN_insert(cnthctl, 0x1); 438 armv8_CNTHCTL_EL2_wr(NULL, cnthctl); 439} 440 441static void configure_el1_traps(void) 442{ 443 /* disable traps for FP/SIMD access */ 444 armv8_CPACR_EL1_FPEN_wrf(NULL, armv8_fpen_trap_none); 445} 446 447static void drop_to_el2(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) 448{ 449 /* write the stack pointer for EL1 */ 450 armv8_SP_EL1_wr(NULL, stack + KERNEL_OFFSET); 451 452 /* Set the jump target */ 453 armv8_ELR_EL3_wr(NULL, (uint64_t)cpu_driver_entry); 454 455 /* call exception return */ 456 eret(magic, pointer + KERNEL_OFFSET, stack + KERNEL_OFFSET, 0); 457} 458 459static void drop_to_el1(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) 460{ 461 /* write the stack pointer for EL1 */ 462 armv8_SP_EL1_wr(NULL, stack + KERNEL_OFFSET); 463 464 /* Set the jump target */ 465 armv8_ELR_EL2_wr(NULL, (uint64_t)cpu_driver_entry); 466 467 /* call exception return */ 468 eret(magic, pointer + KERNEL_OFFSET, stack + KERNEL_OFFSET, 0); 469} 470 471static void jump_to_cpudriver(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) 472{ 473 // We are in EL1, so call arch_init directly. 474 475 // we may need to re set the stack pointer 476 uint64_t sp = sysreg_read_sp(); 477 if (sp < KERNEL_OFFSET) { 478 sysreg_write_sp(sp + KERNEL_OFFSET); 479 } 480 cpu_driver_entry(magic, pointer + KERNEL_OFFSET, stack + KERNEL_OFFSET); 481 482} 483 484 485/* On entry: 486 487 Execution is starting in LOW addresses 488 Pointers to stack and multiboot are LOW addresses 489 Single core running (not guaranteed to be core 0) 490 CPU is in highest implemented exception level 491 MMU enabled, 4k translation granule, 1:1 mapping of all RAM 492 Little-endian mode 493 Core caches (L1&L2) and TLB enabled 494 Non-architectural caches disabled (e.g. L3) 495 Interrupts enabled 496 Generic timer initialized and enabled 497 >= 128KiB stack 498 ACPI tables available 499 Register x0 contains the multiboot magic value 500 Register x1 contains a pointer to multiboot image 501 Register x1 contains a pointer to top entry in the kernel stack 502 */ 503static void boot_generic_init(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) { 504 505 uint8_t el = armv8_CurrentEL_EL_rdf(NULL); 506 507 /* Configure the EL1 translation regime. */ 508 configure_tcr(); 509 510 /* Copy the current TTBR for EL1. */ 511 configure_ttbr1(el); 512 513 /* configure memory attributes */ 514 configure_mair(); 515 516 /* Enable EL0/1 translation. */ 517 configure_sctlr(); 518 519 /* configure spsr */ 520 configure_spsr(el); 521 522 /* configure EL 1 traps*/ 523 configure_el1_traps(); 524 525 switch(el) { 526 case 3: 527 configure_el3_traps(); 528 configure_el2_traps(); 529 drop_to_el2(magic, pointer, stack); 530 break; 531 case 2: 532 configure_el2_traps(); 533 drop_to_el1(magic, pointer, stack); 534 break; 535 case 1: 536 jump_to_cpudriver(magic, pointer, stack); 537 break; 538 default: 539 break; 540 } 541} 542 543/** 544 * @brief initializes an application core 545 * 546 * @param state pointer to the armv8_core_data structure 547 * 548 * This function is intended to bring the core to the same state as if it 549 * has been booted by the UEFI boot loader. 550 */ 551void boot_app_init(lpaddr_t state) 552{ 553 554 debug_uart_initialize(); 555 debug_print_string("APP BOOTING\n"); 556 557 struct armv8_core_data *cd = (struct armv8_core_data *)state; 558 559 cpu_driver_entry = (void *)cd->cpu_driver_entry; 560 561 uint8_t current_el = armv8_CurrentEL_EL_rdf(NULL); 562 563 if (current_el == 2) { 564 uint64_t zero = 0; 565 __asm volatile("MSR CPTR_EL2, %[zero]" : : [zero] "r" (zero)); 566 } 567 568 // /* disable interrupts */ 569 armv8_disable_interrupts(); 570 571 /* set the ttbr0/1 */ 572 armv8_set_ttbr0(current_el, cd->page_table_root); 573 574 /* set the TCR */ 575 armv8_set_tcr(current_el); 576 577 /* enable MMU */ 578 armv8_enable_mmu(current_el); 579 580 /* invalidate TLB */ 581 armv8_invalidate_tlb(current_el); 582 583 /* invalidate icache */ 584 armv8_invalidate_icache(); 585 armv8_instruction_synchronization_barrier(); 586 587 boot_generic_init(cd->boot_magic, state, cd->cpu_driver_stack); 588 589 while(1) { 590 __asm volatile("wfi \n"); 591 } 592} 593 594/* On entry: 595 596 Execution is starting in LOW addresses 597 Pointers to stack and multiboot are LOW addresses 598 Single core running (not guaranteed to be core 0) 599 CPU is in highest implemented exception level 600 MMU enabled, 4k translation granule, 1:1 mapping of all RAM 601 Little-endian mode 602 Core caches (L1&L2) and TLB enabled 603 Non-architectural caches disabled (e.g. L3) 604 Interrupts enabled 605 Generic timer initialized and enabled 606 >= 128KiB stack 607 ACPI tables available 608 Register x0 contains the multiboot magic value 609 Register x1 contains a pointer to multiboot image 610 Register x1 contains a pointer to top entry in the kernel stack 611 */ 612void 613boot_bsp_init(uint32_t magic, lpaddr_t pointer, lpaddr_t stack) { 614 615 debug_uart_initialize(); 616 debug_print_string("BSP BOOTING\n"); 617 618 /* Boot magic must be set */ 619 if (magic != MULTIBOOT2_BOOTLOADER_MAGIC) { 620 goto stop; 621 } 622 623 /* 624 * get the first entry in the multiboot structure, this holds a pointer 625 * to the CPU driver entry point 626 */ 627 struct multiboot_header *mbhdr = (void *)pointer; 628 629 struct multiboot_header_tag *mb; 630 mb = (struct multiboot_header_tag *)(mbhdr + 1); 631 632 if (mb->type == MULTIBOOT_TAG_TYPE_EFI64) { 633 cpu_driver_entry = (void *)((struct multiboot_tag_efi64 *)mb)->pointer; 634 } 635 636 /* disable interrupts */ 637 armv8_disable_interrupts(); 638 639 boot_generic_init(magic, pointer, stack); 640 641 stop: 642 while(1) { 643 __asm volatile("wfi \n"); 644 } 645} 646