1/* 2 * Copyright 2014, General Dynamics C4 Systems 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8#include <assert.h> 9#include <kernel/boot.h> 10#include <machine/io.h> 11#include <model/statedata.h> 12#include <object/interrupt.h> 13#include <arch/machine.h> 14#include <arch/kernel/boot.h> 15#include <arch/kernel/vspace.h> 16#include <arch/benchmark.h> 17#include <arch/user_access.h> 18#include <arch/object/iospace.h> 19#include <linker.h> 20#include <plat/machine/hardware.h> 21#include <machine.h> 22#include <arch/machine/timer.h> 23#include <arch/machine/fpu.h> 24#include <arch/machine/tlb.h> 25 26#ifdef CONFIG_ARM_SMMU 27#include <drivers/smmu/smmuv2.h> 28#endif 29/* pointer to the end of boot code/data in kernel image */ 30/* need a fake array to get the pointer from the linker script */ 31extern char ki_boot_end[1]; 32/* pointer to end of kernel image */ 33extern char ki_end[1]; 34 35#ifdef ENABLE_SMP_SUPPORT 36/* sync variable to prevent other nodes from booting 37 * until kernel data structures initialized */ 38BOOT_DATA static volatile int node_boot_lock = 0; 39#endif /* ENABLE_SMP_SUPPORT */ 40 41#define ARCH_RESERVED 3 // kernel + user image + dtb 42#define MAX_RESERVED (ARCH_RESERVED + MODE_RESERVED) 43BOOT_DATA static region_t reserved[MAX_RESERVED]; 44 45BOOT_CODE static void arch_init_freemem(p_region_t ui_p_reg, p_region_t dtb_p_reg, v_region_t it_v_reg, 46 word_t extra_bi_size_bits) 47{ 48 reserved[0].start = KERNEL_ELF_BASE; 49 reserved[0].end = (pptr_t)ki_end; 50 51 int index = 1; 52 if (dtb_p_reg.start) { 53 /* the dtb region could be empty */ 54 reserved[index].start = (pptr_t) paddr_to_pptr(dtb_p_reg.start); 55 reserved[index].end = (pptr_t) paddr_to_pptr(dtb_p_reg.end); 56 index++; 57 } 58 59 if (MODE_RESERVED > 1) { 60 printf("MODE_RESERVED > 1 unsupported!\n"); 61 halt(); 62 } 63 64 if (ui_p_reg.start < PADDR_TOP) { 65 region_t ui_reg = paddr_to_pptr_reg(ui_p_reg); 66 if (MODE_RESERVED == 1) { 67 if (ui_reg.end > mode_reserved_region[0].start) { 68 reserved[index] = mode_reserved_region[0]; 69 index++; 70 reserved[index].start = ui_reg.start; 71 reserved[index].end = ui_reg.end; 72 } else { 73 reserved[index].start = ui_reg.start; 74 reserved[index].end = ui_reg.end; 75 index++; 76 reserved[index] = mode_reserved_region[0]; 77 } 78 index++; 79 } else { 80 reserved[index].start = ui_reg.start; 81 reserved[index].end = ui_reg.end; 82 index++; 83 } 84 } else if (MODE_RESERVED == 1) { 85 reserved[index] = mode_reserved_region[0]; 86 index++; 87 } 88 89 init_freemem(get_num_avail_p_regs(), get_avail_p_regs(), index, reserved, it_v_reg, extra_bi_size_bits); 90} 91 92 93BOOT_CODE static void init_irqs(cap_t root_cnode_cap) 94{ 95 unsigned i; 96 97 for (i = 0; i <= maxIRQ ; i++) { 98 setIRQState(IRQInactive, CORE_IRQ_TO_IRQT(0, i)); 99 } 100 setIRQState(IRQTimer, CORE_IRQ_TO_IRQT(0, KERNEL_TIMER_IRQ)); 101#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT 102 setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, INTERRUPT_VGIC_MAINTENANCE)); 103 setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, INTERRUPT_VTIMER_EVENT)); 104#endif 105#ifdef CONFIG_TK1_SMMU 106 setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, INTERRUPT_SMMU)); 107#endif 108 109#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT 110#ifdef KERNEL_PMU_IRQ 111 setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(0, KERNEL_PMU_IRQ)); 112#if (defined CONFIG_PLAT_TX1 && defined ENABLE_SMP_SUPPORT) 113//SELFOUR-1252 114#error "This platform doesn't support tracking CPU utilisation on multicore" 115#endif /* CONFIG_PLAT_TX1 && ENABLE_SMP_SUPPORT */ 116#else 117#error "This platform doesn't support tracking CPU utilisation feature" 118#endif /* KERNEL_TIMER_IRQ */ 119#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */ 120 121#ifdef ENABLE_SMP_SUPPORT 122 setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_remote_call_ipi)); 123 setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_reschedule_ipi)); 124#endif 125 126 /* provide the IRQ control cap */ 127 write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIRQControl), cap_irq_control_cap_new()); 128} 129 130#ifdef CONFIG_ARM_SMMU 131BOOT_CODE static void init_smmu(cap_t root_cnode_cap) 132{ 133 plat_smmu_init(); 134 /*provide the SID and CB control cap*/ 135 write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapSMMUSIDControl), cap_sid_control_cap_new()); 136 write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapSMMUCBControl), cap_cb_control_cap_new()); 137} 138 139#endif 140 141BOOT_CODE static bool_t create_untypeds(cap_t root_cnode_cap, region_t boot_mem_reuse_reg) 142{ 143 seL4_SlotPos slot_pos_before; 144 seL4_SlotPos slot_pos_after; 145 146 slot_pos_before = ndks_boot.slot_pos_cur; 147 create_device_untypeds(root_cnode_cap, slot_pos_before); 148 create_kernel_untypeds(root_cnode_cap, boot_mem_reuse_reg, slot_pos_before); 149 150 slot_pos_after = ndks_boot.slot_pos_cur; 151 ndks_boot.bi_frame->untyped = (seL4_SlotRegion) { 152 slot_pos_before, slot_pos_after 153 }; 154 return true; 155 156} 157 158/** This and only this function initialises the CPU. 159 * 160 * It does NOT initialise any kernel state. 161 * @return For the verification build, this currently returns true always. 162 */ 163BOOT_CODE static bool_t init_cpu(void) 164{ 165 bool_t haveHWFPU; 166 167#ifdef CONFIG_ARCH_AARCH64 168 if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) { 169 if (!checkTCR_EL2()) { 170 return false; 171 } 172 } 173#endif 174 175 activate_global_pd(); 176 if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) { 177 vcpu_boot_init(); 178 } 179 180#ifdef CONFIG_HARDWARE_DEBUG_API 181 if (!Arch_initHardwareBreakpoints()) { 182 printf("Kernel built with CONFIG_HARDWARE_DEBUG_API, but this board doesn't " 183 "reliably support it.\n"); 184 return false; 185 } 186#endif 187 188 /* Setup kernel stack pointer. 189 * On ARM SMP, the array index here is the CPU ID 190 */ 191#ifndef CONFIG_ARCH_ARM_V6 192 word_t stack_top = ((word_t) kernel_stack_alloc[SMP_TERNARY(getCurrentCPUIndex(), 0)]) + BIT(CONFIG_KERNEL_STACK_BITS); 193#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_ARCH_AARCH64) 194 /* the least 12 bits are used to store logical core ID */ 195 stack_top |= getCurrentCPUIndex(); 196#endif 197 setKernelStack(stack_top); 198#endif /* CONFIG_ARCH_ARM_V6 */ 199 200#ifdef CONFIG_ARCH_AARCH64 201 /* initialise CPU's exception vector table */ 202 setVtable((pptr_t)arm_vector_table); 203#endif /* CONFIG_ARCH_AARCH64 */ 204 205 haveHWFPU = fpsimd_HWCapTest(); 206 207 /* Disable FPU to avoid channels where a platform has an FPU but doesn't make use of it */ 208 if (haveHWFPU) { 209 disableFpu(); 210 } 211 212#ifdef CONFIG_HAVE_FPU 213 if (haveHWFPU) { 214 if (!fpsimd_init()) { 215 return false; 216 } 217 } else { 218 printf("Platform claims to have FP hardware, but does not!"); 219 return false; 220 } 221#endif /* CONFIG_HAVE_FPU */ 222 223 cpu_initLocalIRQController(); 224 225#ifdef CONFIG_ENABLE_BENCHMARKS 226 arm_init_ccnt(); 227#endif /* CONFIG_ENABLE_BENCHMARKS */ 228 229 /* Export selected CPU features for access by PL0 */ 230 armv_init_user_access(); 231 232 initTimer(); 233 234 return true; 235} 236 237/* This and only this function initialises the platform. It does NOT initialise any kernel state. */ 238 239BOOT_CODE static void init_plat(void) 240{ 241 initIRQController(); 242 initL2Cache(); 243#ifdef CONFIG_ARM_SMMU 244 plat_smmu_init(); 245#endif 246} 247 248#ifdef ENABLE_SMP_SUPPORT 249BOOT_CODE static bool_t try_init_kernel_secondary_core(void) 250{ 251 unsigned i; 252 253 /* need to first wait until some kernel init has been done */ 254 while (!node_boot_lock); 255 256 /* Perform cpu init */ 257 init_cpu(); 258 259 for (i = 0; i < NUM_PPI; i++) { 260 maskInterrupt(true, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), i)); 261 } 262 setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_remote_call_ipi)); 263 setIRQState(IRQIPI, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_reschedule_ipi)); 264 /* Enable per-CPU timer interrupts */ 265 setIRQState(IRQTimer, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), KERNEL_TIMER_IRQ)); 266#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT 267 setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), INTERRUPT_VGIC_MAINTENANCE)); 268 setIRQState(IRQReserved, CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), INTERRUPT_VTIMER_EVENT)); 269#endif /* CONFIG_ARM_HYPERVISOR_SUPPORT */ 270 NODE_LOCK_SYS; 271 272 ksNumCPUs++; 273 274 init_core_state(SchedulerAction_ResumeCurrentThread); 275 276 return true; 277} 278 279BOOT_CODE static void release_secondary_cpus(void) 280{ 281 282 /* release the cpus at the same time */ 283 node_boot_lock = 1; 284 285#ifndef CONFIG_ARCH_AARCH64 286 /* At this point in time the other CPUs do *not* have the seL4 global pd set. 287 * However, they still have a PD from the elfloader (which is mapping mmemory 288 * as strongly ordered uncached, as a result we need to explicitly clean 289 * the cache for it to see the update of node_boot_lock 290 * 291 * For ARMv8, the elfloader sets the page table entries as inner shareable 292 * (so is the attribute of the seL4 global PD) when SMP is enabled, and 293 * turns on the cache. Thus, we do not need to clean and invaliate the cache. 294 */ 295 cleanInvalidateL1Caches(); 296 plat_cleanInvalidateL2Cache(); 297#endif 298 299 /* Wait until all the secondary cores are done initialising */ 300 while (ksNumCPUs != CONFIG_MAX_NUM_NODES) { 301 /* perform a memory release+acquire to get new values of ksNumCPUs */ 302 __atomic_signal_fence(__ATOMIC_ACQ_REL); 303 } 304} 305#endif /* ENABLE_SMP_SUPPORT */ 306 307/* Main kernel initialisation function. */ 308 309static BOOT_CODE bool_t try_init_kernel( 310 paddr_t ui_p_reg_start, 311 paddr_t ui_p_reg_end, 312 sword_t pv_offset, 313 vptr_t v_entry, 314 paddr_t dtb_addr_start, 315 paddr_t dtb_addr_end 316) 317{ 318 cap_t root_cnode_cap; 319 cap_t it_ap_cap; 320 cap_t it_pd_cap; 321 cap_t ipcbuf_cap; 322 p_region_t ui_p_reg = (p_region_t) { 323 ui_p_reg_start, ui_p_reg_end 324 }; 325 region_t ui_reg = paddr_to_pptr_reg(ui_p_reg); 326 region_t dtb_reg; 327 word_t extra_bi_size; 328 pptr_t extra_bi_offset = 0; 329 vptr_t extra_bi_frame_vptr; 330 vptr_t bi_frame_vptr; 331 vptr_t ipcbuf_vptr; 332 create_frames_of_region_ret_t create_frames_ret; 333 create_frames_of_region_ret_t extra_bi_ret; 334 335 /* convert from physical addresses to userland vptrs */ 336 v_region_t ui_v_reg; 337 v_region_t it_v_reg; 338 ui_v_reg.start = ui_p_reg_start - pv_offset; 339 ui_v_reg.end = ui_p_reg_end - pv_offset; 340 341 ipcbuf_vptr = ui_v_reg.end; 342 bi_frame_vptr = ipcbuf_vptr + BIT(PAGE_BITS); 343 extra_bi_frame_vptr = bi_frame_vptr + BIT(PAGE_BITS); 344 345 /* If no DTB was provided, skip allocating extra bootinfo */ 346 p_region_t dtb_p_reg = { 347 dtb_addr_start, ROUND_UP(dtb_addr_end, PAGE_BITS) 348 }; 349 if (dtb_addr_start == 0) { 350 extra_bi_size = 0; 351 dtb_reg = (region_t) { 352 0, 0 353 }; 354 } else { 355 dtb_reg = paddr_to_pptr_reg(dtb_p_reg); 356 extra_bi_size = sizeof(seL4_BootInfoHeader) + (dtb_reg.end - dtb_reg.start); 357 } 358 word_t extra_bi_size_bits = calculate_extra_bi_size_bits(extra_bi_size); 359 360 /* The region of the initial thread is the user image + ipcbuf and boot info */ 361 it_v_reg.start = ui_v_reg.start; 362 it_v_reg.end = extra_bi_frame_vptr + BIT(extra_bi_size_bits); 363 364 if (it_v_reg.end >= USER_TOP) { 365 printf("Userland image virtual end address too high\n"); 366 return false; 367 } 368 369 /* setup virtual memory for the kernel */ 370 map_kernel_window(); 371 372 /* initialise the CPU */ 373 if (!init_cpu()) { 374 return false; 375 } 376 377 /* debug output via serial port is only available from here */ 378 printf("Bootstrapping kernel\n"); 379 380 /* initialise the platform */ 381 init_plat(); 382 383 arch_init_freemem(ui_p_reg, dtb_p_reg, it_v_reg, extra_bi_size_bits); 384 385 /* create the root cnode */ 386 root_cnode_cap = create_root_cnode(); 387 if (cap_get_capType(root_cnode_cap) == cap_null_cap) { 388 return false; 389 } 390 391 /* create the cap for managing thread domains */ 392 create_domain_cap(root_cnode_cap); 393 394 /* initialise the IRQ states and provide the IRQ control cap */ 395 init_irqs(root_cnode_cap); 396 397#ifdef CONFIG_ARM_SMMU 398 /* initialise the SMMU and provide the SMMU control caps*/ 399 init_smmu(root_cnode_cap); 400#endif 401 populate_bi_frame(0, CONFIG_MAX_NUM_NODES, ipcbuf_vptr, extra_bi_size); 402 403 /* put DTB in the bootinfo block, if present. */ 404 seL4_BootInfoHeader header; 405 if (dtb_reg.start) { 406 header.id = SEL4_BOOTINFO_HEADER_FDT; 407 header.len = sizeof(header) + dtb_reg.end - dtb_reg.start; 408 *(seL4_BootInfoHeader *)(rootserver.extra_bi + extra_bi_offset) = header; 409 extra_bi_offset += sizeof(header); 410 memcpy((void *)(rootserver.extra_bi + extra_bi_offset), (void *)dtb_reg.start, 411 dtb_reg.end - dtb_reg.start); 412 extra_bi_offset += (dtb_reg.end - dtb_reg.start); 413 } 414 415 if (extra_bi_size > extra_bi_offset) { 416 /* provde a chunk for any leftover padding in the extended boot info */ 417 header.id = SEL4_BOOTINFO_HEADER_PADDING; 418 header.len = (extra_bi_size - extra_bi_offset); 419 *(seL4_BootInfoHeader *)(rootserver.extra_bi + extra_bi_offset) = header; 420 } 421 422 if (config_set(CONFIG_TK1_SMMU)) { 423 ndks_boot.bi_frame->ioSpaceCaps = create_iospace_caps(root_cnode_cap); 424 if (ndks_boot.bi_frame->ioSpaceCaps.start == 0 && 425 ndks_boot.bi_frame->ioSpaceCaps.end == 0) { 426 return false; 427 } 428 } else { 429 ndks_boot.bi_frame->ioSpaceCaps = S_REG_EMPTY; 430 } 431 432 /* Construct an initial address space with enough virtual addresses 433 * to cover the user image + ipc buffer and bootinfo frames */ 434 it_pd_cap = create_it_address_space(root_cnode_cap, it_v_reg); 435 if (cap_get_capType(it_pd_cap) == cap_null_cap) { 436 return false; 437 } 438 439 /* Create and map bootinfo frame cap */ 440 create_bi_frame_cap( 441 root_cnode_cap, 442 it_pd_cap, 443 bi_frame_vptr 444 ); 445 446 /* create and map extra bootinfo region */ 447 if (extra_bi_size > 0) { 448 region_t extra_bi_region = { 449 .start = rootserver.extra_bi, 450 .end = rootserver.extra_bi + extra_bi_size 451 }; 452 extra_bi_ret = 453 create_frames_of_region( 454 root_cnode_cap, 455 it_pd_cap, 456 extra_bi_region, 457 true, 458 pptr_to_paddr((void *)extra_bi_region.start) - extra_bi_frame_vptr 459 ); 460 if (!extra_bi_ret.success) { 461 return false; 462 } 463 ndks_boot.bi_frame->extraBIPages = extra_bi_ret.region; 464 } 465 466#ifdef CONFIG_KERNEL_MCS 467 init_sched_control(root_cnode_cap, CONFIG_MAX_NUM_NODES); 468#endif 469 470 /* create the initial thread's IPC buffer */ 471 ipcbuf_cap = create_ipcbuf_frame_cap(root_cnode_cap, it_pd_cap, ipcbuf_vptr); 472 if (cap_get_capType(ipcbuf_cap) == cap_null_cap) { 473 return false; 474 } 475 476 /* create all userland image frames */ 477 create_frames_ret = 478 create_frames_of_region( 479 root_cnode_cap, 480 it_pd_cap, 481 ui_reg, 482 true, 483 pv_offset 484 ); 485 if (!create_frames_ret.success) { 486 return false; 487 } 488 ndks_boot.bi_frame->userImageFrames = create_frames_ret.region; 489 490 /* create/initialise the initial thread's ASID pool */ 491 it_ap_cap = create_it_asid_pool(root_cnode_cap); 492 if (cap_get_capType(it_ap_cap) == cap_null_cap) { 493 return false; 494 } 495 write_it_asid_pool(it_ap_cap, it_pd_cap); 496 497#ifdef CONFIG_KERNEL_MCS 498 NODE_STATE(ksCurTime) = getCurrentTime(); 499#endif 500 501 /* create the idle thread */ 502 if (!create_idle_thread()) { 503 return false; 504 } 505 506 /* Before creating the initial thread (which also switches to it) 507 * we clean the cache so that any page table information written 508 * as a result of calling create_frames_of_region will be correctly 509 * read by the hardware page table walker */ 510 cleanInvalidateL1Caches(); 511 512 /* create the initial thread */ 513 tcb_t *initial = create_initial_thread( 514 root_cnode_cap, 515 it_pd_cap, 516 v_entry, 517 bi_frame_vptr, 518 ipcbuf_vptr, 519 ipcbuf_cap 520 ); 521 522 if (initial == NULL) { 523 return false; 524 } 525 526 init_core_state(initial); 527 528 /* create all of the untypeds. Both devices and kernel window memory */ 529 if (!create_untypeds( 530 root_cnode_cap, 531 (region_t) { 532 KERNEL_ELF_BASE, (pptr_t)ki_boot_end 533 } /* reusable boot code/data */ 534 )) { 535 return false; 536 } 537 538 /* no shared-frame caps (ARM has no multikernel support) */ 539 ndks_boot.bi_frame->sharedFrames = S_REG_EMPTY; 540 541 /* finalise the bootinfo frame */ 542 bi_finalise(); 543 544 /* make everything written by the kernel visible to userland. Cleaning to PoC is not 545 * strictly neccessary, but performance is not critical here so clean and invalidate 546 * everything to PoC */ 547 cleanInvalidateL1Caches(); 548 invalidateLocalTLB(); 549 if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) { 550 invalidateHypTLB(); 551 } 552 553 554 ksNumCPUs = 1; 555 556 /* initialize BKL before booting up other cores */ 557 SMP_COND_STATEMENT(clh_lock_init()); 558 SMP_COND_STATEMENT(release_secondary_cpus()); 559 560 /* grab BKL before leaving the kernel */ 561 NODE_LOCK_SYS; 562 563 printf("Booting all finished, dropped to user space\n"); 564 565 /* kernel successfully initialized */ 566 return true; 567} 568 569BOOT_CODE VISIBLE void init_kernel( 570 paddr_t ui_p_reg_start, 571 paddr_t ui_p_reg_end, 572 sword_t pv_offset, 573 vptr_t v_entry, 574 paddr_t dtb_addr_p, 575 uint32_t dtb_size 576) 577{ 578 bool_t result; 579 paddr_t dtb_end_p = 0; 580 581 if (dtb_addr_p) { 582 dtb_end_p = dtb_addr_p + dtb_size; 583 } 584 585#ifdef ENABLE_SMP_SUPPORT 586 /* we assume there exists a cpu with id 0 and will use it for bootstrapping */ 587 if (getCurrentCPUIndex() == 0) { 588 result = try_init_kernel(ui_p_reg_start, 589 ui_p_reg_end, 590 pv_offset, 591 v_entry, 592 dtb_addr_p, dtb_end_p); 593 } else { 594 result = try_init_kernel_secondary_core(); 595 } 596 597#else 598 result = try_init_kernel(ui_p_reg_start, 599 ui_p_reg_end, 600 pv_offset, 601 v_entry, 602 dtb_addr_p, dtb_end_p); 603 604#endif /* ENABLE_SMP_SUPPORT */ 605 606 if (!result) { 607 fail("Kernel init failed for some reason :("); 608 } 609 610#ifdef CONFIG_KERNEL_MCS 611 NODE_STATE(ksCurTime) = getCurrentTime(); 612 NODE_STATE(ksConsumed) = 0; 613#endif 614 schedule(); 615 activateThread(); 616} 617 618