1/* 2 * Copyright 2014, General Dynamics C4 Systems 3 * 4 * This software may be distributed and modified according to the terms of 5 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 6 * See "LICENSE_GPLv2.txt" for details. 7 * 8 * @TAG(GD_GPL) 9 */ 10 11#include <config.h> 12#include <assert.h> 13#include <kernel/boot.h> 14#include <machine/io.h> 15#include <model/statedata.h> 16#include <object/interrupt.h> 17#include <arch/machine.h> 18#include <arch/kernel/boot.h> 19#include <arch/kernel/vspace.h> 20#include <arch/benchmark.h> 21#include <arch/user_access.h> 22#include <arch/object/iospace.h> 23#include <linker.h> 24#include <plat/machine/hardware.h> 25#include <machine.h> 26#include <plat/machine/timer.h> 27#include <arch/machine/timer.h> 28#include <arch/machine/fpu.h> 29#include <arch/machine/tlb.h> 30 31/* pointer to the end of boot code/data in kernel image */ 32/* need a fake array to get the pointer from the linker script */ 33extern char ki_boot_end[1]; 34/* pointer to end of kernel image */ 35extern char ki_end[1]; 36 37#ifdef ENABLE_SMP_SUPPORT 38/* sync variable to prevent other nodes from booting 39 * until kernel data structures initialized */ 40BOOT_DATA static volatile int node_boot_lock = 0; 41#endif /* ENABLE_SMP_SUPPORT */ 42 43/** 44 * Split mem_reg about reserved_reg. If memory exists in the lower 45 * segment, insert it. If memory exists in the upper segment, return it. 46 */ 47BOOT_CODE static region_t 48insert_region_excluded(region_t mem_reg, region_t reserved_reg) 49{ 50 region_t residual_reg = mem_reg; 51 bool_t result UNUSED; 52 53 if (reserved_reg.start < mem_reg.start) { 54 /* Reserved region is below the provided mem_reg. */ 55 mem_reg.end = 0; 56 mem_reg.start = 0; 57 /* Fit the residual around the reserved region */ 58 if (reserved_reg.end > residual_reg.start) { 59 residual_reg.start = reserved_reg.end; 60 } 61 } else if (mem_reg.end > reserved_reg.start) { 62 /* Split mem_reg around reserved_reg */ 63 mem_reg.end = reserved_reg.start; 64 residual_reg.start = reserved_reg.end; 65 } else { 66 /* reserved_reg is completely above mem_reg */ 67 residual_reg.start = 0; 68 residual_reg.end = 0; 69 } 70 /* Add the lower region if it exists */ 71 if (mem_reg.start < mem_reg.end) { 72 result = insert_region(mem_reg); 73 assert(result); 74 } 75 /* Validate the upper region */ 76 if (residual_reg.start > residual_reg.end) { 77 residual_reg.start = residual_reg.end; 78 } 79 80 return residual_reg; 81} 82 83BOOT_CODE static region_t 84get_reserved_region(int i, pptr_t res_reg_end) 85{ 86 region_t res_reg = mode_reserved_region[i]; 87 88 /* Force ordering and exclusivity of reserved regions. */ 89 assert(res_reg.start < res_reg.end); 90 assert(res_reg_end <= res_reg.start); 91 return res_reg; 92} 93 94BOOT_CODE static int 95get_num_reserved_region(void) 96{ 97 return sizeof(mode_reserved_region) / sizeof(region_t); 98} 99 100BOOT_CODE static void 101init_freemem(region_t ui_reg) 102{ 103 word_t i; 104 bool_t result UNUSED; 105 region_t cur_reg; 106 region_t res_reg[] = { 107 { 108 .start = kernelBase, 109 .end = (pptr_t)ki_end 110 }, 111 { 112 .start = ui_reg.start, 113 .end = ui_reg.end 114 }, 115 }; 116 117 for (i = 0; i < MAX_NUM_FREEMEM_REG; i++) { 118 ndks_boot.freemem[i] = REG_EMPTY; 119 } 120 121 /* Force ordering and exclusivity of reserved regions. */ 122 assert(res_reg[0].start < res_reg[0].end); 123 assert(res_reg[1].start < res_reg[1].end); 124 assert(res_reg[0].end <= res_reg[1].start); 125 for (i = 0; i < get_num_avail_p_regs(); i++) { 126 cur_reg = paddr_to_pptr_reg(get_avail_p_reg(i)); 127 /* Adjust region if it exceeds the kernel window 128 * Note that we compare physical address in case of overflow. 129 */ 130 if (pptr_to_paddr((void*)cur_reg.end) > PADDR_TOP) { 131 cur_reg.end = PPTR_TOP; 132 } 133 if (pptr_to_paddr((void*)cur_reg.start) > PADDR_TOP) { 134 cur_reg.start = PPTR_TOP; 135 } 136 137 cur_reg = insert_region_excluded(cur_reg, res_reg[0]); 138 cur_reg = insert_region_excluded(cur_reg, res_reg[1]); 139 140 /* Check any reserved mode specific reagion */ 141 region_t mode_res_reg = res_reg[1]; 142 for (int m = 0; m < get_num_reserved_region(); m++) { 143 mode_res_reg = get_reserved_region(i, mode_res_reg.end); 144 cur_reg = insert_region_excluded(cur_reg, mode_res_reg); 145 } 146 147 if (cur_reg.start != cur_reg.end) { 148 result = insert_region(cur_reg); 149 assert(result); 150 } 151 } 152} 153 154BOOT_CODE static void 155init_irqs(cap_t root_cnode_cap) 156{ 157 irq_t i; 158 159 for (i = 0; i <= maxIRQ; i++) { 160 setIRQState(IRQInactive, i); 161 } 162 setIRQState(IRQTimer, KERNEL_TIMER_IRQ); 163#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT 164 setIRQState(IRQReserved, INTERRUPT_VGIC_MAINTENANCE); 165#endif 166#ifdef CONFIG_ARM_SMMU 167 setIRQState(IRQReserved, INTERRUPT_SMMU); 168#endif 169 170#ifdef CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT 171#ifdef KERNEL_PMU_IRQ 172 setIRQState(IRQReserved, KERNEL_PMU_IRQ); 173#if (defined CONFIG_PLAT_TX1 && defined ENABLE_SMP_SUPPORT) 174//SELFOUR-1252 175#error "This platform doesn't support tracking CPU utilisation on multicore" 176#endif /* CONFIG_PLAT_TX1 && ENABLE_SMP_SUPPORT */ 177#else 178#error "This platform doesn't support tracking CPU utilisation feature" 179#endif /* KERNEL_TIMER_IRQ */ 180#endif /* CONFIG_ARM_ENABLE_PMU_OVERFLOW_INTERRUPT */ 181 182#ifdef ENABLE_SMP_SUPPORT 183 setIRQState(IRQIPI, irq_remote_call_ipi); 184 setIRQState(IRQIPI, irq_reschedule_ipi); 185#endif /* ENABLE_SMP_SUPPORT */ 186 187 /* provide the IRQ control cap */ 188 write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIRQControl), cap_irq_control_cap_new()); 189} 190 191BOOT_CODE static bool_t 192create_untypeds(cap_t root_cnode_cap, region_t boot_mem_reuse_reg) 193{ 194 seL4_SlotPos slot_pos_before; 195 seL4_SlotPos slot_pos_after; 196 region_t dev_reg; 197 word_t i; 198 199 slot_pos_before = ndks_boot.slot_pos_cur; 200 create_kernel_untypeds(root_cnode_cap, boot_mem_reuse_reg, slot_pos_before); 201 for (i = 0; i < get_num_dev_p_regs(); i++) { 202 dev_reg = paddr_to_pptr_reg(get_dev_p_reg(i)); 203 if (!create_untypeds_for_region(root_cnode_cap, true, 204 dev_reg, slot_pos_before)) { 205 return false; 206 } 207 } 208 209 slot_pos_after = ndks_boot.slot_pos_cur; 210 ndks_boot.bi_frame->untyped = (seL4_SlotRegion) { 211 slot_pos_before, slot_pos_after 212 }; 213 return true; 214 215} 216 217/** This and only this function initialises the CPU. 218 * 219 * It does NOT initialise any kernel state. 220 * @return For the verification build, this currently returns true always. 221 */ 222BOOT_CODE static bool_t 223init_cpu(void) 224{ 225 bool_t haveHWFPU; 226 227#ifdef CONFIG_ARCH_AARCH64 228 if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) { 229 if (!checkTCR_EL2()) { 230 return false; 231 } 232 } 233#endif 234 235 activate_global_pd(); 236 if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) { 237 vcpu_boot_init(); 238 } 239 240#ifdef CONFIG_HARDWARE_DEBUG_API 241 if (!Arch_initHardwareBreakpoints()) { 242 printf("Kernel built with CONFIG_HARDWARE_DEBUG_API, but this board doesn't " 243 "reliably support it.\n"); 244 return false; 245 } 246#endif 247 248 /* Setup kernel stack pointer. 249 * On ARM SMP, the array index here is the CPU ID 250 */ 251#ifndef CONFIG_ARCH_ARM_V6 252 word_t stack_top = ((word_t) kernel_stack_alloc[SMP_TERNARY(getCurrentCPUIndex(), 0)]) + BIT(CONFIG_KERNEL_STACK_BITS); 253#if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_ARCH_AARCH64) 254 /* the least 12 bits are used to store logical core ID */ 255 stack_top |= getCurrentCPUIndex(); 256#endif 257 setKernelStack(stack_top); 258#endif /* CONFIG_ARCH_ARM_V6 */ 259 260#ifdef CONFIG_ARCH_AARCH64 261 /* initialise CPU's exception vector table */ 262 setVtable((pptr_t)arm_vector_table); 263#endif /* CONFIG_ARCH_AARCH64 */ 264 265 haveHWFPU = fpsimd_HWCapTest(); 266 267 /* Disable FPU to avoid channels where a platform has an FPU but doesn't make use of it */ 268 if (haveHWFPU) { 269 disableFpu(); 270 } 271 272#ifdef CONFIG_HAVE_FPU 273 if (haveHWFPU) { 274 if (!fpsimd_init()) { 275 return false; 276 } 277 } else { 278 printf("Platform claims to have FP hardware, but does not!"); 279 return false; 280 } 281#endif /* CONFIG_HAVE_FPU */ 282 283 cpu_initLocalIRQController(); 284 285#ifdef CONFIG_ENABLE_BENCHMARKS 286 armv_init_ccnt(); 287#endif /* CONFIG_ENABLE_BENCHMARKS */ 288 289 /* Export selected CPU features for access by PL0 */ 290 armv_init_user_access(); 291 292 initTimer(); 293 294 return true; 295} 296 297/* This and only this function initialises the platform. It does NOT initialise any kernel state. */ 298 299BOOT_CODE static void 300init_plat(void) 301{ 302 initIRQController(); 303 initL2Cache(); 304} 305 306#ifdef ENABLE_SMP_SUPPORT 307BOOT_CODE static bool_t 308try_init_kernel_secondary_core(void) 309{ 310 /* need to first wait until some kernel init has been done */ 311 while (!node_boot_lock); 312 313 /* Perform cpu init */ 314 init_cpu(); 315 316 /* Enable per-CPU timer interrupts */ 317 maskInterrupt(false, KERNEL_TIMER_IRQ); 318 319 NODE_LOCK_SYS; 320 321 ksNumCPUs++; 322 323 init_core_state(SchedulerAction_ResumeCurrentThread); 324 325 return true; 326} 327 328BOOT_CODE static void 329release_secondary_cpus(void) 330{ 331 332 /* release the cpus at the same time */ 333 node_boot_lock = 1; 334 335#ifndef CONFIG_ARCH_AARCH64 336 /* At this point in time the other CPUs do *not* have the seL4 global pd set. 337 * However, they still have a PD from the elfloader (which is mapping mmemory 338 * as strongly ordered uncached, as a result we need to explicitly clean 339 * the cache for it to see the update of node_boot_lock 340 * 341 * For ARMv8, the elfloader sets the page table entries as inner shareable 342 * (so is the attribute of the seL4 global PD) when SMP is enabled, and 343 * turns on the cache. Thus, we do not need to clean and invaliate the cache. 344 */ 345 cleanInvalidateL1Caches(); 346 plat_cleanInvalidateCache(); 347#endif 348 349 /* Wait until all the secondary cores are done initialising */ 350 while (ksNumCPUs != CONFIG_MAX_NUM_NODES) { 351 /* perform a memory release+acquire to get new values of ksNumCPUs */ 352 __atomic_signal_fence(__ATOMIC_ACQ_REL); 353 } 354} 355#endif /* ENABLE_SMP_SUPPORT */ 356 357/* Main kernel initialisation function. */ 358 359static BOOT_CODE bool_t 360try_init_kernel( 361 paddr_t ui_p_reg_start, 362 paddr_t ui_p_reg_end, 363 sword_t pv_offset, 364 vptr_t v_entry 365) 366{ 367 cap_t root_cnode_cap; 368 cap_t it_ap_cap; 369 cap_t it_pd_cap; 370 cap_t ipcbuf_cap; 371 region_t ui_reg = paddr_to_pptr_reg((p_region_t) { 372 ui_p_reg_start, ui_p_reg_end 373 }); 374 pptr_t bi_frame_pptr; 375 vptr_t bi_frame_vptr; 376 vptr_t ipcbuf_vptr; 377 create_frames_of_region_ret_t create_frames_ret; 378 379 /* convert from physical addresses to userland vptrs */ 380 v_region_t ui_v_reg; 381 v_region_t it_v_reg; 382 ui_v_reg.start = ui_p_reg_start - pv_offset; 383 ui_v_reg.end = ui_p_reg_end - pv_offset; 384 385 ipcbuf_vptr = ui_v_reg.end; 386 bi_frame_vptr = ipcbuf_vptr + BIT(PAGE_BITS); 387 388 /* The region of the initial thread is the user image + ipcbuf and boot info */ 389 it_v_reg.start = ui_v_reg.start; 390 it_v_reg.end = bi_frame_vptr + BIT(PAGE_BITS); 391 392 if (it_v_reg.end > kernelBase) { 393 printf("Userland image virtual end address too high\n"); 394 return false; 395 } 396 397 /* setup virtual memory for the kernel */ 398 map_kernel_window(); 399 400 /* initialise the CPU */ 401 if (!init_cpu()) { 402 return false; 403 } 404 405 /* debug output via serial port is only available from here */ 406 printf("Bootstrapping kernel\n"); 407 408 /* initialise the platform */ 409 init_plat(); 410 411 /* make the free memory available to alloc_region() */ 412 init_freemem(ui_reg); 413 414 /* create the root cnode */ 415 root_cnode_cap = create_root_cnode(); 416 if (cap_get_capType(root_cnode_cap) == cap_null_cap) { 417 return false; 418 } 419 420 /* create the cap for managing thread domains */ 421 create_domain_cap(root_cnode_cap); 422 423 /* create the IRQ CNode */ 424 if (!create_irq_cnode()) { 425 return false; 426 } 427 428 /* initialise the IRQ states and provide the IRQ control cap */ 429 init_irqs(root_cnode_cap); 430 431 /* create the bootinfo frame */ 432 bi_frame_pptr = allocate_bi_frame(0, CONFIG_MAX_NUM_NODES, ipcbuf_vptr); 433 if (!bi_frame_pptr) { 434 return false; 435 } 436 437 if (config_set(CONFIG_ARM_SMMU)) { 438 ndks_boot.bi_frame->ioSpaceCaps = create_iospace_caps(root_cnode_cap); 439 if (ndks_boot.bi_frame->ioSpaceCaps.start == 0 && 440 ndks_boot.bi_frame->ioSpaceCaps.end == 0) { 441 return false; 442 } 443 } else { 444 ndks_boot.bi_frame->ioSpaceCaps = S_REG_EMPTY; 445 } 446 447 /* Construct an initial address space with enough virtual addresses 448 * to cover the user image + ipc buffer and bootinfo frames */ 449 it_pd_cap = create_it_address_space(root_cnode_cap, it_v_reg); 450 if (cap_get_capType(it_pd_cap) == cap_null_cap) { 451 return false; 452 } 453 454 /* Create and map bootinfo frame cap */ 455 create_bi_frame_cap( 456 root_cnode_cap, 457 it_pd_cap, 458 bi_frame_pptr, 459 bi_frame_vptr 460 ); 461 462 /* create the initial thread's IPC buffer */ 463 ipcbuf_cap = create_ipcbuf_frame(root_cnode_cap, it_pd_cap, ipcbuf_vptr); 464 if (cap_get_capType(ipcbuf_cap) == cap_null_cap) { 465 return false; 466 } 467 468 /* create all userland image frames */ 469 create_frames_ret = 470 create_frames_of_region( 471 root_cnode_cap, 472 it_pd_cap, 473 ui_reg, 474 true, 475 pv_offset 476 ); 477 if (!create_frames_ret.success) { 478 return false; 479 } 480 ndks_boot.bi_frame->userImageFrames = create_frames_ret.region; 481 482 /* create/initialise the initial thread's ASID pool */ 483 it_ap_cap = create_it_asid_pool(root_cnode_cap); 484 if (cap_get_capType(it_ap_cap) == cap_null_cap) { 485 return false; 486 } 487 write_it_asid_pool(it_ap_cap, it_pd_cap); 488 489 /* create the idle thread */ 490 if (!create_idle_thread()) { 491 return false; 492 } 493 494 /* Before creating the initial thread (which also switches to it) 495 * we clean the cache so that any page table information written 496 * as a result of calling create_frames_of_region will be correctly 497 * read by the hardware page table walker */ 498 cleanInvalidateL1Caches(); 499 500 /* create the initial thread */ 501 tcb_t *initial = create_initial_thread( 502 root_cnode_cap, 503 it_pd_cap, 504 v_entry, 505 bi_frame_vptr, 506 ipcbuf_vptr, 507 ipcbuf_cap 508 ); 509 510 if (initial == NULL) { 511 return false; 512 } 513 514 init_core_state(initial); 515 516 /* create all of the untypeds. Both devices and kernel window memory */ 517 if (!create_untypeds( 518 root_cnode_cap, 519 (region_t) { 520 kernelBase, (pptr_t)ki_boot_end 521 } /* reusable boot code/data */ 522 )) { 523 return false; 524 } 525 526 /* no shared-frame caps (ARM has no multikernel support) */ 527 ndks_boot.bi_frame->sharedFrames = S_REG_EMPTY; 528 529 /* finalise the bootinfo frame */ 530 bi_finalise(); 531 532 /* make everything written by the kernel visible to userland. Cleaning to PoC is not 533 * strictly neccessary, but performance is not critical here so clean and invalidate 534 * everything to PoC */ 535 cleanInvalidateL1Caches(); 536 invalidateLocalTLB(); 537 if (config_set(CONFIG_ARM_HYPERVISOR_SUPPORT)) { 538 invalidateHypTLB(); 539 } 540 541 542 ksNumCPUs = 1; 543 544 /* initialize BKL before booting up other cores */ 545 SMP_COND_STATEMENT(clh_lock_init()); 546 SMP_COND_STATEMENT(release_secondary_cpus()); 547 548 /* grab BKL before leaving the kernel */ 549 NODE_LOCK_SYS; 550 551 printf("Booting all finished, dropped to user space\n"); 552 553 /* kernel successfully initialized */ 554 return true; 555} 556 557BOOT_CODE VISIBLE void 558init_kernel( 559 paddr_t ui_p_reg_start, 560 paddr_t ui_p_reg_end, 561 sword_t pv_offset, 562 vptr_t v_entry 563) 564{ 565 bool_t result; 566 567#ifdef ENABLE_SMP_SUPPORT 568 /* we assume there exists a cpu with id 0 and will use it for bootstrapping */ 569 if (getCurrentCPUIndex() == 0) { 570 result = try_init_kernel(ui_p_reg_start, 571 ui_p_reg_end, 572 pv_offset, 573 v_entry); 574 } else { 575 result = try_init_kernel_secondary_core(); 576 } 577 578#else 579 result = try_init_kernel(ui_p_reg_start, 580 ui_p_reg_end, 581 pv_offset, 582 v_entry); 583 584#endif /* ENABLE_SMP_SUPPORT */ 585 586 if (!result) { 587 fail ("Kernel init failed for some reason :("); 588 } 589 590 schedule(); 591 activateThread(); 592} 593 594