1// Copyright 2016 The Fuchsia Authors 2// Copyright (c) 2015 Travis Geiselbrecht 3// 4// Use of this source code is governed by a MIT-style 5// license that can be found in the LICENSE file or at 6// https://opensource.org/licenses/MIT 7 8#include <debug.h> 9#include <err.h> 10#include <fbl/atomic.h> 11#include <fbl/auto_lock.h> 12#include <fbl/ref_ptr.h> 13#include <reg.h> 14#include <trace.h> 15 16#include <arch.h> 17#include <dev/display.h> 18#include <dev/hw_rng.h> 19#include <dev/interrupt.h> 20#include <dev/power.h> 21#include <dev/psci.h> 22#include <dev/uart.h> 23#include <kernel/cmdline.h> 24#include <kernel/dpc.h> 25#include <kernel/spinlock.h> 26#include <lk/init.h> 27#include <object/resource_dispatcher.h> 28#include <vm/kstack.h> 29#include <vm/physmap.h> 30#include <vm/vm.h> 31 32#include <mexec.h> 33#include <platform.h> 34 35#include <target.h> 36 37#include <arch/arch_ops.h> 38#include <arch/arm64.h> 39#include <arch/arm64/mmu.h> 40#include <arch/arm64/mp.h> 41#include <arch/arm64/periphmap.h> 42#include <arch/mp.h> 43 44#include <vm/bootreserve.h> 45#include <vm/vm_aspace.h> 46 47#include <lib/console.h> 48#include <lib/memory_limit.h> 49#include <lib/debuglog.h> 50#if WITH_PANIC_BACKTRACE 51#include <kernel/thread.h> 52#endif 53 54#include <libzbi/zbi-cpp.h> 55#include <pdev/pdev.h> 56#include <zircon/boot/image.h> 57#include <zircon/rights.h> 58#include <zircon/types.h> 59 60// Defined in start.S. 61extern paddr_t kernel_entry_paddr; 62extern paddr_t zbi_paddr; 63 64static void* ramdisk_base; 65static size_t ramdisk_size; 66 67static zbi_nvram_t lastlog_nvram; 68 69static uint cpu_cluster_count = 0; 70static uint cpu_cluster_cpus[SMP_CPU_MAX_CLUSTERS] = {0}; 71 72static bool halt_on_panic = false; 73static bool uart_disabled = false; 74 75// all of the configured memory arenas from the zbi 76// at the moment, only support 1 arena 77static pmm_arena_info_t mem_arena = { 78 /* .name */ "sdram", 79 /* .flags */ 0, 80 /* .priority */ 0, 81 /* .base */ 0, // filled in by zbi 82 /* .size */ 0, // filled in by zbi 83}; 84 85// boot items to save for mexec 86// TODO(voydanoff): more generic way of doing this that can be shared with PC platform 87static uint8_t mexec_zbi[4096]; 88static size_t mexec_zbi_length = 0; 89 90static volatile int panic_started; 91 92static void halt_other_cpus(void) { 93 static volatile int halted = 0; 94 95 if (atomic_swap(&halted, 1) == 0) { 96 // stop the other cpus 97 printf("stopping other cpus\n"); 98 arch_mp_send_ipi(MP_IPI_TARGET_ALL_BUT_LOCAL, 0, MP_IPI_HALT); 99 100 // spin for a while 101 // TODO: find a better way to spin at this low level 102 for (volatile int i = 0; i < 100000000; i++) { 103 __asm volatile("nop"); 104 } 105 } 106} 107 108void platform_panic_start(void) { 109 arch_disable_ints(); 110 111 halt_other_cpus(); 112 113 if (atomic_swap(&panic_started, 1) == 0) { 114 dlog_bluescreen_init(); 115 } 116} 117 118void* platform_get_ramdisk(size_t* size) { 119 if (ramdisk_base) { 120 *size = ramdisk_size; 121 return ramdisk_base; 122 } else { 123 *size = 0; 124 return nullptr; 125 } 126} 127 128void platform_halt_cpu(void) { 129 uint32_t result = psci_cpu_off(); 130 // should have never returned 131 panic("psci_cpu_off returned %u\n", result); 132} 133 134void platform_halt_secondary_cpus(void) { 135 // Ensure the current thread is pinned to the boot CPU. 136 DEBUG_ASSERT(get_current_thread()->cpu_affinity == cpu_num_to_mask(BOOT_CPU_ID)); 137 138 // "Unplug" online secondary CPUs before halting them. 139 cpu_mask_t primary = cpu_num_to_mask(BOOT_CPU_ID); 140 cpu_mask_t mask = mp_get_online_mask() & ~primary; 141 zx_status_t result = mp_unplug_cpu_mask(mask); 142 DEBUG_ASSERT(result == ZX_OK); 143} 144 145static zx_status_t platform_start_cpu(uint cluster, uint cpu) { 146 // Issue memory barrier before starting to ensure previous stores will be visible to new CPU. 147 smp_mb(); 148 149 uint32_t ret = psci_cpu_on(cluster, cpu, kernel_entry_paddr); 150 dprintf(INFO, "Trying to start cpu %u:%u returned: %d\n", cluster, cpu, (int)ret); 151 if (ret != 0) { 152 return ZX_ERR_INTERNAL; 153 } 154 return ZX_OK; 155} 156 157static void platform_cpu_init(void) { 158 for (uint cluster = 0; cluster < cpu_cluster_count; cluster++) { 159 for (uint cpu = 0; cpu < cpu_cluster_cpus[cluster]; cpu++) { 160 if (cluster != 0 || cpu != 0) { 161 // create a stack for the cpu we're about to start 162 zx_status_t status = arm64_create_secondary_stack(cluster, cpu); 163 DEBUG_ASSERT(status == ZX_OK); 164 165 // start the cpu 166 status = platform_start_cpu(cluster, cpu); 167 168 if (status != ZX_OK) { 169 // TODO(maniscalco): Is continuing really the right thing to do here? 170 171 // start failed, free the stack 172 zx_status_t status = arm64_free_secondary_stack(cluster, cpu); 173 DEBUG_ASSERT(status == ZX_OK); 174 continue; 175 } 176 177 // the cpu booted 178 // 179 // bootstrap thread is now responsible for freeing its stack 180 } 181 } 182 } 183} 184 185static inline bool is_zbi_container(void* addr) { 186 DEBUG_ASSERT(addr); 187 188 zbi_header_t* item = (zbi_header_t*)addr; 189 return item->type == ZBI_TYPE_CONTAINER; 190} 191 192static void save_mexec_zbi(zbi_header_t* item) { 193 size_t length = ZBI_ALIGN(item->length + sizeof(zbi_header_t)); 194 ASSERT(sizeof(mexec_zbi) - mexec_zbi_length >= length); 195 196 memcpy(&mexec_zbi[mexec_zbi_length], item, length); 197 mexec_zbi_length += length; 198} 199 200static void process_mem_range(const zbi_mem_range_t* mem_range) { 201 switch (mem_range->type) { 202 case ZBI_MEM_RANGE_RAM: 203 if (mem_arena.size == 0) { 204 mem_arena.base = mem_range->paddr; 205 mem_arena.size = mem_range->length; 206 dprintf(INFO, "mem_arena.base %#" PRIx64 " size %#" PRIx64 "\n", mem_arena.base, 207 mem_arena.size); 208 } else { 209 if (mem_range->paddr) { 210 mem_arena.base = mem_range->paddr; 211 dprintf(INFO, "overriding mem arena 0 base from FDT: %#zx\n", mem_arena.base); 212 } 213 // if mem_area.base is already set, then just update the size 214 mem_arena.size = mem_range->length; 215 dprintf(INFO, "overriding mem arena 0 size from FDT: %#zx\n", mem_arena.size); 216 } 217 break; 218 case ZBI_MEM_RANGE_PERIPHERAL: { 219 auto status = add_periph_range(mem_range->paddr, mem_range->length); 220 ASSERT(status == ZX_OK); 221 break; 222 } 223 case ZBI_MEM_RANGE_RESERVED: 224 dprintf(INFO, "boot reserve mem range: phys base %#" PRIx64 " length %#" PRIx64 "\n", 225 mem_range->paddr, mem_range->length); 226 boot_reserve_add_range(mem_range->paddr, mem_range->length); 227 break; 228 default: 229 panic("bad mem_range->type in process_mem_range\n"); 230 break; 231 } 232} 233 234static zbi_result_t process_zbi_item(zbi_header_t* item, void* payload, void* cookie) { 235 if (ZBI_TYPE_DRV_METADATA(item->type)) { 236 save_mexec_zbi(item); 237 return ZBI_RESULT_OK; 238 } 239 switch (item->type) { 240 case ZBI_TYPE_KERNEL_DRIVER: 241 case ZBI_TYPE_PLATFORM_ID: 242 // we don't process these here, but we need to save them for mexec 243 save_mexec_zbi(item); 244 break; 245 case ZBI_TYPE_CMDLINE: { 246 if (item->length < 1) { 247 break; 248 } 249 char* contents = reinterpret_cast<char*>(payload); 250 contents[item->length - 1] = '\0'; 251 cmdline_append(contents); 252 break; 253 } 254 case ZBI_TYPE_MEM_CONFIG: { 255 zbi_mem_range_t* mem_range = reinterpret_cast<zbi_mem_range_t*>(payload); 256 uint32_t count = item->length / (uint32_t)sizeof(zbi_mem_range_t); 257 for (uint32_t i = 0; i < count; i++) { 258 process_mem_range(mem_range++); 259 } 260 save_mexec_zbi(item); 261 break; 262 } 263 case ZBI_TYPE_CPU_CONFIG: { 264 zbi_cpu_config_t* cpu_config = reinterpret_cast<zbi_cpu_config_t*>(payload); 265 cpu_cluster_count = cpu_config->cluster_count; 266 for (uint32_t i = 0; i < cpu_cluster_count; i++) { 267 cpu_cluster_cpus[i] = cpu_config->clusters[i].cpu_count; 268 } 269 arch_init_cpu_map(cpu_cluster_count, cpu_cluster_cpus); 270 save_mexec_zbi(item); 271 break; 272 } 273 case ZBI_TYPE_NVRAM: { 274 zbi_nvram_t* nvram = reinterpret_cast<zbi_nvram_t*>(payload); 275 memcpy(&lastlog_nvram, nvram, sizeof(lastlog_nvram)); 276 boot_reserve_add_range(nvram->base, nvram->length); 277 save_mexec_zbi(item); 278 break; 279 } 280 } 281 282 return ZBI_RESULT_OK; 283} 284 285static void process_zbi(zbi_header_t* root) { 286 DEBUG_ASSERT(root); 287 zbi_result_t result; 288 289 uint8_t* zbi_base = reinterpret_cast<uint8_t*>(root); 290 zbi::Zbi image(zbi_base); 291 292 // Make sure the image looks valid. 293 result = image.Check(nullptr); 294 if (result != ZBI_RESULT_OK) { 295 // TODO(gkalsi): Print something informative here? 296 return; 297 } 298 299 image.ForEach(process_zbi_item, nullptr); 300} 301 302void platform_early_init(void) { 303 // if the zbi_paddr variable is -1, it was not set 304 // in start.S, so we are in a bad place. 305 if (zbi_paddr == -1UL) { 306 panic("no zbi_paddr!\n"); 307 } 308 309 void* zbi_vaddr = paddr_to_physmap(zbi_paddr); 310 311 // initialize the boot memory reservation system 312 boot_reserve_init(); 313 314 if (zbi_vaddr && is_zbi_container(zbi_vaddr)) { 315 zbi_header_t* header = (zbi_header_t*)zbi_vaddr; 316 317 ramdisk_base = header; 318 ramdisk_size = ROUNDUP(header->length + sizeof(*header), PAGE_SIZE); 319 } else { 320 panic("no bootdata!\n"); 321 } 322 323 if (!ramdisk_base || !ramdisk_size) { 324 panic("no ramdisk!\n"); 325 } 326 327 zbi_header_t* zbi = reinterpret_cast<zbi_header_t*>(ramdisk_base); 328 // walk the zbi structure and process all the items 329 process_zbi(zbi); 330 331 // is the cmdline option to bypass dlog set ? 332 dlog_bypass_init(); 333 334 // bring up kernel drivers after we have mapped our peripheral ranges 335 pdev_init(zbi); 336 337 // Serial port should be active now 338 339 // Read cmdline after processing zbi, which may contain cmdline data. 340 halt_on_panic = cmdline_get_bool("kernel.halt-on-panic", false); 341 342 // Check if serial should be enabled 343 const char* serial_mode = cmdline_get("kernel.serial"); 344 uart_disabled = (serial_mode != NULL && !strcmp(serial_mode, "none")); 345 346 // add the ramdisk to the boot reserve memory list 347 paddr_t ramdisk_start_phys = physmap_to_paddr(ramdisk_base); 348 paddr_t ramdisk_end_phys = ramdisk_start_phys + ramdisk_size; 349 dprintf(INFO, "reserving ramdisk phys range [%#" PRIx64 ", %#" PRIx64 "]\n", 350 ramdisk_start_phys, ramdisk_end_phys - 1); 351 boot_reserve_add_range(ramdisk_start_phys, ramdisk_size); 352 353 // check if a memory limit was passed in via kernel.memory-limit-mb and 354 // find memory ranges to use if one is found. 355 mem_limit_ctx_t ctx; 356 zx_status_t status = mem_limit_init(&ctx); 357 if (status == ZX_OK) { 358 // For these ranges we're using the base physical values 359 ctx.kernel_base = get_kernel_base_phys(); 360 ctx.kernel_size = get_kernel_size(); 361 ctx.ramdisk_base = ramdisk_start_phys; 362 ctx.ramdisk_size = ramdisk_end_phys - ramdisk_start_phys; 363 364 // Figure out and add arenas based on the memory limit and our range of DRAM 365 status = mem_limit_add_arenas_from_range(&ctx, mem_arena.base, mem_arena.size, mem_arena); 366 } 367 368 // If no memory limit was found, or adding arenas from the range failed, then add 369 // the existing global arena. 370 if (status != ZX_OK) { 371 pmm_add_arena(&mem_arena); 372 } 373 374 // tell the boot allocator to mark ranges we've reserved as off limits 375 boot_reserve_wire(); 376} 377 378void platform_init(void) { 379 platform_cpu_init(); 380} 381 382// after the fact create a region to reserve the peripheral map(s) 383static void platform_init_postvm(uint level) { 384 reserve_periph_ranges(); 385} 386 387LK_INIT_HOOK(platform_postvm, platform_init_postvm, LK_INIT_LEVEL_VM); 388 389void platform_dputs_thread(const char* str, size_t len) { 390 if (uart_disabled) { 391 return; 392 } 393 uart_puts(str, len, true, true); 394} 395 396void platform_dputs_irq(const char* str, size_t len) { 397 if (uart_disabled) { 398 return; 399 } 400 uart_puts(str, len, false, true); 401} 402 403int platform_dgetc(char* c, bool wait) { 404 if (uart_disabled) { 405 return ZX_ERR_NOT_SUPPORTED; 406 } 407 int ret = uart_getc(wait); 408 if (ret < 0) 409 return ret; 410 *c = static_cast<char>(ret); 411 return 0; 412} 413 414void platform_pputc(char c) { 415 if (uart_disabled) { 416 return; 417 } 418 uart_pputc(c); 419} 420 421int platform_pgetc(char* c, bool wait) { 422 if (uart_disabled) { 423 return ZX_ERR_NOT_SUPPORTED; 424 } 425 int r = uart_pgetc(); 426 if (r < 0) { 427 return r; 428 } 429 430 *c = static_cast<char>(r); 431 return 0; 432} 433 434/* stub out the hardware rng entropy generator, which doesn't exist on this platform */ 435size_t hw_rng_get_entropy(void* buf, size_t len, bool block) { 436 return 0; 437} 438 439/* no built in framebuffer */ 440zx_status_t display_get_info(struct display_info* info) { 441 return ZX_ERR_NOT_FOUND; 442} 443 444void platform_halt(platform_halt_action suggested_action, platform_halt_reason reason) { 445 446 if (suggested_action == HALT_ACTION_REBOOT) { 447 power_reboot(REBOOT_NORMAL); 448 printf("reboot failed\n"); 449 } else if (suggested_action == HALT_ACTION_REBOOT_BOOTLOADER) { 450 power_reboot(REBOOT_BOOTLOADER); 451 printf("reboot-bootloader failed\n"); 452 } else if (suggested_action == HALT_ACTION_REBOOT_RECOVERY) { 453 power_reboot(REBOOT_RECOVERY); 454 printf("reboot-recovery failed\n"); 455 } else if (suggested_action == HALT_ACTION_SHUTDOWN) { 456 power_shutdown(); 457 } 458 459 if (reason == HALT_REASON_SW_PANIC) { 460 thread_print_current_backtrace(); 461 dlog_bluescreen_halt(); 462 if (!halt_on_panic) { 463 power_reboot(REBOOT_NORMAL); 464 printf("reboot failed\n"); 465 } 466#if ENABLE_PANIC_SHELL 467 dprintf(ALWAYS, "CRASH: starting debug shell... (reason = %d)\n", reason); 468 arch_disable_ints(); 469 panic_shell_start(); 470#endif // ENABLE_PANIC_SHELL 471 } 472 473 dprintf(ALWAYS, "HALT: spinning forever... (reason = %d)\n", reason); 474 475 // catch all fallthrough cases 476 arch_disable_ints(); 477 for (;;) 478 ; 479} 480 481typedef struct { 482 //TODO: combine with x86 nvram crashlog handling 483 //TODO: ECC for more robust crashlogs 484 uint64_t magic; 485 uint64_t length; 486 uint64_t nmagic; 487 uint64_t nlength; 488} log_hdr_t; 489 490#define NVRAM_MAGIC (0x6f8962d66b28504fULL) 491 492size_t platform_stow_crashlog(void* log, size_t len) { 493 size_t max = lastlog_nvram.length - sizeof(log_hdr_t); 494 void* nvram = paddr_to_physmap(lastlog_nvram.base); 495 if (nvram == NULL) { 496 return 0; 497 } 498 499 if (log == NULL) { 500 return max; 501 } 502 if (len > max) { 503 len = max; 504 } 505 506 log_hdr_t hdr = { 507 .magic = NVRAM_MAGIC, 508 .length = len, 509 .nmagic = ~NVRAM_MAGIC, 510 .nlength = ~len, 511 }; 512 memcpy(nvram, &hdr, sizeof(hdr)); 513 memcpy(static_cast<char*>(nvram) + sizeof(hdr), log, len); 514 arch_clean_cache_range((uintptr_t)nvram, sizeof(hdr) + len); 515 return len; 516} 517 518size_t platform_recover_crashlog(size_t len, void* cookie, 519 void (*func)(const void* data, size_t, size_t len, void* cookie)) { 520 size_t max = lastlog_nvram.length - sizeof(log_hdr_t); 521 void* nvram = paddr_to_physmap(lastlog_nvram.base); 522 if (nvram == NULL) { 523 return 0; 524 } 525 log_hdr_t hdr; 526 memcpy(&hdr, nvram, sizeof(hdr)); 527 if ((hdr.magic != NVRAM_MAGIC) || (hdr.length > max) || 528 (hdr.nmagic != ~NVRAM_MAGIC) || (hdr.nlength != ~hdr.length)) { 529 printf("nvram-crashlog: bad header: %016lx %016lx %016lx %016lx\n", 530 hdr.magic, hdr.length, hdr.nmagic, hdr.nlength); 531 return 0; 532 } 533 if (len == 0) { 534 return hdr.length; 535 } 536 if (len > hdr.length) { 537 len = hdr.length; 538 } 539 func(static_cast<char*>(nvram) + sizeof(hdr), 0, len, cookie); 540 541 // invalidate header so we don't get a stale crashlog 542 // on future boots 543 hdr.magic = 0; 544 memcpy(nvram, &hdr, sizeof(hdr)); 545 return hdr.length; 546} 547 548zx_status_t platform_mexec_patch_zbi(uint8_t* zbi, const size_t len) { 549 size_t offset = 0; 550 551 // copy certain boot items provided by the bootloader or boot shim 552 // to the mexec zbi 553 zbi::Zbi image(zbi, len); 554 while (offset < mexec_zbi_length) { 555 zbi_header_t* item = reinterpret_cast<zbi_header_t*>(mexec_zbi + offset); 556 557 zbi_result_t status; 558 status = image.AppendSection(item->length, item->type, item->extra, 559 item->flags, 560 reinterpret_cast<uint8_t*>(item + 1)); 561 562 if (status != ZBI_RESULT_OK) 563 return ZX_ERR_INTERNAL; 564 565 offset += ZBI_ALIGN(sizeof(zbi_header_t) + item->length); 566 } 567 568 return ZX_OK; 569} 570 571void platform_mexec_prep(uintptr_t new_bootimage_addr, size_t new_bootimage_len) { 572 DEBUG_ASSERT(!arch_ints_disabled()); 573 DEBUG_ASSERT(mp_get_online_mask() == cpu_num_to_mask(BOOT_CPU_ID)); 574} 575 576void platform_mexec(mexec_asm_func mexec_assembly, memmov_ops_t* ops, 577 uintptr_t new_bootimage_addr, size_t new_bootimage_len, 578 uintptr_t entry64_addr) { 579 DEBUG_ASSERT(arch_ints_disabled()); 580 DEBUG_ASSERT(mp_get_online_mask() == cpu_num_to_mask(BOOT_CPU_ID)); 581 582 paddr_t kernel_src_phys = (paddr_t)ops[0].src; 583 paddr_t kernel_dst_phys = (paddr_t)ops[0].dst; 584 585 // check to see if the kernel is packaged as a zbi container 586 zbi_header_t* header = (zbi_header_t*)paddr_to_physmap(kernel_src_phys); 587 if (header[0].type == ZBI_TYPE_CONTAINER && header[1].type == ZBI_TYPE_KERNEL_ARM64) { 588 zbi_kernel_t* kernel_header = (zbi_kernel_t*)&header[2]; 589 // add offset from kernel header to entry point 590 kernel_dst_phys += kernel_header->entry; 591 } 592 // else just jump to beginning of kernel image 593 594 mexec_assembly((uintptr_t)new_bootimage_addr, 0, 0, arm64_get_boot_el(), ops, 595 (void*)kernel_dst_phys); 596} 597 598bool platform_serial_enabled(void) { 599 return !uart_disabled && uart_present(); 600} 601 602bool platform_early_console_enabled() { 603 return false; 604} 605 606// Initialize Resource system after the heap is initialized. 607static void arm_resource_dispatcher_init_hook(unsigned int rl) { 608 // 64 bit address space for MMIO on ARM64 609 zx_status_t status = ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_MMIO, 0, 610 UINT64_MAX); 611 if (status != ZX_OK) { 612 printf("Resources: Failed to initialize MMIO allocator: %d\n", status); 613 } 614 // Set up IRQs based on values from the GIC 615 status = ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_IRQ, 616 interrupt_get_base_vector(), 617 interrupt_get_max_vector()); 618 if (status != ZX_OK) { 619 printf("Resources: Failed to initialize IRQ allocator: %d\n", status); 620 } 621} 622 623LK_INIT_HOOK(arm_resource_init, arm_resource_dispatcher_init_hook, LK_INIT_LEVEL_HEAP); 624