1// Copyright 2016 The Fuchsia Authors
2// Copyright (c) 2015 Travis Geiselbrecht
3//
4// Use of this source code is governed by a MIT-style
5// license that can be found in the LICENSE file or at
6// https://opensource.org/licenses/MIT
7
8#include <debug.h>
9#include <err.h>
10#include <fbl/atomic.h>
11#include <fbl/auto_lock.h>
12#include <fbl/ref_ptr.h>
13#include <reg.h>
14#include <trace.h>
15
16#include <arch.h>
17#include <dev/display.h>
18#include <dev/hw_rng.h>
19#include <dev/interrupt.h>
20#include <dev/power.h>
21#include <dev/psci.h>
22#include <dev/uart.h>
23#include <kernel/cmdline.h>
24#include <kernel/dpc.h>
25#include <kernel/spinlock.h>
26#include <lk/init.h>
27#include <object/resource_dispatcher.h>
28#include <vm/kstack.h>
29#include <vm/physmap.h>
30#include <vm/vm.h>
31
32#include <mexec.h>
33#include <platform.h>
34
35#include <target.h>
36
37#include <arch/arch_ops.h>
38#include <arch/arm64.h>
39#include <arch/arm64/mmu.h>
40#include <arch/arm64/mp.h>
41#include <arch/arm64/periphmap.h>
42#include <arch/mp.h>
43
44#include <vm/bootreserve.h>
45#include <vm/vm_aspace.h>
46
47#include <lib/console.h>
48#include <lib/memory_limit.h>
49#include <lib/debuglog.h>
50#if WITH_PANIC_BACKTRACE
51#include <kernel/thread.h>
52#endif
53
54#include <libzbi/zbi-cpp.h>
55#include <pdev/pdev.h>
56#include <zircon/boot/image.h>
57#include <zircon/rights.h>
58#include <zircon/types.h>
59
60// Defined in start.S.
61extern paddr_t kernel_entry_paddr;
62extern paddr_t zbi_paddr;
63
64static void* ramdisk_base;
65static size_t ramdisk_size;
66
67static zbi_nvram_t lastlog_nvram;
68
69static uint cpu_cluster_count = 0;
70static uint cpu_cluster_cpus[SMP_CPU_MAX_CLUSTERS] = {0};
71
72static bool halt_on_panic = false;
73static bool uart_disabled = false;
74
75// all of the configured memory arenas from the zbi
76// at the moment, only support 1 arena
77static pmm_arena_info_t mem_arena = {
78    /* .name */ "sdram",
79    /* .flags */ 0,
80    /* .priority */ 0,
81    /* .base */ 0, // filled in by zbi
82    /* .size */ 0, // filled in by zbi
83};
84
85// boot items to save for mexec
86// TODO(voydanoff): more generic way of doing this that can be shared with PC platform
87static uint8_t mexec_zbi[4096];
88static size_t mexec_zbi_length = 0;
89
90static volatile int panic_started;
91
92static void halt_other_cpus(void) {
93    static volatile int halted = 0;
94
95    if (atomic_swap(&halted, 1) == 0) {
96        // stop the other cpus
97        printf("stopping other cpus\n");
98        arch_mp_send_ipi(MP_IPI_TARGET_ALL_BUT_LOCAL, 0, MP_IPI_HALT);
99
100        // spin for a while
101        // TODO: find a better way to spin at this low level
102        for (volatile int i = 0; i < 100000000; i++) {
103            __asm volatile("nop");
104        }
105    }
106}
107
108void platform_panic_start(void) {
109    arch_disable_ints();
110
111    halt_other_cpus();
112
113    if (atomic_swap(&panic_started, 1) == 0) {
114        dlog_bluescreen_init();
115    }
116}
117
118void* platform_get_ramdisk(size_t* size) {
119    if (ramdisk_base) {
120        *size = ramdisk_size;
121        return ramdisk_base;
122    } else {
123        *size = 0;
124        return nullptr;
125    }
126}
127
128void platform_halt_cpu(void) {
129    uint32_t result = psci_cpu_off();
130    // should have never returned
131    panic("psci_cpu_off returned %u\n", result);
132}
133
134void platform_halt_secondary_cpus(void) {
135    // Ensure the current thread is pinned to the boot CPU.
136    DEBUG_ASSERT(get_current_thread()->cpu_affinity == cpu_num_to_mask(BOOT_CPU_ID));
137
138    // "Unplug" online secondary CPUs before halting them.
139    cpu_mask_t primary = cpu_num_to_mask(BOOT_CPU_ID);
140    cpu_mask_t mask = mp_get_online_mask() & ~primary;
141    zx_status_t result = mp_unplug_cpu_mask(mask);
142    DEBUG_ASSERT(result == ZX_OK);
143}
144
145static zx_status_t platform_start_cpu(uint cluster, uint cpu) {
146    // Issue memory barrier before starting to ensure previous stores will be visible to new CPU.
147    smp_mb();
148
149    uint32_t ret = psci_cpu_on(cluster, cpu, kernel_entry_paddr);
150    dprintf(INFO, "Trying to start cpu %u:%u returned: %d\n", cluster, cpu, (int)ret);
151    if (ret != 0) {
152        return ZX_ERR_INTERNAL;
153    }
154    return ZX_OK;
155}
156
157static void platform_cpu_init(void) {
158    for (uint cluster = 0; cluster < cpu_cluster_count; cluster++) {
159        for (uint cpu = 0; cpu < cpu_cluster_cpus[cluster]; cpu++) {
160            if (cluster != 0 || cpu != 0) {
161                // create a stack for the cpu we're about to start
162                zx_status_t status = arm64_create_secondary_stack(cluster, cpu);
163                DEBUG_ASSERT(status == ZX_OK);
164
165                // start the cpu
166                status = platform_start_cpu(cluster, cpu);
167
168                if (status != ZX_OK) {
169                    // TODO(maniscalco): Is continuing really the right thing to do here?
170
171                    // start failed, free the stack
172                    zx_status_t status = arm64_free_secondary_stack(cluster, cpu);
173                    DEBUG_ASSERT(status == ZX_OK);
174                    continue;
175                }
176
177                // the cpu booted
178                //
179                // bootstrap thread is now responsible for freeing its stack
180            }
181        }
182    }
183}
184
185static inline bool is_zbi_container(void* addr) {
186    DEBUG_ASSERT(addr);
187
188    zbi_header_t* item = (zbi_header_t*)addr;
189    return item->type == ZBI_TYPE_CONTAINER;
190}
191
192static void save_mexec_zbi(zbi_header_t* item) {
193    size_t length = ZBI_ALIGN(item->length + sizeof(zbi_header_t));
194    ASSERT(sizeof(mexec_zbi) - mexec_zbi_length >= length);
195
196    memcpy(&mexec_zbi[mexec_zbi_length], item, length);
197    mexec_zbi_length += length;
198}
199
200static void process_mem_range(const zbi_mem_range_t* mem_range) {
201    switch (mem_range->type) {
202    case ZBI_MEM_RANGE_RAM:
203        if (mem_arena.size == 0) {
204            mem_arena.base = mem_range->paddr;
205            mem_arena.size = mem_range->length;
206            dprintf(INFO, "mem_arena.base %#" PRIx64 " size %#" PRIx64 "\n", mem_arena.base,
207                    mem_arena.size);
208        } else {
209            if (mem_range->paddr) {
210                mem_arena.base = mem_range->paddr;
211                dprintf(INFO, "overriding mem arena 0 base from FDT: %#zx\n", mem_arena.base);
212            }
213            // if mem_area.base is already set, then just update the size
214            mem_arena.size = mem_range->length;
215            dprintf(INFO, "overriding mem arena 0 size from FDT: %#zx\n", mem_arena.size);
216        }
217        break;
218    case ZBI_MEM_RANGE_PERIPHERAL: {
219        auto status = add_periph_range(mem_range->paddr, mem_range->length);
220        ASSERT(status == ZX_OK);
221        break;
222    }
223    case ZBI_MEM_RANGE_RESERVED:
224        dprintf(INFO, "boot reserve mem range: phys base %#" PRIx64 " length %#" PRIx64 "\n",
225                mem_range->paddr, mem_range->length);
226        boot_reserve_add_range(mem_range->paddr, mem_range->length);
227        break;
228    default:
229        panic("bad mem_range->type in process_mem_range\n");
230        break;
231    }
232}
233
234static zbi_result_t process_zbi_item(zbi_header_t* item, void* payload, void* cookie) {
235    if (ZBI_TYPE_DRV_METADATA(item->type)) {
236        save_mexec_zbi(item);
237        return ZBI_RESULT_OK;
238    }
239    switch (item->type) {
240    case ZBI_TYPE_KERNEL_DRIVER:
241    case ZBI_TYPE_PLATFORM_ID:
242        // we don't process these here, but we need to save them for mexec
243        save_mexec_zbi(item);
244        break;
245    case ZBI_TYPE_CMDLINE: {
246        if (item->length < 1) {
247            break;
248        }
249        char* contents = reinterpret_cast<char*>(payload);
250        contents[item->length - 1] = '\0';
251        cmdline_append(contents);
252        break;
253    }
254    case ZBI_TYPE_MEM_CONFIG: {
255        zbi_mem_range_t* mem_range = reinterpret_cast<zbi_mem_range_t*>(payload);
256        uint32_t count = item->length / (uint32_t)sizeof(zbi_mem_range_t);
257        for (uint32_t i = 0; i < count; i++) {
258            process_mem_range(mem_range++);
259        }
260        save_mexec_zbi(item);
261        break;
262    }
263    case ZBI_TYPE_CPU_CONFIG: {
264        zbi_cpu_config_t* cpu_config = reinterpret_cast<zbi_cpu_config_t*>(payload);
265        cpu_cluster_count = cpu_config->cluster_count;
266        for (uint32_t i = 0; i < cpu_cluster_count; i++) {
267            cpu_cluster_cpus[i] = cpu_config->clusters[i].cpu_count;
268        }
269        arch_init_cpu_map(cpu_cluster_count, cpu_cluster_cpus);
270        save_mexec_zbi(item);
271        break;
272    }
273    case ZBI_TYPE_NVRAM: {
274        zbi_nvram_t* nvram = reinterpret_cast<zbi_nvram_t*>(payload);
275        memcpy(&lastlog_nvram, nvram, sizeof(lastlog_nvram));
276        boot_reserve_add_range(nvram->base, nvram->length);
277        save_mexec_zbi(item);
278        break;
279    }
280    }
281
282    return ZBI_RESULT_OK;
283}
284
285static void process_zbi(zbi_header_t* root) {
286    DEBUG_ASSERT(root);
287    zbi_result_t result;
288
289    uint8_t* zbi_base = reinterpret_cast<uint8_t*>(root);
290    zbi::Zbi image(zbi_base);
291
292    // Make sure the image looks valid.
293    result = image.Check(nullptr);
294    if (result != ZBI_RESULT_OK) {
295        // TODO(gkalsi): Print something informative here?
296        return;
297    }
298
299    image.ForEach(process_zbi_item, nullptr);
300}
301
302void platform_early_init(void) {
303    // if the zbi_paddr variable is -1, it was not set
304    // in start.S, so we are in a bad place.
305    if (zbi_paddr == -1UL) {
306        panic("no zbi_paddr!\n");
307    }
308
309    void* zbi_vaddr = paddr_to_physmap(zbi_paddr);
310
311    // initialize the boot memory reservation system
312    boot_reserve_init();
313
314    if (zbi_vaddr && is_zbi_container(zbi_vaddr)) {
315        zbi_header_t* header = (zbi_header_t*)zbi_vaddr;
316
317        ramdisk_base = header;
318        ramdisk_size = ROUNDUP(header->length + sizeof(*header), PAGE_SIZE);
319    } else {
320        panic("no bootdata!\n");
321    }
322
323    if (!ramdisk_base || !ramdisk_size) {
324        panic("no ramdisk!\n");
325    }
326
327    zbi_header_t* zbi = reinterpret_cast<zbi_header_t*>(ramdisk_base);
328    // walk the zbi structure and process all the items
329    process_zbi(zbi);
330
331    // is the cmdline option to bypass dlog set ?
332    dlog_bypass_init();
333
334    // bring up kernel drivers after we have mapped our peripheral ranges
335    pdev_init(zbi);
336
337    // Serial port should be active now
338
339    // Read cmdline after processing zbi, which may contain cmdline data.
340    halt_on_panic = cmdline_get_bool("kernel.halt-on-panic", false);
341
342    // Check if serial should be enabled
343    const char* serial_mode = cmdline_get("kernel.serial");
344    uart_disabled = (serial_mode != NULL && !strcmp(serial_mode, "none"));
345
346    // add the ramdisk to the boot reserve memory list
347    paddr_t ramdisk_start_phys = physmap_to_paddr(ramdisk_base);
348    paddr_t ramdisk_end_phys = ramdisk_start_phys + ramdisk_size;
349    dprintf(INFO, "reserving ramdisk phys range [%#" PRIx64 ", %#" PRIx64 "]\n",
350            ramdisk_start_phys, ramdisk_end_phys - 1);
351    boot_reserve_add_range(ramdisk_start_phys, ramdisk_size);
352
353    // check if a memory limit was passed in via kernel.memory-limit-mb and
354    // find memory ranges to use if one is found.
355    mem_limit_ctx_t ctx;
356    zx_status_t status = mem_limit_init(&ctx);
357    if (status == ZX_OK) {
358        // For these ranges we're using the base physical values
359        ctx.kernel_base = get_kernel_base_phys();
360        ctx.kernel_size = get_kernel_size();
361        ctx.ramdisk_base = ramdisk_start_phys;
362        ctx.ramdisk_size = ramdisk_end_phys - ramdisk_start_phys;
363
364        // Figure out and add arenas based on the memory limit and our range of DRAM
365        status = mem_limit_add_arenas_from_range(&ctx, mem_arena.base, mem_arena.size, mem_arena);
366    }
367
368    // If no memory limit was found, or adding arenas from the range failed, then add
369    // the existing global arena.
370    if (status != ZX_OK) {
371        pmm_add_arena(&mem_arena);
372    }
373
374    // tell the boot allocator to mark ranges we've reserved as off limits
375    boot_reserve_wire();
376}
377
378void platform_init(void) {
379    platform_cpu_init();
380}
381
382// after the fact create a region to reserve the peripheral map(s)
383static void platform_init_postvm(uint level) {
384    reserve_periph_ranges();
385}
386
387LK_INIT_HOOK(platform_postvm, platform_init_postvm, LK_INIT_LEVEL_VM);
388
389void platform_dputs_thread(const char* str, size_t len) {
390    if (uart_disabled) {
391        return;
392    }
393    uart_puts(str, len, true, true);
394}
395
396void platform_dputs_irq(const char* str, size_t len) {
397    if (uart_disabled) {
398        return;
399    }
400    uart_puts(str, len, false, true);
401}
402
403int platform_dgetc(char* c, bool wait) {
404    if (uart_disabled) {
405        return ZX_ERR_NOT_SUPPORTED;
406    }
407    int ret = uart_getc(wait);
408    if (ret < 0)
409        return ret;
410    *c = static_cast<char>(ret);
411    return 0;
412}
413
414void platform_pputc(char c) {
415    if (uart_disabled) {
416        return;
417    }
418    uart_pputc(c);
419}
420
421int platform_pgetc(char* c, bool wait) {
422    if (uart_disabled) {
423        return ZX_ERR_NOT_SUPPORTED;
424    }
425    int r = uart_pgetc();
426    if (r < 0) {
427        return r;
428    }
429
430    *c = static_cast<char>(r);
431    return 0;
432}
433
434/* stub out the hardware rng entropy generator, which doesn't exist on this platform */
435size_t hw_rng_get_entropy(void* buf, size_t len, bool block) {
436    return 0;
437}
438
439/* no built in framebuffer */
440zx_status_t display_get_info(struct display_info* info) {
441    return ZX_ERR_NOT_FOUND;
442}
443
444void platform_halt(platform_halt_action suggested_action, platform_halt_reason reason) {
445
446    if (suggested_action == HALT_ACTION_REBOOT) {
447        power_reboot(REBOOT_NORMAL);
448        printf("reboot failed\n");
449    } else if (suggested_action == HALT_ACTION_REBOOT_BOOTLOADER) {
450        power_reboot(REBOOT_BOOTLOADER);
451        printf("reboot-bootloader failed\n");
452    } else if (suggested_action == HALT_ACTION_REBOOT_RECOVERY) {
453        power_reboot(REBOOT_RECOVERY);
454        printf("reboot-recovery failed\n");
455    } else if (suggested_action == HALT_ACTION_SHUTDOWN) {
456        power_shutdown();
457    }
458
459    if (reason == HALT_REASON_SW_PANIC) {
460        thread_print_current_backtrace();
461        dlog_bluescreen_halt();
462        if (!halt_on_panic) {
463            power_reboot(REBOOT_NORMAL);
464            printf("reboot failed\n");
465        }
466#if ENABLE_PANIC_SHELL
467        dprintf(ALWAYS, "CRASH: starting debug shell... (reason = %d)\n", reason);
468        arch_disable_ints();
469        panic_shell_start();
470#endif // ENABLE_PANIC_SHELL
471    }
472
473    dprintf(ALWAYS, "HALT: spinning forever... (reason = %d)\n", reason);
474
475    // catch all fallthrough cases
476    arch_disable_ints();
477    for (;;)
478        ;
479}
480
481typedef struct {
482    //TODO: combine with x86 nvram crashlog handling
483    //TODO: ECC for more robust crashlogs
484    uint64_t magic;
485    uint64_t length;
486    uint64_t nmagic;
487    uint64_t nlength;
488} log_hdr_t;
489
490#define NVRAM_MAGIC (0x6f8962d66b28504fULL)
491
492size_t platform_stow_crashlog(void* log, size_t len) {
493    size_t max = lastlog_nvram.length - sizeof(log_hdr_t);
494    void* nvram = paddr_to_physmap(lastlog_nvram.base);
495    if (nvram == NULL) {
496        return 0;
497    }
498
499    if (log == NULL) {
500        return max;
501    }
502    if (len > max) {
503        len = max;
504    }
505
506    log_hdr_t hdr = {
507        .magic = NVRAM_MAGIC,
508        .length = len,
509        .nmagic = ~NVRAM_MAGIC,
510        .nlength = ~len,
511    };
512    memcpy(nvram, &hdr, sizeof(hdr));
513    memcpy(static_cast<char*>(nvram) + sizeof(hdr), log, len);
514    arch_clean_cache_range((uintptr_t)nvram, sizeof(hdr) + len);
515    return len;
516}
517
518size_t platform_recover_crashlog(size_t len, void* cookie,
519                                 void (*func)(const void* data, size_t, size_t len, void* cookie)) {
520    size_t max = lastlog_nvram.length - sizeof(log_hdr_t);
521    void* nvram = paddr_to_physmap(lastlog_nvram.base);
522    if (nvram == NULL) {
523        return 0;
524    }
525    log_hdr_t hdr;
526    memcpy(&hdr, nvram, sizeof(hdr));
527    if ((hdr.magic != NVRAM_MAGIC) || (hdr.length > max) ||
528        (hdr.nmagic != ~NVRAM_MAGIC) || (hdr.nlength != ~hdr.length)) {
529        printf("nvram-crashlog: bad header: %016lx %016lx %016lx %016lx\n",
530               hdr.magic, hdr.length, hdr.nmagic, hdr.nlength);
531        return 0;
532    }
533    if (len == 0) {
534        return hdr.length;
535    }
536    if (len > hdr.length) {
537        len = hdr.length;
538    }
539    func(static_cast<char*>(nvram) + sizeof(hdr), 0, len, cookie);
540
541    // invalidate header so we don't get a stale crashlog
542    // on future boots
543    hdr.magic = 0;
544    memcpy(nvram, &hdr, sizeof(hdr));
545    return hdr.length;
546}
547
548zx_status_t platform_mexec_patch_zbi(uint8_t* zbi, const size_t len) {
549    size_t offset = 0;
550
551    // copy certain boot items provided by the bootloader or boot shim
552    // to the mexec zbi
553    zbi::Zbi image(zbi, len);
554    while (offset < mexec_zbi_length) {
555        zbi_header_t* item = reinterpret_cast<zbi_header_t*>(mexec_zbi + offset);
556
557        zbi_result_t status;
558        status = image.AppendSection(item->length, item->type, item->extra,
559                                     item->flags,
560                                     reinterpret_cast<uint8_t*>(item + 1));
561
562        if (status != ZBI_RESULT_OK)
563            return ZX_ERR_INTERNAL;
564
565        offset += ZBI_ALIGN(sizeof(zbi_header_t) + item->length);
566    }
567
568    return ZX_OK;
569}
570
571void platform_mexec_prep(uintptr_t new_bootimage_addr, size_t new_bootimage_len) {
572    DEBUG_ASSERT(!arch_ints_disabled());
573    DEBUG_ASSERT(mp_get_online_mask() == cpu_num_to_mask(BOOT_CPU_ID));
574}
575
576void platform_mexec(mexec_asm_func mexec_assembly, memmov_ops_t* ops,
577                    uintptr_t new_bootimage_addr, size_t new_bootimage_len,
578                    uintptr_t entry64_addr) {
579    DEBUG_ASSERT(arch_ints_disabled());
580    DEBUG_ASSERT(mp_get_online_mask() == cpu_num_to_mask(BOOT_CPU_ID));
581
582    paddr_t kernel_src_phys = (paddr_t)ops[0].src;
583    paddr_t kernel_dst_phys = (paddr_t)ops[0].dst;
584
585    // check to see if the kernel is packaged as a zbi container
586    zbi_header_t* header = (zbi_header_t*)paddr_to_physmap(kernel_src_phys);
587    if (header[0].type == ZBI_TYPE_CONTAINER && header[1].type == ZBI_TYPE_KERNEL_ARM64) {
588        zbi_kernel_t* kernel_header = (zbi_kernel_t*)&header[2];
589        // add offset from kernel header to entry point
590        kernel_dst_phys += kernel_header->entry;
591    }
592    // else just jump to beginning of kernel image
593
594    mexec_assembly((uintptr_t)new_bootimage_addr, 0, 0, arm64_get_boot_el(), ops,
595                   (void*)kernel_dst_phys);
596}
597
598bool platform_serial_enabled(void) {
599    return !uart_disabled && uart_present();
600}
601
602bool platform_early_console_enabled() {
603    return false;
604}
605
606// Initialize Resource system after the heap is initialized.
607static void arm_resource_dispatcher_init_hook(unsigned int rl) {
608    // 64 bit address space for MMIO on ARM64
609    zx_status_t status = ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_MMIO, 0,
610                                                                 UINT64_MAX);
611    if (status != ZX_OK) {
612        printf("Resources: Failed to initialize MMIO allocator: %d\n", status);
613    }
614    // Set up IRQs based on values from the GIC
615    status = ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_IRQ,
616                                                     interrupt_get_base_vector(),
617                                                     interrupt_get_max_vector());
618    if (status != ZX_OK) {
619        printf("Resources: Failed to initialize IRQ allocator: %d\n", status);
620    }
621}
622
623LK_INIT_HOOK(arm_resource_init, arm_resource_dispatcher_init_hook, LK_INIT_LEVEL_HEAP);
624