1// Copyright 2016 The Fuchsia Authors
2// Copyright (c) 2014-2016 Travis Geiselbrecht
3//
4// Use of this source code is governed by a MIT-style
5// license that can be found in the LICENSE file or at
6// https://opensource.org/licenses/MIT
7
8#include <arch.h>
9#include <arch/arm64.h>
10#include <arch/arm64/feature.h>
11#include <arch/arm64/mmu.h>
12#include <arch/mp.h>
13#include <arch/ops.h>
14#include <assert.h>
15#include <bits.h>
16#include <debug.h>
17#include <inttypes.h>
18#include <kernel/cmdline.h>
19#include <kernel/thread.h>
20#include <lk/init.h>
21#include <lk/main.h>
22#include <platform.h>
23#include <stdlib.h>
24#include <string.h>
25#include <trace.h>
26#include <zircon/errors.h>
27#include <zircon/types.h>
28
29#define LOCAL_TRACE 0
30
31// Counter-timer Kernel Control Register, EL1.
32static constexpr uint64_t CNTKCTL_EL1_ENABLE_VIRTUAL_COUNTER = 1 << 1;
33
34// Monitor Debug System Control Register, EL1.
35static constexpr uint32_t MDSCR_EL1_ENABLE_DEBUG_EXCEPTIONS = 1 << 13;
36static constexpr uint32_t MDSCR_EL1_ENABLE_DEBUG_BREAKPOINTS = 1 << 15;
37
38// Initial value for MSDCR_EL1 when starting userspace.
39static constexpr uint32_t MSDCR_EL1_INITIAL_VALUE =
40    MDSCR_EL1_ENABLE_DEBUG_EXCEPTIONS | MDSCR_EL1_ENABLE_DEBUG_BREAKPOINTS;
41
42// Performance Monitors Count Enable Set, EL0.
43static constexpr uint64_t PMCNTENSET_EL0_ENABLE = 1UL << 31;  // Enable cycle count register.
44
45// Performance Monitor Control Register, EL0.
46static constexpr uint64_t PMCR_EL0_ENABLE_BIT = 1 << 0;
47static constexpr uint64_t PMCR_EL0_LONG_COUNTER_BIT = 1 << 6;
48
49// Performance Monitors User Enable Regiser, EL0.
50static constexpr uint64_t PMUSERENR_EL0_ENABLE = 1 << 0;  // Enable EL0 access to cycle counter.
51
52// System Control Register, EL1.
53static constexpr uint64_t SCTLR_EL1_UCI = 1 << 26; // Allow certain cache ops in EL0.
54static constexpr uint64_t SCTLR_EL1_UCT = 1 << 15; // Allow EL0 access to CTR register.
55static constexpr uint64_t SCTLR_EL1_DZE = 1 << 14; // Allow EL0 to use DC ZVA.
56static constexpr uint64_t SCTLR_EL1_SA0 = 1 << 4;  // Enable Stack Alignment Check EL0.
57static constexpr uint64_t SCTLR_EL1_SA = 1 << 3;   // Enable Stack Alignment Check EL1.
58static constexpr uint64_t SCTLR_EL1_AC = 1 << 1;   // Enable Alignment Checking for EL1 EL0.
59
60struct arm64_sp_info_t {
61    uint64_t mpid;
62    void* sp;
63
64    // This part of the struct itself will serve temporarily as the
65    // fake arch_thread in the thread pointer, so that safe-stack
66    // and stack-protector code can work early.  The thread pointer
67    // (TPIDR_EL1) points just past arm64_sp_info_t.
68    uintptr_t stack_guard;
69    void* unsafe_sp;
70};
71
72static_assert(sizeof(arm64_sp_info_t) == 32,
73              "check arm64_get_secondary_sp assembly");
74static_assert(offsetof(arm64_sp_info_t, sp) == 8,
75              "check arm64_get_secondary_sp assembly");
76static_assert(offsetof(arm64_sp_info_t, mpid) == 0,
77              "check arm64_get_secondary_sp assembly");
78
79#define TP_OFFSET(field) \
80    ((int)offsetof(arm64_sp_info_t, field) - (int)sizeof(arm64_sp_info_t))
81static_assert(TP_OFFSET(stack_guard) == ZX_TLS_STACK_GUARD_OFFSET, "");
82static_assert(TP_OFFSET(unsafe_sp) == ZX_TLS_UNSAFE_SP_OFFSET, "");
83#undef TP_OFFSET
84
85// SMP boot lock.
86static spin_lock_t arm_boot_cpu_lock = (spin_lock_t){1};
87static volatile int secondaries_to_init = 0;
88
89// one for each secondary CPU, indexed by (cpu_num - 1).
90static thread_t _init_thread[SMP_MAX_CPUS - 1];
91
92// one for each CPU
93arm64_sp_info_t arm64_secondary_sp_list[SMP_MAX_CPUS];
94
95extern uint64_t arch_boot_el;  // Defined in start.S.
96
97uint64_t arm64_get_boot_el() {
98    return arch_boot_el >> 2;
99}
100
101zx_status_t arm64_create_secondary_stack(uint cluster, uint cpu) {
102    // Allocate a stack, indexed by CPU num so that |arm64_secondary_entry| can find it.
103    cpu_num_t cpu_num = arch_mpid_to_cpu_num(cluster, cpu);
104    DEBUG_ASSERT(cpu_num > 0 && cpu_num < SMP_MAX_CPUS);
105    kstack_t* stack = &_init_thread[cpu_num - 1].stack;
106    DEBUG_ASSERT(stack->base == 0);
107    zx_status_t status = vm_allocate_kstack(stack);
108    if (status != ZX_OK) {
109        return status;
110    }
111
112    // Get the stack pointers.
113    void* sp = reinterpret_cast<void*>(stack->top);
114    void* unsafe_sp = nullptr;
115#if __has_feature(safe_stack)
116    DEBUG_ASSERT(stack->unsafe_base != 0);
117    unsafe_sp = reinterpret_cast<void*>(stack->unsafe_base + stack->size);
118#endif
119
120    // Find an empty slot for the low-level stack info.
121    uint32_t i = 0;
122    while ((i < SMP_MAX_CPUS) && (arm64_secondary_sp_list[i].mpid != 0)) {
123        i++;
124    }
125    if (i == SMP_MAX_CPUS)
126        return ZX_ERR_NO_RESOURCES;
127
128    // Store it.
129    uint64_t mpid = ARM64_MPID(cluster, cpu);
130    LTRACEF("set mpid 0x%lx sp to %p\n", mpid, sp);
131#if __has_feature(safe_stack)
132    LTRACEF("set mpid 0x%lx unsafe-sp to %p\n", mpid, unsafe_sp);
133#endif
134    arm64_secondary_sp_list[i].mpid = mpid;
135    arm64_secondary_sp_list[i].sp = sp;
136    arm64_secondary_sp_list[i].stack_guard = get_current_thread()->arch.stack_guard;
137    arm64_secondary_sp_list[i].unsafe_sp = unsafe_sp;
138
139    return ZX_OK;
140}
141
142zx_status_t arm64_free_secondary_stack(uint cluster, uint cpu) {
143    cpu_num_t cpu_num = arch_mpid_to_cpu_num(cluster, cpu);
144    DEBUG_ASSERT(cpu_num > 0 && cpu_num < SMP_MAX_CPUS);
145    kstack_t* stack = &_init_thread[cpu_num - 1].stack;
146    zx_status_t status = vm_free_kstack(stack);
147    return status;
148}
149
150static void arm64_cpu_early_init() {
151    // Make sure the per cpu pointer is set up.
152    arm64_init_percpu_early();
153
154    // Set the vector base.
155    ARM64_WRITE_SYSREG(VBAR_EL1, (uint64_t)&arm64_el1_exception_base);
156
157    // Set some control bits in sctlr.
158    uint64_t sctlr = ARM64_READ_SYSREG(sctlr_el1);
159    sctlr |= SCTLR_EL1_UCI | SCTLR_EL1_UCT | SCTLR_EL1_DZE | SCTLR_EL1_SA0 | SCTLR_EL1_SA;
160    sctlr &= ~SCTLR_EL1_AC;  // Disable alignment checking for EL1, EL0.
161    ARM64_WRITE_SYSREG(sctlr_el1, sctlr);
162
163    // Save all of the features of the cpu.
164    arm64_feature_init();
165
166    // Enable cycle counter.
167    ARM64_WRITE_SYSREG(pmcr_el0, PMCR_EL0_ENABLE_BIT | PMCR_EL0_LONG_COUNTER_BIT);
168    ARM64_WRITE_SYSREG(pmcntenset_el0, PMCNTENSET_EL0_ENABLE);
169
170    // Enable user space access to cycle counter.
171    ARM64_WRITE_SYSREG(pmuserenr_el0, PMUSERENR_EL0_ENABLE);
172
173    // Enable Debug Exceptions by Disabling the OS Lock. The OSLAR_EL1 is a WO
174    // register with only the low bit defined as OSLK. Write 0 to disable.
175    ARM64_WRITE_SYSREG(oslar_el1, 0x0);
176
177    // Enable user space access to virtual counter (CNTVCT_EL0).
178    ARM64_WRITE_SYSREG(cntkctl_el1, CNTKCTL_EL1_ENABLE_VIRTUAL_COUNTER);
179
180    ARM64_WRITE_SYSREG(mdscr_el1, MSDCR_EL1_INITIAL_VALUE);
181
182    arch_enable_fiqs();
183}
184
185void arch_early_init() {
186    arm64_cpu_early_init();
187
188    platform_init_mmu_mappings();
189}
190
191void arch_init() TA_NO_THREAD_SAFETY_ANALYSIS {
192    arch_mp_init_percpu();
193
194    dprintf(INFO, "ARM boot EL%lu\n", arm64_get_boot_el());
195
196    arm64_feature_debug(true);
197
198    uint32_t max_cpus = arch_max_num_cpus();
199    uint32_t cmdline_max_cpus = cmdline_get_uint32("kernel.smp.maxcpus", max_cpus);
200    if (cmdline_max_cpus > max_cpus || cmdline_max_cpus <= 0) {
201        printf("invalid kernel.smp.maxcpus value, defaulting to %u\n", max_cpus);
202        cmdline_max_cpus = max_cpus;
203    }
204
205    secondaries_to_init = cmdline_max_cpus - 1;
206
207    lk_init_secondary_cpus(secondaries_to_init);
208
209    LTRACEF("releasing %d secondary cpus\n", secondaries_to_init);
210
211    // Release the secondary cpus.
212    spin_unlock(&arm_boot_cpu_lock);
213
214    // Flush the release of the lock, since the secondary cpus are running without cache on.
215    arch_clean_cache_range((addr_t)&arm_boot_cpu_lock, sizeof(arm_boot_cpu_lock));
216}
217
218__NO_RETURN int arch_idle_thread_routine(void*) {
219    for (;;)
220        __asm__ volatile("wfi");
221}
222
223// Switch to user mode, set the user stack pointer to user_stack_top, put the svc stack pointer to
224// the top of the kernel stack.
225void arch_enter_uspace(uintptr_t pc, uintptr_t sp, uintptr_t arg1, uintptr_t arg2) {
226    thread_t* ct = get_current_thread();
227
228    // Set up a default spsr to get into 64bit user space:
229    //  - Zeroed NZCV.
230    //  - No SS, no IL, no D.
231    //  - All interrupts enabled.
232    //  - Mode 0: EL0t.
233    //
234    // TODO: (hollande,travisg) Need to determine why some platforms throw an
235    //         SError exception when first switching to uspace.
236    uint32_t spsr = 1 << 8;  // Mask SError exceptions (currently unhandled).
237
238    arch_disable_ints();
239
240    LTRACEF("arm_uspace_entry(%#" PRIxPTR ", %#" PRIxPTR ", %#x, %#" PRIxPTR
241            ", %#" PRIxPTR ", 0, %#" PRIxPTR ")\n",
242            arg1, arg2, spsr, ct->stack.top, sp, pc);
243    arm64_uspace_entry(arg1, arg2, pc, sp, ct->stack.top, spsr, MSDCR_EL1_INITIAL_VALUE);
244    __UNREACHABLE;
245}
246
247// called from assembly.
248extern "C" void arm64_secondary_entry() {
249    arm64_cpu_early_init();
250
251    spin_lock(&arm_boot_cpu_lock);
252    spin_unlock(&arm_boot_cpu_lock);
253
254    uint cpu = arch_curr_cpu_num();
255    thread_secondary_cpu_init_early(&_init_thread[cpu - 1]);
256    // Run early secondary cpu init routines up to the threading level.
257    lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
258
259    arch_mp_init_percpu();
260
261    arm64_feature_debug(false);
262
263    lk_secondary_cpu_entry();
264}
265