1// Copyright 2016 The Fuchsia Authors 2// Copyright (c) 2014-2016 Travis Geiselbrecht 3// 4// Use of this source code is governed by a MIT-style 5// license that can be found in the LICENSE file or at 6// https://opensource.org/licenses/MIT 7 8#include <arch.h> 9#include <arch/arm64.h> 10#include <arch/arm64/feature.h> 11#include <arch/arm64/mmu.h> 12#include <arch/mp.h> 13#include <arch/ops.h> 14#include <assert.h> 15#include <bits.h> 16#include <debug.h> 17#include <inttypes.h> 18#include <kernel/cmdline.h> 19#include <kernel/thread.h> 20#include <lk/init.h> 21#include <lk/main.h> 22#include <platform.h> 23#include <stdlib.h> 24#include <string.h> 25#include <trace.h> 26#include <zircon/errors.h> 27#include <zircon/types.h> 28 29#define LOCAL_TRACE 0 30 31// Counter-timer Kernel Control Register, EL1. 32static constexpr uint64_t CNTKCTL_EL1_ENABLE_VIRTUAL_COUNTER = 1 << 1; 33 34// Monitor Debug System Control Register, EL1. 35static constexpr uint32_t MDSCR_EL1_ENABLE_DEBUG_EXCEPTIONS = 1 << 13; 36static constexpr uint32_t MDSCR_EL1_ENABLE_DEBUG_BREAKPOINTS = 1 << 15; 37 38// Initial value for MSDCR_EL1 when starting userspace. 39static constexpr uint32_t MSDCR_EL1_INITIAL_VALUE = 40 MDSCR_EL1_ENABLE_DEBUG_EXCEPTIONS | MDSCR_EL1_ENABLE_DEBUG_BREAKPOINTS; 41 42// Performance Monitors Count Enable Set, EL0. 43static constexpr uint64_t PMCNTENSET_EL0_ENABLE = 1UL << 31; // Enable cycle count register. 44 45// Performance Monitor Control Register, EL0. 46static constexpr uint64_t PMCR_EL0_ENABLE_BIT = 1 << 0; 47static constexpr uint64_t PMCR_EL0_LONG_COUNTER_BIT = 1 << 6; 48 49// Performance Monitors User Enable Regiser, EL0. 50static constexpr uint64_t PMUSERENR_EL0_ENABLE = 1 << 0; // Enable EL0 access to cycle counter. 51 52// System Control Register, EL1. 53static constexpr uint64_t SCTLR_EL1_UCI = 1 << 26; // Allow certain cache ops in EL0. 54static constexpr uint64_t SCTLR_EL1_UCT = 1 << 15; // Allow EL0 access to CTR register. 55static constexpr uint64_t SCTLR_EL1_DZE = 1 << 14; // Allow EL0 to use DC ZVA. 56static constexpr uint64_t SCTLR_EL1_SA0 = 1 << 4; // Enable Stack Alignment Check EL0. 57static constexpr uint64_t SCTLR_EL1_SA = 1 << 3; // Enable Stack Alignment Check EL1. 58static constexpr uint64_t SCTLR_EL1_AC = 1 << 1; // Enable Alignment Checking for EL1 EL0. 59 60struct arm64_sp_info_t { 61 uint64_t mpid; 62 void* sp; 63 64 // This part of the struct itself will serve temporarily as the 65 // fake arch_thread in the thread pointer, so that safe-stack 66 // and stack-protector code can work early. The thread pointer 67 // (TPIDR_EL1) points just past arm64_sp_info_t. 68 uintptr_t stack_guard; 69 void* unsafe_sp; 70}; 71 72static_assert(sizeof(arm64_sp_info_t) == 32, 73 "check arm64_get_secondary_sp assembly"); 74static_assert(offsetof(arm64_sp_info_t, sp) == 8, 75 "check arm64_get_secondary_sp assembly"); 76static_assert(offsetof(arm64_sp_info_t, mpid) == 0, 77 "check arm64_get_secondary_sp assembly"); 78 79#define TP_OFFSET(field) \ 80 ((int)offsetof(arm64_sp_info_t, field) - (int)sizeof(arm64_sp_info_t)) 81static_assert(TP_OFFSET(stack_guard) == ZX_TLS_STACK_GUARD_OFFSET, ""); 82static_assert(TP_OFFSET(unsafe_sp) == ZX_TLS_UNSAFE_SP_OFFSET, ""); 83#undef TP_OFFSET 84 85// SMP boot lock. 86static spin_lock_t arm_boot_cpu_lock = (spin_lock_t){1}; 87static volatile int secondaries_to_init = 0; 88 89// one for each secondary CPU, indexed by (cpu_num - 1). 90static thread_t _init_thread[SMP_MAX_CPUS - 1]; 91 92// one for each CPU 93arm64_sp_info_t arm64_secondary_sp_list[SMP_MAX_CPUS]; 94 95extern uint64_t arch_boot_el; // Defined in start.S. 96 97uint64_t arm64_get_boot_el() { 98 return arch_boot_el >> 2; 99} 100 101zx_status_t arm64_create_secondary_stack(uint cluster, uint cpu) { 102 // Allocate a stack, indexed by CPU num so that |arm64_secondary_entry| can find it. 103 cpu_num_t cpu_num = arch_mpid_to_cpu_num(cluster, cpu); 104 DEBUG_ASSERT(cpu_num > 0 && cpu_num < SMP_MAX_CPUS); 105 kstack_t* stack = &_init_thread[cpu_num - 1].stack; 106 DEBUG_ASSERT(stack->base == 0); 107 zx_status_t status = vm_allocate_kstack(stack); 108 if (status != ZX_OK) { 109 return status; 110 } 111 112 // Get the stack pointers. 113 void* sp = reinterpret_cast<void*>(stack->top); 114 void* unsafe_sp = nullptr; 115#if __has_feature(safe_stack) 116 DEBUG_ASSERT(stack->unsafe_base != 0); 117 unsafe_sp = reinterpret_cast<void*>(stack->unsafe_base + stack->size); 118#endif 119 120 // Find an empty slot for the low-level stack info. 121 uint32_t i = 0; 122 while ((i < SMP_MAX_CPUS) && (arm64_secondary_sp_list[i].mpid != 0)) { 123 i++; 124 } 125 if (i == SMP_MAX_CPUS) 126 return ZX_ERR_NO_RESOURCES; 127 128 // Store it. 129 uint64_t mpid = ARM64_MPID(cluster, cpu); 130 LTRACEF("set mpid 0x%lx sp to %p\n", mpid, sp); 131#if __has_feature(safe_stack) 132 LTRACEF("set mpid 0x%lx unsafe-sp to %p\n", mpid, unsafe_sp); 133#endif 134 arm64_secondary_sp_list[i].mpid = mpid; 135 arm64_secondary_sp_list[i].sp = sp; 136 arm64_secondary_sp_list[i].stack_guard = get_current_thread()->arch.stack_guard; 137 arm64_secondary_sp_list[i].unsafe_sp = unsafe_sp; 138 139 return ZX_OK; 140} 141 142zx_status_t arm64_free_secondary_stack(uint cluster, uint cpu) { 143 cpu_num_t cpu_num = arch_mpid_to_cpu_num(cluster, cpu); 144 DEBUG_ASSERT(cpu_num > 0 && cpu_num < SMP_MAX_CPUS); 145 kstack_t* stack = &_init_thread[cpu_num - 1].stack; 146 zx_status_t status = vm_free_kstack(stack); 147 return status; 148} 149 150static void arm64_cpu_early_init() { 151 // Make sure the per cpu pointer is set up. 152 arm64_init_percpu_early(); 153 154 // Set the vector base. 155 ARM64_WRITE_SYSREG(VBAR_EL1, (uint64_t)&arm64_el1_exception_base); 156 157 // Set some control bits in sctlr. 158 uint64_t sctlr = ARM64_READ_SYSREG(sctlr_el1); 159 sctlr |= SCTLR_EL1_UCI | SCTLR_EL1_UCT | SCTLR_EL1_DZE | SCTLR_EL1_SA0 | SCTLR_EL1_SA; 160 sctlr &= ~SCTLR_EL1_AC; // Disable alignment checking for EL1, EL0. 161 ARM64_WRITE_SYSREG(sctlr_el1, sctlr); 162 163 // Save all of the features of the cpu. 164 arm64_feature_init(); 165 166 // Enable cycle counter. 167 ARM64_WRITE_SYSREG(pmcr_el0, PMCR_EL0_ENABLE_BIT | PMCR_EL0_LONG_COUNTER_BIT); 168 ARM64_WRITE_SYSREG(pmcntenset_el0, PMCNTENSET_EL0_ENABLE); 169 170 // Enable user space access to cycle counter. 171 ARM64_WRITE_SYSREG(pmuserenr_el0, PMUSERENR_EL0_ENABLE); 172 173 // Enable Debug Exceptions by Disabling the OS Lock. The OSLAR_EL1 is a WO 174 // register with only the low bit defined as OSLK. Write 0 to disable. 175 ARM64_WRITE_SYSREG(oslar_el1, 0x0); 176 177 // Enable user space access to virtual counter (CNTVCT_EL0). 178 ARM64_WRITE_SYSREG(cntkctl_el1, CNTKCTL_EL1_ENABLE_VIRTUAL_COUNTER); 179 180 ARM64_WRITE_SYSREG(mdscr_el1, MSDCR_EL1_INITIAL_VALUE); 181 182 arch_enable_fiqs(); 183} 184 185void arch_early_init() { 186 arm64_cpu_early_init(); 187 188 platform_init_mmu_mappings(); 189} 190 191void arch_init() TA_NO_THREAD_SAFETY_ANALYSIS { 192 arch_mp_init_percpu(); 193 194 dprintf(INFO, "ARM boot EL%lu\n", arm64_get_boot_el()); 195 196 arm64_feature_debug(true); 197 198 uint32_t max_cpus = arch_max_num_cpus(); 199 uint32_t cmdline_max_cpus = cmdline_get_uint32("kernel.smp.maxcpus", max_cpus); 200 if (cmdline_max_cpus > max_cpus || cmdline_max_cpus <= 0) { 201 printf("invalid kernel.smp.maxcpus value, defaulting to %u\n", max_cpus); 202 cmdline_max_cpus = max_cpus; 203 } 204 205 secondaries_to_init = cmdline_max_cpus - 1; 206 207 lk_init_secondary_cpus(secondaries_to_init); 208 209 LTRACEF("releasing %d secondary cpus\n", secondaries_to_init); 210 211 // Release the secondary cpus. 212 spin_unlock(&arm_boot_cpu_lock); 213 214 // Flush the release of the lock, since the secondary cpus are running without cache on. 215 arch_clean_cache_range((addr_t)&arm_boot_cpu_lock, sizeof(arm_boot_cpu_lock)); 216} 217 218__NO_RETURN int arch_idle_thread_routine(void*) { 219 for (;;) 220 __asm__ volatile("wfi"); 221} 222 223// Switch to user mode, set the user stack pointer to user_stack_top, put the svc stack pointer to 224// the top of the kernel stack. 225void arch_enter_uspace(uintptr_t pc, uintptr_t sp, uintptr_t arg1, uintptr_t arg2) { 226 thread_t* ct = get_current_thread(); 227 228 // Set up a default spsr to get into 64bit user space: 229 // - Zeroed NZCV. 230 // - No SS, no IL, no D. 231 // - All interrupts enabled. 232 // - Mode 0: EL0t. 233 // 234 // TODO: (hollande,travisg) Need to determine why some platforms throw an 235 // SError exception when first switching to uspace. 236 uint32_t spsr = 1 << 8; // Mask SError exceptions (currently unhandled). 237 238 arch_disable_ints(); 239 240 LTRACEF("arm_uspace_entry(%#" PRIxPTR ", %#" PRIxPTR ", %#x, %#" PRIxPTR 241 ", %#" PRIxPTR ", 0, %#" PRIxPTR ")\n", 242 arg1, arg2, spsr, ct->stack.top, sp, pc); 243 arm64_uspace_entry(arg1, arg2, pc, sp, ct->stack.top, spsr, MSDCR_EL1_INITIAL_VALUE); 244 __UNREACHABLE; 245} 246 247// called from assembly. 248extern "C" void arm64_secondary_entry() { 249 arm64_cpu_early_init(); 250 251 spin_lock(&arm_boot_cpu_lock); 252 spin_unlock(&arm_boot_cpu_lock); 253 254 uint cpu = arch_curr_cpu_num(); 255 thread_secondary_cpu_init_early(&_init_thread[cpu - 1]); 256 // Run early secondary cpu init routines up to the threading level. 257 lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); 258 259 arch_mp_init_percpu(); 260 261 arm64_feature_debug(false); 262 263 lk_secondary_cpu_entry(); 264} 265