1// Copyright 2016 The Fuchsia Authors 2// 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file or at 5// https://opensource.org/licenses/MIT 6 7#include <asm.h> 8#include <arch/arm64/mmu.h> 9#include <arch/arm64.h> 10#include <arch/asm_macros.h> 11#include <arch/defines.h> 12#include <zircon/tls.h> 13 14#ifndef __has_feature 15#define __has_feature(x) 0 16#endif 17 18/* 19 * Register use: 20 * x0-x3 Arguments 21 * x9-x15 Scratch 22 * x19-x28 Globals 23 */ 24tmp .req x9 25tmp2 .req x10 26wtmp2 .req w10 27page_table .req x11 28 29cpuid .req x19 30page_table0 .req x20 31page_table1 .req x21 32kernel_vaddr .req x22 33 34// This code is purely position-independent and generates no relocations 35// that need boot-time fixup; gen-kaslr-fixup.sh ensures this (and would 36// ignore it if this code were in .text.boot, so don't put it there). 37.text 38FUNCTION(_start) 39 /* Save the Boot info for the primary CPU only */ 40 mrs cpuid, mpidr_el1 41 ubfx cpuid, cpuid, #0, #15 /* mask Aff0 and Aff1 fields */ 42 cbnz cpuid, .Lno_save_bootinfo 43 /* save x0 in zbi_paddr */ 44 adrp tmp, zbi_paddr 45 str x0, [tmp, #:lo12:zbi_paddr] 46 /* save entry point physical address in kernel_entry_paddr */ 47 adrp tmp, kernel_entry_paddr 48 adr tmp2, _start 49 str tmp2, [tmp, #:lo12:kernel_entry_paddr] 50 adrp tmp2, arch_boot_el 51 mrs x2, CurrentEL 52 str x2, [tmp2, #:lo12:arch_boot_el] 53.Lno_save_bootinfo: 54 55 bl arm64_elX_to_el1 56 bl arch_invalidate_cache_all 57 58 /* enable caches so atomics and spinlocks work */ 59 mrs tmp, sctlr_el1 60 orr tmp, tmp, #(1<<12) /* Enable icache */ 61 orr tmp, tmp, #(1<<2) /* Enable dcache/ucache */ 62 msr sctlr_el1, tmp 63 64 // This can be any arbitrary (page-aligned) address >= KERNEL_ASPACE_BASE. 65 // TODO(SEC-31): Choose it randomly. 66 adr_global tmp, kernel_relocated_base 67 ldr kernel_vaddr, [tmp] 68 69 // Load the base of the translation tables. 70 adr_global page_table0, tt_trampoline 71 adr_global page_table1, arm64_kernel_translation_table 72 73 // Send secondary cpus over to a waiting spot for the primary to finish. 74 cbnz cpuid, .Lmmu_enable_secondary 75 76 // The fixup code appears right after the kernel image (at __data_end in 77 // our view). Note this code overlaps with the kernel's bss! It 78 // expects x0 to contain the actual runtime address of __code_start. 79 mov x0, kernel_vaddr 80 bl __data_end 81 82 /* clear out the kernel's bss using current physical location */ 83 /* NOTE: Relies on __bss_start and _end being 16 byte aligned */ 84.Ldo_bss: 85 adr_global tmp, __bss_start 86 adr_global tmp2, _end 87 sub tmp2, tmp2, tmp 88 cbz tmp2, .Lbss_loop_done 89.Lbss_loop: 90 sub tmp2, tmp2, #16 91 stp xzr, xzr, [tmp], #16 92 cbnz tmp2, .Lbss_loop 93.Lbss_loop_done: 94 95 /* set up a functional stack pointer */ 96 adr_global tmp, boot_cpu_kstack_end 97 mov sp, tmp 98 99 /* make sure the boot allocator is given a chance to figure out where 100 * we are loaded in physical memory. */ 101 bl boot_alloc_init 102 103 /* save the physical address the kernel is loaded at */ 104 adr_global x0, __code_start 105 adr_global x1, kernel_base_phys 106 str x0, [x1] 107 108 /* set up the mmu according to mmu_initial_mappings */ 109 110 /* clear out the kernel translation table */ 111 mov tmp, #0 112.Lclear_top_page_table_loop: 113 str xzr, [page_table1, tmp, lsl #3] 114 add tmp, tmp, #1 115 cmp tmp, #MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP 116 bne .Lclear_top_page_table_loop 117 118 /* void arm64_boot_map(pte_t* kernel_table0, vaddr_t vaddr, paddr_t paddr, size_t len, pte_t flags); */ 119 120 /* map a large run of physical memory at the base of the kernel's address space */ 121 mov x0, page_table1 122 mov x1, KERNEL_ASPACE_BASE 123 mov x2, 0 124 mov x3, ARCH_PHYSMAP_SIZE 125 movlit x4, MMU_PTE_KERNEL_DATA_FLAGS 126 bl arm64_boot_map 127 128 /* map the kernel to a fixed address */ 129 /* note: mapping the kernel here with full rwx, this will get locked down later in vm initialization; */ 130 mov x0, page_table1 131 mov x1, kernel_vaddr 132 adr_global x2, __code_start 133 adr_global x3, _end 134 sub x3, x3, x2 135 mov x4, MMU_PTE_KERNEL_RWX_FLAGS 136 bl arm64_boot_map 137 138 /* Prepare tt_trampoline page table. 139 * this will identity map the 1GB page holding the physical address of this code. 140 * Used to temporarily help us get switched to the upper virtual address. */ 141 142 /* Zero tt_trampoline translation tables */ 143 mov tmp, #0 144.Lclear_tt_trampoline: 145 str xzr, [page_table0, tmp, lsl#3] 146 add tmp, tmp, #1 147 cmp tmp, #MMU_PAGE_TABLE_ENTRIES_IDENT 148 blt .Lclear_tt_trampoline 149 150 /* Setup mapping at phys -> phys */ 151 adr tmp, .Lmmu_on_pc 152 lsr tmp, tmp, #MMU_IDENT_TOP_SHIFT /* tmp = paddr index */ 153 movlit tmp2, MMU_PTE_IDENT_FLAGS 154 add tmp2, tmp2, tmp, lsl #MMU_IDENT_TOP_SHIFT /* tmp2 = pt entry */ 155 156 str tmp2, [page_table0, tmp, lsl #3] /* tt_trampoline[paddr index] = pt entry */ 157 158 /* mark page tables as set up, so secondary cpus can fall through */ 159 adr_global tmp, page_tables_not_ready 160 str wzr, [tmp] 161 b .Lpage_tables_ready 162 163.Lmmu_enable_secondary: 164 adr_global tmp, page_tables_not_ready 165 /* trap any secondary cpus until the primary has set up the page tables */ 166.Lpage_tables_not_ready: 167 ldr wtmp2, [tmp] 168 cbnz wtmp2, .Lpage_tables_not_ready 169.Lpage_tables_ready: 170 171 /* set up the mmu */ 172 173 /* Invalidate TLB */ 174 tlbi vmalle1is 175 isb 176 dsb sy 177 178 /* Initialize Memory Attribute Indirection Register */ 179 movlit tmp, MMU_MAIR_VAL 180 msr mair_el1, tmp 181 182 /* Initialize TCR_EL1 */ 183 /* set cacheable attributes on translation walk */ 184 /* (SMP extensions) non-shareable, inner write-back write-allocate */ 185 movlit tmp, MMU_TCR_FLAGS_IDENT 186 msr tcr_el1, tmp 187 188 isb 189 190 /* Write ttbr with phys addr of the translation table */ 191 msr ttbr0_el1, page_table0 192 msr ttbr1_el1, page_table1 193 isb 194 195 /* Read SCTLR */ 196 mrs tmp, sctlr_el1 197 198 /* Turn on the MMU */ 199 orr tmp, tmp, #0x1 200 201 /* Write back SCTLR */ 202 msr sctlr_el1, tmp 203.Lmmu_on_pc: 204 isb 205 206 // Map our current physical PC to the virtual PC and jump there. 207 // PC = next_PC - __code_start + kernel_vaddr 208 adr tmp, .Lmmu_on_vaddr 209 adr tmp2, __code_start 210 sub tmp, tmp, tmp2 211 add tmp, tmp, kernel_vaddr 212 br tmp 213 214.Lmmu_on_vaddr: 215 216 /* Disable trampoline page-table in ttbr0 */ 217 movlit tmp, MMU_TCR_FLAGS_KERNEL 218 msr tcr_el1, tmp 219 isb 220 221 /* Invalidate TLB */ 222 tlbi vmalle1 223 isb 224 225 cbnz cpuid, .Lsecondary_boot 226 227 // set up the boot stack for real 228 adr_global tmp, boot_cpu_kstack_end 229 mov sp, tmp 230 231 // Set the thread pointer early so compiler-generated references 232 // to the stack-guard and unsafe-sp slots work. This is not a 233 // real 'struct thread' yet, just a pointer to (past, actually) 234 // the two slots used by the ABI known to the compiler. This avoids 235 // having to compile-time disable safe-stack and stack-protector 236 // code generation features for all the C code in the bootstrap 237 // path, which (unlike on x86, e.g.) is enough to get annoying. 238 adr_global tmp, boot_cpu_fake_thread_pointer_location 239 msr tpidr_el1, tmp 240 241 // set the per cpu pointer for cpu 0 242 adr_global x18, arm64_percpu_array 243 244 // Choose a good (ideally random) stack-guard value as early as possible. 245 bl choose_stack_guard 246 mrs tmp, tpidr_el1 247 str x0, [tmp, #ZX_TLS_STACK_GUARD_OFFSET] 248 // Don't leak the value to other code. 249 mov x0, xzr 250 251 bl lk_main 252 b . 253 254.Lsecondary_boot: 255 bl arm64_get_secondary_sp 256 cbz x0, .Lunsupported_cpu_trap 257 mov sp, x0 258 msr tpidr_el1, x1 259 260 bl arm64_secondary_entry 261 262.Lunsupported_cpu_trap: 263 wfe 264 b .Lunsupported_cpu_trap 265END_FUNCTION(_start) 266 267.ltorg 268 269// These are logically .bss (uninitialized data). But they're set before 270// clearing the .bss, so put them in .data so they don't get zeroed. 271.data 272 .balign 64 273DATA(arch_boot_el) 274 .quad 0xdeadbeef00ff00ff 275END_DATA(arch_boot_el) 276DATA(zbi_paddr) 277 .quad -1 278END_DATA(zbi_paddr) 279DATA(kernel_entry_paddr) 280 .quad -1 281END_DATA(kernel_entry_paddr) 282 283DATA(page_tables_not_ready) 284 .long 1 285END_DATA(page_tables_not_ready) 286 287 .balign 8 288LOCAL_DATA(boot_cpu_fake_arch_thread) 289 .quad 0xdeadbeef1ee2d00d // stack_guard 290#if __has_feature(safe_stack) 291 .quad boot_cpu_unsafe_kstack_end 292#else 293 .quad 0 294#endif 295LOCAL_DATA(boot_cpu_fake_thread_pointer_location) 296END_DATA(boot_cpu_fake_arch_thread) 297 298.bss 299LOCAL_DATA(boot_cpu_kstack) 300 .skip ARCH_DEFAULT_STACK_SIZE 301 .balign 16 302LOCAL_DATA(boot_cpu_kstack_end) 303END_DATA(boot_cpu_kstack) 304 305#if __has_feature(safe_stack) 306LOCAL_DATA(boot_cpu_unsafe_kstack) 307 .skip ARCH_DEFAULT_STACK_SIZE 308 .balign 16 309LOCAL_DATA(boot_cpu_unsafe_kstack_end) 310END_DATA(boot_cpu_unsafe_kstack) 311#endif 312 313.section .bss.prebss.translation_table, "aw", @nobits 314.align 3 + MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT 315DATA(tt_trampoline) 316 .skip 8 * MMU_PAGE_TABLE_ENTRIES_IDENT 317END_DATA(tt_trampoline) 318 319// This symbol is used by image.S 320.global IMAGE_ELF_ENTRY 321IMAGE_ELF_ENTRY = _start 322 323// This symbol is used by gdb python to know the base of the kernel module 324.global KERNEL_BASE_ADDRESS 325KERNEL_BASE_ADDRESS = KERNEL_BASE 326