1// Copyright 2016 The Fuchsia Authors
2//
3// Use of this source code is governed by a MIT-style
4// license that can be found in the LICENSE file or at
5// https://opensource.org/licenses/MIT
6
7#include <asm.h>
8#include <arch/arm64/mmu.h>
9#include <arch/arm64.h>
10#include <arch/asm_macros.h>
11#include <arch/defines.h>
12#include <zircon/tls.h>
13
14#ifndef __has_feature
15#define __has_feature(x) 0
16#endif
17
18/*
19 * Register use:
20 *  x0-x3   Arguments
21 *  x9-x15  Scratch
22 *  x19-x28 Globals
23 */
24tmp                     .req x9
25tmp2                    .req x10
26wtmp2                   .req w10
27page_table              .req x11
28
29cpuid                   .req x19
30page_table0             .req x20
31page_table1             .req x21
32kernel_vaddr            .req x22
33
34// This code is purely position-independent and generates no relocations
35// that need boot-time fixup; gen-kaslr-fixup.sh ensures this (and would
36// ignore it if this code were in .text.boot, so don't put it there).
37.text
38FUNCTION(_start)
39    /* Save the Boot info for the primary CPU only */
40    mrs     cpuid, mpidr_el1
41    ubfx    cpuid, cpuid, #0, #15 /* mask Aff0 and Aff1 fields */
42    cbnz    cpuid, .Lno_save_bootinfo
43    /* save x0 in zbi_paddr */
44    adrp    tmp, zbi_paddr
45    str     x0, [tmp, #:lo12:zbi_paddr]
46    /* save entry point physical address in kernel_entry_paddr */
47    adrp    tmp, kernel_entry_paddr
48    adr     tmp2, _start
49    str     tmp2, [tmp, #:lo12:kernel_entry_paddr]
50    adrp    tmp2, arch_boot_el
51    mrs     x2, CurrentEL
52    str     x2, [tmp2, #:lo12:arch_boot_el]
53.Lno_save_bootinfo:
54
55    bl      arm64_elX_to_el1
56    bl      arch_invalidate_cache_all
57
58    /* enable caches so atomics and spinlocks work */
59    mrs     tmp, sctlr_el1
60    orr     tmp, tmp, #(1<<12) /* Enable icache */
61    orr     tmp, tmp, #(1<<2)  /* Enable dcache/ucache */
62    msr     sctlr_el1, tmp
63
64    // This can be any arbitrary (page-aligned) address >= KERNEL_ASPACE_BASE.
65    // TODO(SEC-31): Choose it randomly.
66    adr_global  tmp, kernel_relocated_base
67    ldr     kernel_vaddr, [tmp]
68
69    // Load the base of the translation tables.
70    adr_global page_table0, tt_trampoline
71    adr_global page_table1, arm64_kernel_translation_table
72
73    // Send secondary cpus over to a waiting spot for the primary to finish.
74    cbnz    cpuid, .Lmmu_enable_secondary
75
76    // The fixup code appears right after the kernel image (at __data_end in
77    // our view).  Note this code overlaps with the kernel's bss!  It
78    // expects x0 to contain the actual runtime address of __code_start.
79    mov     x0, kernel_vaddr
80    bl      __data_end
81
82    /* clear out the kernel's bss using current physical location */
83    /* NOTE: Relies on __bss_start and _end being 16 byte aligned */
84.Ldo_bss:
85    adr_global tmp, __bss_start
86    adr_global tmp2, _end
87    sub     tmp2, tmp2, tmp
88    cbz     tmp2, .Lbss_loop_done
89.Lbss_loop:
90    sub     tmp2, tmp2, #16
91    stp     xzr, xzr, [tmp], #16
92    cbnz    tmp2, .Lbss_loop
93.Lbss_loop_done:
94
95    /* set up a functional stack pointer */
96    adr_global tmp, boot_cpu_kstack_end
97    mov     sp, tmp
98
99    /* make sure the boot allocator is given a chance to figure out where
100     * we are loaded in physical memory. */
101    bl      boot_alloc_init
102
103    /* save the physical address the kernel is loaded at */
104    adr_global x0, __code_start
105    adr_global x1, kernel_base_phys
106    str     x0, [x1]
107
108    /* set up the mmu according to mmu_initial_mappings */
109
110    /* clear out the kernel translation table */
111    mov     tmp, #0
112.Lclear_top_page_table_loop:
113    str     xzr, [page_table1, tmp, lsl #3]
114    add     tmp, tmp, #1
115    cmp     tmp, #MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP
116    bne     .Lclear_top_page_table_loop
117
118    /* void arm64_boot_map(pte_t* kernel_table0, vaddr_t vaddr, paddr_t paddr, size_t len, pte_t flags); */
119
120    /* map a large run of physical memory at the base of the kernel's address space */
121    mov     x0, page_table1
122    mov     x1, KERNEL_ASPACE_BASE
123    mov     x2, 0
124    mov     x3, ARCH_PHYSMAP_SIZE
125    movlit  x4, MMU_PTE_KERNEL_DATA_FLAGS
126    bl      arm64_boot_map
127
128    /* map the kernel to a fixed address */
129    /* note: mapping the kernel here with full rwx, this will get locked down later in vm initialization; */
130    mov     x0, page_table1
131    mov     x1, kernel_vaddr
132    adr_global x2, __code_start
133    adr_global x3, _end
134    sub     x3, x3, x2
135    mov     x4, MMU_PTE_KERNEL_RWX_FLAGS
136    bl      arm64_boot_map
137
138    /* Prepare tt_trampoline page table.
139     * this will identity map the 1GB page holding the physical address of this code.
140     * Used to temporarily help us get switched to the upper virtual address. */
141
142    /* Zero tt_trampoline translation tables */
143    mov     tmp, #0
144.Lclear_tt_trampoline:
145    str     xzr, [page_table0, tmp, lsl#3]
146    add     tmp, tmp, #1
147    cmp     tmp, #MMU_PAGE_TABLE_ENTRIES_IDENT
148    blt     .Lclear_tt_trampoline
149
150    /* Setup mapping at phys -> phys */
151    adr     tmp, .Lmmu_on_pc
152    lsr     tmp, tmp, #MMU_IDENT_TOP_SHIFT    /* tmp = paddr index */
153    movlit  tmp2, MMU_PTE_IDENT_FLAGS
154    add     tmp2, tmp2, tmp, lsl #MMU_IDENT_TOP_SHIFT  /* tmp2 = pt entry */
155
156    str     tmp2, [page_table0, tmp, lsl #3]  /* tt_trampoline[paddr index] = pt entry */
157
158    /* mark page tables as set up, so secondary cpus can fall through */
159    adr_global tmp, page_tables_not_ready
160    str     wzr, [tmp]
161    b       .Lpage_tables_ready
162
163.Lmmu_enable_secondary:
164    adr_global tmp, page_tables_not_ready
165    /* trap any secondary cpus until the primary has set up the page tables */
166.Lpage_tables_not_ready:
167    ldr     wtmp2, [tmp]
168    cbnz    wtmp2, .Lpage_tables_not_ready
169.Lpage_tables_ready:
170
171    /* set up the mmu */
172
173    /* Invalidate TLB */
174    tlbi    vmalle1is
175    isb
176    dsb     sy
177
178    /* Initialize Memory Attribute Indirection Register */
179    movlit  tmp, MMU_MAIR_VAL
180    msr     mair_el1, tmp
181
182    /* Initialize TCR_EL1 */
183    /* set cacheable attributes on translation walk */
184    /* (SMP extensions) non-shareable, inner write-back write-allocate */
185    movlit  tmp, MMU_TCR_FLAGS_IDENT
186    msr     tcr_el1, tmp
187
188    isb
189
190    /* Write ttbr with phys addr of the translation table */
191    msr     ttbr0_el1, page_table0
192    msr     ttbr1_el1, page_table1
193    isb
194
195    /* Read SCTLR */
196    mrs     tmp, sctlr_el1
197
198    /* Turn on the MMU */
199    orr     tmp, tmp, #0x1
200
201    /* Write back SCTLR */
202    msr     sctlr_el1, tmp
203.Lmmu_on_pc:
204    isb
205
206    // Map our current physical PC to the virtual PC and jump there.
207    // PC = next_PC - __code_start + kernel_vaddr
208    adr     tmp, .Lmmu_on_vaddr
209    adr     tmp2, __code_start
210    sub     tmp, tmp, tmp2
211    add     tmp, tmp, kernel_vaddr
212    br      tmp
213
214.Lmmu_on_vaddr:
215
216    /* Disable trampoline page-table in ttbr0 */
217    movlit  tmp, MMU_TCR_FLAGS_KERNEL
218    msr     tcr_el1, tmp
219    isb
220
221    /* Invalidate TLB */
222    tlbi    vmalle1
223    isb
224
225    cbnz    cpuid, .Lsecondary_boot
226
227    // set up the boot stack for real
228    adr_global tmp, boot_cpu_kstack_end
229    mov     sp, tmp
230
231    // Set the thread pointer early so compiler-generated references
232    // to the stack-guard and unsafe-sp slots work.  This is not a
233    // real 'struct thread' yet, just a pointer to (past, actually)
234    // the two slots used by the ABI known to the compiler.  This avoids
235    // having to compile-time disable safe-stack and stack-protector
236    // code generation features for all the C code in the bootstrap
237    // path, which (unlike on x86, e.g.) is enough to get annoying.
238    adr_global tmp, boot_cpu_fake_thread_pointer_location
239    msr     tpidr_el1, tmp
240
241    // set the per cpu pointer for cpu 0
242    adr_global x18, arm64_percpu_array
243
244    // Choose a good (ideally random) stack-guard value as early as possible.
245    bl      choose_stack_guard
246    mrs     tmp, tpidr_el1
247    str     x0, [tmp, #ZX_TLS_STACK_GUARD_OFFSET]
248    // Don't leak the value to other code.
249    mov     x0, xzr
250
251    bl  lk_main
252    b   .
253
254.Lsecondary_boot:
255    bl      arm64_get_secondary_sp
256    cbz     x0, .Lunsupported_cpu_trap
257    mov     sp, x0
258    msr     tpidr_el1, x1
259
260    bl      arm64_secondary_entry
261
262.Lunsupported_cpu_trap:
263    wfe
264    b       .Lunsupported_cpu_trap
265END_FUNCTION(_start)
266
267.ltorg
268
269// These are logically .bss (uninitialized data).  But they're set before
270// clearing the .bss, so put them in .data so they don't get zeroed.
271.data
272    .balign 64
273DATA(arch_boot_el)
274    .quad 0xdeadbeef00ff00ff
275END_DATA(arch_boot_el)
276DATA(zbi_paddr)
277    .quad -1
278END_DATA(zbi_paddr)
279DATA(kernel_entry_paddr)
280    .quad -1
281END_DATA(kernel_entry_paddr)
282
283DATA(page_tables_not_ready)
284    .long       1
285END_DATA(page_tables_not_ready)
286
287    .balign 8
288LOCAL_DATA(boot_cpu_fake_arch_thread)
289    .quad 0xdeadbeef1ee2d00d // stack_guard
290#if __has_feature(safe_stack)
291    .quad boot_cpu_unsafe_kstack_end
292#else
293    .quad 0
294#endif
295LOCAL_DATA(boot_cpu_fake_thread_pointer_location)
296END_DATA(boot_cpu_fake_arch_thread)
297
298.bss
299LOCAL_DATA(boot_cpu_kstack)
300    .skip ARCH_DEFAULT_STACK_SIZE
301    .balign 16
302LOCAL_DATA(boot_cpu_kstack_end)
303END_DATA(boot_cpu_kstack)
304
305#if __has_feature(safe_stack)
306LOCAL_DATA(boot_cpu_unsafe_kstack)
307    .skip ARCH_DEFAULT_STACK_SIZE
308    .balign 16
309LOCAL_DATA(boot_cpu_unsafe_kstack_end)
310END_DATA(boot_cpu_unsafe_kstack)
311#endif
312
313.section .bss.prebss.translation_table, "aw", @nobits
314.align 3 + MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT
315DATA(tt_trampoline)
316    .skip 8 * MMU_PAGE_TABLE_ENTRIES_IDENT
317END_DATA(tt_trampoline)
318
319// This symbol is used by image.S
320.global IMAGE_ELF_ENTRY
321IMAGE_ELF_ENTRY = _start
322
323// This symbol is used by gdb python to know the base of the kernel module
324.global KERNEL_BASE_ADDRESS
325KERNEL_BASE_ADDRESS = KERNEL_BASE
326