1// Copyright 2016 The Fuchsia Authors
2// Copyright (c) 2009 Corey Tabaka
3// Copyright (c) 2015 Intel Corporation
4// Copyright (c) 2016 Travis Geiselbrecht
5//
6// Use of this source code is governed by a MIT-style
7// license that can be found in the LICENSE file or at
8// https://opensource.org/licenses/MIT
9
10#include <asm.h>
11#include <arch/x86/asm.h>
12#include <arch/x86/descriptor.h>
13#include <arch/x86/mmu.h>
14#include <arch/x86/registers.h>
15#include <zircon/boot/multiboot.h>
16#include <zircon/tls.h>
17
18// TODO(mcgrathr): This is in a macro because it's used in both 32-bit mode
19// and in 64-bit mode with addresses known to be <4GB.  The assembly text
20// is the same either way, but it needs to be assembled separately for each.
21// In future, the 32-bit boot path should trampoline into the 64-bit path so
22// there will only be a 64-bit version of this.
23// NOTE: Clobbers %rax, %rcx, %rsi, %rdi.
24.macro move_fixups_and_zero_bss
25    // The fixup code in image.S runs in 64-bit mode with paging enabled,
26    // so we can't run it too early.  But it overlaps the bss, so we move
27    // it before zeroing the bss.  We can't delay zeroing the bss because
28    // the page tables we're about to set up are themselves in bss.
29
30    // The first word after the kernel image (at __data_end in our view)
31    // gives the size of the following code.  Copy it to _end.
32    mov PHYS(__data_end), %ecx
33    mov $PHYS(__data_end+4), %esi
34    mov $PHYS(_end), %edi
35    rep movsb // while (ecx-- > 0) *edi++ = *esi++;
36
37    // Now it's safe to zero the bss.
38    movl $PHYS(__bss_start), %edi
39    movl $PHYS(_end), %ecx
40    sub %edi, %ecx              // Compute the length of the bss in bytes.
41    xor %eax, %eax
42    rep stosb // while (ecx-- > 0) *edi++ = al;
43.endm
44
45#define ADDR_OFFSET_MASK ((1 << ADDR_OFFSET)-1)
46#define SHIFT_OFFSET(_s) ((_s) >> 3)
47#define SHIFT_REMAIN(_s) ((_s) - (SHIFT_OFFSET(_s) << 3))
48
49// Set a page table entry for the kernel module relocated 64-bit virtual
50// address in 32-bit code. Clobbers the %ecx register.
51.macro set_relocated_page_table_entry table, shift, value
52    // Extract 32-bit chunk of kernel_relocated_base containing the index bits
53    // for this page level shift.
54    mov PHYS(kernel_relocated_base + SHIFT_OFFSET(\shift)), %ecx
55
56    // Get the exact portion of the 32-bit value that is the index
57    shrl $SHIFT_REMAIN(\shift), %ecx
58    andl $ADDR_OFFSET_MASK, %ecx
59
60    // Get the adddress on the page table of index * 8 and set the value
61    shll $3, %ecx
62    addl $PHYS(\table), %ecx
63    movl \value, (%ecx)
64.endm
65
66/* shared code to set up a default 64bit page table structure */
67.macro page_table_init
68    /* Setting the First PML4E with a PDP table reference*/
69    movl $PHYS(pdp), %eax
70    orl  $X86_KERNEL_PD_FLAGS, %eax
71    movl %eax, PHYS(pml4)
72
73    /* Setting the First PDPTE with a Page table reference*/
74    movl $PHYS(pte), %eax
75    orl  $X86_KERNEL_PD_FLAGS, %eax
76    movl %eax, PHYS(pdp)
77
78    /* point the pml4e at the second high PDP (for -2GB mapping) */
79    movl $PHYS(pdp_high),   %eax
80    orl  $X86_KERNEL_PD_FLAGS, %eax
81    set_relocated_page_table_entry pml4, PML4_SHIFT, %eax
82
83    /* point the second pdp at the same low level page table */
84    movl $PHYS(pte), %eax
85    orl  $X86_KERNEL_PD_FLAGS, %eax
86    set_relocated_page_table_entry pdp_high, PDP_SHIFT, %eax
87
88    /* map the first 1GB in this table */
89    movl $PHYS(pte), %esi
90    movl $0x200, %ecx
91    xor  %eax, %eax
92
930:
94    mov  %eax, %ebx
95    shll $21, %ebx
96    orl  $X86_KERNEL_PD_LP_FLAGS, %ebx
97    movl %ebx, (%esi)
98    addl $8,%esi
99    inc  %eax
100    loop 0b
101
102    /* set up a linear map of the first 64GB at 0xffffff8000000000 */
103    movl $PHYS(linear_map_pdp), %esi
104    movl $32768, %ecx
105    xor  %eax, %eax
106
107    /* loop across these page tables, incrementing the address by 2MB */
1080:
109    mov  %eax, %ebx
110    shll $21, %ebx
111    orl  $X86_KERNEL_PD_LP_FLAGS, %ebx    // lower word of the entry
112    movl %ebx, (%esi)
113    mov  %eax, %ebx
114    shrl $11, %ebx      // upper word of the entry
115    movl %ebx, 4(%esi)
116    addl $8,%esi
117    inc  %eax
118    loop 0b
119
120    /* point the high pdp at our linear mapping page tables */
121    movl $PHYS(pdp_high), %esi
122    movl $64, %ecx
123    movl $PHYS(linear_map_pdp), %eax
124    orl  $X86_KERNEL_PD_FLAGS, %eax
125
1260:
127    movl %eax, (%esi)
128    add  $8, %esi
129    addl $4096, %eax
130    loop 0b
131.endm
132
133// Load our new GDT by physical pointer.
134// _temp_gdtr has it as a virtual pointer, so copy it and adjust.
135// Clobbers %eax and %ecx.
136.macro load_phys_gdtr
137    movw PHYS(_temp_gdtr), %ax
138    movl PHYS(_temp_gdtr+2), %ecx
139    sub $PHYS_ADDR_DELTA, %ecx
140    movw %ax, -6(%esp)
141    movl %ecx, -4(%esp)
142    lgdt -6(%esp)
143.endm
144
145// This section name is known specially to kernel.ld and gen-kaslr-fixups.sh.
146// This code has relocations for absolute physical addresses, which do not get
147// adjusted by the boot-time fixups (which this code calls at the end).
148.section .text.boot, "ax", @progbits
149.code32
150FUNCTION_LABEL(_multiboot_start)
151    cmpl $MULTIBOOT_BOOTLOADER_MAGIC, %eax
152    je 0f
153    xor %ebx,%ebx
1540:  // multiboot_info_t pointer is in %ebx.
155
156    /* load our gdtr */
157    load_phys_gdtr
158
159    /* load our data selectors */
160    movw $DATA_SELECTOR, %ax
161    movw %ax, %ds
162    movw %ax, %es
163    movw %ax, %fs
164    movw %ax, %gs
165    movw %ax, %ss
166
167    /* We need to jump to our sane 32 bit CS */
168    pushl $CODE_SELECTOR
169    pushl $PHYS(.Lfarjump)
170    lret
171
172.Lfarjump:
173    move_fixups_and_zero_bss
174
175    // _multiboot_info is in bss, so now it's safe to set it.
176    movl %ebx, PHYS(_multiboot_info)
177
178    /* Start using the zircon stack from its physical address. */
179    mov $PHYS(_kstack_end), %esp
180
181paging_setup:
182    /* Preparing 64 bit paging, we will use 2MB pages covering 1GB
183     * for initial bootstrap, this page table will be 1 to 1
184     */
185
186    /* Set the PAE bit to enable 64bit paging
187     * Set PGE to enable global kernel pages
188     */
189    mov %cr4, %eax
190    or $(X86_CR4_PAE|X86_CR4_PGE), %eax
191    mov %eax, %cr4
192
193    /* Long Mode Enabled at this point */
194    movl $X86_MSR_IA32_EFER, %ecx
195    rdmsr
196    orl $X86_EFER_LME,%eax
197    wrmsr
198
199    /* initialize the default page tables */
200    page_table_init
201
202    /* load the physical pointer to the top level page table */
203    movl $PHYS(pml4), %eax
204    mov %eax, %cr3
205
206    /* Enabling Paging and from this point we are in
207    32 bit compatibility mode*/
208    mov %cr0,  %eax
209    btsl $(31), %eax
210    mov %eax,  %cr0
211
212    /* Using another long jump to be on 64 bit mode
213    after this we will be on real 64 bit mode */
214    pushl $CODE_64_SELECTOR     /*Need to put it in a the right CS*/
215    pushl $PHYS(farjump64)
216    lret
217
218.align 8
219.code64
220farjump64:
221    /* give the boot allocator a chance to compute the physical address of the kernel */
222    call boot_alloc_init
223
224    /* branch to our high address, with the relocated offset */
225    mov $PHYS(high_entry), %rax
226    addq PHYS(kernel_relocated_base), %rax
227    jmp  *%rax
228
229// The 64-bit entry path also gets here; see below.
230// This code runs at the final virtual address, so it should be pure PIC.
231.pushsection .text
232high_entry:
233    /* zero our kernel segment data registers */
234    xor %eax, %eax
235    mov %eax, %ds
236    mov %eax, %es
237    mov %eax, %fs
238    mov %eax, %gs
239    mov %eax, %ss
240
241    /* load the high kernel stack */
242    lea _kstack_end(%rip), %rsp
243
244    // move_fixups_and_zero_bss copied the fixup code to _end.
245    // It expects %rdi to contain the actual runtime address of __code_start.
246    lea __code_start(%rip), %rdi
247    call _end
248    // The fixup code won't be used again, so the memory can be reused now.
249
250    /* reload the gdtr after relocations as it relies on relocated VAs */
251    lgdt _temp_gdtr(%rip)
252
253    // Set %gs.base to &bp_percpu.  It's statically initialized
254    // with kernel_unsafe_sp set, so after this it's safe to call
255    // into C code that might use safe-stack and/or stack-protector.
256    lea bp_percpu(%rip), %rax
257    mov %rax, %rdx
258    shr $32, %rdx
259    mov $X86_MSR_IA32_GS_BASE, %ecx
260    wrmsr
261
262    /* set up the idt */
263    lea _idt_startup(%rip), %rdi
264    call idt_setup
265    call load_startup_idt
266
267    /* assign this core CPU# 0 and initialize its per cpu state */
268    xor %edi, %edi
269    call x86_init_percpu
270
271    // Fill the stack canary with a random value as early as possible.
272    // This isn't done in x86_init_percpu because the hw_rng_get_entropy
273    // call would make it eligible for stack-guard checking itself.  But
274    // %gs is not set up yet in the prologue of the function, so it would
275    // crash if it tried to use the stack-guard.
276    call choose_stack_guard
277
278    // Move it into place.
279    mov %rcx, %gs:ZX_TLS_STACK_GUARD_OFFSET
280    // Don't leak that value to other code.
281    xor %ecx, %ecx
282
283    // configure the kernel base address
284    // TODO: dynamically figure this out once we allow the x86 kernel to be loaded anywhere
285    movl $PHYS_LOAD_ADDRESS, kernel_base_phys(%rip)
286
287    /* call the main module */
288    call lk_main
289
2900:                          /* just sit around waiting for interrupts */
291    hlt                     /* interrupts will unhalt the processor */
292    pause
293    jmp 0b                  /* so jump back to halt to conserve power */
294.popsection
295
296    /* 64bit entry point from a secondary loader */
297.align 8
298FUNCTION_LABEL(_start)
299    /* set up a temporary stack pointer */
300    mov $PHYS(_kstack_end), %rsp
301
302    // Save off the bootdata pointer in a register that won't get clobbered.
303    mov %esi, %ebx
304
305    move_fixups_and_zero_bss
306
307    // _zbi_base is in bss, so now it's safe to set it.
308    mov %ebx, PHYS(_zbi_base)
309
310    /* give the boot allocator a chance to compute the physical address of the kernel */
311    call boot_alloc_init
312
313.Lpaging_setup64:
314    /* initialize the default page tables */
315    page_table_init
316
317    /*
318     * Set PGE to enable global kernel pages
319     */
320    mov   %cr4, %rax
321    or    $(X86_CR4_PGE), %rax
322    mov   %rax, %cr4
323
324    /* load the physical pointer to the top level page table */
325    mov  $PHYS(pml4), %rax
326    mov  %rax, %cr3
327
328    /* load our gdtr */
329    load_phys_gdtr
330
331    /* long jump to our code selector and the high address relocated */
332    push  $CODE_64_SELECTOR
333    mov  $PHYS(high_entry), %rax
334    addq PHYS(kernel_relocated_base), %rax
335    pushq %rax
336    lretq
337
338.bss
339.align 16
340DATA(_kstack)
341    .skip 4096
342DATA(_kstack_end)
343
344// These symbols are used by image.S
345.global IMAGE_ELF_ENTRY
346.global IMAGE_MULTIBOOT_ENTRY
347IMAGE_ELF_ENTRY = _start
348IMAGE_MULTIBOOT_ENTRY = _multiboot_start
349
350// This symbol is used by gdb python to know the base of the kernel module
351.global KERNEL_BASE_ADDRESS
352KERNEL_BASE_ADDRESS = KERNEL_BASE - KERNEL_LOAD_OFFSET
353