1// Copyright 2016 The Fuchsia Authors 2// Copyright (c) 2009 Corey Tabaka 3// Copyright (c) 2015 Intel Corporation 4// Copyright (c) 2016 Travis Geiselbrecht 5// 6// Use of this source code is governed by a MIT-style 7// license that can be found in the LICENSE file or at 8// https://opensource.org/licenses/MIT 9 10#include <asm.h> 11#include <arch/x86/asm.h> 12#include <arch/x86/descriptor.h> 13#include <arch/x86/mmu.h> 14#include <arch/x86/registers.h> 15#include <zircon/boot/multiboot.h> 16#include <zircon/tls.h> 17 18// TODO(mcgrathr): This is in a macro because it's used in both 32-bit mode 19// and in 64-bit mode with addresses known to be <4GB. The assembly text 20// is the same either way, but it needs to be assembled separately for each. 21// In future, the 32-bit boot path should trampoline into the 64-bit path so 22// there will only be a 64-bit version of this. 23// NOTE: Clobbers %rax, %rcx, %rsi, %rdi. 24.macro move_fixups_and_zero_bss 25 // The fixup code in image.S runs in 64-bit mode with paging enabled, 26 // so we can't run it too early. But it overlaps the bss, so we move 27 // it before zeroing the bss. We can't delay zeroing the bss because 28 // the page tables we're about to set up are themselves in bss. 29 30 // The first word after the kernel image (at __data_end in our view) 31 // gives the size of the following code. Copy it to _end. 32 mov PHYS(__data_end), %ecx 33 mov $PHYS(__data_end+4), %esi 34 mov $PHYS(_end), %edi 35 rep movsb // while (ecx-- > 0) *edi++ = *esi++; 36 37 // Now it's safe to zero the bss. 38 movl $PHYS(__bss_start), %edi 39 movl $PHYS(_end), %ecx 40 sub %edi, %ecx // Compute the length of the bss in bytes. 41 xor %eax, %eax 42 rep stosb // while (ecx-- > 0) *edi++ = al; 43.endm 44 45#define ADDR_OFFSET_MASK ((1 << ADDR_OFFSET)-1) 46#define SHIFT_OFFSET(_s) ((_s) >> 3) 47#define SHIFT_REMAIN(_s) ((_s) - (SHIFT_OFFSET(_s) << 3)) 48 49// Set a page table entry for the kernel module relocated 64-bit virtual 50// address in 32-bit code. Clobbers the %ecx register. 51.macro set_relocated_page_table_entry table, shift, value 52 // Extract 32-bit chunk of kernel_relocated_base containing the index bits 53 // for this page level shift. 54 mov PHYS(kernel_relocated_base + SHIFT_OFFSET(\shift)), %ecx 55 56 // Get the exact portion of the 32-bit value that is the index 57 shrl $SHIFT_REMAIN(\shift), %ecx 58 andl $ADDR_OFFSET_MASK, %ecx 59 60 // Get the adddress on the page table of index * 8 and set the value 61 shll $3, %ecx 62 addl $PHYS(\table), %ecx 63 movl \value, (%ecx) 64.endm 65 66/* shared code to set up a default 64bit page table structure */ 67.macro page_table_init 68 /* Setting the First PML4E with a PDP table reference*/ 69 movl $PHYS(pdp), %eax 70 orl $X86_KERNEL_PD_FLAGS, %eax 71 movl %eax, PHYS(pml4) 72 73 /* Setting the First PDPTE with a Page table reference*/ 74 movl $PHYS(pte), %eax 75 orl $X86_KERNEL_PD_FLAGS, %eax 76 movl %eax, PHYS(pdp) 77 78 /* point the pml4e at the second high PDP (for -2GB mapping) */ 79 movl $PHYS(pdp_high), %eax 80 orl $X86_KERNEL_PD_FLAGS, %eax 81 set_relocated_page_table_entry pml4, PML4_SHIFT, %eax 82 83 /* point the second pdp at the same low level page table */ 84 movl $PHYS(pte), %eax 85 orl $X86_KERNEL_PD_FLAGS, %eax 86 set_relocated_page_table_entry pdp_high, PDP_SHIFT, %eax 87 88 /* map the first 1GB in this table */ 89 movl $PHYS(pte), %esi 90 movl $0x200, %ecx 91 xor %eax, %eax 92 930: 94 mov %eax, %ebx 95 shll $21, %ebx 96 orl $X86_KERNEL_PD_LP_FLAGS, %ebx 97 movl %ebx, (%esi) 98 addl $8,%esi 99 inc %eax 100 loop 0b 101 102 /* set up a linear map of the first 64GB at 0xffffff8000000000 */ 103 movl $PHYS(linear_map_pdp), %esi 104 movl $32768, %ecx 105 xor %eax, %eax 106 107 /* loop across these page tables, incrementing the address by 2MB */ 1080: 109 mov %eax, %ebx 110 shll $21, %ebx 111 orl $X86_KERNEL_PD_LP_FLAGS, %ebx // lower word of the entry 112 movl %ebx, (%esi) 113 mov %eax, %ebx 114 shrl $11, %ebx // upper word of the entry 115 movl %ebx, 4(%esi) 116 addl $8,%esi 117 inc %eax 118 loop 0b 119 120 /* point the high pdp at our linear mapping page tables */ 121 movl $PHYS(pdp_high), %esi 122 movl $64, %ecx 123 movl $PHYS(linear_map_pdp), %eax 124 orl $X86_KERNEL_PD_FLAGS, %eax 125 1260: 127 movl %eax, (%esi) 128 add $8, %esi 129 addl $4096, %eax 130 loop 0b 131.endm 132 133// Load our new GDT by physical pointer. 134// _temp_gdtr has it as a virtual pointer, so copy it and adjust. 135// Clobbers %eax and %ecx. 136.macro load_phys_gdtr 137 movw PHYS(_temp_gdtr), %ax 138 movl PHYS(_temp_gdtr+2), %ecx 139 sub $PHYS_ADDR_DELTA, %ecx 140 movw %ax, -6(%esp) 141 movl %ecx, -4(%esp) 142 lgdt -6(%esp) 143.endm 144 145// This section name is known specially to kernel.ld and gen-kaslr-fixups.sh. 146// This code has relocations for absolute physical addresses, which do not get 147// adjusted by the boot-time fixups (which this code calls at the end). 148.section .text.boot, "ax", @progbits 149.code32 150FUNCTION_LABEL(_multiboot_start) 151 cmpl $MULTIBOOT_BOOTLOADER_MAGIC, %eax 152 je 0f 153 xor %ebx,%ebx 1540: // multiboot_info_t pointer is in %ebx. 155 156 /* load our gdtr */ 157 load_phys_gdtr 158 159 /* load our data selectors */ 160 movw $DATA_SELECTOR, %ax 161 movw %ax, %ds 162 movw %ax, %es 163 movw %ax, %fs 164 movw %ax, %gs 165 movw %ax, %ss 166 167 /* We need to jump to our sane 32 bit CS */ 168 pushl $CODE_SELECTOR 169 pushl $PHYS(.Lfarjump) 170 lret 171 172.Lfarjump: 173 move_fixups_and_zero_bss 174 175 // _multiboot_info is in bss, so now it's safe to set it. 176 movl %ebx, PHYS(_multiboot_info) 177 178 /* Start using the zircon stack from its physical address. */ 179 mov $PHYS(_kstack_end), %esp 180 181paging_setup: 182 /* Preparing 64 bit paging, we will use 2MB pages covering 1GB 183 * for initial bootstrap, this page table will be 1 to 1 184 */ 185 186 /* Set the PAE bit to enable 64bit paging 187 * Set PGE to enable global kernel pages 188 */ 189 mov %cr4, %eax 190 or $(X86_CR4_PAE|X86_CR4_PGE), %eax 191 mov %eax, %cr4 192 193 /* Long Mode Enabled at this point */ 194 movl $X86_MSR_IA32_EFER, %ecx 195 rdmsr 196 orl $X86_EFER_LME,%eax 197 wrmsr 198 199 /* initialize the default page tables */ 200 page_table_init 201 202 /* load the physical pointer to the top level page table */ 203 movl $PHYS(pml4), %eax 204 mov %eax, %cr3 205 206 /* Enabling Paging and from this point we are in 207 32 bit compatibility mode*/ 208 mov %cr0, %eax 209 btsl $(31), %eax 210 mov %eax, %cr0 211 212 /* Using another long jump to be on 64 bit mode 213 after this we will be on real 64 bit mode */ 214 pushl $CODE_64_SELECTOR /*Need to put it in a the right CS*/ 215 pushl $PHYS(farjump64) 216 lret 217 218.align 8 219.code64 220farjump64: 221 /* give the boot allocator a chance to compute the physical address of the kernel */ 222 call boot_alloc_init 223 224 /* branch to our high address, with the relocated offset */ 225 mov $PHYS(high_entry), %rax 226 addq PHYS(kernel_relocated_base), %rax 227 jmp *%rax 228 229// The 64-bit entry path also gets here; see below. 230// This code runs at the final virtual address, so it should be pure PIC. 231.pushsection .text 232high_entry: 233 /* zero our kernel segment data registers */ 234 xor %eax, %eax 235 mov %eax, %ds 236 mov %eax, %es 237 mov %eax, %fs 238 mov %eax, %gs 239 mov %eax, %ss 240 241 /* load the high kernel stack */ 242 lea _kstack_end(%rip), %rsp 243 244 // move_fixups_and_zero_bss copied the fixup code to _end. 245 // It expects %rdi to contain the actual runtime address of __code_start. 246 lea __code_start(%rip), %rdi 247 call _end 248 // The fixup code won't be used again, so the memory can be reused now. 249 250 /* reload the gdtr after relocations as it relies on relocated VAs */ 251 lgdt _temp_gdtr(%rip) 252 253 // Set %gs.base to &bp_percpu. It's statically initialized 254 // with kernel_unsafe_sp set, so after this it's safe to call 255 // into C code that might use safe-stack and/or stack-protector. 256 lea bp_percpu(%rip), %rax 257 mov %rax, %rdx 258 shr $32, %rdx 259 mov $X86_MSR_IA32_GS_BASE, %ecx 260 wrmsr 261 262 /* set up the idt */ 263 lea _idt_startup(%rip), %rdi 264 call idt_setup 265 call load_startup_idt 266 267 /* assign this core CPU# 0 and initialize its per cpu state */ 268 xor %edi, %edi 269 call x86_init_percpu 270 271 // Fill the stack canary with a random value as early as possible. 272 // This isn't done in x86_init_percpu because the hw_rng_get_entropy 273 // call would make it eligible for stack-guard checking itself. But 274 // %gs is not set up yet in the prologue of the function, so it would 275 // crash if it tried to use the stack-guard. 276 call choose_stack_guard 277 278 // Move it into place. 279 mov %rcx, %gs:ZX_TLS_STACK_GUARD_OFFSET 280 // Don't leak that value to other code. 281 xor %ecx, %ecx 282 283 // configure the kernel base address 284 // TODO: dynamically figure this out once we allow the x86 kernel to be loaded anywhere 285 movl $PHYS_LOAD_ADDRESS, kernel_base_phys(%rip) 286 287 /* call the main module */ 288 call lk_main 289 2900: /* just sit around waiting for interrupts */ 291 hlt /* interrupts will unhalt the processor */ 292 pause 293 jmp 0b /* so jump back to halt to conserve power */ 294.popsection 295 296 /* 64bit entry point from a secondary loader */ 297.align 8 298FUNCTION_LABEL(_start) 299 /* set up a temporary stack pointer */ 300 mov $PHYS(_kstack_end), %rsp 301 302 // Save off the bootdata pointer in a register that won't get clobbered. 303 mov %esi, %ebx 304 305 move_fixups_and_zero_bss 306 307 // _zbi_base is in bss, so now it's safe to set it. 308 mov %ebx, PHYS(_zbi_base) 309 310 /* give the boot allocator a chance to compute the physical address of the kernel */ 311 call boot_alloc_init 312 313.Lpaging_setup64: 314 /* initialize the default page tables */ 315 page_table_init 316 317 /* 318 * Set PGE to enable global kernel pages 319 */ 320 mov %cr4, %rax 321 or $(X86_CR4_PGE), %rax 322 mov %rax, %cr4 323 324 /* load the physical pointer to the top level page table */ 325 mov $PHYS(pml4), %rax 326 mov %rax, %cr3 327 328 /* load our gdtr */ 329 load_phys_gdtr 330 331 /* long jump to our code selector and the high address relocated */ 332 push $CODE_64_SELECTOR 333 mov $PHYS(high_entry), %rax 334 addq PHYS(kernel_relocated_base), %rax 335 pushq %rax 336 lretq 337 338.bss 339.align 16 340DATA(_kstack) 341 .skip 4096 342DATA(_kstack_end) 343 344// These symbols are used by image.S 345.global IMAGE_ELF_ENTRY 346.global IMAGE_MULTIBOOT_ENTRY 347IMAGE_ELF_ENTRY = _start 348IMAGE_MULTIBOOT_ENTRY = _multiboot_start 349 350// This symbol is used by gdb python to know the base of the kernel module 351.global KERNEL_BASE_ADDRESS 352KERNEL_BASE_ADDRESS = KERNEL_BASE - KERNEL_LOAD_OFFSET 353