1/* 2 * Copyright (c) 2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58 59#include <platforms.h> 60#include <debug.h> 61 62#include <i386/asm.h> 63#include <i386/proc_reg.h> 64#include <i386/postcode.h> 65#include <assym.s> 66 67#include <i386/mp.h> 68#include <i386/cpuid.h> 69#include <i386/acpi.h> 70 71.code32 72 73 74/* 75 * Interrupt and bootup stack for initial processor. 76 * Note: we switch to a dynamically allocated interrupt stack once VM is up. 77 */ 78 79/* in the __HIB section since the hibernate restore code uses this stack. */ 80 .section __HIB, __data 81 .align 12 82 83 .globl EXT(low_intstack) 84EXT(low_intstack): 85 .globl EXT(gIOHibernateRestoreStack) 86EXT(gIOHibernateRestoreStack): 87 88 .space INTSTACK_SIZE 89 90 .globl EXT(low_eintstack) 91EXT(low_eintstack:) 92 .globl EXT(gIOHibernateRestoreStackEnd) 93EXT(gIOHibernateRestoreStackEnd): 94 95 /* back to the regular __DATA section. */ 96 97 .section __DATA, __data 98 99/* 100 * Stack for machine-check handler. 101 */ 102 .align 12 103 .globl EXT(mc_task_stack) 104EXT(mc_task_stack): 105 .space INTSTACK_SIZE 106 .globl EXT(mc_task_stack_end) 107EXT(mc_task_stack_end): 108 109 /* Must not clobber EDI */ 110#define SWITCH_TO_64BIT_MODE \ 111 movl $(CR4_PAE),%eax /* enable PAE */ ;\ 112 movl %eax,%cr4 ;\ 113 movl $MSR_IA32_EFER,%ecx ;\ 114 rdmsr ;\ 115 /* enable long mode, NX */ ;\ 116 orl $(MSR_IA32_EFER_LME | MSR_IA32_EFER_NXE),%eax ;\ 117 wrmsr ;\ 118 movl $EXT(BootPML4),%eax ;\ 119 movl %eax,%cr3 ;\ 120 movl %cr0,%eax ;\ 121 orl $(CR0_PG|CR0_WP),%eax /* enable paging */ ;\ 122 movl %eax,%cr0 ;\ 123 ljmpl $KERNEL64_CS,$64f ;\ 12464: ;\ 125 .code64 126 127/* 128 * BSP CPU start here. 129 * eax points to kernbootstruct 130 * 131 * Environment: 132 * protected mode, no paging, flat 32-bit address space. 133 * (Code/data/stack segments have base == 0, limit == 4G) 134 */ 135 136.code32 137 .text 138 .section __HIB, __text 139 .align ALIGN 140 .globl EXT(_start) 141 .globl EXT(pstart) 142LEXT(_start) 143LEXT(pstart) 144 145/* 146 * Here we do the minimal setup to switch from 32 bit mode to 64 bit long mode. 147 * 148 * Initial memory layout: 149 * 150 * ------------------------- 151 * | | 152 * | Kernel text/data | 153 * | | 154 * |-----------------------| Kernel text base addr - 2MB-aligned 155 * | padding | 156 * |-----------------------| 157 * | __HIB section | 158 * |-----------------------| Page-aligned 159 * | | 160 * | padding | 161 * | | 162 * ------------------------- 0 163 * 164 */ 165 mov %eax, %edi /* save kernbootstruct */ 166 167 /* Use low 32-bits of address as 32-bit stack */ 168 movl $EXT(low_eintstack), %esp 169 170 POSTCODE(PSTART_ENTRY) 171 172 /* 173 * Set up segmentation 174 */ 175 movl $EXT(protected_mode_gdtr), %eax 176 lgdtl (%eax) 177 178 /* 179 * Rebase Boot page tables to kernel base address. 180 */ 181 movl $EXT(BootPML4), %eax // Level 4: 182 add %eax, 0*8+0(%eax) // - 1:1 183 add %eax, KERNEL_PML4_INDEX*8+0(%eax) // - kernel space 184 185 movl $EXT(BootPDPT), %edx // Level 3: 186 add %eax, 0*8+0(%edx) 187 add %eax, 1*8+0(%edx) 188 add %eax, 2*8+0(%edx) 189 add %eax, 3*8+0(%edx) 190 191 POSTCODE(PSTART_REBASE) 192 193/* the following code is shared by the master CPU and all slave CPUs */ 194L_pstart_common: 195 /* 196 * switch to 64 bit mode 197 */ 198 SWITCH_TO_64BIT_MODE 199 200 /* Flush data segment selectors */ 201 xor %eax, %eax 202 mov %ax, %ss 203 mov %ax, %ds 204 mov %ax, %es 205 mov %ax, %fs 206 mov %ax, %gs 207 208 test %edi, %edi /* Populate stack canary on BSP */ 209 jz Lvstartshim 210 211 mov $1, %eax 212 cpuid 213 test $(1 << 30), %ecx 214 jz Lnon_rdrand 215 RDRAND_RAX /* RAX := 64 bits of DRBG entropy */ 216 jnc Lnon_rdrand /* TODO: complain if DRBG fails at this stage */ 217 218Lstore_random_guard: 219 xor %ah, %ah /* Security: zero second byte of stack canary */ 220 movq %rax, ___stack_chk_guard(%rip) 221 /* %edi = boot_args_start if BSP */ 222Lvstartshim: 223 224 POSTCODE(PSTART_VSTART) 225 226 /* %edi = boot_args_start */ 227 228 leaq _vstart(%rip), %rcx 229 movq $0xffffff8000000000, %rax /* adjust pointer up high */ 230 or %rax, %rsp /* and stack pointer up there */ 231 or %rcx, %rax 232 andq $0xfffffffffffffff0, %rsp /* align stack */ 233 xorq %rbp, %rbp /* zero frame pointer */ 234 callq *%rax 235 236Lnon_rdrand: 237 rdtsc /* EDX:EAX := TSC */ 238 /* Distribute low order bits */ 239 mov %eax, %ecx 240 xor %al, %ah 241 shl $16, %rcx 242 xor %rcx, %rax 243 xor %eax, %edx 244 245 /* Incorporate ASLR entropy, if any */ 246 lea (%rip), %rcx 247 shr $21, %rcx 248 movzbl %cl, %ecx 249 shl $16, %ecx 250 xor %ecx, %edx 251 252 mov %ah, %cl 253 ror %cl, %edx /* Right rotate EDX (TSC&0xFF ^ (TSC>>8 & 0xFF))&1F */ 254 shl $32, %rdx 255 xor %rdx, %rax 256 mov %cl, %al 257 jmp Lstore_random_guard 258/* 259 * AP (slave) CPUs enter here. 260 * 261 * Environment: 262 * protected mode, no paging, flat 32-bit address space. 263 * (Code/data/stack segments have base == 0, limit == 4G) 264 */ 265 .align ALIGN 266 .globl EXT(slave_pstart) 267LEXT(slave_pstart) 268 .code32 269 cli /* disable interrupts, so we don`t */ 270 /* need IDT for a while */ 271 POSTCODE(SLAVE_PSTART) 272 273 movl $EXT(mp_slave_stack) + PAGE_SIZE, %esp 274 275 xor %edi, %edi /* AP, no "kernbootstruct" */ 276 277 jmp L_pstart_common /* hop a ride to vstart() */ 278 279 280/* BEGIN HIBERNATE CODE */ 281 282.section __HIB, __text 283/* 284 * This code is linked into the kernel but part of the "__HIB" section, 285 * which means it's used by code running in the special context of restoring 286 * the kernel text and data from the hibernation image read by the booter. 287 * hibernate_kernel_entrypoint() and everything it calls or references 288 * (ie. hibernate_restore_phys_page()) needs to be careful to only touch 289 * memory also in the "__HIB" section. 290 */ 291 292 .align ALIGN 293 .globl EXT(hibernate_machine_entrypoint) 294.code32 295LEXT(hibernate_machine_entrypoint) 296 movl %eax, %edi /* regparm(1) calling convention */ 297 298 /* Use low 32-bits of address as 32-bit stack */ 299 movl $EXT(low_eintstack), %esp 300 301 /* 302 * Set up GDT 303 */ 304 movl $EXT(master_gdtr), %eax 305 lgdtl (%eax) 306 307 /* Switch to 64-bit on the Boot PTs */ 308 SWITCH_TO_64BIT_MODE 309 310 leaq EXT(hibernate_kernel_entrypoint)(%rip),%rcx 311 312 /* adjust the pointers to be up high */ 313 movq $0xffffff8000000000, %rax 314 orq %rax, %rsp 315 orq %rcx, %rax 316 317 /* %edi is already filled with header pointer */ 318 xorl %esi, %esi /* zero 2nd arg */ 319 xorl %edx, %edx /* zero 3rd arg */ 320 xorl %ecx, %ecx /* zero 4th arg */ 321 andq $0xfffffffffffffff0, %rsp /* align stack */ 322 323 /* call instead of jmp to keep the required stack alignment */ 324 xorq %rbp, %rbp /* zero frame pointer */ 325 call *%rax 326 327 /* NOTREACHED */ 328 hlt 329 330/* END HIBERNATE CODE */ 331 332#if CONFIG_SLEEP 333/* BEGIN ACPI WAKEUP CODE */ 334 335#include <i386/acpi.h> 336 337 338/* 339 * acpi_wake_start 340 */ 341 342.section __TEXT,__text 343.code64 344 345/* 346 * acpi_sleep_cpu(acpi_sleep_callback func, void * refcon) 347 * 348 * Save CPU state before platform sleep. Restore CPU state 349 * following wake up. 350 */ 351 352ENTRY(acpi_sleep_cpu) 353 push %rbp 354 mov %rsp, %rbp 355 356 /* save flags */ 357 pushf 358 359 /* save general purpose registers */ 360 push %rax 361 push %rbx 362 push %rcx 363 push %rdx 364 push %rbp 365 push %rsi 366 push %rdi 367 push %r8 368 push %r9 369 push %r10 370 push %r11 371 push %r12 372 push %r13 373 push %r14 374 push %r15 375 376 mov %rsp, saved_rsp(%rip) 377 378 /* make sure tlb is flushed */ 379 mov %cr3,%rax 380 mov %rax,%cr3 381 382 /* save control registers */ 383 mov %cr0, %rax 384 mov %rax, saved_cr0(%rip) 385 mov %cr2, %rax 386 mov %rax, saved_cr2(%rip) 387 mov %cr3, %rax 388 mov %rax, saved_cr3(%rip) 389 mov %cr4, %rax 390 mov %rax, saved_cr4(%rip) 391 392 /* save segment registers */ 393 movw %es, saved_es(%rip) 394 movw %fs, saved_fs(%rip) 395 movw %gs, saved_gs(%rip) 396 movw %ss, saved_ss(%rip) 397 398 /* save the 64bit user and kernel gs base */ 399 /* note: user's curently swapped into kernel base MSR */ 400 mov $MSR_IA32_KERNEL_GS_BASE, %rcx 401 rdmsr 402 movl %eax, saved_ugs_base(%rip) 403 movl %edx, saved_ugs_base+4(%rip) 404 swapgs 405 rdmsr 406 movl %eax, saved_kgs_base(%rip) 407 movl %edx, saved_kgs_base+4(%rip) 408 swapgs 409 410 /* save descriptor table registers */ 411 sgdt saved_gdt(%rip) 412 sldt saved_ldt(%rip) 413 sidt saved_idt(%rip) 414 str saved_tr(%rip) 415 416 /* 417 * Call ACPI function provided by the caller to sleep the platform. 418 * This call will not return on success. 419 */ 420 421 xchgq %rdi, %rsi 422 call *%rsi 423 424 /* sleep failed, no cpu context lost */ 425 jmp wake_restore 426 427.section __HIB, __text 428.code32 429.globl EXT(acpi_wake_prot) 430EXT(acpi_wake_prot): 431 /* protected mode, paging disabled */ 432 movl $EXT(low_eintstack), %esp 433 434 SWITCH_TO_64BIT_MODE 435 436 jmp Lwake_64 437 438.section __TEXT,__text 439.code64 440 441.globl EXT(acpi_wake_prot_entry) 442EXT(acpi_wake_prot_entry): 443 POSTCODE(ACPI_WAKE_PROT_ENTRY) 444 /* Return from hibernate code in iokit/Kernel/IOHibernateRestoreKernel.c 445 */ 446Lwake_64: 447 /* 448 * restore cr4, PAE and NXE states in an orderly fashion 449 */ 450 mov saved_cr4(%rip), %rcx 451 mov %rcx, %cr4 452 453 mov $(MSR_IA32_EFER), %ecx /* MSR number in ecx */ 454 rdmsr /* MSR value in edx:eax */ 455 or $(MSR_IA32_EFER_NXE), %eax /* Set NXE bit in low 32-bits */ 456 wrmsr /* Update */ 457 458 movq saved_cr2(%rip), %rax 459 mov %rax, %cr2 460 461 /* restore CR0, paging enabled */ 462 mov saved_cr0(%rip), %rax 463 mov %rax, %cr0 464 465 /* restore the page tables */ 466 mov saved_cr3(%rip), %rax 467 mov %rax, %cr3 468 469 /* protected mode, paging enabled */ 470 POSTCODE(ACPI_WAKE_PAGED_ENTRY) 471 472 /* load null segment selectors */ 473 xor %eax, %eax 474 movw %ax, %ss 475 movw %ax, %ds 476 477 /* restore descriptor tables */ 478 lgdt saved_gdt(%rip) 479 lldt saved_ldt(%rip) 480 lidt saved_idt(%rip) 481 482 /* restore segment registers */ 483 movw saved_es(%rip), %es 484 movw saved_fs(%rip), %fs 485 movw saved_gs(%rip), %gs 486 movw saved_ss(%rip), %ss 487 488 /* restore the 64bit kernel and user gs base */ 489 mov $MSR_IA32_KERNEL_GS_BASE, %rcx 490 movl saved_kgs_base(%rip), %eax 491 movl saved_kgs_base+4(%rip), %edx 492 wrmsr 493 swapgs 494 movl saved_ugs_base(%rip), %eax 495 movl saved_ugs_base+4(%rip), %edx 496 wrmsr 497 498 /* 499 * Restore task register. Before doing this, clear the busy flag 500 * in the TSS descriptor set by the CPU. 501 */ 502 lea saved_gdt(%rip), %rax 503 movq 2(%rax), %rdx /* GDT base, skip limit word */ 504 movl $(KERNEL_TSS), %eax /* TSS segment selector */ 505 movb $(K_TSS), 5(%rdx, %rax) /* clear busy flag */ 506 507 ltr saved_tr(%rip) /* restore TR */ 508 509wake_restore: 510 mov saved_rsp(%rip), %rsp 511 512 /* restore general purpose registers */ 513 pop %r15 514 pop %r14 515 pop %r13 516 pop %r12 517 pop %r11 518 pop %r10 519 pop %r9 520 pop %r8 521 pop %rdi 522 pop %rsi 523 pop %rbp 524 pop %rdx 525 pop %rcx 526 pop %rbx 527 pop %rax 528 529 /* restore flags */ 530 popf 531 532 leave 533 ret 534 535/* END ACPI WAKEUP CODE */ 536#endif /* CONFIG_SLEEP */ 537 538/* Code to get from real mode to protected mode */ 539 540#define operand_size_prefix .byte 0x66 541#define address_size_prefix .byte 0x67 542#define cs_base_prefix .byte 0x2e 543 544#define LJMP(segment,address) \ 545 operand_size_prefix ;\ 546 .byte 0xea ;\ 547 .long address-EXT(real_mode_bootstrap_base) ;\ 548 .word segment 549 550#define LGDT(address) \ 551 cs_base_prefix ;\ 552 address_size_prefix ;\ 553 operand_size_prefix ;\ 554 .word 0x010f ;\ 555 .byte 0x15 ;\ 556 .long address-EXT(real_mode_bootstrap_base) 557 558.section __HIB, __text 559.align 12 /* Page align for single bcopy_phys() */ 560.code32 561Entry(real_mode_bootstrap_base) 562 cli 563 564 LGDT(EXT(protected_mode_gdtr)) 565 566 /* set the PE bit of CR0 */ 567 mov %cr0, %eax 568 inc %eax 569 mov %eax, %cr0 570 571 /* reload CS register */ 572 LJMP(KERNEL32_CS, 1f + REAL_MODE_BOOTSTRAP_OFFSET) 5731: 574 575 /* we are in protected mode now */ 576 /* set up the segment registers */ 577 mov $KERNEL_DS, %eax 578 movw %ax, %ds 579 movw %ax, %es 580 movw %ax, %ss 581 xor %eax,%eax 582 movw %ax, %fs 583 movw %ax, %gs 584 585 POSTCODE(SLAVE_STARTPROG_ENTRY); 586 587 mov PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, %ecx 588 jmp *%ecx 589 590Entry(protected_mode_gdtr) 591 .short 160 /* limit (8*20 segs) */ 592 .quad EXT(master_gdt) 593 594Entry(real_mode_bootstrap_end) 595 596/* Save area used across sleep/wake */ 597.section __HIB, __data 598.align 2 599 600/* gdtr for real address of master_gdt in HIB (not the aliased address) */ 601Entry(master_gdtr) 602 .word 160 /* limit (8*20 segs) */ 603 .quad EXT(master_gdt) 604 605saved_gdt: .word 0 606 .quad 0 607saved_rsp: .quad 0 608saved_es: .word 0 609saved_fs: .word 0 610saved_gs: .word 0 611saved_ss: .word 0 612saved_cr0: .quad 0 613saved_cr2: .quad 0 614saved_cr3: .quad 0 615saved_cr4: .quad 0 616saved_idt: .word 0 617 .quad 0 618saved_ldt: .word 0 619saved_tr: .word 0 620saved_kgs_base: .quad 0 621saved_ugs_base: .quad 0 622 623