1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
49 *  School of Computer Science
50 *  Carnegie Mellon University
51 *  Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58
59#include <platforms.h>
60#include <debug.h>
61
62#include <i386/asm.h>
63#include <i386/proc_reg.h>
64#include <i386/postcode.h>
65#include <assym.s>
66
67#include <i386/mp.h>
68#include <i386/cpuid.h>
69#include <i386/acpi.h>
70
71.code32
72
73
74/*
75 * Interrupt and bootup stack for initial processor.
76 * Note: we switch to a dynamically allocated interrupt stack once VM is up.
77 */
78
79/* in the __HIB section since the hibernate restore code uses this stack. */
80	.section __HIB, __data
81	.align	12
82
83	.globl	EXT(low_intstack)
84EXT(low_intstack):
85	.globl  EXT(gIOHibernateRestoreStack)
86EXT(gIOHibernateRestoreStack):
87
88	.space	INTSTACK_SIZE
89
90	.globl	EXT(low_eintstack)
91EXT(low_eintstack:)
92	.globl  EXT(gIOHibernateRestoreStackEnd)
93EXT(gIOHibernateRestoreStackEnd):
94
95	/* back to the regular __DATA section. */
96
97	.section __DATA, __data
98
99/*
100 * Stack for machine-check handler.
101 */
102	.align	12
103	.globl	EXT(mc_task_stack)
104EXT(mc_task_stack):
105	.space	INTSTACK_SIZE
106	.globl	EXT(mc_task_stack_end)
107EXT(mc_task_stack_end):
108
109	/* Must not clobber EDI */
110#define SWITCH_TO_64BIT_MODE					 \
111	movl	$(CR4_PAE),%eax		/* enable PAE */	;\
112	movl	%eax,%cr4					;\
113	movl    $MSR_IA32_EFER,%ecx				;\
114	rdmsr							;\
115	/* enable long mode, NX */				;\
116	orl	$(MSR_IA32_EFER_LME | MSR_IA32_EFER_NXE),%eax	;\
117	wrmsr							;\
118	movl	$EXT(BootPML4),%eax				;\
119	movl	%eax,%cr3					;\
120	movl	%cr0,%eax					;\
121	orl	$(CR0_PG|CR0_WP),%eax	/* enable paging */	;\
122	movl	%eax,%cr0					;\
123	/* "The Aussie Maneuver" ("Myria" variant) */ 		;\
124	pushl $(0xcb<<24)|KERNEL64_CS /* reload CS with 0x08 */ ;\
125	call .-1						;\
126	.code64
127
128/*
129 * BSP CPU start here.
130 *	eax points to kernbootstruct
131 *
132 * Environment:
133 *	protected mode, no paging, flat 32-bit address space.
134 *	(Code/data/stack segments have base == 0, limit == 4G)
135 */
136
137.code32
138	.text
139	.section __HIB, __text
140	.align	ALIGN
141	.globl	EXT(_start)
142	.globl	EXT(pstart)
143LEXT(_start)
144LEXT(pstart)
145
146/*
147 * Here we do the minimal setup to switch from 32 bit mode to 64 bit long mode.
148 *
149 * Initial memory layout:
150 *
151 *	-------------------------
152 *	|			|
153 *	| Kernel text/data	|
154 *	|			|
155 *	|-----------------------| Kernel text base addr - 2MB-aligned
156 *	| padding		|
157 *	|-----------------------|
158 *	| __HIB section		|
159 *	|-----------------------| Page-aligned
160 *	|			|
161 *	| padding		|
162 *	|			|
163 *	------------------------- 0
164 *
165 */
166	mov	%eax, %edi	/* save kernbootstruct */
167
168	/* Use low 32-bits of address as 32-bit stack */
169	movl	$EXT(low_eintstack), %esp
170
171	POSTCODE(PSTART_ENTRY)
172
173	/*
174	 * Set up segmentation
175	 */
176	movl	$EXT(protected_mode_gdtr), %eax
177	lgdtl	(%eax)
178
179	/*
180	 * Rebase Boot page tables to kernel base address.
181	 */
182	movl	$EXT(BootPML4), %eax			// Level 4:
183	add	%eax, 0*8+0(%eax)			//  - 1:1
184	add	%eax, KERNEL_PML4_INDEX*8+0(%eax)	//  - kernel space
185
186	movl	$EXT(BootPDPT), %edx			// Level 3:
187	add	%eax, 0*8+0(%edx)
188	add	%eax, 1*8+0(%edx)
189	add	%eax, 2*8+0(%edx)
190	add	%eax, 3*8+0(%edx)
191
192	POSTCODE(PSTART_REBASE)
193
194/* the following code is shared by the master CPU and all slave CPUs */
195L_pstart_common:
196	/*
197	 * switch to 64 bit mode
198	 */
199	SWITCH_TO_64BIT_MODE
200
201	/* Flush data segment selectors */
202	xor	%eax, %eax
203	mov	%ax, %ss
204	mov	%ax, %ds
205	mov	%ax, %es
206	mov	%ax, %fs
207	mov	%ax, %gs
208
209	test	%edi, %edi /* Populate stack canary on BSP */
210	jz	Lvstartshim
211
212	mov	$1, %eax
213	cpuid
214	test	$(1 << 30), %ecx
215	jz	Lnon_rdrand
216	RDRAND_RAX		/* RAX := 64 bits of DRBG entropy */
217	jnc	Lnon_rdrand	/* TODO: complain if DRBG fails at this stage */
218
219Lstore_random_guard:
220	xor	%ah, %ah	/* Security: zero second byte of stack canary */
221	movq	%rax, ___stack_chk_guard(%rip)
222	/* %edi = boot_args_start if BSP */
223Lvstartshim:
224
225	POSTCODE(PSTART_VSTART)
226
227	/* %edi = boot_args_start */
228
229	leaq	_vstart(%rip), %rcx
230	movq	$0xffffff8000000000, %rax	/* adjust pointer up high */
231	or	%rax, %rsp			/* and stack pointer up there */
232	or	%rcx, %rax
233	andq	$0xfffffffffffffff0, %rsp	/* align stack */
234	xorq	%rbp, %rbp			/* zero frame pointer */
235	callq	*%rax
236
237Lnon_rdrand:
238	rdtsc /* EDX:EAX := TSC */
239	/* Distribute low order bits */
240	mov	%eax, %ecx
241	xor	%al, %ah
242	shl	$16, %rcx
243	xor	%rcx, %rax
244	xor	%eax, %edx
245
246	/* Incorporate ASLR entropy, if any */
247	lea	(%rip), %rcx
248	shr	$21, %rcx
249	movzbl	%cl, %ecx
250	shl	$16, %ecx
251	xor	%ecx, %edx
252
253	mov	%ah, %cl
254	ror	%cl, %edx /* Right rotate EDX (TSC&0xFF ^ (TSC>>8 & 0xFF))&1F */
255	shl	$32, %rdx
256	xor	%rdx, %rax
257	mov	%cl, %al
258	jmp	Lstore_random_guard
259/*
260 * AP (slave) CPUs enter here.
261 *
262 * Environment:
263 *	protected mode, no paging, flat 32-bit address space.
264 *	(Code/data/stack segments have base == 0, limit == 4G)
265 */
266	.align	ALIGN
267	.globl	EXT(slave_pstart)
268LEXT(slave_pstart)
269	.code32
270	cli				/* disable interrupts, so we don`t */
271					/* need IDT for a while */
272	POSTCODE(SLAVE_PSTART)
273
274	movl	$EXT(mp_slave_stack) + PAGE_SIZE, %esp
275
276	xor 	%edi, %edi		/* AP, no "kernbootstruct" */
277
278	jmp	L_pstart_common		/* hop a ride to vstart() */
279
280
281/* BEGIN HIBERNATE CODE */
282
283.section __HIB, __text
284/*
285 * This code is linked into the kernel but part of the "__HIB" section,
286 * which means it's used by code running in the special context of restoring
287 * the kernel text and data from the hibernation image read by the booter.
288 * hibernate_kernel_entrypoint() and everything it calls or references
289 * (ie. hibernate_restore_phys_page()) needs to be careful to only touch
290 * memory also in the "__HIB" section.
291 */
292
293	.align	ALIGN
294	.globl	EXT(hibernate_machine_entrypoint)
295.code32
296LEXT(hibernate_machine_entrypoint)
297	movl    %eax, %edi /* regparm(1) calling convention */
298
299	/* Use low 32-bits of address as 32-bit stack */
300	movl $EXT(low_eintstack), %esp
301
302	/*
303	 * Set up GDT
304	 */
305	movl	$EXT(master_gdtr), %eax
306	lgdtl	(%eax)
307
308	/* Switch to 64-bit on the Boot PTs */
309	SWITCH_TO_64BIT_MODE
310
311	leaq	EXT(hibernate_kernel_entrypoint)(%rip),%rcx
312
313	/* adjust the pointers to be up high */
314	movq	$0xffffff8000000000, %rax
315	orq	%rax, %rsp
316	orq	%rcx, %rax
317
318	/* %edi is already filled with header pointer */
319	xorl	%esi, %esi			/* zero 2nd arg */
320	xorl	%edx, %edx			/* zero 3rd arg */
321	xorl	%ecx, %ecx			/* zero 4th arg */
322	andq	$0xfffffffffffffff0, %rsp	/* align stack */
323
324	/* call instead of jmp to keep the required stack alignment */
325	xorq	%rbp, %rbp			/* zero frame pointer */
326	call	*%rax
327
328	/* NOTREACHED */
329	hlt
330
331/* END HIBERNATE CODE */
332
333#if CONFIG_SLEEP
334/* BEGIN ACPI WAKEUP CODE */
335
336#include <i386/acpi.h>
337
338
339/*
340 * acpi_wake_start
341 */
342
343.section __TEXT,__text
344.code64
345
346/*
347 * acpi_sleep_cpu(acpi_sleep_callback func, void * refcon)
348 *
349 * Save CPU state before platform sleep. Restore CPU state
350 * following wake up.
351 */
352
353ENTRY(acpi_sleep_cpu)
354	push	%rbp
355	mov	%rsp, %rbp
356
357	/* save flags */
358	pushf
359
360	/* save general purpose registers */
361	push %rax
362	push %rbx
363	push %rcx
364	push %rdx
365	push %rbp
366	push %rsi
367	push %rdi
368	push %r8
369	push %r9
370	push %r10
371	push %r11
372	push %r12
373	push %r13
374	push %r14
375	push %r15
376
377	mov	%rsp, saved_rsp(%rip)
378
379	/* make sure tlb is flushed */
380	mov	%cr3,%rax
381	mov	%rax,%cr3
382
383	/* save control registers */
384	mov	%cr0, %rax
385	mov	%rax, saved_cr0(%rip)
386	mov	%cr2, %rax
387	mov	%rax, saved_cr2(%rip)
388	mov	%cr3, %rax
389	mov	%rax, saved_cr3(%rip)
390	mov	%cr4, %rax
391	mov	%rax, saved_cr4(%rip)
392
393	/* save segment registers */
394	movw	%es, saved_es(%rip)
395	movw	%fs, saved_fs(%rip)
396	movw	%gs, saved_gs(%rip)
397	movw	%ss, saved_ss(%rip)
398
399	/* save the 64bit user and kernel gs base */
400	/* note: user's curently swapped into kernel base MSR */
401	mov	$MSR_IA32_KERNEL_GS_BASE, %rcx
402	rdmsr
403	movl	%eax, saved_ugs_base(%rip)
404	movl	%edx, saved_ugs_base+4(%rip)
405	swapgs
406	rdmsr
407	movl	%eax, saved_kgs_base(%rip)
408	movl	%edx, saved_kgs_base+4(%rip)
409	swapgs
410
411	/* save descriptor table registers */
412	sgdt	saved_gdt(%rip)
413	sldt	saved_ldt(%rip)
414	sidt	saved_idt(%rip)
415	str	saved_tr(%rip)
416
417	/*
418	 * Call ACPI function provided by the caller to sleep the platform.
419	 * This call will not return on success.
420	 */
421
422	xchgq %rdi, %rsi
423	call	*%rsi
424
425	/* sleep failed, no cpu context lost */
426	jmp	wake_restore
427
428.section __HIB, __text
429.code32
430.globl EXT(acpi_wake_prot)
431EXT(acpi_wake_prot):
432	/* protected mode, paging disabled */
433	movl	$EXT(low_eintstack), %esp
434
435	SWITCH_TO_64BIT_MODE
436
437	jmp	Lwake_64
438
439.section __TEXT,__text
440.code64
441
442.globl EXT(acpi_wake_prot_entry)
443EXT(acpi_wake_prot_entry):
444	POSTCODE(ACPI_WAKE_PROT_ENTRY)
445	/* Return from hibernate code in iokit/Kernel/IOHibernateRestoreKernel.c
446	 */
447Lwake_64:
448	/*
449	 * restore cr4, PAE and NXE states in an orderly fashion
450	 */
451	mov	saved_cr4(%rip), %rcx
452	mov	%rcx, %cr4
453
454	mov	$(MSR_IA32_EFER), %ecx		/* MSR number in ecx */
455	rdmsr					/* MSR value in edx:eax */
456	or	$(MSR_IA32_EFER_NXE), %eax	/* Set NXE bit in low 32-bits */
457	wrmsr					/* Update */
458
459	movq	saved_cr2(%rip), %rax
460	mov	%rax, %cr2
461
462	/* restore CR0, paging enabled */
463	mov	saved_cr0(%rip), %rax
464	mov	%rax, %cr0
465
466	/* restore the page tables */
467	mov	saved_cr3(%rip), %rax
468	mov	%rax, %cr3
469
470	/* protected mode, paging enabled */
471	POSTCODE(ACPI_WAKE_PAGED_ENTRY)
472
473	/* load null segment selectors */
474	xor	%eax, %eax
475	movw	%ax, %ss
476	movw	%ax, %ds
477
478	/* restore descriptor tables */
479	lgdt	saved_gdt(%rip)
480	lldt	saved_ldt(%rip)
481	lidt	saved_idt(%rip)
482
483	/* restore segment registers */
484	movw	saved_es(%rip), %es
485	movw	saved_fs(%rip), %fs
486	movw	saved_gs(%rip), %gs
487	movw	saved_ss(%rip), %ss
488
489	/* restore the 64bit kernel and user gs base */
490	mov	$MSR_IA32_KERNEL_GS_BASE, %rcx
491	movl	saved_kgs_base(%rip),   %eax
492	movl	saved_kgs_base+4(%rip), %edx
493	wrmsr
494	swapgs
495	movl	saved_ugs_base(%rip),   %eax
496	movl	saved_ugs_base+4(%rip), %edx
497	wrmsr
498
499	/*
500	 * Restore task register. Before doing this, clear the busy flag
501	 * in the TSS descriptor set by the CPU.
502	 */
503	lea	saved_gdt(%rip), %rax
504	movq	2(%rax), %rdx			/* GDT base, skip limit word */
505	movl	$(KERNEL_TSS), %eax		/* TSS segment selector */
506	movb	$(K_TSS), 5(%rdx, %rax)		/* clear busy flag */
507
508	ltr	saved_tr(%rip)			/* restore TR */
509
510wake_restore:
511	mov	saved_rsp(%rip), %rsp
512
513	/* restore general purpose registers */
514	pop %r15
515	pop %r14
516	pop %r13
517	pop %r12
518	pop %r11
519	pop %r10
520	pop %r9
521	pop %r8
522	pop %rdi
523	pop %rsi
524	pop %rbp
525	pop %rdx
526	pop %rcx
527	pop %rbx
528	pop %rax
529
530	/* restore flags */
531	popf
532
533	leave
534	ret
535
536/* END ACPI WAKEUP CODE */
537#endif /* CONFIG_SLEEP */
538
539/* Code to get from real mode to protected mode */
540
541#define	operand_size_prefix	.byte 0x66
542#define	address_size_prefix	.byte 0x67
543#define	cs_base_prefix		.byte 0x2e
544
545#define	LJMP(segment,address)			\
546	operand_size_prefix			;\
547	.byte	0xea				;\
548	.long	address-EXT(real_mode_bootstrap_base)	;\
549	.word	segment
550
551#define	LGDT(address)				\
552	cs_base_prefix				;\
553	address_size_prefix			;\
554	operand_size_prefix			;\
555	.word	0x010f				;\
556	.byte	0x15				;\
557	.long	address-EXT(real_mode_bootstrap_base)
558
559.section __HIB, __text
560.align	12	/* Page align for single bcopy_phys() */
561.code32
562Entry(real_mode_bootstrap_base)
563	cli
564
565	LGDT(EXT(protected_mode_gdtr))
566
567	/* set the PE bit of CR0 */
568	mov	%cr0, %eax
569	inc %eax
570	mov	%eax, %cr0
571
572	/* reload CS register */
573	LJMP(KERNEL32_CS, 1f + REAL_MODE_BOOTSTRAP_OFFSET)
5741:
575
576	/* we are in protected mode now */
577	/* set up the segment registers */
578	mov	$KERNEL_DS, %eax
579	movw	%ax, %ds
580	movw	%ax, %es
581	movw	%ax, %ss
582	xor	%eax,%eax
583	movw	%ax, %fs
584	movw	%ax, %gs
585
586	POSTCODE(SLAVE_STARTPROG_ENTRY);
587
588	mov	PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, %ecx
589	jmp 	*%ecx
590
591Entry(protected_mode_gdtr)
592	.short	160		/* limit (8*20 segs) */
593	.quad	EXT(master_gdt)
594
595Entry(real_mode_bootstrap_end)
596
597/* Save area used across sleep/wake */
598.section __HIB, __data
599.align	2
600
601/* gdtr for real address of master_gdt in HIB (not the aliased address) */
602Entry(master_gdtr)
603		.word 160		/* limit (8*20 segs) */
604		.quad EXT(master_gdt)
605
606saved_gdt:	.word 0
607		.quad 0
608saved_rsp:	.quad 0
609saved_es:	.word 0
610saved_fs:	.word 0
611saved_gs:	.word 0
612saved_ss:	.word 0
613saved_cr0:	.quad 0
614saved_cr2:	.quad 0
615saved_cr3:	.quad 0
616saved_cr4:	.quad 0
617saved_idt:	.word 0
618		.quad 0
619saved_ldt:	.word 0
620saved_tr:	.word 0
621saved_kgs_base:	.quad 0
622saved_ugs_base:	.quad 0
623
624