1/*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30
31#include <mach/machine/vm_types.h>
32
33#include <mach/boolean.h>
34#include <kern/thread.h>
35#include <kern/zalloc.h>
36
37#include <kern/lock.h>
38#include <kern/kalloc.h>
39#include <kern/spl.h>
40
41#include <vm/pmap.h>
42#include <vm/vm_map.h>
43#include <vm/vm_kern.h>
44#include <mach/vm_param.h>
45#include <mach/vm_prot.h>
46#include <vm/vm_object.h>
47#include <vm/vm_page.h>
48
49#include <mach/machine/vm_param.h>
50#include <machine/thread.h>
51
52#include <kern/misc_protos.h>			/* prototyping */
53#include <i386/misc_protos.h>
54
55#include <i386/cpuid.h>
56#include <i386/cpu_data.h>
57#include <i386/mp.h>
58#include <i386/cpu_number.h>
59#include <i386/machine_cpu.h>
60#include <i386/mp_slave_boot.h>
61#include <i386/seg.h>
62
63#include <vm/vm_protos.h>
64
65#include <sys/kdebug.h>
66
67#include <i386/postcode.h>
68
69void
70cpu_IA32e_enable(cpu_data_t *cdp)
71{
72	assert(!ml_get_interrupts_enabled());
73
74	if (!cdp->cpu_is64bit ||
75	    (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
76		return;
77
78	postcode(CPU_IA32_ENABLE_ENTRY);
79
80	/*
81	 * The following steps are performed by inlines so that
82	 * we can be assured we don't use the stack or any other
83	 * non-identity mapped data while paging is turned off...
84	 */
85	/* Turn paging off */
86	asm volatile(
87		"mov	%%cr0, %%eax	\n\t"
88		"andl	%0, %%eax	\n\t"
89		"mov	%%eax, %%cr0	\n\t"
90		:
91		: "i" (~CR0_PG)
92		: "eax" );
93
94	/* Pop new top level phys pg addr into CR3 */
95	asm volatile(
96		"mov	%%eax, %%cr3	\n\t"
97		:
98		: "a" ((uint32_t) kernel64_cr3));
99
100	/* Turn on the 64-bit mode bit */
101	asm volatile(
102		"rdmsr			\n\t"
103		"orl	%1, %%eax	\n\t"
104		"wrmsr			\n\t"
105		:
106		: "c" (MSR_IA32_EFER), "i" (MSR_IA32_EFER_LME)
107		: "eax", "edx");
108
109	/* Turn paging on again */
110	asm volatile(
111		"mov	%%cr0, %%eax	\n\t"
112		"orl	%0, %%eax	\n\t"
113		"mov	%%eax, %%cr0	\n\t"
114		:
115		: "i" (CR0_PG)
116		: "eax" );
117
118#if ONLY_SAFE_FOR_LINDA_SERIAL
119	kprintf("cpu_IA32e_enable(%p)\n", cdp);
120#endif
121
122	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
123		panic("cpu_IA32e_enable() MSR_IA32_EFER_LMA not asserted");
124
125	cdp->cpu_kernel_cr3 = kernel64_cr3;
126
127	postcode(CPU_IA32_ENABLE_EXIT);
128}
129
130void
131cpu_IA32e_disable(cpu_data_t *cdp)
132{
133	assert(!ml_get_interrupts_enabled());
134
135	postcode(CPU_IA32_DISABLE_ENTRY);
136
137	if (!cdp->cpu_is64bit ||
138	    (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
139		return;
140
141	/*
142	 * The following steps are performed by inlines so that
143	 * we can be assured we don't use the stack or any other
144	 * non-identity mapped data while paging is turned off...
145	 */
146	/* Turn paging off */
147	asm volatile(
148		"mov	%%cr0, %%eax	\n\t"
149		"andl	%0, %%eax	\n\t"
150		"mov	%%eax, %%cr0	\n\t"
151		:
152		: "i" (~CR0_PG)
153		: "eax" );
154
155	/* Pop legacy top level phys pg addr into CR3 */
156	asm volatile(
157		"mov	%%eax, %%cr3	\n\t"
158		:
159		: "a" ((uint32_t) lo_kernel_cr3));
160
161	/* Turn off the 64-bit mode bit */
162	asm volatile(
163		"rdmsr			\n\t"
164		"andl	%1, %%eax	\n\t"
165		"wrmsr			\n\t"
166		:
167		: "c" (MSR_IA32_EFER), "i" (~MSR_IA32_EFER_LME)
168		: "eax", "edx");
169
170	/* Turn paging on again */
171	asm volatile(
172		"mov	%%cr0, %%eax	\n\t"
173		"orl	%0, %%eax	\n\t"
174		"mov	%%eax, %%cr0	\n\t"
175		:
176		: "i" (CR0_PG)
177		: "eax" );
178
179	kprintf("cpu_IA32e_disable(%p)\n", cdp);
180
181	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
182		panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA not cleared");
183
184	cdp->cpu_kernel_cr3 = 0ULL;
185
186	postcode(CPU_IA32_DISABLE_EXIT);
187}
188
189void
190fix_desc64(void *descp, int count)
191{
192	struct fake_descriptor64	*fakep;
193	union {
194		struct real_gate64		gate;
195		struct real_descriptor64	desc;
196	}				real;
197	int				i;
198
199	fakep = (struct fake_descriptor64 *) descp;
200
201	for (i = 0; i < count; i++, fakep++) {
202		/*
203		 * Construct the real decriptor locally.
204		 */
205
206		bzero((void *) &real, sizeof(real));
207
208		switch (fakep->access & ACC_TYPE) {
209		case 0:
210			break;
211		case ACC_CALL_GATE:
212		case ACC_INTR_GATE:
213		case ACC_TRAP_GATE:
214			real.gate.offset_low16 = fakep->offset[0] & 0xFFFF;
215			real.gate.selector16 = fakep->lim_or_seg & 0xFFFF;
216			real.gate.IST = fakep->size_or_IST & 0x7;
217			real.gate.access8 = fakep->access;
218			real.gate.offset_high16 = (fakep->offset[0]>>16)&0xFFFF;
219			real.gate.offset_top32 = (uint32_t)fakep->offset[1];
220			break;
221		default:	/* Otherwise */
222			real.desc.limit_low16 = fakep->lim_or_seg & 0xFFFF;
223			real.desc.base_low16 = fakep->offset[0] & 0xFFFF;
224			real.desc.base_med8 = (fakep->offset[0] >> 16) & 0xFF;
225			real.desc.access8 = fakep->access;
226			real.desc.limit_high4 = (fakep->lim_or_seg >> 16) & 0xFF;
227			real.desc.granularity4 = fakep->size_or_IST;
228			real.desc.base_high8 = (fakep->offset[0] >> 24) & 0xFF;
229			real.desc.base_top32 = (uint32_t) fakep->offset[1];
230		}
231
232		/*
233		 * Now copy back over the fake structure.
234		 */
235		bcopy((void *) &real, (void *) fakep, sizeof(real));
236	}
237}
238
239#if DEBUG
240extern void dump_gdt(void *);
241extern void dump_ldt(void *);
242extern void dump_idt(void *);
243extern void dump_tss(void *);
244extern void dump_frame32(x86_saved_state_compat32_t *scp);
245extern void dump_frame64(x86_saved_state64_t *sp);
246extern void dump_frame(x86_saved_state_t *sp);
247
248void
249dump_frame(x86_saved_state_t *sp)
250{
251	if (is_saved_state32(sp))
252		dump_frame32((x86_saved_state_compat32_t *) sp);
253	else if (is_saved_state64(sp))
254		dump_frame64(&sp->ss_64);
255	else
256		kprintf("dump_frame(%p) unknown type %d\n", sp, sp->flavor);
257}
258
259void
260dump_frame32(x86_saved_state_compat32_t *scp)
261{
262	unsigned int	i;
263	uint32_t	*ip = (uint32_t *) scp;
264
265	kprintf("dump_frame32(%p):\n", scp);
266
267	for (i = 0;
268	     i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t);
269	     i++, ip++)
270		kprintf("%p: 0x%08x\n", ip, *ip);
271
272	kprintf("scp->isf64.err:    0x%016llx\n", scp->isf64.err);
273	kprintf("scp->isf64.rip:    0x%016llx\n", scp->isf64.rip);
274	kprintf("scp->isf64.cs:     0x%016llx\n", scp->isf64.cs);
275	kprintf("scp->isf64.rflags: 0x%016llx\n", scp->isf64.rflags);
276	kprintf("scp->isf64.rsp:    0x%016llx\n", scp->isf64.rsp);
277	kprintf("scp->isf64.ss:     0x%016llx\n", scp->isf64.ss);
278
279	kprintf("scp->iss32.tag:    0x%08x\n", scp->iss32.tag);
280	kprintf("scp->iss32.state.gs:     0x%08x\n", scp->iss32.state.gs);
281	kprintf("scp->iss32.state.fs:     0x%08x\n", scp->iss32.state.fs);
282	kprintf("scp->iss32.state.es:     0x%08x\n", scp->iss32.state.es);
283	kprintf("scp->iss32.state.ds:     0x%08x\n", scp->iss32.state.ds);
284	kprintf("scp->iss32.state.edi:    0x%08x\n", scp->iss32.state.edi);
285	kprintf("scp->iss32.state.esi:    0x%08x\n", scp->iss32.state.esi);
286	kprintf("scp->iss32.state.ebp:    0x%08x\n", scp->iss32.state.ebp);
287	kprintf("scp->iss32.state.cr2:    0x%08x\n", scp->iss32.state.cr2);
288	kprintf("scp->iss32.state.ebx:    0x%08x\n", scp->iss32.state.ebx);
289	kprintf("scp->iss32.state.edx:    0x%08x\n", scp->iss32.state.edx);
290	kprintf("scp->iss32.state.ecx:    0x%08x\n", scp->iss32.state.ecx);
291	kprintf("scp->iss32.state.eax:    0x%08x\n", scp->iss32.state.eax);
292	kprintf("scp->iss32.state.trapno: 0x%08x\n", scp->iss32.state.eax);
293	kprintf("scp->iss32.state.eip:    0x%08x\n", scp->iss32.state.eip);
294	kprintf("scp->iss32.state.cs:     0x%08x\n", scp->iss32.state.cs);
295	kprintf("scp->iss32.state.efl:    0x%08x\n", scp->iss32.state.efl);
296	kprintf("scp->iss32.state.uesp:   0x%08x\n", scp->iss32.state.uesp);
297	kprintf("scp->iss32.state.ss:     0x%08x\n", scp->iss32.state.ss);
298
299	postcode(0x99);
300}
301
302void
303dump_frame64(x86_saved_state64_t *sp)
304{
305	unsigned int	i;
306	uint64_t	*ip = (uint64_t *) sp;
307
308	kprintf("dump_frame64(%p):\n", sp);
309
310	for (i = 0;
311	     i < sizeof(x86_saved_state64_t)/sizeof(uint64_t);
312	     i++, ip++)
313		kprintf("%p: 0x%016llx\n", ip, *ip);
314
315	kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno);
316	kprintf("sp->isf.trapfn: 0x%08x\n", sp->isf.trapfn);
317	kprintf("sp->isf.err:    0x%016llx\n", sp->isf.err);
318	kprintf("sp->isf.rip:    0x%016llx\n", sp->isf.rip);
319	kprintf("sp->isf.cs:     0x%016llx\n", sp->isf.cs);
320	kprintf("sp->isf.rflags: 0x%016llx\n", sp->isf.rflags);
321	kprintf("sp->isf.rsp:    0x%016llx\n", sp->isf.rsp);
322	kprintf("sp->isf.ss:     0x%016llx\n", sp->isf.ss);
323
324	kprintf("sp->fs:         0x%016x\n", sp->fs);
325	kprintf("sp->gs:         0x%016x\n", sp->gs);
326	kprintf("sp->rax:        0x%016llx\n", sp->rax);
327	kprintf("sp->rcx:        0x%016llx\n", sp->rcx);
328	kprintf("sp->rbx:        0x%016llx\n", sp->rbx);
329	kprintf("sp->rbp:        0x%016llx\n", sp->rbp);
330	kprintf("sp->r11:        0x%016llx\n", sp->r11);
331	kprintf("sp->r12:        0x%016llx\n", sp->r12);
332	kprintf("sp->r13:        0x%016llx\n", sp->r13);
333	kprintf("sp->r14:        0x%016llx\n", sp->r14);
334	kprintf("sp->r15:        0x%016llx\n", sp->r15);
335	kprintf("sp->cr2:        0x%016llx\n", sp->cr2);
336	kprintf("sp->v_arg8:     0x%016llx\n", sp->v_arg8);
337	kprintf("sp->v_arg7:     0x%016llx\n", sp->v_arg7);
338	kprintf("sp->v_arg6:     0x%016llx\n", sp->v_arg6);
339	kprintf("sp->r9:         0x%016llx\n", sp->r9);
340	kprintf("sp->r8:         0x%016llx\n", sp->r8);
341	kprintf("sp->r10:        0x%016llx\n", sp->r10);
342	kprintf("sp->rdx:        0x%016llx\n", sp->rdx);
343	kprintf("sp->rsi:        0x%016llx\n", sp->rsi);
344	kprintf("sp->rdi:        0x%016llx\n", sp->rdi);
345
346	postcode(0x98);
347}
348
349void
350dump_gdt(void *gdtp)
351{
352	unsigned int	i;
353	uint32_t	*ip = (uint32_t *) gdtp;
354
355	kprintf("GDT:\n");
356	for (i = 0; i < GDTSZ; i++, ip += 2) {
357		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
358		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
359	}
360}
361
362void
363dump_ldt(void *ldtp)
364{
365	unsigned int	i;
366	uint32_t	*ip = (uint32_t *) ldtp;
367
368	kprintf("LDT:\n");
369	for (i = 0; i < LDTSZ_MIN; i++, ip += 2) {
370		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
371		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
372	}
373}
374
375void
376dump_idt(void *idtp)
377{
378	unsigned int	i;
379	uint32_t	*ip = (uint32_t *) idtp;
380
381	kprintf("IDT64:\n");
382	for (i = 0; i < 16; i++, ip += 4) {
383		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
384		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
385		kprintf("%p: 0x%08x\n", ip+2, *(ip+2));
386		kprintf("%p: 0x%08x\n", ip+3, *(ip+3));
387	}
388}
389
390void
391dump_tss(void *tssp)
392{
393	unsigned int	i;
394	uint32_t	*ip = (uint32_t *) tssp;
395
396	kprintf("TSS64:\n");
397	for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) {
398		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
399	}
400}
401#endif /* DEBUG */
402