1/*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30
31#include <mach/machine/vm_types.h>
32
33#include <mach/boolean.h>
34#include <kern/thread.h>
35#include <kern/zalloc.h>
36
37#include <kern/lock.h>
38#include <kern/kalloc.h>
39#include <kern/spl.h>
40
41#include <vm/pmap.h>
42#include <vm/vm_map.h>
43#include <vm/vm_kern.h>
44#include <mach/vm_param.h>
45#include <mach/vm_prot.h>
46#include <vm/vm_object.h>
47#include <vm/vm_page.h>
48
49#include <mach/machine/vm_param.h>
50#include <machine/thread.h>
51
52#include <kern/misc_protos.h>			/* prototyping */
53#include <i386/misc_protos.h>
54
55#include <i386/cpuid.h>
56#include <i386/cpu_data.h>
57#include <i386/mp.h>
58#include <i386/cpu_number.h>
59#include <i386/machine_cpu.h>
60#include <i386/seg.h>
61
62#include <vm/vm_protos.h>
63
64#include <sys/kdebug.h>
65
66#include <i386/postcode.h>
67
68#ifdef __i386__
69void
70cpu_IA32e_enable(cpu_data_t *cdp)
71{
72	assert(!ml_get_interrupts_enabled());
73
74	if (!cdp->cpu_is64bit ||
75	    (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
76		return;
77
78	postcode(CPU_IA32_ENABLE_ENTRY);
79
80	/*
81	 * The following steps are performed by inlines so that
82	 * we can be assured we don't use the stack or any other
83	 * non-identity mapped data while paging is turned off...
84	 */
85	/* Turn paging off */
86	asm volatile(
87		"mov	%%cr0, %%eax	\n\t"
88		"andl	%0, %%eax	\n\t"
89		"mov	%%eax, %%cr0	\n\t"
90		:
91		: "i" (~CR0_PG)
92		: "eax" );
93
94	/* Pop new top level phys pg addr into CR3 */
95	asm volatile(
96		"mov	%%eax, %%cr3	\n\t"
97		:
98		: "a" ((uint32_t) kernel64_cr3));
99
100	/* Turn on the 64-bit mode bit */
101	asm volatile(
102		"rdmsr			\n\t"
103		"orl	%1, %%eax	\n\t"
104		"wrmsr			\n\t"
105		:
106		: "c" (MSR_IA32_EFER), "i" (MSR_IA32_EFER_LME)
107		: "eax", "edx");
108
109	/* Turn paging on again */
110	asm volatile(
111		"mov	%%cr0, %%eax	\n\t"
112		"orl	%0, %%eax	\n\t"
113		"mov	%%eax, %%cr0	\n\t"
114		:
115		: "i" (CR0_PG)
116		: "eax" );
117
118#if ONLY_SAFE_FOR_LINDA_SERIAL
119	kprintf("cpu_IA32e_enable(%p)\n", cdp);
120#endif
121
122	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
123		panic("cpu_IA32e_enable() MSR_IA32_EFER_LMA not asserted");
124
125	cdp->cpu_kernel_cr3 = kernel64_cr3;
126
127	postcode(CPU_IA32_ENABLE_EXIT);
128}
129
130void
131cpu_IA32e_disable(cpu_data_t *cdp)
132{
133	assert(!ml_get_interrupts_enabled());
134
135	postcode(CPU_IA32_DISABLE_ENTRY);
136
137	if (!cdp->cpu_is64bit ||
138	    (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
139		return;
140
141	/*
142	 * The following steps are performed by inlines so that
143	 * we can be assured we don't use the stack or any other
144	 * non-identity mapped data while paging is turned off...
145	 */
146	/* Turn paging off */
147	asm volatile(
148		"mov	%%cr0, %%eax	\n\t"
149		"andl	%0, %%eax	\n\t"
150		"mov	%%eax, %%cr0	\n\t"
151		:
152		: "i" (~CR0_PG)
153		: "eax" );
154
155	/* Pop legacy top level phys pg addr into CR3 */
156	asm volatile(
157		"mov	%%eax, %%cr3	\n\t"
158		:
159		: "a" ((uint32_t) lo_kernel_cr3));
160
161	/* Turn off the 64-bit mode bit */
162	asm volatile(
163		"rdmsr			\n\t"
164		"andl	%1, %%eax	\n\t"
165		"wrmsr			\n\t"
166		:
167		: "c" (MSR_IA32_EFER), "i" (~MSR_IA32_EFER_LME)
168		: "eax", "edx");
169
170	/* Turn paging on again */
171	asm volatile(
172		"mov	%%cr0, %%eax	\n\t"
173		"orl	%0, %%eax	\n\t"
174		"mov	%%eax, %%cr0	\n\t"
175		:
176		: "i" (CR0_PG)
177		: "eax" );
178
179	kprintf("cpu_IA32e_disable(%p)\n", cdp);
180
181	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
182		panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA not cleared");
183
184	cdp->cpu_kernel_cr3 = 0ULL;
185
186	postcode(CPU_IA32_DISABLE_EXIT);
187}
188#endif
189
190#if DEBUG
191extern void dump_regs64(void);
192extern void dump_gdt(void *);
193extern void dump_ldt(void *);
194extern void dump_idt(void *);
195extern void dump_tss(void *);
196extern void dump_frame32(x86_saved_state_compat32_t *scp);
197extern void dump_frame64(x86_saved_state64_t *sp);
198extern void dump_frame(x86_saved_state_t *sp);
199
200void
201dump_frame(x86_saved_state_t *sp)
202{
203	if (is_saved_state32(sp))
204		dump_frame32((x86_saved_state_compat32_t *) sp);
205	else if (is_saved_state64(sp))
206		dump_frame64(&sp->ss_64);
207	else
208		kprintf("dump_frame(%p) unknown type %d\n", sp, sp->flavor);
209}
210
211void
212dump_frame32(x86_saved_state_compat32_t *scp)
213{
214	unsigned int	i;
215	uint32_t	*ip = (uint32_t *) scp;
216
217	kprintf("dump_frame32(%p):\n", scp);
218
219	for (i = 0;
220	     i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t);
221	     i++, ip++)
222		kprintf("%p: 0x%08x\n", ip, *ip);
223
224	kprintf("scp->isf64.err:    0x%016llx\n", scp->isf64.err);
225	kprintf("scp->isf64.rip:    0x%016llx\n", scp->isf64.rip);
226	kprintf("scp->isf64.cs:     0x%016llx\n", scp->isf64.cs);
227	kprintf("scp->isf64.rflags: 0x%016llx\n", scp->isf64.rflags);
228	kprintf("scp->isf64.rsp:    0x%016llx\n", scp->isf64.rsp);
229	kprintf("scp->isf64.ss:     0x%016llx\n", scp->isf64.ss);
230
231	kprintf("scp->iss32.tag:    0x%08x\n", scp->iss32.tag);
232	kprintf("scp->iss32.state.gs:     0x%08x\n", scp->iss32.state.gs);
233	kprintf("scp->iss32.state.fs:     0x%08x\n", scp->iss32.state.fs);
234	kprintf("scp->iss32.state.es:     0x%08x\n", scp->iss32.state.es);
235	kprintf("scp->iss32.state.ds:     0x%08x\n", scp->iss32.state.ds);
236	kprintf("scp->iss32.state.edi:    0x%08x\n", scp->iss32.state.edi);
237	kprintf("scp->iss32.state.esi:    0x%08x\n", scp->iss32.state.esi);
238	kprintf("scp->iss32.state.ebp:    0x%08x\n", scp->iss32.state.ebp);
239	kprintf("scp->iss32.state.cr2:    0x%08x\n", scp->iss32.state.cr2);
240	kprintf("scp->iss32.state.ebx:    0x%08x\n", scp->iss32.state.ebx);
241	kprintf("scp->iss32.state.edx:    0x%08x\n", scp->iss32.state.edx);
242	kprintf("scp->iss32.state.ecx:    0x%08x\n", scp->iss32.state.ecx);
243	kprintf("scp->iss32.state.eax:    0x%08x\n", scp->iss32.state.eax);
244	kprintf("scp->iss32.state.trapno: 0x%08x\n", scp->iss32.state.eax);
245	kprintf("scp->iss32.state.eip:    0x%08x\n", scp->iss32.state.eip);
246	kprintf("scp->iss32.state.cs:     0x%08x\n", scp->iss32.state.cs);
247	kprintf("scp->iss32.state.efl:    0x%08x\n", scp->iss32.state.efl);
248	kprintf("scp->iss32.state.uesp:   0x%08x\n", scp->iss32.state.uesp);
249	kprintf("scp->iss32.state.ss:     0x%08x\n", scp->iss32.state.ss);
250
251	postcode(0x99);
252}
253
254void
255dump_frame64(x86_saved_state64_t *sp)
256{
257	unsigned int	i;
258	uint64_t	*ip = (uint64_t *) sp;
259
260	kprintf("dump_frame64(%p):\n", sp);
261
262	for (i = 0;
263	     i < sizeof(x86_saved_state64_t)/sizeof(uint64_t);
264	     i++, ip++)
265		kprintf("%p: 0x%016llx\n", ip, *ip);
266
267	kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno);
268	kprintf("sp->isf.trapfn: 0x%016llx\n", sp->isf.trapfn);
269	kprintf("sp->isf.err:    0x%016llx\n", sp->isf.err);
270	kprintf("sp->isf.rip:    0x%016llx\n", sp->isf.rip);
271	kprintf("sp->isf.cs:     0x%016llx\n", sp->isf.cs);
272	kprintf("sp->isf.rflags: 0x%016llx\n", sp->isf.rflags);
273	kprintf("sp->isf.rsp:    0x%016llx\n", sp->isf.rsp);
274	kprintf("sp->isf.ss:     0x%016llx\n", sp->isf.ss);
275
276	kprintf("sp->fs:         0x%016x\n", sp->fs);
277	kprintf("sp->gs:         0x%016x\n", sp->gs);
278	kprintf("sp->rax:        0x%016llx\n", sp->rax);
279	kprintf("sp->rcx:        0x%016llx\n", sp->rcx);
280	kprintf("sp->rbx:        0x%016llx\n", sp->rbx);
281	kprintf("sp->rbp:        0x%016llx\n", sp->rbp);
282	kprintf("sp->r11:        0x%016llx\n", sp->r11);
283	kprintf("sp->r12:        0x%016llx\n", sp->r12);
284	kprintf("sp->r13:        0x%016llx\n", sp->r13);
285	kprintf("sp->r14:        0x%016llx\n", sp->r14);
286	kprintf("sp->r15:        0x%016llx\n", sp->r15);
287	kprintf("sp->cr2:        0x%016llx\n", sp->cr2);
288	kprintf("sp->v_arg8:     0x%016llx\n", sp->v_arg8);
289	kprintf("sp->v_arg7:     0x%016llx\n", sp->v_arg7);
290	kprintf("sp->v_arg6:     0x%016llx\n", sp->v_arg6);
291	kprintf("sp->r9:         0x%016llx\n", sp->r9);
292	kprintf("sp->r8:         0x%016llx\n", sp->r8);
293	kprintf("sp->r10:        0x%016llx\n", sp->r10);
294	kprintf("sp->rdx:        0x%016llx\n", sp->rdx);
295	kprintf("sp->rsi:        0x%016llx\n", sp->rsi);
296	kprintf("sp->rdi:        0x%016llx\n", sp->rdi);
297
298	postcode(0x98);
299}
300
301void
302dump_gdt(void *gdtp)
303{
304	unsigned int	i;
305	uint32_t	*ip = (uint32_t *) gdtp;
306
307	kprintf("GDT:\n");
308	for (i = 0; i < GDTSZ; i++, ip += 2) {
309		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
310		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
311	}
312}
313
314void
315dump_ldt(void *ldtp)
316{
317	unsigned int	i;
318	uint32_t	*ip = (uint32_t *) ldtp;
319
320	kprintf("LDT:\n");
321	for (i = 0; i < LDTSZ_MIN; i++, ip += 2) {
322		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
323		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
324	}
325}
326
327void
328dump_idt(void *idtp)
329{
330	unsigned int	i;
331	uint32_t	*ip = (uint32_t *) idtp;
332
333	kprintf("IDT64:\n");
334	for (i = 0; i < 16; i++, ip += 4) {
335		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
336		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
337		kprintf("%p: 0x%08x\n", ip+2, *(ip+2));
338		kprintf("%p: 0x%08x\n", ip+3, *(ip+3));
339	}
340}
341
342void
343dump_tss(void *tssp)
344{
345	unsigned int	i;
346	uint32_t	*ip = (uint32_t *) tssp;
347
348	kprintf("TSS64:\n");
349	for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) {
350		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
351	}
352}
353
354#if defined(__x86_64__)
355void dump_regs64(void)
356{
357
358#define SNAP_REG(reg)						\
359	uint64_t	reg;					\
360	__asm__ volatile("mov %%" #reg ", %0" : "=m" (reg))
361
362#define KPRINT_REG(reg)						\
363	kprintf("%3s: %p\n", #reg, (void *) reg)
364
365	SNAP_REG(rsp);
366	SNAP_REG(rbp);
367	SNAP_REG(rax);
368	SNAP_REG(rbx);
369	SNAP_REG(rcx);
370	SNAP_REG(rdx);
371	SNAP_REG(rsi);
372	SNAP_REG(rdi);
373	SNAP_REG(r8);
374	SNAP_REG(r9);
375	SNAP_REG(r10);
376	SNAP_REG(r11);
377	SNAP_REG(r12);
378	SNAP_REG(r13);
379	SNAP_REG(r14);
380
381	KPRINT_REG(rsp);
382	KPRINT_REG(rbp);
383	KPRINT_REG(rax);
384	KPRINT_REG(rbx);
385	KPRINT_REG(rcx);
386	KPRINT_REG(rdx);
387	KPRINT_REG(rsi);
388	KPRINT_REG(rdi);
389	KPRINT_REG(r8);
390	KPRINT_REG(r9);
391	KPRINT_REG(r10);
392	KPRINT_REG(r11);
393	KPRINT_REG(r12);
394	KPRINT_REG(r13);
395	KPRINT_REG(r14);
396}
397#endif /* __x86_64__ */
398#endif /* DEBUG */
399