1/* 2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <string.h> 30 31#include <mach/machine/vm_types.h> 32 33#include <mach/boolean.h> 34#include <kern/thread.h> 35#include <kern/zalloc.h> 36 37#include <kern/lock.h> 38#include <kern/kalloc.h> 39#include <kern/spl.h> 40 41#include <vm/pmap.h> 42#include <vm/vm_map.h> 43#include <vm/vm_kern.h> 44#include <mach/vm_param.h> 45#include <mach/vm_prot.h> 46#include <vm/vm_object.h> 47#include <vm/vm_page.h> 48 49#include <mach/machine/vm_param.h> 50#include <machine/thread.h> 51 52#include <kern/misc_protos.h> /* prototyping */ 53#include <i386/misc_protos.h> 54 55#include <i386/cpuid.h> 56#include <i386/cpu_data.h> 57#include <i386/mp.h> 58#include <i386/cpu_number.h> 59#include <i386/machine_cpu.h> 60#include <i386/seg.h> 61 62#include <vm/vm_protos.h> 63 64#include <sys/kdebug.h> 65 66#include <i386/postcode.h> 67 68#ifdef __i386__ 69void 70cpu_IA32e_enable(cpu_data_t *cdp) 71{ 72 assert(!ml_get_interrupts_enabled()); 73 74 if (!cdp->cpu_is64bit || 75 (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0) 76 return; 77 78 postcode(CPU_IA32_ENABLE_ENTRY); 79 80 /* 81 * The following steps are performed by inlines so that 82 * we can be assured we don't use the stack or any other 83 * non-identity mapped data while paging is turned off... 84 */ 85 /* Turn paging off */ 86 asm volatile( 87 "mov %%cr0, %%eax \n\t" 88 "andl %0, %%eax \n\t" 89 "mov %%eax, %%cr0 \n\t" 90 : 91 : "i" (~CR0_PG) 92 : "eax" ); 93 94 /* Pop new top level phys pg addr into CR3 */ 95 asm volatile( 96 "mov %%eax, %%cr3 \n\t" 97 : 98 : "a" ((uint32_t) kernel64_cr3)); 99 100 /* Turn on the 64-bit mode bit */ 101 asm volatile( 102 "rdmsr \n\t" 103 "orl %1, %%eax \n\t" 104 "wrmsr \n\t" 105 : 106 : "c" (MSR_IA32_EFER), "i" (MSR_IA32_EFER_LME) 107 : "eax", "edx"); 108 109 /* Turn paging on again */ 110 asm volatile( 111 "mov %%cr0, %%eax \n\t" 112 "orl %0, %%eax \n\t" 113 "mov %%eax, %%cr0 \n\t" 114 : 115 : "i" (CR0_PG) 116 : "eax" ); 117 118#if ONLY_SAFE_FOR_LINDA_SERIAL 119 kprintf("cpu_IA32e_enable(%p)\n", cdp); 120#endif 121 122 if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0) 123 panic("cpu_IA32e_enable() MSR_IA32_EFER_LMA not asserted"); 124 125 cdp->cpu_kernel_cr3 = kernel64_cr3; 126 127 postcode(CPU_IA32_ENABLE_EXIT); 128} 129 130void 131cpu_IA32e_disable(cpu_data_t *cdp) 132{ 133 assert(!ml_get_interrupts_enabled()); 134 135 postcode(CPU_IA32_DISABLE_ENTRY); 136 137 if (!cdp->cpu_is64bit || 138 (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0) 139 return; 140 141 /* 142 * The following steps are performed by inlines so that 143 * we can be assured we don't use the stack or any other 144 * non-identity mapped data while paging is turned off... 145 */ 146 /* Turn paging off */ 147 asm volatile( 148 "mov %%cr0, %%eax \n\t" 149 "andl %0, %%eax \n\t" 150 "mov %%eax, %%cr0 \n\t" 151 : 152 : "i" (~CR0_PG) 153 : "eax" ); 154 155 /* Pop legacy top level phys pg addr into CR3 */ 156 asm volatile( 157 "mov %%eax, %%cr3 \n\t" 158 : 159 : "a" ((uint32_t) lo_kernel_cr3)); 160 161 /* Turn off the 64-bit mode bit */ 162 asm volatile( 163 "rdmsr \n\t" 164 "andl %1, %%eax \n\t" 165 "wrmsr \n\t" 166 : 167 : "c" (MSR_IA32_EFER), "i" (~MSR_IA32_EFER_LME) 168 : "eax", "edx"); 169 170 /* Turn paging on again */ 171 asm volatile( 172 "mov %%cr0, %%eax \n\t" 173 "orl %0, %%eax \n\t" 174 "mov %%eax, %%cr0 \n\t" 175 : 176 : "i" (CR0_PG) 177 : "eax" ); 178 179 kprintf("cpu_IA32e_disable(%p)\n", cdp); 180 181 if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0) 182 panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA not cleared"); 183 184 cdp->cpu_kernel_cr3 = 0ULL; 185 186 postcode(CPU_IA32_DISABLE_EXIT); 187} 188#endif 189 190#if DEBUG 191extern void dump_regs64(void); 192extern void dump_gdt(void *); 193extern void dump_ldt(void *); 194extern void dump_idt(void *); 195extern void dump_tss(void *); 196extern void dump_frame32(x86_saved_state_compat32_t *scp); 197extern void dump_frame64(x86_saved_state64_t *sp); 198extern void dump_frame(x86_saved_state_t *sp); 199 200void 201dump_frame(x86_saved_state_t *sp) 202{ 203 if (is_saved_state32(sp)) 204 dump_frame32((x86_saved_state_compat32_t *) sp); 205 else if (is_saved_state64(sp)) 206 dump_frame64(&sp->ss_64); 207 else 208 kprintf("dump_frame(%p) unknown type %d\n", sp, sp->flavor); 209} 210 211void 212dump_frame32(x86_saved_state_compat32_t *scp) 213{ 214 unsigned int i; 215 uint32_t *ip = (uint32_t *) scp; 216 217 kprintf("dump_frame32(%p):\n", scp); 218 219 for (i = 0; 220 i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t); 221 i++, ip++) 222 kprintf("%p: 0x%08x\n", ip, *ip); 223 224 kprintf("scp->isf64.err: 0x%016llx\n", scp->isf64.err); 225 kprintf("scp->isf64.rip: 0x%016llx\n", scp->isf64.rip); 226 kprintf("scp->isf64.cs: 0x%016llx\n", scp->isf64.cs); 227 kprintf("scp->isf64.rflags: 0x%016llx\n", scp->isf64.rflags); 228 kprintf("scp->isf64.rsp: 0x%016llx\n", scp->isf64.rsp); 229 kprintf("scp->isf64.ss: 0x%016llx\n", scp->isf64.ss); 230 231 kprintf("scp->iss32.tag: 0x%08x\n", scp->iss32.tag); 232 kprintf("scp->iss32.state.gs: 0x%08x\n", scp->iss32.state.gs); 233 kprintf("scp->iss32.state.fs: 0x%08x\n", scp->iss32.state.fs); 234 kprintf("scp->iss32.state.es: 0x%08x\n", scp->iss32.state.es); 235 kprintf("scp->iss32.state.ds: 0x%08x\n", scp->iss32.state.ds); 236 kprintf("scp->iss32.state.edi: 0x%08x\n", scp->iss32.state.edi); 237 kprintf("scp->iss32.state.esi: 0x%08x\n", scp->iss32.state.esi); 238 kprintf("scp->iss32.state.ebp: 0x%08x\n", scp->iss32.state.ebp); 239 kprintf("scp->iss32.state.cr2: 0x%08x\n", scp->iss32.state.cr2); 240 kprintf("scp->iss32.state.ebx: 0x%08x\n", scp->iss32.state.ebx); 241 kprintf("scp->iss32.state.edx: 0x%08x\n", scp->iss32.state.edx); 242 kprintf("scp->iss32.state.ecx: 0x%08x\n", scp->iss32.state.ecx); 243 kprintf("scp->iss32.state.eax: 0x%08x\n", scp->iss32.state.eax); 244 kprintf("scp->iss32.state.trapno: 0x%08x\n", scp->iss32.state.eax); 245 kprintf("scp->iss32.state.eip: 0x%08x\n", scp->iss32.state.eip); 246 kprintf("scp->iss32.state.cs: 0x%08x\n", scp->iss32.state.cs); 247 kprintf("scp->iss32.state.efl: 0x%08x\n", scp->iss32.state.efl); 248 kprintf("scp->iss32.state.uesp: 0x%08x\n", scp->iss32.state.uesp); 249 kprintf("scp->iss32.state.ss: 0x%08x\n", scp->iss32.state.ss); 250 251 postcode(0x99); 252} 253 254void 255dump_frame64(x86_saved_state64_t *sp) 256{ 257 unsigned int i; 258 uint64_t *ip = (uint64_t *) sp; 259 260 kprintf("dump_frame64(%p):\n", sp); 261 262 for (i = 0; 263 i < sizeof(x86_saved_state64_t)/sizeof(uint64_t); 264 i++, ip++) 265 kprintf("%p: 0x%016llx\n", ip, *ip); 266 267 kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno); 268 kprintf("sp->isf.trapfn: 0x%016llx\n", sp->isf.trapfn); 269 kprintf("sp->isf.err: 0x%016llx\n", sp->isf.err); 270 kprintf("sp->isf.rip: 0x%016llx\n", sp->isf.rip); 271 kprintf("sp->isf.cs: 0x%016llx\n", sp->isf.cs); 272 kprintf("sp->isf.rflags: 0x%016llx\n", sp->isf.rflags); 273 kprintf("sp->isf.rsp: 0x%016llx\n", sp->isf.rsp); 274 kprintf("sp->isf.ss: 0x%016llx\n", sp->isf.ss); 275 276 kprintf("sp->fs: 0x%016x\n", sp->fs); 277 kprintf("sp->gs: 0x%016x\n", sp->gs); 278 kprintf("sp->rax: 0x%016llx\n", sp->rax); 279 kprintf("sp->rcx: 0x%016llx\n", sp->rcx); 280 kprintf("sp->rbx: 0x%016llx\n", sp->rbx); 281 kprintf("sp->rbp: 0x%016llx\n", sp->rbp); 282 kprintf("sp->r11: 0x%016llx\n", sp->r11); 283 kprintf("sp->r12: 0x%016llx\n", sp->r12); 284 kprintf("sp->r13: 0x%016llx\n", sp->r13); 285 kprintf("sp->r14: 0x%016llx\n", sp->r14); 286 kprintf("sp->r15: 0x%016llx\n", sp->r15); 287 kprintf("sp->cr2: 0x%016llx\n", sp->cr2); 288 kprintf("sp->v_arg8: 0x%016llx\n", sp->v_arg8); 289 kprintf("sp->v_arg7: 0x%016llx\n", sp->v_arg7); 290 kprintf("sp->v_arg6: 0x%016llx\n", sp->v_arg6); 291 kprintf("sp->r9: 0x%016llx\n", sp->r9); 292 kprintf("sp->r8: 0x%016llx\n", sp->r8); 293 kprintf("sp->r10: 0x%016llx\n", sp->r10); 294 kprintf("sp->rdx: 0x%016llx\n", sp->rdx); 295 kprintf("sp->rsi: 0x%016llx\n", sp->rsi); 296 kprintf("sp->rdi: 0x%016llx\n", sp->rdi); 297 298 postcode(0x98); 299} 300 301void 302dump_gdt(void *gdtp) 303{ 304 unsigned int i; 305 uint32_t *ip = (uint32_t *) gdtp; 306 307 kprintf("GDT:\n"); 308 for (i = 0; i < GDTSZ; i++, ip += 2) { 309 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 310 kprintf("%p: 0x%08x\n", ip+1, *(ip+1)); 311 } 312} 313 314void 315dump_ldt(void *ldtp) 316{ 317 unsigned int i; 318 uint32_t *ip = (uint32_t *) ldtp; 319 320 kprintf("LDT:\n"); 321 for (i = 0; i < LDTSZ_MIN; i++, ip += 2) { 322 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 323 kprintf("%p: 0x%08x\n", ip+1, *(ip+1)); 324 } 325} 326 327void 328dump_idt(void *idtp) 329{ 330 unsigned int i; 331 uint32_t *ip = (uint32_t *) idtp; 332 333 kprintf("IDT64:\n"); 334 for (i = 0; i < 16; i++, ip += 4) { 335 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 336 kprintf("%p: 0x%08x\n", ip+1, *(ip+1)); 337 kprintf("%p: 0x%08x\n", ip+2, *(ip+2)); 338 kprintf("%p: 0x%08x\n", ip+3, *(ip+3)); 339 } 340} 341 342void 343dump_tss(void *tssp) 344{ 345 unsigned int i; 346 uint32_t *ip = (uint32_t *) tssp; 347 348 kprintf("TSS64:\n"); 349 for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) { 350 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 351 } 352} 353 354#if defined(__x86_64__) 355void dump_regs64(void) 356{ 357 358#define SNAP_REG(reg) \ 359 uint64_t reg; \ 360 __asm__ volatile("mov %%" #reg ", %0" : "=m" (reg)) 361 362#define KPRINT_REG(reg) \ 363 kprintf("%3s: %p\n", #reg, (void *) reg) 364 365 SNAP_REG(rsp); 366 SNAP_REG(rbp); 367 SNAP_REG(rax); 368 SNAP_REG(rbx); 369 SNAP_REG(rcx); 370 SNAP_REG(rdx); 371 SNAP_REG(rsi); 372 SNAP_REG(rdi); 373 SNAP_REG(r8); 374 SNAP_REG(r9); 375 SNAP_REG(r10); 376 SNAP_REG(r11); 377 SNAP_REG(r12); 378 SNAP_REG(r13); 379 SNAP_REG(r14); 380 381 KPRINT_REG(rsp); 382 KPRINT_REG(rbp); 383 KPRINT_REG(rax); 384 KPRINT_REG(rbx); 385 KPRINT_REG(rcx); 386 KPRINT_REG(rdx); 387 KPRINT_REG(rsi); 388 KPRINT_REG(rdi); 389 KPRINT_REG(r8); 390 KPRINT_REG(r9); 391 KPRINT_REG(r10); 392 KPRINT_REG(r11); 393 KPRINT_REG(r12); 394 KPRINT_REG(r13); 395 KPRINT_REG(r14); 396} 397#endif /* __x86_64__ */ 398#endif /* DEBUG */ 399