1/* 2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <string.h> 30 31#include <mach/machine/vm_types.h> 32 33#include <mach/boolean.h> 34#include <kern/thread.h> 35#include <kern/zalloc.h> 36 37#include <kern/lock.h> 38#include <kern/kalloc.h> 39#include <kern/spl.h> 40 41#include <vm/pmap.h> 42#include <vm/vm_map.h> 43#include <vm/vm_kern.h> 44#include <mach/vm_param.h> 45#include <mach/vm_prot.h> 46#include <vm/vm_object.h> 47#include <vm/vm_page.h> 48 49#include <mach/machine/vm_param.h> 50#include <machine/thread.h> 51 52#include <kern/misc_protos.h> /* prototyping */ 53#include <i386/misc_protos.h> 54 55#include <i386/cpuid.h> 56#include <i386/cpu_data.h> 57#include <i386/mp.h> 58#include <i386/cpu_number.h> 59#include <i386/machine_cpu.h> 60#include <i386/mp_slave_boot.h> 61#include <i386/seg.h> 62 63#include <vm/vm_protos.h> 64 65#include <sys/kdebug.h> 66 67#include <i386/postcode.h> 68 69void 70cpu_IA32e_enable(cpu_data_t *cdp) 71{ 72 assert(!ml_get_interrupts_enabled()); 73 74 if (!cdp->cpu_is64bit || 75 (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0) 76 return; 77 78 postcode(CPU_IA32_ENABLE_ENTRY); 79 80 /* 81 * The following steps are performed by inlines so that 82 * we can be assured we don't use the stack or any other 83 * non-identity mapped data while paging is turned off... 84 */ 85 /* Turn paging off */ 86 asm volatile( 87 "mov %%cr0, %%eax \n\t" 88 "andl %0, %%eax \n\t" 89 "mov %%eax, %%cr0 \n\t" 90 : 91 : "i" (~CR0_PG) 92 : "eax" ); 93 94 /* Pop new top level phys pg addr into CR3 */ 95 asm volatile( 96 "mov %%eax, %%cr3 \n\t" 97 : 98 : "a" ((uint32_t) kernel64_cr3)); 99 100 /* Turn on the 64-bit mode bit */ 101 asm volatile( 102 "rdmsr \n\t" 103 "orl %1, %%eax \n\t" 104 "wrmsr \n\t" 105 : 106 : "c" (MSR_IA32_EFER), "i" (MSR_IA32_EFER_LME) 107 : "eax", "edx"); 108 109 /* Turn paging on again */ 110 asm volatile( 111 "mov %%cr0, %%eax \n\t" 112 "orl %0, %%eax \n\t" 113 "mov %%eax, %%cr0 \n\t" 114 : 115 : "i" (CR0_PG) 116 : "eax" ); 117 118#if ONLY_SAFE_FOR_LINDA_SERIAL 119 kprintf("cpu_IA32e_enable(%p)\n", cdp); 120#endif 121 122 if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0) 123 panic("cpu_IA32e_enable() MSR_IA32_EFER_LMA not asserted"); 124 125 cdp->cpu_kernel_cr3 = kernel64_cr3; 126 127 postcode(CPU_IA32_ENABLE_EXIT); 128} 129 130void 131cpu_IA32e_disable(cpu_data_t *cdp) 132{ 133 assert(!ml_get_interrupts_enabled()); 134 135 postcode(CPU_IA32_DISABLE_ENTRY); 136 137 if (!cdp->cpu_is64bit || 138 (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0) 139 return; 140 141 /* 142 * The following steps are performed by inlines so that 143 * we can be assured we don't use the stack or any other 144 * non-identity mapped data while paging is turned off... 145 */ 146 /* Turn paging off */ 147 asm volatile( 148 "mov %%cr0, %%eax \n\t" 149 "andl %0, %%eax \n\t" 150 "mov %%eax, %%cr0 \n\t" 151 : 152 : "i" (~CR0_PG) 153 : "eax" ); 154 155 /* Pop legacy top level phys pg addr into CR3 */ 156 asm volatile( 157 "mov %%eax, %%cr3 \n\t" 158 : 159 : "a" ((uint32_t) lo_kernel_cr3)); 160 161 /* Turn off the 64-bit mode bit */ 162 asm volatile( 163 "rdmsr \n\t" 164 "andl %1, %%eax \n\t" 165 "wrmsr \n\t" 166 : 167 : "c" (MSR_IA32_EFER), "i" (~MSR_IA32_EFER_LME) 168 : "eax", "edx"); 169 170 /* Turn paging on again */ 171 asm volatile( 172 "mov %%cr0, %%eax \n\t" 173 "orl %0, %%eax \n\t" 174 "mov %%eax, %%cr0 \n\t" 175 : 176 : "i" (CR0_PG) 177 : "eax" ); 178 179 kprintf("cpu_IA32e_disable(%p)\n", cdp); 180 181 if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0) 182 panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA not cleared"); 183 184 cdp->cpu_kernel_cr3 = 0ULL; 185 186 postcode(CPU_IA32_DISABLE_EXIT); 187} 188 189void 190fix_desc64(void *descp, int count) 191{ 192 struct fake_descriptor64 *fakep; 193 union { 194 struct real_gate64 gate; 195 struct real_descriptor64 desc; 196 } real; 197 int i; 198 199 fakep = (struct fake_descriptor64 *) descp; 200 201 for (i = 0; i < count; i++, fakep++) { 202 /* 203 * Construct the real decriptor locally. 204 */ 205 206 bzero((void *) &real, sizeof(real)); 207 208 switch (fakep->access & ACC_TYPE) { 209 case 0: 210 break; 211 case ACC_CALL_GATE: 212 case ACC_INTR_GATE: 213 case ACC_TRAP_GATE: 214 real.gate.offset_low16 = fakep->offset[0] & 0xFFFF; 215 real.gate.selector16 = fakep->lim_or_seg & 0xFFFF; 216 real.gate.IST = fakep->size_or_IST & 0x7; 217 real.gate.access8 = fakep->access; 218 real.gate.offset_high16 = (fakep->offset[0]>>16)&0xFFFF; 219 real.gate.offset_top32 = (uint32_t)fakep->offset[1]; 220 break; 221 default: /* Otherwise */ 222 real.desc.limit_low16 = fakep->lim_or_seg & 0xFFFF; 223 real.desc.base_low16 = fakep->offset[0] & 0xFFFF; 224 real.desc.base_med8 = (fakep->offset[0] >> 16) & 0xFF; 225 real.desc.access8 = fakep->access; 226 real.desc.limit_high4 = (fakep->lim_or_seg >> 16) & 0xFF; 227 real.desc.granularity4 = fakep->size_or_IST; 228 real.desc.base_high8 = (fakep->offset[0] >> 24) & 0xFF; 229 real.desc.base_top32 = (uint32_t) fakep->offset[1]; 230 } 231 232 /* 233 * Now copy back over the fake structure. 234 */ 235 bcopy((void *) &real, (void *) fakep, sizeof(real)); 236 } 237} 238 239#if DEBUG 240extern void dump_gdt(void *); 241extern void dump_ldt(void *); 242extern void dump_idt(void *); 243extern void dump_tss(void *); 244extern void dump_frame32(x86_saved_state_compat32_t *scp); 245extern void dump_frame64(x86_saved_state64_t *sp); 246extern void dump_frame(x86_saved_state_t *sp); 247 248void 249dump_frame(x86_saved_state_t *sp) 250{ 251 if (is_saved_state32(sp)) 252 dump_frame32((x86_saved_state_compat32_t *) sp); 253 else if (is_saved_state64(sp)) 254 dump_frame64(&sp->ss_64); 255 else 256 kprintf("dump_frame(%p) unknown type %d\n", sp, sp->flavor); 257} 258 259void 260dump_frame32(x86_saved_state_compat32_t *scp) 261{ 262 unsigned int i; 263 uint32_t *ip = (uint32_t *) scp; 264 265 kprintf("dump_frame32(%p):\n", scp); 266 267 for (i = 0; 268 i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t); 269 i++, ip++) 270 kprintf("%p: 0x%08x\n", ip, *ip); 271 272 kprintf("scp->isf64.err: 0x%016llx\n", scp->isf64.err); 273 kprintf("scp->isf64.rip: 0x%016llx\n", scp->isf64.rip); 274 kprintf("scp->isf64.cs: 0x%016llx\n", scp->isf64.cs); 275 kprintf("scp->isf64.rflags: 0x%016llx\n", scp->isf64.rflags); 276 kprintf("scp->isf64.rsp: 0x%016llx\n", scp->isf64.rsp); 277 kprintf("scp->isf64.ss: 0x%016llx\n", scp->isf64.ss); 278 279 kprintf("scp->iss32.tag: 0x%08x\n", scp->iss32.tag); 280 kprintf("scp->iss32.state.gs: 0x%08x\n", scp->iss32.state.gs); 281 kprintf("scp->iss32.state.fs: 0x%08x\n", scp->iss32.state.fs); 282 kprintf("scp->iss32.state.es: 0x%08x\n", scp->iss32.state.es); 283 kprintf("scp->iss32.state.ds: 0x%08x\n", scp->iss32.state.ds); 284 kprintf("scp->iss32.state.edi: 0x%08x\n", scp->iss32.state.edi); 285 kprintf("scp->iss32.state.esi: 0x%08x\n", scp->iss32.state.esi); 286 kprintf("scp->iss32.state.ebp: 0x%08x\n", scp->iss32.state.ebp); 287 kprintf("scp->iss32.state.cr2: 0x%08x\n", scp->iss32.state.cr2); 288 kprintf("scp->iss32.state.ebx: 0x%08x\n", scp->iss32.state.ebx); 289 kprintf("scp->iss32.state.edx: 0x%08x\n", scp->iss32.state.edx); 290 kprintf("scp->iss32.state.ecx: 0x%08x\n", scp->iss32.state.ecx); 291 kprintf("scp->iss32.state.eax: 0x%08x\n", scp->iss32.state.eax); 292 kprintf("scp->iss32.state.trapno: 0x%08x\n", scp->iss32.state.eax); 293 kprintf("scp->iss32.state.eip: 0x%08x\n", scp->iss32.state.eip); 294 kprintf("scp->iss32.state.cs: 0x%08x\n", scp->iss32.state.cs); 295 kprintf("scp->iss32.state.efl: 0x%08x\n", scp->iss32.state.efl); 296 kprintf("scp->iss32.state.uesp: 0x%08x\n", scp->iss32.state.uesp); 297 kprintf("scp->iss32.state.ss: 0x%08x\n", scp->iss32.state.ss); 298 299 postcode(0x99); 300} 301 302void 303dump_frame64(x86_saved_state64_t *sp) 304{ 305 unsigned int i; 306 uint64_t *ip = (uint64_t *) sp; 307 308 kprintf("dump_frame64(%p):\n", sp); 309 310 for (i = 0; 311 i < sizeof(x86_saved_state64_t)/sizeof(uint64_t); 312 i++, ip++) 313 kprintf("%p: 0x%016llx\n", ip, *ip); 314 315 kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno); 316 kprintf("sp->isf.trapfn: 0x%08x\n", sp->isf.trapfn); 317 kprintf("sp->isf.err: 0x%016llx\n", sp->isf.err); 318 kprintf("sp->isf.rip: 0x%016llx\n", sp->isf.rip); 319 kprintf("sp->isf.cs: 0x%016llx\n", sp->isf.cs); 320 kprintf("sp->isf.rflags: 0x%016llx\n", sp->isf.rflags); 321 kprintf("sp->isf.rsp: 0x%016llx\n", sp->isf.rsp); 322 kprintf("sp->isf.ss: 0x%016llx\n", sp->isf.ss); 323 324 kprintf("sp->fs: 0x%016x\n", sp->fs); 325 kprintf("sp->gs: 0x%016x\n", sp->gs); 326 kprintf("sp->rax: 0x%016llx\n", sp->rax); 327 kprintf("sp->rcx: 0x%016llx\n", sp->rcx); 328 kprintf("sp->rbx: 0x%016llx\n", sp->rbx); 329 kprintf("sp->rbp: 0x%016llx\n", sp->rbp); 330 kprintf("sp->r11: 0x%016llx\n", sp->r11); 331 kprintf("sp->r12: 0x%016llx\n", sp->r12); 332 kprintf("sp->r13: 0x%016llx\n", sp->r13); 333 kprintf("sp->r14: 0x%016llx\n", sp->r14); 334 kprintf("sp->r15: 0x%016llx\n", sp->r15); 335 kprintf("sp->cr2: 0x%016llx\n", sp->cr2); 336 kprintf("sp->v_arg8: 0x%016llx\n", sp->v_arg8); 337 kprintf("sp->v_arg7: 0x%016llx\n", sp->v_arg7); 338 kprintf("sp->v_arg6: 0x%016llx\n", sp->v_arg6); 339 kprintf("sp->r9: 0x%016llx\n", sp->r9); 340 kprintf("sp->r8: 0x%016llx\n", sp->r8); 341 kprintf("sp->r10: 0x%016llx\n", sp->r10); 342 kprintf("sp->rdx: 0x%016llx\n", sp->rdx); 343 kprintf("sp->rsi: 0x%016llx\n", sp->rsi); 344 kprintf("sp->rdi: 0x%016llx\n", sp->rdi); 345 346 postcode(0x98); 347} 348 349void 350dump_gdt(void *gdtp) 351{ 352 unsigned int i; 353 uint32_t *ip = (uint32_t *) gdtp; 354 355 kprintf("GDT:\n"); 356 for (i = 0; i < GDTSZ; i++, ip += 2) { 357 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 358 kprintf("%p: 0x%08x\n", ip+1, *(ip+1)); 359 } 360} 361 362void 363dump_ldt(void *ldtp) 364{ 365 unsigned int i; 366 uint32_t *ip = (uint32_t *) ldtp; 367 368 kprintf("LDT:\n"); 369 for (i = 0; i < LDTSZ_MIN; i++, ip += 2) { 370 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 371 kprintf("%p: 0x%08x\n", ip+1, *(ip+1)); 372 } 373} 374 375void 376dump_idt(void *idtp) 377{ 378 unsigned int i; 379 uint32_t *ip = (uint32_t *) idtp; 380 381 kprintf("IDT64:\n"); 382 for (i = 0; i < 16; i++, ip += 4) { 383 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 384 kprintf("%p: 0x%08x\n", ip+1, *(ip+1)); 385 kprintf("%p: 0x%08x\n", ip+2, *(ip+2)); 386 kprintf("%p: 0x%08x\n", ip+3, *(ip+3)); 387 } 388} 389 390void 391dump_tss(void *tssp) 392{ 393 unsigned int i; 394 uint32_t *ip = (uint32_t *) tssp; 395 396 kprintf("TSS64:\n"); 397 for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) { 398 kprintf("%p: 0x%08x\n", ip+0, *(ip+0)); 399 } 400} 401#endif /* DEBUG */ 402