1/* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 57/* 58 */ 59 60/* 61 * File: model_dep.c 62 * Author: Avadis Tevanian, Jr., Michael Wayne Young 63 * 64 * Copyright (C) 1986, Avadis Tevanian, Jr., Michael Wayne Young 65 * 66 * Basic initialization for I386 - ISA bus machines. 67 */ 68 69 70#include <mach/i386/vm_param.h> 71 72#include <string.h> 73#include <mach/vm_param.h> 74#include <mach/vm_prot.h> 75#include <mach/machine.h> 76#include <mach/time_value.h> 77#include <kern/spl.h> 78#include <kern/assert.h> 79#include <kern/debug.h> 80#include <kern/misc_protos.h> 81#include <kern/startup.h> 82#include <kern/clock.h> 83#include <kern/cpu_data.h> 84#include <kern/machine.h> 85#include <i386/postcode.h> 86#include <i386/mp_desc.h> 87#include <i386/misc_protos.h> 88#include <i386/thread.h> 89#include <i386/trap.h> 90#include <i386/machine_routines.h> 91#include <i386/mp.h> /* mp_rendezvous_break_lock */ 92#include <i386/cpuid.h> 93#include <i386/fpu.h> 94#include <i386/machine_cpu.h> 95#include <i386/pmap.h> 96#if CONFIG_MTRR 97#include <i386/mtrr.h> 98#endif 99#include <i386/ucode.h> 100#include <i386/pmCPU.h> 101#include <i386/panic_hooks.h> 102 103#include <architecture/i386/pio.h> /* inb() */ 104#include <pexpert/i386/boot.h> 105 106#include <vm/pmap.h> 107#include <vm/vm_map.h> 108#include <vm/vm_kern.h> 109 110#include <IOKit/IOPlatformExpert.h> 111#include <IOKit/IOHibernatePrivate.h> 112 113#include <pexpert/i386/efi.h> 114 115#include <kern/thread.h> 116#include <kern/sched.h> 117#include <mach-o/loader.h> 118#include <mach-o/nlist.h> 119 120#include <libkern/kernel_mach_header.h> 121#include <libkern/OSKextLibPrivate.h> 122 123#if DEBUG 124#define DPRINTF(x...) kprintf(x) 125#else 126#define DPRINTF(x...) 127#endif 128 129static void machine_conf(void); 130 131extern int max_unsafe_quanta; 132extern int max_poll_quanta; 133extern unsigned int panic_is_inited; 134 135int db_run_mode; 136 137volatile int pbtcpu = -1; 138hw_lock_data_t pbtlock; /* backtrace print lock */ 139uint32_t pbtcnt = 0; 140 141volatile int panic_double_fault_cpu = -1; 142 143#define PRINT_ARGS_FROM_STACK_FRAME 0 144 145typedef struct _cframe_t { 146 struct _cframe_t *prev; 147 uintptr_t caller; 148#if PRINT_ARGS_FROM_STACK_FRAME 149 unsigned args[0]; 150#endif 151} cframe_t; 152 153static unsigned panic_io_port; 154static unsigned commit_paniclog_to_nvram; 155 156unsigned int debug_boot_arg; 157 158void 159machine_startup(void) 160{ 161 int boot_arg; 162 163#if 0 164 if( PE_get_hotkey( kPEControlKey )) 165 halt_in_debugger = halt_in_debugger ? 0 : 1; 166#endif 167 168 if (PE_parse_boot_argn("debug", &debug_boot_arg, sizeof (debug_boot_arg))) { 169 panicDebugging = TRUE; 170 if (debug_boot_arg & DB_HALT) halt_in_debugger=1; 171 if (debug_boot_arg & DB_PRT) disable_debug_output=FALSE; 172 if (debug_boot_arg & DB_SLOG) systemLogDiags=TRUE; 173 if (debug_boot_arg & DB_LOG_PI_SCRN) logPanicDataToScreen=TRUE; 174 } else { 175 debug_boot_arg = 0; 176 } 177 178 if (!PE_parse_boot_argn("nvram_paniclog", &commit_paniclog_to_nvram, sizeof (commit_paniclog_to_nvram))) 179 commit_paniclog_to_nvram = 1; 180 181 /* 182 * Entering the debugger will put the CPUs into a "safe" 183 * power mode. 184 */ 185 if (PE_parse_boot_argn("pmsafe_debug", &boot_arg, sizeof (boot_arg))) 186 pmsafe_debug = boot_arg; 187 188#if NOTYET 189 hw_lock_init(&debugger_lock); /* initialize debugger lock */ 190#endif 191 hw_lock_init(&pbtlock); /* initialize print backtrace lock */ 192 193 if (PE_parse_boot_argn("preempt", &boot_arg, sizeof (boot_arg))) { 194 default_preemption_rate = boot_arg; 195 } 196 if (PE_parse_boot_argn("unsafe", &boot_arg, sizeof (boot_arg))) { 197 max_unsafe_quanta = boot_arg; 198 } 199 if (PE_parse_boot_argn("poll", &boot_arg, sizeof (boot_arg))) { 200 max_poll_quanta = boot_arg; 201 } 202 if (PE_parse_boot_argn("yield", &boot_arg, sizeof (boot_arg))) { 203 sched_poll_yield_shift = boot_arg; 204 } 205/* The I/O port to issue a read from, in the event of a panic. Useful for 206 * triggering logic analyzers. 207 */ 208 if (PE_parse_boot_argn("panic_io_port", &boot_arg, sizeof (boot_arg))) { 209 /*I/O ports range from 0 through 0xFFFF */ 210 panic_io_port = boot_arg & 0xffff; 211 } 212 213 machine_conf(); 214 215 panic_hooks_init(); 216 217 /* 218 * Start the system. 219 */ 220 kernel_bootstrap(); 221 /*NOTREACHED*/ 222} 223 224 225static void 226machine_conf(void) 227{ 228 machine_info.memory_size = (typeof(machine_info.memory_size))mem_size; 229} 230 231 232extern void *gPEEFIRuntimeServices; 233extern void *gPEEFISystemTable; 234 235/*- 236 * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or 237 * code or tables extracted from it, as desired without restriction. 238 * 239 * First, the polynomial itself and its table of feedback terms. The 240 * polynomial is 241 * X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 242 * 243 * Note that we take it "backwards" and put the highest-order term in 244 * the lowest-order bit. The X^32 term is "implied"; the LSB is the 245 * X^31 term, etc. The X^0 term (usually shown as "+1") results in 246 * the MSB being 1 247 * 248 * Note that the usual hardware shift register implementation, which 249 * is what we're using (we're merely optimizing it by doing eight-bit 250 * chunks at a time) shifts bits into the lowest-order term. In our 251 * implementation, that means shifting towards the right. Why do we 252 * do it this way? Because the calculated CRC must be transmitted in 253 * order from highest-order term to lowest-order term. UARTs transmit 254 * characters in order from LSB to MSB. By storing the CRC this way 255 * we hand it to the UART in the order low-byte to high-byte; the UART 256 * sends each low-bit to hight-bit; and the result is transmission bit 257 * by bit from highest- to lowest-order term without requiring any bit 258 * shuffling on our part. Reception works similarly 259 * 260 * The feedback terms table consists of 256, 32-bit entries. Notes 261 * 262 * The table can be generated at runtime if desired; code to do so 263 * is shown later. It might not be obvious, but the feedback 264 * terms simply represent the results of eight shift/xor opera 265 * tions for all combinations of data and CRC register values 266 * 267 * The values must be right-shifted by eight bits by the "updcrc 268 * logic; the shift must be unsigned (bring in zeroes). On some 269 * hardware you could probably optimize the shift in assembler by 270 * using byte-swap instructions 271 * polynomial $edb88320 272 * 273 * 274 * CRC32 code derived from work by Gary S. Brown. 275 */ 276 277static uint32_t crc32_tab[] = { 278 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 279 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 280 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 281 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 282 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 283 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 284 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 285 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 286 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 287 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 288 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 289 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 290 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 291 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 292 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 293 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 294 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 295 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 296 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 297 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 298 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 299 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 300 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 301 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 302 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 303 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 304 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 305 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 306 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 307 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 308 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 309 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 310 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 311 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 312 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 313 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 314 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 315 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 316 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 317 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 318 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 319 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 320 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d 321}; 322 323static uint32_t 324crc32(uint32_t crc, const void *buf, size_t size) 325{ 326 const uint8_t *p; 327 328 p = buf; 329 crc = crc ^ ~0U; 330 331 while (size--) 332 crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); 333 334 return crc ^ ~0U; 335} 336 337static void 338efi_set_tables_64(EFI_SYSTEM_TABLE_64 * system_table) 339{ 340 EFI_RUNTIME_SERVICES_64 *runtime; 341 uint32_t hdr_cksum; 342 uint32_t cksum; 343 344 DPRINTF("Processing 64-bit EFI tables at %p\n", system_table); 345 do { 346 DPRINTF("Header:\n"); 347 DPRINTF(" Signature: 0x%016llx\n", system_table->Hdr.Signature); 348 DPRINTF(" Revision: 0x%08x\n", system_table->Hdr.Revision); 349 DPRINTF(" HeaderSize: 0x%08x\n", system_table->Hdr.HeaderSize); 350 DPRINTF(" CRC32: 0x%08x\n", system_table->Hdr.CRC32); 351 DPRINTF("RuntimeServices: 0x%016llx\n", system_table->RuntimeServices); 352 if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) { 353 kprintf("Bad EFI system table signature\n"); 354 break; 355 } 356 // Verify signature of the system table 357 hdr_cksum = system_table->Hdr.CRC32; 358 system_table->Hdr.CRC32 = 0; 359 cksum = crc32(0L, system_table, system_table->Hdr.HeaderSize); 360 361 DPRINTF("System table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum); 362 system_table->Hdr.CRC32 = hdr_cksum; 363 if (cksum != hdr_cksum) { 364 kprintf("Bad EFI system table checksum\n"); 365 break; 366 } 367 368 gPEEFISystemTable = system_table; 369 370 if(system_table->RuntimeServices == 0) { 371 kprintf("No runtime table present\n"); 372 break; 373 } 374 DPRINTF("RuntimeServices table at 0x%qx\n", system_table->RuntimeServices); 375 // 64-bit virtual address is OK for 64-bit EFI and 64/32-bit kernel. 376 runtime = (EFI_RUNTIME_SERVICES_64 *) (uintptr_t)system_table->RuntimeServices; 377 DPRINTF("Checking runtime services table %p\n", runtime); 378 if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) { 379 kprintf("Bad EFI runtime table signature\n"); 380 break; 381 } 382 383 // Verify signature of runtime services table 384 hdr_cksum = runtime->Hdr.CRC32; 385 runtime->Hdr.CRC32 = 0; 386 cksum = crc32(0L, runtime, runtime->Hdr.HeaderSize); 387 388 DPRINTF("Runtime table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum); 389 runtime->Hdr.CRC32 = hdr_cksum; 390 if (cksum != hdr_cksum) { 391 kprintf("Bad EFI runtime table checksum\n"); 392 break; 393 } 394 395 gPEEFIRuntimeServices = runtime; 396 } 397 while (FALSE); 398} 399 400static void 401efi_set_tables_32(EFI_SYSTEM_TABLE_32 * system_table) 402{ 403 EFI_RUNTIME_SERVICES_32 *runtime; 404 uint32_t hdr_cksum; 405 uint32_t cksum; 406 407 DPRINTF("Processing 32-bit EFI tables at %p\n", system_table); 408 do { 409 DPRINTF("Header:\n"); 410 DPRINTF(" Signature: 0x%016llx\n", system_table->Hdr.Signature); 411 DPRINTF(" Revision: 0x%08x\n", system_table->Hdr.Revision); 412 DPRINTF(" HeaderSize: 0x%08x\n", system_table->Hdr.HeaderSize); 413 DPRINTF(" CRC32: 0x%08x\n", system_table->Hdr.CRC32); 414 DPRINTF("RuntimeServices: 0x%08x\n", system_table->RuntimeServices); 415 if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) { 416 kprintf("Bad EFI system table signature\n"); 417 break; 418 } 419 // Verify signature of the system table 420 hdr_cksum = system_table->Hdr.CRC32; 421 system_table->Hdr.CRC32 = 0; 422 DPRINTF("System table at %p HeaderSize 0x%x\n", system_table, system_table->Hdr.HeaderSize); 423 cksum = crc32(0L, system_table, system_table->Hdr.HeaderSize); 424 425 DPRINTF("System table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum); 426 system_table->Hdr.CRC32 = hdr_cksum; 427 if (cksum != hdr_cksum) { 428 kprintf("Bad EFI system table checksum\n"); 429 break; 430 } 431 432 gPEEFISystemTable = system_table; 433 434 if(system_table->RuntimeServices == 0) { 435 kprintf("No runtime table present\n"); 436 break; 437 } 438 DPRINTF("RuntimeServices table at 0x%x\n", system_table->RuntimeServices); 439 // 32-bit virtual address is OK for 32-bit EFI and 32-bit kernel. 440 // For a 64-bit kernel, booter provides a virtual address mod 4G 441 runtime = (EFI_RUNTIME_SERVICES_32 *) 442 (system_table->RuntimeServices | VM_MIN_KERNEL_ADDRESS); 443 DPRINTF("Runtime table addressed at %p\n", runtime); 444 if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) { 445 kprintf("Bad EFI runtime table signature\n"); 446 break; 447 } 448 449 // Verify signature of runtime services table 450 hdr_cksum = runtime->Hdr.CRC32; 451 runtime->Hdr.CRC32 = 0; 452 cksum = crc32(0L, runtime, runtime->Hdr.HeaderSize); 453 454 DPRINTF("Runtime table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum); 455 runtime->Hdr.CRC32 = hdr_cksum; 456 if (cksum != hdr_cksum) { 457 kprintf("Bad EFI runtime table checksum\n"); 458 break; 459 } 460 461 DPRINTF("Runtime functions\n"); 462 DPRINTF(" GetTime : 0x%x\n", runtime->GetTime); 463 DPRINTF(" SetTime : 0x%x\n", runtime->SetTime); 464 DPRINTF(" GetWakeupTime : 0x%x\n", runtime->GetWakeupTime); 465 DPRINTF(" SetWakeupTime : 0x%x\n", runtime->SetWakeupTime); 466 DPRINTF(" SetVirtualAddressMap : 0x%x\n", runtime->SetVirtualAddressMap); 467 DPRINTF(" ConvertPointer : 0x%x\n", runtime->ConvertPointer); 468 DPRINTF(" GetVariable : 0x%x\n", runtime->GetVariable); 469 DPRINTF(" GetNextVariableName : 0x%x\n", runtime->GetNextVariableName); 470 DPRINTF(" SetVariable : 0x%x\n", runtime->SetVariable); 471 DPRINTF(" GetNextHighMonotonicCount: 0x%x\n", runtime->GetNextHighMonotonicCount); 472 DPRINTF(" ResetSystem : 0x%x\n", runtime->ResetSystem); 473 474 gPEEFIRuntimeServices = runtime; 475 } 476 while (FALSE); 477} 478 479 480/* Map in EFI runtime areas. */ 481static void 482efi_init(void) 483{ 484 boot_args *args = (boot_args *)PE_state.bootArgs; 485 486 kprintf("Initializing EFI runtime services\n"); 487 488 do 489 { 490 vm_offset_t vm_size, vm_addr; 491 vm_map_offset_t phys_addr; 492 EfiMemoryRange *mptr; 493 unsigned int msize, mcount; 494 unsigned int i; 495 496 msize = args->MemoryMapDescriptorSize; 497 mcount = args->MemoryMapSize / msize; 498 499 DPRINTF("efi_init() kernel base: 0x%x size: 0x%x\n", 500 args->kaddr, args->ksize); 501 DPRINTF(" efiSystemTable physical: 0x%x virtual: %p\n", 502 args->efiSystemTable, 503 (void *) ml_static_ptovirt(args->efiSystemTable)); 504 DPRINTF(" efiRuntimeServicesPageStart: 0x%x\n", 505 args->efiRuntimeServicesPageStart); 506 DPRINTF(" efiRuntimeServicesPageCount: 0x%x\n", 507 args->efiRuntimeServicesPageCount); 508 DPRINTF(" efiRuntimeServicesVirtualPageStart: 0x%016llx\n", 509 args->efiRuntimeServicesVirtualPageStart); 510 mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap); 511 for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) { 512 if (((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) ) { 513 vm_size = (vm_offset_t)i386_ptob((uint32_t)mptr->NumberOfPages); 514 vm_addr = (vm_offset_t) mptr->VirtualStart; 515 /* For K64 on EFI32, shadow-map into high KVA */ 516 if (vm_addr < VM_MIN_KERNEL_ADDRESS) 517 vm_addr |= VM_MIN_KERNEL_ADDRESS; 518 phys_addr = (vm_map_offset_t) mptr->PhysicalStart; 519 DPRINTF(" Type: %x phys: %p EFIv: %p kv: %p size: %p\n", 520 mptr->Type, 521 (void *) (uintptr_t) phys_addr, 522 (void *) (uintptr_t) mptr->VirtualStart, 523 (void *) vm_addr, 524 (void *) vm_size); 525 pmap_map_bd(vm_addr, phys_addr, phys_addr + round_page(vm_size), 526 (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE, 527 (mptr->Type == EfiMemoryMappedIO) ? VM_WIMG_IO : VM_WIMG_USE_DEFAULT); 528 } 529 } 530 531 if (args->Version != kBootArgsVersion2) 532 panic("Incompatible boot args version %d revision %d\n", args->Version, args->Revision); 533 534 DPRINTF("Boot args version %d revision %d mode %d\n", args->Version, args->Revision, args->efiMode); 535 if (args->efiMode == kBootArgsEfiMode64) { 536 efi_set_tables_64((EFI_SYSTEM_TABLE_64 *) ml_static_ptovirt(args->efiSystemTable)); 537 } else { 538 efi_set_tables_32((EFI_SYSTEM_TABLE_32 *) ml_static_ptovirt(args->efiSystemTable)); 539 } 540 } 541 while (FALSE); 542 543 return; 544} 545 546/* Remap EFI runtime areas. */ 547void 548hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_offset) 549{ 550 boot_args *args = (boot_args *)PE_state.bootArgs; 551 552 kprintf("Reinitializing EFI runtime services\n"); 553 554 do 555 { 556 vm_offset_t vm_size, vm_addr; 557 vm_map_offset_t phys_addr; 558 EfiMemoryRange *mptr; 559 unsigned int msize, mcount; 560 unsigned int i; 561 562 gPEEFISystemTable = 0; 563 gPEEFIRuntimeServices = 0; 564 565 system_table_offset += ptoa_32(args->efiRuntimeServicesPageStart); 566 567 kprintf("Old system table 0x%x, new 0x%x\n", 568 (uint32_t)args->efiSystemTable, system_table_offset); 569 570 args->efiSystemTable = system_table_offset; 571 572 kprintf("Old map:\n"); 573 msize = args->MemoryMapDescriptorSize; 574 mcount = args->MemoryMapSize / msize; 575 mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap); 576 for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) { 577 if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) { 578 579 vm_size = (vm_offset_t)i386_ptob((uint32_t)mptr->NumberOfPages); 580 vm_addr = (vm_offset_t) mptr->VirtualStart; 581 /* K64 on EFI32 */ 582 if (vm_addr < VM_MIN_KERNEL_ADDRESS) 583 vm_addr |= VM_MIN_KERNEL_ADDRESS; 584 phys_addr = (vm_map_offset_t) mptr->PhysicalStart; 585 586 kprintf("mapping[%u] %qx @ %lx, %llu\n", mptr->Type, phys_addr, (unsigned long)vm_addr, mptr->NumberOfPages); 587 } 588 } 589 590 pmap_remove(kernel_pmap, i386_ptob(args->efiRuntimeServicesPageStart), 591 i386_ptob(args->efiRuntimeServicesPageStart + args->efiRuntimeServicesPageCount)); 592 593 kprintf("New map:\n"); 594 msize = args->MemoryMapDescriptorSize; 595 mcount = (unsigned int )(map_size / msize); 596 mptr = map; 597 for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) { 598 if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) { 599 600 vm_size = (vm_offset_t)i386_ptob((uint32_t)mptr->NumberOfPages); 601 vm_addr = (vm_offset_t) mptr->VirtualStart; 602 if (vm_addr < VM_MIN_KERNEL_ADDRESS) 603 vm_addr |= VM_MIN_KERNEL_ADDRESS; 604 phys_addr = (vm_map_offset_t) mptr->PhysicalStart; 605 606 kprintf("mapping[%u] %qx @ %lx, %llu\n", mptr->Type, phys_addr, (unsigned long)vm_addr, mptr->NumberOfPages); 607 608 pmap_map(vm_addr, phys_addr, phys_addr + round_page(vm_size), 609 (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE, 610 (mptr->Type == EfiMemoryMappedIO) ? VM_WIMG_IO : VM_WIMG_USE_DEFAULT); 611 } 612 } 613 614 if (args->Version != kBootArgsVersion2) 615 panic("Incompatible boot args version %d revision %d\n", args->Version, args->Revision); 616 617 kprintf("Boot args version %d revision %d mode %d\n", args->Version, args->Revision, args->efiMode); 618 if (args->efiMode == kBootArgsEfiMode64) { 619 efi_set_tables_64((EFI_SYSTEM_TABLE_64 *) ml_static_ptovirt(args->efiSystemTable)); 620 } else { 621 efi_set_tables_32((EFI_SYSTEM_TABLE_32 *) ml_static_ptovirt(args->efiSystemTable)); 622 } 623 } 624 while (FALSE); 625 626 kprintf("Done reinitializing EFI runtime services\n"); 627 628 return; 629} 630 631/* 632 * Find devices. The system is alive. 633 */ 634void 635machine_init(void) 636{ 637 /* Now with VM up, switch to dynamically allocated cpu data */ 638 cpu_data_realloc(); 639 640 /* Ensure panic buffer is initialized. */ 641 debug_log_init(); 642 643 /* 644 * Display CPU identification 645 */ 646 cpuid_cpu_display("CPU identification"); 647 cpuid_feature_display("CPU features"); 648 cpuid_extfeature_display("CPU extended features"); 649 650 /* 651 * Initialize EFI runtime services. 652 */ 653 efi_init(); 654 655 smp_init(); 656 657 /* 658 * Set up to use floating point. 659 */ 660 init_fpu(); 661 662 /* 663 * Configure clock devices. 664 */ 665 clock_config(); 666 667#if CONFIG_MTRR 668 /* 669 * Initialize MTRR from boot processor. 670 */ 671 mtrr_init(); 672 673 /* 674 * Set up PAT for boot processor. 675 */ 676 pat_init(); 677#endif 678 679 /* 680 * Free lowmem pages and complete other setup 681 */ 682 pmap_lowmem_finalize(); 683} 684 685/* 686 * Halt a cpu. 687 */ 688void 689halt_cpu(void) 690{ 691 halt_all_cpus(FALSE); 692} 693 694int reset_mem_on_reboot = 1; 695 696/* 697 * Halt the system or reboot. 698 */ 699void 700halt_all_cpus(boolean_t reboot) 701{ 702 if (reboot) { 703 printf("MACH Reboot\n"); 704 PEHaltRestart( kPERestartCPU ); 705 } else { 706 printf("CPU halted\n"); 707 PEHaltRestart( kPEHaltCPU ); 708 } 709 while(1); 710} 711 712 713/* Issue an I/O port read if one has been requested - this is an event logic 714 * analyzers can use as a trigger point. 715 */ 716 717void 718panic_io_port_read(void) { 719 if (panic_io_port) 720 (void)inb(panic_io_port); 721} 722 723/* For use with the MP rendezvous mechanism 724 */ 725 726uint64_t panic_restart_timeout = ~(0ULL); 727 728#define PANIC_RESTART_TIMEOUT (3ULL * NSEC_PER_SEC) 729 730static void 731machine_halt_cpu(void) { 732 uint64_t deadline; 733 734 panic_io_port_read(); 735 736 /* Halt here forever if we're not rebooting */ 737 if (!PE_reboot_on_panic() && panic_restart_timeout == ~(0ULL)) { 738 pmCPUHalt(PM_HALT_DEBUG); 739 return; 740 } 741 742 if (PE_reboot_on_panic()) 743 deadline = mach_absolute_time() + PANIC_RESTART_TIMEOUT; 744 else 745 deadline = mach_absolute_time() + panic_restart_timeout; 746 747 while (mach_absolute_time() < deadline) 748 cpu_pause(); 749 750 kprintf("Invoking PE_halt_restart\n"); 751 /* Attempt restart via ACPI RESET_REG; at the time of this 752 * writing, this is routine is chained through AppleSMC-> 753 * AppleACPIPlatform 754 */ 755 if (PE_halt_restart) 756 (*PE_halt_restart)(kPERestartCPU); 757 pmCPUHalt(PM_HALT_DEBUG); 758} 759 760void 761DebuggerWithContext( 762 __unused unsigned int reason, 763 __unused void *ctx, 764 const char *message) 765{ 766 Debugger(message); 767} 768 769void 770Debugger( 771 const char *message) 772{ 773 unsigned long pi_size = 0; 774 void *stackptr; 775 int cn = cpu_number(); 776 777 hw_atomic_add(&debug_mode, 1); 778 if (!panic_is_inited) { 779 postcode(PANIC_HLT); 780 asm("hlt"); 781 } 782 783 printf("Debugger called: <%s>\n", message); 784 kprintf("Debugger called: <%s>\n", message); 785 786 /* 787 * Skip the graphical panic box if no panic string. 788 * This is the case if we're being called from 789 * host_reboot(,HOST_REBOOT_DEBUGGER) 790 * as a quiet way into the debugger. 791 */ 792 793 if (panicstr) { 794 disable_preemption(); 795 796/* Issue an I/O port read if one has been requested - this is an event logic 797 * analyzers can use as a trigger point. 798 */ 799 panic_io_port_read(); 800 801 /* Obtain current frame pointer */ 802 __asm__ volatile("movq %%rbp, %0" : "=m" (stackptr)); 803 804 /* Print backtrace - callee is internally synchronized */ 805 panic_i386_backtrace(stackptr, ((panic_double_fault_cpu == cn) ? 80: 48), NULL, FALSE, NULL); 806 807 /* everything should be printed now so copy to NVRAM 808 */ 809 810 if( debug_buf_size > 0) { 811 /* Optionally sync the panic log, if any, to NVRAM 812 * This is the default. 813 */ 814 if (commit_paniclog_to_nvram) { 815 unsigned int bufpos; 816 uintptr_t cr0; 817 818 debug_putc(0); 819 820 /* Now call the compressor */ 821 /* XXX Consider using the WKdm compressor in the 822 * future, rather than just packing - would need to 823 * be co-ordinated with crashreporter, which decodes 824 * this post-restart. The compressor should be 825 * capable of in-place compression. 826 */ 827 bufpos = packA(debug_buf, 828 (unsigned int) (debug_buf_ptr - debug_buf), debug_buf_size); 829 /* If compression was successful, 830 * use the compressed length 831 */ 832 pi_size = bufpos ? bufpos : (unsigned) (debug_buf_ptr - debug_buf); 833 834 /* Save panic log to non-volatile store 835 * Panic info handler must truncate data that is 836 * too long for this platform. 837 * This call must save data synchronously, 838 * since we can subsequently halt the system. 839 */ 840 841 842/* The following sequence is a workaround for: 843 * <rdar://problem/5915669> SnowLeopard10A67: AppleEFINVRAM should not invoke 844 * any routines that use floating point (MMX in this case) when saving panic 845 * logs to nvram/flash. 846 */ 847 cr0 = get_cr0(); 848 clear_ts(); 849 850 kprintf("Attempting to commit panic log to NVRAM\n"); 851 pi_size = PESavePanicInfo((unsigned char *)debug_buf, 852 (uint32_t)pi_size ); 853 set_cr0(cr0); 854 855 /* Uncompress in-place, to permit examination of 856 * the panic log by debuggers. 857 */ 858 859 if (bufpos) { 860 unpackA(debug_buf, bufpos); 861 } 862 } 863 } 864 865 if (!panicDebugging) { 866 unsigned cnum; 867 /* Clear the MP rendezvous function lock, in the event 868 * that a panic occurred while in that codepath. 869 */ 870 mp_rendezvous_break_lock(); 871 872 /* Non-maskably interrupt all other processors 873 * If a restart timeout is specified, this processor 874 * will attempt a restart. 875 */ 876 kprintf("Invoking machine_halt_cpu on CPU %d\n", cn); 877 for (cnum = 0; cnum < real_ncpus; cnum++) { 878 if (cnum != (unsigned) cn) { 879 cpu_NMI_interrupt(cnum); 880 } 881 } 882 machine_halt_cpu(); 883 /* NOT REACHED */ 884 } 885 } 886 887 __asm__("int3"); 888 hw_atomic_sub(&debug_mode, 1); 889} 890 891char * 892machine_boot_info(char *buf, __unused vm_size_t size) 893{ 894 *buf ='\0'; 895 return buf; 896} 897 898/* Routines for address - symbol translation. Not called unless the "keepsyms" 899 * boot-arg is supplied. 900 */ 901 902static int 903panic_print_macho_symbol_name(kernel_mach_header_t *mh, vm_address_t search, const char *module_name) 904{ 905 kernel_nlist_t *sym = NULL; 906 struct load_command *cmd; 907 kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; 908 struct symtab_command *orig_st = NULL; 909 unsigned int i; 910 char *strings, *bestsym = NULL; 911 vm_address_t bestaddr = 0, diff, curdiff; 912 913 /* Assume that if it's loaded and linked into the kernel, it's a valid Mach-O */ 914 915 cmd = (struct load_command *) &mh[1]; 916 for (i = 0; i < mh->ncmds; i++) { 917 if (cmd->cmd == LC_SEGMENT_KERNEL) { 918 kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; 919 920 if (strncmp(SEG_TEXT, orig_sg->segname, 921 sizeof(orig_sg->segname)) == 0) 922 orig_ts = orig_sg; 923 else if (strncmp(SEG_LINKEDIT, orig_sg->segname, 924 sizeof(orig_sg->segname)) == 0) 925 orig_le = orig_sg; 926 else if (strncmp("", orig_sg->segname, 927 sizeof(orig_sg->segname)) == 0) 928 orig_ts = orig_sg; /* pre-Lion i386 kexts have a single unnamed segment */ 929 } 930 else if (cmd->cmd == LC_SYMTAB) 931 orig_st = (struct symtab_command *) cmd; 932 933 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize); 934 } 935 936 if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) 937 return 0; 938 939 if ((search < orig_ts->vmaddr) || 940 (search >= orig_ts->vmaddr + orig_ts->vmsize)) { 941 /* search out of range for this mach header */ 942 return 0; 943 } 944 945 sym = (kernel_nlist_t *)(uintptr_t)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); 946 strings = (char *)(uintptr_t)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); 947 diff = search; 948 949 for (i = 0; i < orig_st->nsyms; i++) { 950 if (sym[i].n_type & N_STAB) continue; 951 952 if (sym[i].n_value <= search) { 953 curdiff = search - (vm_address_t)sym[i].n_value; 954 if (curdiff < diff) { 955 diff = curdiff; 956 bestaddr = sym[i].n_value; 957 bestsym = strings + sym[i].n_un.n_strx; 958 } 959 } 960 } 961 962 if (bestsym != NULL) { 963 if (diff != 0) { 964 kdb_printf("%s : %s + 0x%lx", module_name, bestsym, (unsigned long)diff); 965 } else { 966 kdb_printf("%s : %s", module_name, bestsym); 967 } 968 return 1; 969 } 970 return 0; 971} 972 973extern kmod_info_t * kmod; /* the list of modules */ 974 975static void 976panic_print_kmod_symbol_name(vm_address_t search) 977{ 978 u_int i; 979 980 if (gLoadedKextSummaries == NULL) 981 return; 982 for (i = 0; i < gLoadedKextSummaries->numSummaries; ++i) { 983 OSKextLoadedKextSummary *summary = gLoadedKextSummaries->summaries + i; 984 985 if ((search >= summary->address) && 986 (search < (summary->address + summary->size))) 987 { 988 kernel_mach_header_t *header = (kernel_mach_header_t *)(uintptr_t) summary->address; 989 if (panic_print_macho_symbol_name(header, search, summary->name) == 0) { 990 kdb_printf("%s + %llu", summary->name, (unsigned long)search - summary->address); 991 } 992 break; 993 } 994 } 995} 996 997static void 998panic_print_symbol_name(vm_address_t search) 999{ 1000 /* try searching in the kernel */ 1001 if (panic_print_macho_symbol_name(&_mh_execute_header, search, "mach_kernel") == 0) { 1002 /* that failed, now try to search for the right kext */ 1003 panic_print_kmod_symbol_name(search); 1004 } 1005} 1006 1007/* Generate a backtrace, given a frame pointer - this routine 1008 * should walk the stack safely. The trace is appended to the panic log 1009 * and conditionally, to the console. If the trace contains kernel module 1010 * addresses, display the module name, load address and dependencies. 1011 */ 1012 1013#define DUMPFRAMES 32 1014#define PBT_TIMEOUT_CYCLES (5 * 1000 * 1000 * 1000ULL) 1015void 1016panic_i386_backtrace(void *_frame, int nframes, const char *msg, boolean_t regdump, x86_saved_state_t *regs) 1017{ 1018 cframe_t *frame = (cframe_t *)_frame; 1019 vm_offset_t raddrs[DUMPFRAMES]; 1020 vm_offset_t PC = 0; 1021 int frame_index; 1022 volatile uint32_t *ppbtcnt = &pbtcnt; 1023 uint64_t bt_tsc_timeout; 1024 boolean_t keepsyms = FALSE; 1025 int cn = cpu_number(); 1026 1027 if(pbtcpu != cn) { 1028 hw_atomic_add(&pbtcnt, 1); 1029 /* Spin on print backtrace lock, which serializes output 1030 * Continue anyway if a timeout occurs. 1031 */ 1032 hw_lock_to(&pbtlock, ~0U); 1033 pbtcpu = cn; 1034 } 1035 1036 panic_check_hook(); 1037 1038 PE_parse_boot_argn("keepsyms", &keepsyms, sizeof (keepsyms)); 1039 1040 if (msg != NULL) { 1041 kdb_printf("%s", msg); 1042 } 1043 1044 if ((regdump == TRUE) && (regs != NULL)) { 1045 x86_saved_state64_t *ss64p = saved_state64(regs); 1046 kdb_printf( 1047 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n" 1048 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n" 1049 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n" 1050 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n" 1051 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n", 1052 ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx, 1053 ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi, 1054 ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11, 1055 ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15, 1056 ss64p->isf.rflags, ss64p->isf.rip, ss64p->isf.cs, 1057 ss64p->isf.ss); 1058 PC = ss64p->isf.rip; 1059 } 1060 1061 kdb_printf("Backtrace (CPU %d), " 1062#if PRINT_ARGS_FROM_STACK_FRAME 1063 "Frame : Return Address (4 potential args on stack)\n", cn); 1064#else 1065 "Frame : Return Address\n", cn); 1066#endif 1067 1068 for (frame_index = 0; frame_index < nframes; frame_index++) { 1069 vm_offset_t curframep = (vm_offset_t) frame; 1070 1071 if (!curframep) 1072 break; 1073 1074 if (curframep & 0x3) { 1075 kdb_printf("Unaligned frame\n"); 1076 goto invalid; 1077 } 1078 1079 if (!kvtophys(curframep) || 1080 !kvtophys(curframep + sizeof(cframe_t) - 1)) { 1081 kdb_printf("No mapping exists for frame pointer\n"); 1082 goto invalid; 1083 } 1084 1085 kdb_printf("%p : 0x%lx ", frame, frame->caller); 1086 if (frame_index < DUMPFRAMES) 1087 raddrs[frame_index] = frame->caller; 1088 1089#if PRINT_ARGS_FROM_STACK_FRAME 1090 if (kvtophys((vm_offset_t)&(frame->args[3]))) 1091 kdb_printf("(0x%x 0x%x 0x%x 0x%x) ", 1092 frame->args[0], frame->args[1], 1093 frame->args[2], frame->args[3]); 1094#endif 1095 1096 /* Display address-symbol translation only if the "keepsyms" 1097 * boot-arg is suppplied, since we unload LINKEDIT otherwise. 1098 * This routine is potentially unsafe; also, function 1099 * boundary identification is unreliable after a strip -x. 1100 */ 1101 if (keepsyms) 1102 panic_print_symbol_name((vm_address_t)frame->caller); 1103 1104 kdb_printf("\n"); 1105 1106 frame = frame->prev; 1107 } 1108 1109 if (frame_index >= nframes) 1110 kdb_printf("\tBacktrace continues...\n"); 1111 1112 goto out; 1113 1114invalid: 1115 kdb_printf("Backtrace terminated-invalid frame pointer %p\n",frame); 1116out: 1117 1118 /* Identify kernel modules in the backtrace and display their 1119 * load addresses and dependencies. This routine should walk 1120 * the kmod list safely. 1121 */ 1122 if (frame_index) 1123 kmod_panic_dump((vm_offset_t *)&raddrs[0], frame_index); 1124 1125 if (PC != 0) 1126 kmod_panic_dump(&PC, 1); 1127 1128 panic_display_system_configuration(); 1129 1130 /* Release print backtrace lock, to permit other callers in the 1131 * event of panics on multiple processors. 1132 */ 1133 hw_lock_unlock(&pbtlock); 1134 hw_atomic_sub(&pbtcnt, 1); 1135 /* Wait for other processors to complete output 1136 * Timeout and continue after PBT_TIMEOUT_CYCLES. 1137 */ 1138 bt_tsc_timeout = rdtsc64() + PBT_TIMEOUT_CYCLES; 1139 while(*ppbtcnt && (rdtsc64() < bt_tsc_timeout)); 1140} 1141