1/*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
49 *  School of Computer Science
50 *  Carnegie Mellon University
51 *  Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57/*
58 */
59
60/*
61 *	File:	model_dep.c
62 *	Author:	Avadis Tevanian, Jr., Michael Wayne Young
63 *
64 *	Copyright (C) 1986, Avadis Tevanian, Jr., Michael Wayne Young
65 *
66 *	Basic initialization for I386 - ISA bus machines.
67 */
68
69#include <platforms.h>
70
71#include <mach/i386/vm_param.h>
72
73#include <string.h>
74#include <mach/vm_param.h>
75#include <mach/vm_prot.h>
76#include <mach/machine.h>
77#include <mach/time_value.h>
78#include <kern/spl.h>
79#include <kern/assert.h>
80#include <kern/debug.h>
81#include <kern/misc_protos.h>
82#include <kern/startup.h>
83#include <kern/clock.h>
84#include <kern/cpu_data.h>
85#include <kern/machine.h>
86#include <i386/postcode.h>
87#include <i386/mp_desc.h>
88#include <i386/misc_protos.h>
89#include <i386/thread.h>
90#include <i386/trap.h>
91#include <i386/machine_routines.h>
92#include <i386/mp.h>		/* mp_rendezvous_break_lock */
93#include <i386/cpuid.h>
94#include <i386/fpu.h>
95#include <i386/machine_cpu.h>
96#include <i386/pmap.h>
97#if CONFIG_MTRR
98#include <i386/mtrr.h>
99#endif
100#include <i386/ucode.h>
101#include <i386/pmCPU.h>
102#include <architecture/i386/pio.h> /* inb() */
103#include <pexpert/i386/boot.h>
104
105#include <vm/pmap.h>
106#include <vm/vm_map.h>
107#include <vm/vm_kern.h>
108
109#include <IOKit/IOPlatformExpert.h>
110#include <IOKit/IOHibernatePrivate.h>
111
112#include <pexpert/i386/efi.h>
113
114#include <kern/thread.h>
115#include <kern/sched.h>
116#include <mach-o/loader.h>
117#include <mach-o/nlist.h>
118
119#include <libkern/kernel_mach_header.h>
120#include <libkern/OSKextLibPrivate.h>
121
122#if	DEBUG
123#define DPRINTF(x...)	kprintf(x)
124#else
125#define DPRINTF(x...)
126#endif
127
128static void machine_conf(void);
129
130extern int		max_unsafe_quanta;
131extern int		max_poll_quanta;
132extern unsigned int	panic_is_inited;
133
134int db_run_mode;
135
136volatile int pbtcpu = -1;
137hw_lock_data_t pbtlock;		/* backtrace print lock */
138uint32_t pbtcnt = 0;
139
140volatile int panic_double_fault_cpu = -1;
141
142#define PRINT_ARGS_FROM_STACK_FRAME	0
143
144typedef struct _cframe_t {
145    struct _cframe_t	*prev;
146    uintptr_t		caller;
147#if PRINT_ARGS_FROM_STACK_FRAME
148    unsigned		args[0];
149#endif
150} cframe_t;
151
152static unsigned panic_io_port;
153static unsigned	commit_paniclog_to_nvram;
154
155unsigned int debug_boot_arg;
156
157void
158machine_startup(void)
159{
160	int	boot_arg;
161
162#if 0
163	if( PE_get_hotkey( kPEControlKey ))
164            halt_in_debugger = halt_in_debugger ? 0 : 1;
165#endif
166
167	if (PE_parse_boot_argn("debug", &debug_boot_arg, sizeof (debug_boot_arg))) {
168		panicDebugging = TRUE;
169		if (debug_boot_arg & DB_HALT) halt_in_debugger=1;
170		if (debug_boot_arg & DB_PRT) disable_debug_output=FALSE;
171		if (debug_boot_arg & DB_SLOG) systemLogDiags=TRUE;
172		if (debug_boot_arg & DB_LOG_PI_SCRN) logPanicDataToScreen=TRUE;
173	} else {
174		debug_boot_arg = 0;
175	}
176
177	if (!PE_parse_boot_argn("nvram_paniclog", &commit_paniclog_to_nvram, sizeof (commit_paniclog_to_nvram)))
178		commit_paniclog_to_nvram = 1;
179
180	/*
181	 * Entering the debugger will put the CPUs into a "safe"
182	 * power mode.
183	 */
184	if (PE_parse_boot_argn("pmsafe_debug", &boot_arg, sizeof (boot_arg)))
185	    pmsafe_debug = boot_arg;
186
187#if NOTYET
188	hw_lock_init(&debugger_lock);	/* initialize debugger lock */
189#endif
190	hw_lock_init(&pbtlock);		/* initialize print backtrace lock */
191
192	if (PE_parse_boot_argn("preempt", &boot_arg, sizeof (boot_arg))) {
193		default_preemption_rate = boot_arg;
194	}
195	if (PE_parse_boot_argn("unsafe", &boot_arg, sizeof (boot_arg))) {
196		max_unsafe_quanta = boot_arg;
197	}
198	if (PE_parse_boot_argn("poll", &boot_arg, sizeof (boot_arg))) {
199		max_poll_quanta = boot_arg;
200	}
201	if (PE_parse_boot_argn("yield", &boot_arg, sizeof (boot_arg))) {
202		sched_poll_yield_shift = boot_arg;
203	}
204/* The I/O port to issue a read from, in the event of a panic. Useful for
205 * triggering logic analyzers.
206 */
207	if (PE_parse_boot_argn("panic_io_port", &boot_arg, sizeof (boot_arg))) {
208		/*I/O ports range from 0 through 0xFFFF */
209		panic_io_port = boot_arg & 0xffff;
210	}
211
212	machine_conf();
213
214	/*
215	 * Start the system.
216	 */
217	kernel_bootstrap();
218	/*NOTREACHED*/
219}
220
221
222static void
223machine_conf(void)
224{
225	machine_info.memory_size = (typeof(machine_info.memory_size))mem_size;
226}
227
228
229extern void *gPEEFIRuntimeServices;
230extern void *gPEEFISystemTable;
231
232/*-
233 *  COPYRIGHT (C) 1986 Gary S. Brown.  You may use this program, or
234 *  code or tables extracted from it, as desired without restriction.
235 *
236 *  First, the polynomial itself and its table of feedback terms.  The
237 *  polynomial is
238 *  X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0
239 *
240 *  Note that we take it "backwards" and put the highest-order term in
241 *  the lowest-order bit.  The X^32 term is "implied"; the LSB is the
242 *  X^31 term, etc.  The X^0 term (usually shown as "+1") results in
243 *  the MSB being 1
244 *
245 *  Note that the usual hardware shift register implementation, which
246 *  is what we're using (we're merely optimizing it by doing eight-bit
247 *  chunks at a time) shifts bits into the lowest-order term.  In our
248 *  implementation, that means shifting towards the right.  Why do we
249 *  do it this way?  Because the calculated CRC must be transmitted in
250 *  order from highest-order term to lowest-order term.  UARTs transmit
251 *  characters in order from LSB to MSB.  By storing the CRC this way
252 *  we hand it to the UART in the order low-byte to high-byte; the UART
253 *  sends each low-bit to hight-bit; and the result is transmission bit
254 *  by bit from highest- to lowest-order term without requiring any bit
255 *  shuffling on our part.  Reception works similarly
256 *
257 *  The feedback terms table consists of 256, 32-bit entries.  Notes
258 *
259 *      The table can be generated at runtime if desired; code to do so
260 *      is shown later.  It might not be obvious, but the feedback
261 *      terms simply represent the results of eight shift/xor opera
262 *      tions for all combinations of data and CRC register values
263 *
264 *      The values must be right-shifted by eight bits by the "updcrc
265 *      logic; the shift must be unsigned (bring in zeroes).  On some
266 *      hardware you could probably optimize the shift in assembler by
267 *      using byte-swap instructions
268 *      polynomial $edb88320
269 *
270 *
271 * CRC32 code derived from work by Gary S. Brown.
272 */
273
274static uint32_t crc32_tab[] = {
275	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
276	0xe963a535, 0x9e6495a3,	0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
277	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
278	0xf3b97148, 0x84be41de,	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
279	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,	0x14015c4f, 0x63066cd9,
280	0xfa0f3d63, 0x8d080df5,	0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
281	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,	0x35b5a8fa, 0x42b2986c,
282	0xdbbbc9d6, 0xacbcf940,	0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
283	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
284	0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
285	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,	0x76dc4190, 0x01db7106,
286	0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
287	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
288	0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
289	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
290	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
291	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
292	0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
293	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
294	0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
295	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
296	0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
297	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
298	0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
299	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
300	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
301	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
302	0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
303	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
304	0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
305	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
306	0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
307	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
308	0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
309	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
310	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
311	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
312	0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
313	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
314	0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
315	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
316	0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
317	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
318};
319
320static uint32_t
321crc32(uint32_t crc, const void *buf, size_t size)
322{
323	const uint8_t *p;
324
325	p = buf;
326	crc = crc ^ ~0U;
327
328	while (size--)
329		crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
330
331	return crc ^ ~0U;
332}
333
334static void
335efi_set_tables_64(EFI_SYSTEM_TABLE_64 * system_table)
336{
337    EFI_RUNTIME_SERVICES_64 *runtime;
338    uint32_t hdr_cksum;
339    uint32_t cksum;
340
341    DPRINTF("Processing 64-bit EFI tables at %p\n", system_table);
342    do {
343	DPRINTF("Header:\n");
344	DPRINTF("  Signature:   0x%016llx\n", system_table->Hdr.Signature);
345	DPRINTF("  Revision:    0x%08x\n", system_table->Hdr.Revision);
346	DPRINTF("  HeaderSize:  0x%08x\n", system_table->Hdr.HeaderSize);
347	DPRINTF("  CRC32:       0x%08x\n", system_table->Hdr.CRC32);
348	DPRINTF("RuntimeServices: 0x%016llx\n", system_table->RuntimeServices);
349        if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) {
350	    kprintf("Bad EFI system table signature\n");
351            break;
352        }
353        // Verify signature of the system table
354        hdr_cksum = system_table->Hdr.CRC32;
355        system_table->Hdr.CRC32 = 0;
356        cksum = crc32(0L, system_table, system_table->Hdr.HeaderSize);
357
358        DPRINTF("System table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
359        system_table->Hdr.CRC32 = hdr_cksum;
360        if (cksum != hdr_cksum) {
361            kprintf("Bad EFI system table checksum\n");
362            break;
363        }
364
365        gPEEFISystemTable     = system_table;
366
367        if(system_table->RuntimeServices == 0) {
368            kprintf("No runtime table present\n");
369            break;
370        }
371        DPRINTF("RuntimeServices table at 0x%qx\n", system_table->RuntimeServices);
372        // 64-bit virtual address is OK for 64-bit EFI and 64/32-bit kernel.
373        runtime = (EFI_RUNTIME_SERVICES_64 *) (uintptr_t)system_table->RuntimeServices;
374        DPRINTF("Checking runtime services table %p\n", runtime);
375        if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) {
376            kprintf("Bad EFI runtime table signature\n");
377            break;
378        }
379
380	// Verify signature of runtime services table
381	hdr_cksum = runtime->Hdr.CRC32;
382	runtime->Hdr.CRC32 = 0;
383	cksum = crc32(0L, runtime, runtime->Hdr.HeaderSize);
384
385	DPRINTF("Runtime table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
386	runtime->Hdr.CRC32 = hdr_cksum;
387	if (cksum != hdr_cksum) {
388	    kprintf("Bad EFI runtime table checksum\n");
389	    break;
390	}
391
392	gPEEFIRuntimeServices = runtime;
393    }
394    while (FALSE);
395}
396
397static void
398efi_set_tables_32(EFI_SYSTEM_TABLE_32 * system_table)
399{
400    EFI_RUNTIME_SERVICES_32 *runtime;
401    uint32_t hdr_cksum;
402    uint32_t cksum;
403
404    DPRINTF("Processing 32-bit EFI tables at %p\n", system_table);
405    do {
406	DPRINTF("Header:\n");
407	DPRINTF("  Signature:   0x%016llx\n", system_table->Hdr.Signature);
408	DPRINTF("  Revision:    0x%08x\n", system_table->Hdr.Revision);
409	DPRINTF("  HeaderSize:  0x%08x\n", system_table->Hdr.HeaderSize);
410	DPRINTF("  CRC32:       0x%08x\n", system_table->Hdr.CRC32);
411	DPRINTF("RuntimeServices: 0x%08x\n", system_table->RuntimeServices);
412        if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) {
413            kprintf("Bad EFI system table signature\n");
414            break;
415        }
416        // Verify signature of the system table
417        hdr_cksum = system_table->Hdr.CRC32;
418        system_table->Hdr.CRC32 = 0;
419        DPRINTF("System table at %p HeaderSize 0x%x\n", system_table, system_table->Hdr.HeaderSize);
420        cksum = crc32(0L, system_table, system_table->Hdr.HeaderSize);
421
422        DPRINTF("System table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
423        system_table->Hdr.CRC32 = hdr_cksum;
424        if (cksum != hdr_cksum) {
425            kprintf("Bad EFI system table checksum\n");
426            break;
427        }
428
429        gPEEFISystemTable     = system_table;
430
431        if(system_table->RuntimeServices == 0) {
432            kprintf("No runtime table present\n");
433            break;
434        }
435        DPRINTF("RuntimeServices table at 0x%x\n", system_table->RuntimeServices);
436        // 32-bit virtual address is OK for 32-bit EFI and 32-bit kernel.
437        // For a 64-bit kernel, booter provides a virtual address mod 4G
438        runtime = (EFI_RUNTIME_SERVICES_32 *)
439			(system_table->RuntimeServices | VM_MIN_KERNEL_ADDRESS);
440	DPRINTF("Runtime table addressed at %p\n", runtime);
441        if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) {
442            kprintf("Bad EFI runtime table signature\n");
443            break;
444        }
445
446	// Verify signature of runtime services table
447	hdr_cksum = runtime->Hdr.CRC32;
448	runtime->Hdr.CRC32 = 0;
449	cksum = crc32(0L, runtime, runtime->Hdr.HeaderSize);
450
451	DPRINTF("Runtime table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
452	runtime->Hdr.CRC32 = hdr_cksum;
453	if (cksum != hdr_cksum) {
454	    kprintf("Bad EFI runtime table checksum\n");
455	    break;
456	}
457
458	DPRINTF("Runtime functions\n");
459	DPRINTF("  GetTime                  : 0x%x\n", runtime->GetTime);
460	DPRINTF("  SetTime                  : 0x%x\n", runtime->SetTime);
461	DPRINTF("  GetWakeupTime            : 0x%x\n", runtime->GetWakeupTime);
462	DPRINTF("  SetWakeupTime            : 0x%x\n", runtime->SetWakeupTime);
463	DPRINTF("  SetVirtualAddressMap     : 0x%x\n", runtime->SetVirtualAddressMap);
464	DPRINTF("  ConvertPointer           : 0x%x\n", runtime->ConvertPointer);
465	DPRINTF("  GetVariable              : 0x%x\n", runtime->GetVariable);
466	DPRINTF("  GetNextVariableName      : 0x%x\n", runtime->GetNextVariableName);
467	DPRINTF("  SetVariable              : 0x%x\n", runtime->SetVariable);
468	DPRINTF("  GetNextHighMonotonicCount: 0x%x\n", runtime->GetNextHighMonotonicCount);
469	DPRINTF("  ResetSystem              : 0x%x\n", runtime->ResetSystem);
470
471	gPEEFIRuntimeServices = runtime;
472    }
473    while (FALSE);
474}
475
476
477/* Map in EFI runtime areas. */
478static void
479efi_init(void)
480{
481    boot_args *args = (boot_args *)PE_state.bootArgs;
482
483    kprintf("Initializing EFI runtime services\n");
484
485    do
486    {
487	vm_offset_t vm_size, vm_addr;
488	vm_map_offset_t phys_addr;
489	EfiMemoryRange *mptr;
490	unsigned int msize, mcount;
491	unsigned int i;
492
493	msize = args->MemoryMapDescriptorSize;
494	mcount = args->MemoryMapSize / msize;
495
496	DPRINTF("efi_init() kernel base: 0x%x size: 0x%x\n",
497		args->kaddr, args->ksize);
498	DPRINTF("           efiSystemTable physical: 0x%x virtual: %p\n",
499		args->efiSystemTable,
500		(void *) ml_static_ptovirt(args->efiSystemTable));
501	DPRINTF("           efiRuntimeServicesPageStart: 0x%x\n",
502		args->efiRuntimeServicesPageStart);
503	DPRINTF("           efiRuntimeServicesPageCount: 0x%x\n",
504		args->efiRuntimeServicesPageCount);
505	DPRINTF("           efiRuntimeServicesVirtualPageStart: 0x%016llx\n",
506		args->efiRuntimeServicesVirtualPageStart);
507	mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap);
508	for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
509	    if (((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) ) {
510		vm_size = (vm_offset_t)i386_ptob((uint32_t)mptr->NumberOfPages);
511		vm_addr =   (vm_offset_t) mptr->VirtualStart;
512		/* For K64 on EFI32, shadow-map into high KVA */
513		if (vm_addr < VM_MIN_KERNEL_ADDRESS)
514			vm_addr |= VM_MIN_KERNEL_ADDRESS;
515		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
516		DPRINTF(" Type: %x phys: %p EFIv: %p kv: %p size: %p\n",
517			mptr->Type,
518			(void *) (uintptr_t) phys_addr,
519			(void *) (uintptr_t) mptr->VirtualStart,
520			(void *) vm_addr,
521			(void *) vm_size);
522		pmap_map_bd(vm_addr, phys_addr, phys_addr + round_page(vm_size),
523		     (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE,
524		     (mptr->Type == EfiMemoryMappedIO)       ? VM_WIMG_IO   : VM_WIMG_USE_DEFAULT);
525	    }
526	}
527
528        if (args->Version != kBootArgsVersion2)
529            panic("Incompatible boot args version %d revision %d\n", args->Version, args->Revision);
530
531	DPRINTF("Boot args version %d revision %d mode %d\n", args->Version, args->Revision, args->efiMode);
532        if (args->efiMode == kBootArgsEfiMode64) {
533            efi_set_tables_64((EFI_SYSTEM_TABLE_64 *) ml_static_ptovirt(args->efiSystemTable));
534        } else {
535            efi_set_tables_32((EFI_SYSTEM_TABLE_32 *) ml_static_ptovirt(args->efiSystemTable));
536        }
537    }
538    while (FALSE);
539
540    return;
541}
542
543/* Remap EFI runtime areas. */
544void
545hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_offset)
546{
547    boot_args *args = (boot_args *)PE_state.bootArgs;
548
549    kprintf("Reinitializing EFI runtime services\n");
550
551    do
552    {
553        vm_offset_t vm_size, vm_addr;
554	vm_map_offset_t phys_addr;
555	EfiMemoryRange *mptr;
556	unsigned int msize, mcount;
557	unsigned int i;
558
559	gPEEFISystemTable     = 0;
560	gPEEFIRuntimeServices = 0;
561
562	system_table_offset += ptoa_32(args->efiRuntimeServicesPageStart);
563
564	kprintf("Old system table 0x%x, new 0x%x\n",
565	    (uint32_t)args->efiSystemTable,    system_table_offset);
566
567	args->efiSystemTable    = system_table_offset;
568
569	kprintf("Old map:\n");
570	msize = args->MemoryMapDescriptorSize;
571	mcount = args->MemoryMapSize / msize;
572	mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap);
573	for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
574	    if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) {
575
576		vm_size = (vm_offset_t)i386_ptob((uint32_t)mptr->NumberOfPages);
577		vm_addr =   (vm_offset_t) mptr->VirtualStart;
578		/* K64 on EFI32 */
579		if (vm_addr < VM_MIN_KERNEL_ADDRESS)
580			vm_addr |= VM_MIN_KERNEL_ADDRESS;
581		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
582
583		kprintf("mapping[%u] %qx @ %lx, %llu\n", mptr->Type, phys_addr, (unsigned long)vm_addr, mptr->NumberOfPages);
584	    }
585	}
586
587	pmap_remove(kernel_pmap, i386_ptob(args->efiRuntimeServicesPageStart),
588				 i386_ptob(args->efiRuntimeServicesPageStart + args->efiRuntimeServicesPageCount));
589
590	kprintf("New map:\n");
591	msize = args->MemoryMapDescriptorSize;
592	mcount = (unsigned int )(map_size / msize);
593	mptr = map;
594	for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
595	    if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) {
596
597		vm_size = (vm_offset_t)i386_ptob((uint32_t)mptr->NumberOfPages);
598		vm_addr =   (vm_offset_t) mptr->VirtualStart;
599		if (vm_addr < VM_MIN_KERNEL_ADDRESS)
600			vm_addr |= VM_MIN_KERNEL_ADDRESS;
601		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
602
603		kprintf("mapping[%u] %qx @ %lx, %llu\n", mptr->Type, phys_addr, (unsigned long)vm_addr, mptr->NumberOfPages);
604
605		pmap_map(vm_addr, phys_addr, phys_addr + round_page(vm_size),
606			 (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE,
607			 (mptr->Type == EfiMemoryMappedIO)       ? VM_WIMG_IO   : VM_WIMG_USE_DEFAULT);
608	    }
609	}
610
611        if (args->Version != kBootArgsVersion2)
612            panic("Incompatible boot args version %d revision %d\n", args->Version, args->Revision);
613
614        kprintf("Boot args version %d revision %d mode %d\n", args->Version, args->Revision, args->efiMode);
615        if (args->efiMode == kBootArgsEfiMode64) {
616	    efi_set_tables_64((EFI_SYSTEM_TABLE_64 *) ml_static_ptovirt(args->efiSystemTable));
617        } else {
618	    efi_set_tables_32((EFI_SYSTEM_TABLE_32 *) ml_static_ptovirt(args->efiSystemTable));
619        }
620    }
621    while (FALSE);
622
623    kprintf("Done reinitializing EFI runtime services\n");
624
625    return;
626}
627
628/*
629 * Find devices.  The system is alive.
630 */
631void
632machine_init(void)
633{
634	/* Now with VM up, switch to dynamically allocated cpu data */
635	cpu_data_realloc();
636
637        /* Ensure panic buffer is initialized. */
638        debug_log_init();
639
640	/*
641	 * Display CPU identification
642	 */
643	cpuid_cpu_display("CPU identification");
644	cpuid_feature_display("CPU features");
645	cpuid_extfeature_display("CPU extended features");
646
647        /*
648         * Initialize EFI runtime services.
649         */
650        efi_init();
651
652	smp_init();
653
654	/*
655	 * Set up to use floating point.
656	 */
657	init_fpu();
658
659	/*
660	 * Configure clock devices.
661	 */
662	clock_config();
663
664#if CONFIG_MTRR
665	/*
666	 * Initialize MTRR from boot processor.
667	 */
668	mtrr_init();
669
670	/*
671	 * Set up PAT for boot processor.
672	 */
673	pat_init();
674#endif
675
676	/*
677	 * Free lowmem pages and complete other setup
678	 */
679	pmap_lowmem_finalize();
680}
681
682/*
683 * Halt a cpu.
684 */
685void
686halt_cpu(void)
687{
688	halt_all_cpus(FALSE);
689}
690
691int reset_mem_on_reboot = 1;
692
693/*
694 * Halt the system or reboot.
695 */
696void
697halt_all_cpus(boolean_t reboot)
698{
699	if (reboot) {
700		printf("MACH Reboot\n");
701		PEHaltRestart( kPERestartCPU );
702	} else {
703		printf("CPU halted\n");
704		PEHaltRestart( kPEHaltCPU );
705	}
706	while(1);
707}
708
709
710/* Issue an I/O port read if one has been requested - this is an event logic
711 * analyzers can use as a trigger point.
712 */
713
714void
715panic_io_port_read(void) {
716	if (panic_io_port)
717		(void)inb(panic_io_port);
718}
719
720/* For use with the MP rendezvous mechanism
721 */
722
723uint64_t panic_restart_timeout = ~(0ULL);
724
725#define PANIC_RESTART_TIMEOUT (3ULL * NSEC_PER_SEC)
726
727static void
728machine_halt_cpu(void) {
729	uint64_t deadline;
730
731	panic_io_port_read();
732
733	/* Halt here forever if we're not rebooting */
734	if (!PE_reboot_on_panic() && panic_restart_timeout == ~(0ULL)) {
735		pmCPUHalt(PM_HALT_DEBUG);
736		return;
737	}
738
739	if (PE_reboot_on_panic())
740		deadline = mach_absolute_time() + PANIC_RESTART_TIMEOUT;
741	else
742		deadline = mach_absolute_time() + panic_restart_timeout;
743
744	while (mach_absolute_time() < deadline)
745		cpu_pause();
746
747	kprintf("Invoking PE_halt_restart\n");
748	/* Attempt restart via ACPI RESET_REG; at the time of this
749	 * writing, this is routine is chained through AppleSMC->
750	 * AppleACPIPlatform
751	 */
752	if (PE_halt_restart)
753		(*PE_halt_restart)(kPERestartCPU);
754	pmCPUHalt(PM_HALT_DEBUG);
755}
756
757void
758DebuggerWithContext(
759	__unused unsigned int	reason,
760	__unused void 		*ctx,
761	const char		*message)
762{
763	Debugger(message);
764}
765
766void
767Debugger(
768	const char	*message)
769{
770	unsigned long pi_size = 0;
771	void *stackptr;
772	int cn = cpu_number();
773
774	hw_atomic_add(&debug_mode, 1);
775	if (!panic_is_inited) {
776		postcode(PANIC_HLT);
777		asm("hlt");
778	}
779
780	printf("Debugger called: <%s>\n", message);
781	kprintf("Debugger called: <%s>\n", message);
782
783	/*
784	 * Skip the graphical panic box if no panic string.
785	 * This is the case if we're being called from
786	 *   host_reboot(,HOST_REBOOT_DEBUGGER)
787	 * as a quiet way into the debugger.
788	 */
789
790	if (panicstr) {
791		disable_preemption();
792
793/* Issue an I/O port read if one has been requested - this is an event logic
794 * analyzers can use as a trigger point.
795 */
796		panic_io_port_read();
797
798		/* Obtain current frame pointer */
799		__asm__ volatile("movq %%rbp, %0" : "=m" (stackptr));
800
801		/* Print backtrace - callee is internally synchronized */
802		panic_i386_backtrace(stackptr, ((panic_double_fault_cpu == cn) ? 80: 48), NULL, FALSE, NULL);
803
804		/* everything should be printed now so copy to NVRAM
805		 */
806
807		if( debug_buf_size > 0) {
808		  /* Optionally sync the panic log, if any, to NVRAM
809		   * This is the default.
810		   */
811		    if (commit_paniclog_to_nvram) {
812			unsigned int bufpos;
813			uintptr_t cr0;
814
815			debug_putc(0);
816
817			/* Now call the compressor */
818			/* XXX Consider using the WKdm compressor in the
819			 * future, rather than just packing - would need to
820			 * be co-ordinated with crashreporter, which decodes
821			 * this post-restart. The compressor should be
822			 * capable of in-place compression.
823			 */
824			bufpos = packA(debug_buf,
825			    (unsigned int) (debug_buf_ptr - debug_buf), debug_buf_size);
826			/* If compression was successful,
827			 * use the compressed length
828			 */
829			pi_size = bufpos ? bufpos : (unsigned) (debug_buf_ptr - debug_buf);
830
831			/* Save panic log to non-volatile store
832			 * Panic info handler must truncate data that is
833			 * too long for this platform.
834			 * This call must save data synchronously,
835			 * since we can subsequently halt the system.
836			 */
837
838
839/* The following sequence is a workaround for:
840 * <rdar://problem/5915669> SnowLeopard10A67: AppleEFINVRAM should not invoke
841 * any routines that use floating point (MMX in this case) when saving panic
842 * logs to nvram/flash.
843 */
844			cr0 = get_cr0();
845			clear_ts();
846
847			kprintf("Attempting to commit panic log to NVRAM\n");
848			pi_size = PESavePanicInfo((unsigned char *)debug_buf,
849					(uint32_t)pi_size );
850			set_cr0(cr0);
851
852			/* Uncompress in-place, to permit examination of
853			 * the panic log by debuggers.
854			 */
855
856			if (bufpos) {
857			  unpackA(debug_buf, bufpos);
858			}
859                    }
860                }
861
862		if (!panicDebugging) {
863			unsigned cnum;
864			/* Clear the MP rendezvous function lock, in the event
865			 * that a panic occurred while in that codepath.
866			 */
867			mp_rendezvous_break_lock();
868
869			/* Non-maskably interrupt all other processors
870			 * If a restart timeout is specified, this processor
871			 * will attempt a restart.
872			 */
873			kprintf("Invoking machine_halt_cpu on CPU %d\n", cn);
874			for (cnum = 0; cnum < real_ncpus; cnum++) {
875				if (cnum != (unsigned) cn) {
876					cpu_NMI_interrupt(cnum);
877				}
878			}
879			machine_halt_cpu();
880			/* NOT REACHED */
881		}
882        }
883
884	__asm__("int3");
885	hw_atomic_sub(&debug_mode, 1);
886}
887
888char *
889machine_boot_info(char *buf, __unused vm_size_t size)
890{
891	*buf ='\0';
892	return buf;
893}
894
895/* Routines for address - symbol translation. Not called unless the "keepsyms"
896 * boot-arg is supplied.
897 */
898
899static int
900panic_print_macho_symbol_name(kernel_mach_header_t *mh, vm_address_t search, const char *module_name)
901{
902    kernel_nlist_t	*sym = NULL;
903    struct load_command		*cmd;
904    kernel_segment_command_t	*orig_ts = NULL, *orig_le = NULL;
905    struct symtab_command	*orig_st = NULL;
906    unsigned int			i;
907    char					*strings, *bestsym = NULL;
908    vm_address_t			bestaddr = 0, diff, curdiff;
909
910    /* Assume that if it's loaded and linked into the kernel, it's a valid Mach-O */
911
912    cmd = (struct load_command *) &mh[1];
913    for (i = 0; i < mh->ncmds; i++) {
914        if (cmd->cmd == LC_SEGMENT_KERNEL) {
915            kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
916
917            if (strncmp(SEG_TEXT, orig_sg->segname,
918				    sizeof(orig_sg->segname)) == 0)
919                orig_ts = orig_sg;
920            else if (strncmp(SEG_LINKEDIT, orig_sg->segname,
921				    sizeof(orig_sg->segname)) == 0)
922                orig_le = orig_sg;
923            else if (strncmp("", orig_sg->segname,
924				    sizeof(orig_sg->segname)) == 0)
925                orig_ts = orig_sg; /* pre-Lion i386 kexts have a single unnamed segment */
926        }
927        else if (cmd->cmd == LC_SYMTAB)
928            orig_st = (struct symtab_command *) cmd;
929
930        cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
931    }
932
933    if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
934        return 0;
935
936    if ((search < orig_ts->vmaddr) ||
937        (search >= orig_ts->vmaddr + orig_ts->vmsize)) {
938        /* search out of range for this mach header */
939        return 0;
940    }
941
942    sym = (kernel_nlist_t *)(uintptr_t)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
943    strings = (char *)(uintptr_t)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
944    diff = search;
945
946    for (i = 0; i < orig_st->nsyms; i++) {
947        if (sym[i].n_type & N_STAB) continue;
948
949        if (sym[i].n_value <= search) {
950            curdiff = search - (vm_address_t)sym[i].n_value;
951            if (curdiff < diff) {
952                diff = curdiff;
953                bestaddr = sym[i].n_value;
954                bestsym = strings + sym[i].n_un.n_strx;
955            }
956        }
957    }
958
959    if (bestsym != NULL) {
960        if (diff != 0) {
961            kdb_printf("%s : %s + 0x%lx", module_name, bestsym, (unsigned long)diff);
962        } else {
963            kdb_printf("%s : %s", module_name, bestsym);
964        }
965        return 1;
966    }
967    return 0;
968}
969
970extern kmod_info_t * kmod; /* the list of modules */
971
972static void
973panic_print_kmod_symbol_name(vm_address_t search)
974{
975    u_int i;
976
977    if (gLoadedKextSummaries == NULL)
978	    return;
979    for (i = 0; i < gLoadedKextSummaries->numSummaries; ++i) {
980        OSKextLoadedKextSummary *summary = gLoadedKextSummaries->summaries + i;
981
982        if ((search >= summary->address) &&
983            (search < (summary->address + summary->size)))
984        {
985            kernel_mach_header_t *header = (kernel_mach_header_t *)(uintptr_t) summary->address;
986            if (panic_print_macho_symbol_name(header, search, summary->name) == 0) {
987                kdb_printf("%s + %llu", summary->name, (unsigned long)search - summary->address);
988            }
989            break;
990        }
991    }
992}
993
994static void
995panic_print_symbol_name(vm_address_t search)
996{
997    /* try searching in the kernel */
998    if (panic_print_macho_symbol_name(&_mh_execute_header, search, "mach_kernel") == 0) {
999        /* that failed, now try to search for the right kext */
1000        panic_print_kmod_symbol_name(search);
1001    }
1002}
1003
1004/* Generate a backtrace, given a frame pointer - this routine
1005 * should walk the stack safely. The trace is appended to the panic log
1006 * and conditionally, to the console. If the trace contains kernel module
1007 * addresses, display the module name, load address and dependencies.
1008 */
1009
1010#define DUMPFRAMES 32
1011#define PBT_TIMEOUT_CYCLES (5 * 1000 * 1000 * 1000ULL)
1012void
1013panic_i386_backtrace(void *_frame, int nframes, const char *msg, boolean_t regdump, x86_saved_state_t *regs)
1014{
1015	cframe_t	*frame = (cframe_t *)_frame;
1016	vm_offset_t raddrs[DUMPFRAMES];
1017	vm_offset_t PC = 0;
1018	int frame_index;
1019	volatile uint32_t *ppbtcnt = &pbtcnt;
1020	uint64_t bt_tsc_timeout;
1021	boolean_t keepsyms = FALSE;
1022	int cn = cpu_number();
1023
1024	if(pbtcpu != cn) {
1025		hw_atomic_add(&pbtcnt, 1);
1026		/* Spin on print backtrace lock, which serializes output
1027		 * Continue anyway if a timeout occurs.
1028		 */
1029		hw_lock_to(&pbtlock, ~0U);
1030		pbtcpu = cn;
1031	}
1032
1033	PE_parse_boot_argn("keepsyms", &keepsyms, sizeof (keepsyms));
1034
1035	if (msg != NULL) {
1036		kdb_printf("%s", msg);
1037	}
1038
1039	if ((regdump == TRUE) && (regs != NULL)) {
1040		x86_saved_state64_t	*ss64p = saved_state64(regs);
1041		kdb_printf(
1042		    "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n"
1043		    "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n"
1044		    "R8:  0x%016llx, R9:  0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n"
1045		    "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n"
1046		    "RFL: 0x%016llx, RIP: 0x%016llx, CS:  0x%016llx, SS:  0x%016llx\n",
1047		    ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
1048		    ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
1049		    ss64p->r8,  ss64p->r9,  ss64p->r10, ss64p->r11,
1050		    ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
1051		    ss64p->isf.rflags, ss64p->isf.rip, ss64p->isf.cs,
1052		    ss64p->isf.ss);
1053		PC = ss64p->isf.rip;
1054	}
1055
1056	kdb_printf("Backtrace (CPU %d), "
1057#if PRINT_ARGS_FROM_STACK_FRAME
1058	"Frame : Return Address (4 potential args on stack)\n", cn);
1059#else
1060	"Frame : Return Address\n", cn);
1061#endif
1062
1063	for (frame_index = 0; frame_index < nframes; frame_index++) {
1064		vm_offset_t curframep = (vm_offset_t) frame;
1065
1066		if (!curframep)
1067			break;
1068
1069		if (curframep & 0x3) {
1070			kdb_printf("Unaligned frame\n");
1071			goto invalid;
1072		}
1073
1074		if (!kvtophys(curframep) ||
1075		    !kvtophys(curframep + sizeof(cframe_t) - 1)) {
1076			kdb_printf("No mapping exists for frame pointer\n");
1077			goto invalid;
1078		}
1079
1080		kdb_printf("%p : 0x%lx ", frame, frame->caller);
1081		if (frame_index < DUMPFRAMES)
1082			raddrs[frame_index] = frame->caller;
1083
1084#if PRINT_ARGS_FROM_STACK_FRAME
1085		if (kvtophys((vm_offset_t)&(frame->args[3])))
1086			kdb_printf("(0x%x 0x%x 0x%x 0x%x) ",
1087			    frame->args[0], frame->args[1],
1088			    frame->args[2], frame->args[3]);
1089#endif
1090
1091		/* Display address-symbol translation only if the "keepsyms"
1092		 * boot-arg is suppplied, since we unload LINKEDIT otherwise.
1093		 * This routine is potentially unsafe; also, function
1094		 * boundary identification is unreliable after a strip -x.
1095		 */
1096		if (keepsyms)
1097			panic_print_symbol_name((vm_address_t)frame->caller);
1098
1099		kdb_printf("\n");
1100
1101		frame = frame->prev;
1102	}
1103
1104	if (frame_index >= nframes)
1105		kdb_printf("\tBacktrace continues...\n");
1106
1107	goto out;
1108
1109invalid:
1110	kdb_printf("Backtrace terminated-invalid frame pointer %p\n",frame);
1111out:
1112
1113	/* Identify kernel modules in the backtrace and display their
1114	 * load addresses and dependencies. This routine should walk
1115	 * the kmod list safely.
1116	 */
1117	if (frame_index)
1118		kmod_panic_dump((vm_offset_t *)&raddrs[0], frame_index);
1119
1120	if (PC != 0)
1121		kmod_panic_dump(&PC, 1);
1122
1123	panic_display_system_configuration();
1124
1125	/* Release print backtrace lock, to permit other callers in the
1126	 * event of panics on multiple processors.
1127	 */
1128	hw_lock_unlock(&pbtlock);
1129	hw_atomic_sub(&pbtcnt, 1);
1130	/* Wait for other processors to complete output
1131	 * Timeout and continue after PBT_TIMEOUT_CYCLES.
1132	 */
1133	bt_tsc_timeout = rdtsc64() + PBT_TIMEOUT_CYCLES;
1134	while(*ppbtcnt && (rdtsc64() < bt_tsc_timeout));
1135}
1136