1/* 2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <i386/pmap.h> 30#include <i386/proc_reg.h> 31#include <i386/mp_desc.h> 32#include <i386/misc_protos.h> 33#include <i386/mp.h> 34#include <i386/cpu_data.h> 35#if CONFIG_MTRR 36#include <i386/mtrr.h> 37#endif 38#if CONFIG_VMX 39#include <i386/vmx/vmx_cpu.h> 40#endif 41#include <i386/ucode.h> 42#include <i386/acpi.h> 43#include <i386/fpu.h> 44#include <i386/lapic.h> 45#include <i386/mp.h> 46#include <i386/mp_desc.h> 47#include <i386/serial_io.h> 48#if CONFIG_MCA 49#include <i386/machine_check.h> 50#endif 51#include <i386/pmCPU.h> 52 53#include <i386/tsc.h> 54 55#include <kern/cpu_data.h> 56#include <kern/etimer.h> 57#include <kern/machine.h> 58#include <kern/timer_queue.h> 59#include <console/serial_protos.h> 60#include <machine/pal_routines.h> 61#include <vm/vm_page.h> 62 63#if HIBERNATION 64#include <IOKit/IOHibernatePrivate.h> 65#endif 66#include <IOKit/IOPlatformExpert.h> 67#include <sys/kdebug.h> 68 69#if CONFIG_SLEEP 70extern void acpi_sleep_cpu(acpi_sleep_callback, void * refcon); 71extern void acpi_wake_prot(void); 72#endif 73extern kern_return_t IOCPURunPlatformQuiesceActions(void); 74extern kern_return_t IOCPURunPlatformActiveActions(void); 75 76extern void fpinit(void); 77 78vm_offset_t 79acpi_install_wake_handler(void) 80{ 81#if CONFIG_SLEEP 82 install_real_mode_bootstrap(acpi_wake_prot); 83 return REAL_MODE_BOOTSTRAP_OFFSET; 84#else 85 return 0; 86#endif 87} 88 89#if HIBERNATION 90struct acpi_hibernate_callback_data { 91 acpi_sleep_callback func; 92 void *refcon; 93}; 94typedef struct acpi_hibernate_callback_data acpi_hibernate_callback_data_t; 95 96unsigned int save_kdebug_enable = 0; 97static uint64_t acpi_sleep_abstime; 98static uint64_t acpi_idle_abstime; 99static uint64_t acpi_wake_abstime; 100boolean_t deep_idle_rebase = TRUE; 101 102#if CONFIG_SLEEP 103static void 104acpi_hibernate(void *refcon) 105{ 106 uint32_t mode; 107 108 acpi_hibernate_callback_data_t *data = 109 (acpi_hibernate_callback_data_t *)refcon; 110 111 if (current_cpu_datap()->cpu_hibernate) 112 { 113#if defined(__i386__) 114 cpu_IA32e_enable(current_cpu_datap()); 115#endif 116 mode = hibernate_write_image(); 117 118 if( mode == kIOHibernatePostWriteHalt ) 119 { 120 // off 121 HIBLOG("power off\n"); 122 if (PE_halt_restart) (*PE_halt_restart)(kPEHaltCPU); 123 } 124 else if( mode == kIOHibernatePostWriteRestart ) 125 { 126 // restart 127 HIBLOG("restart\n"); 128 if (PE_halt_restart) (*PE_halt_restart)(kPERestartCPU); 129 } 130 else 131 { 132 // sleep 133 HIBLOG("sleep\n"); 134 135 // should we come back via regular wake, set the state in memory. 136 cpu_datap(0)->cpu_hibernate = 0; 137 } 138 139#if defined(__i386__) 140 /* 141 * If we're in 64-bit mode, drop back into legacy mode during sleep. 142 */ 143 cpu_IA32e_disable(current_cpu_datap()); 144#endif 145 } 146 kdebug_enable = 0; 147 148 IOCPURunPlatformQuiesceActions(); 149 150 acpi_sleep_abstime = mach_absolute_time(); 151 152 (data->func)(data->refcon); 153 154 /* should never get here! */ 155} 156#endif /* CONFIG_SLEEP */ 157#endif /* HIBERNATION */ 158 159extern void slave_pstart(void); 160 161extern unsigned int wake_nkdbufs; 162 163void 164acpi_sleep_kernel(acpi_sleep_callback func, void *refcon) 165{ 166#if HIBERNATION 167 acpi_hibernate_callback_data_t data; 168#endif 169 boolean_t did_hibernate; 170 unsigned int cpu; 171 kern_return_t rc; 172 unsigned int my_cpu; 173 uint64_t now; 174 uint64_t my_tsc; 175 uint64_t my_abs; 176 177 kprintf("acpi_sleep_kernel hib=%d, cpu=%d\n", 178 current_cpu_datap()->cpu_hibernate, cpu_number()); 179 180 /* Get all CPUs to be in the "off" state */ 181 my_cpu = cpu_number(); 182 for (cpu = 0; cpu < real_ncpus; cpu += 1) { 183 if (cpu == my_cpu) 184 continue; 185 rc = pmCPUExitHaltToOff(cpu); 186 if (rc != KERN_SUCCESS) 187 panic("Error %d trying to transition CPU %d to OFF", 188 rc, cpu); 189 } 190 191 /* shutdown local APIC before passing control to firmware */ 192 lapic_shutdown(); 193 194#if HIBERNATION 195 data.func = func; 196 data.refcon = refcon; 197#endif 198 199 /* Save power management timer state */ 200 pmTimerSave(); 201 202#if CONFIG_VMX 203 /* 204 * Turn off VT, otherwise switching to legacy mode will fail 205 */ 206 vmx_suspend(); 207#endif 208 209#if defined(__i386__) 210 /* 211 * If we're in 64-bit mode, drop back into legacy mode during sleep. 212 */ 213 cpu_IA32e_disable(current_cpu_datap()); 214#endif 215 /* 216 * Enable FPU/SIMD unit for potential hibernate acceleration 217 */ 218 clear_ts(); 219 220 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_START, 0, 0, 0, 0, 0); 221 222 save_kdebug_enable = kdebug_enable; 223 kdebug_enable = 0; 224 225 acpi_sleep_abstime = mach_absolute_time(); 226 227#if CONFIG_SLEEP 228 /* 229 * Save master CPU state and sleep platform. 230 * Will not return until platform is woken up, 231 * or if sleep failed. 232 */ 233#ifdef __x86_64__ 234 uint64_t old_cr3 = x86_64_pre_sleep(); 235#endif 236#if HIBERNATION 237 acpi_sleep_cpu(acpi_hibernate, &data); 238#else 239 acpi_sleep_cpu(func, refcon); 240#endif 241 242#ifdef __x86_64__ 243 x86_64_post_sleep(old_cr3); 244#endif 245 246#endif /* CONFIG_SLEEP */ 247 248 /* Reset UART if kprintf is enabled. 249 * However kprintf should not be used before rtc_sleep_wakeup() 250 * for compatibility with firewire kprintf. 251 */ 252 253 if (FALSE == disable_serial_output) 254 pal_serial_init(); 255 256#if HIBERNATION 257 if (current_cpu_datap()->cpu_hibernate) { 258#if defined(__i386__) 259 int i; 260 for (i = 0; i < PMAP_NWINDOWS; i++) 261 *current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0; 262#endif 263 did_hibernate = TRUE; 264 265 } else 266#endif 267 { 268 did_hibernate = FALSE; 269 } 270 271 /* Re-enable mode (including 64-bit if applicable) */ 272 cpu_mode_init(current_cpu_datap()); 273 274#if CONFIG_MCA 275 /* Re-enable machine check handling */ 276 mca_cpu_init(); 277#endif 278 279#if CONFIG_MTRR 280 /* restore MTRR settings */ 281 mtrr_update_cpu(); 282#endif 283 284 /* update CPU microcode */ 285 ucode_update_wake(); 286 287#if CONFIG_VMX 288 /* 289 * Restore VT mode 290 */ 291 vmx_resume(); 292#endif 293 294#if CONFIG_MTRR 295 /* set up PAT following boot processor power up */ 296 pat_init(); 297#endif 298 299 /* 300 * Go through all of the CPUs and mark them as requiring 301 * a full restart. 302 */ 303 pmMarkAllCPUsOff(); 304 305 ml_get_timebase(&now); 306 307 /* re-enable and re-init local apic (prior to starting timers) */ 308 if (lapic_probe()) 309 lapic_configure(); 310 311 acpi_wake_abstime = mach_absolute_time(); 312 313 /* let the realtime clock reset */ 314 rtc_sleep_wakeup(acpi_sleep_abstime); 315 316 kdebug_enable = save_kdebug_enable; 317 318 if (kdebug_enable == 0) { 319 if (wake_nkdbufs) 320 start_kern_tracing(wake_nkdbufs, TRUE); 321 } 322 323 /* Reconfigure FP/SIMD unit */ 324 init_fpu(); 325 clear_ts(); 326 327 IOCPURunPlatformActiveActions(); 328 329 if (did_hibernate) { 330 331 my_tsc = (now >> 32) | (now << 32); 332 my_abs = tmrCvt(my_tsc, tscFCvtt2n); 333 334 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_START, 335 (uint32_t)(my_abs >> 32), (uint32_t)my_abs, 0, 0, 0); 336 hibernate_machine_init(); 337 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_END, 0, 0, 0, 0, 0); 338 339 current_cpu_datap()->cpu_hibernate = 0; 340 341 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0); 342 } else 343 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0); 344 345 /* Restore power management register state */ 346 pmCPUMarkRunning(current_cpu_datap()); 347 348 /* Restore power management timer state */ 349 pmTimerRestore(); 350 351 /* Restart timer interrupts */ 352 rtc_timer_start(); 353 354 355 356#if HIBERNATION 357#ifdef __i386__ 358 /* The image is written out using the copy engine, which disables 359 * preemption. Since the copy engine writes out the page which contains 360 * the preemption variable when it is disabled, we need to explicitly 361 * enable it here */ 362 if (did_hibernate) 363 enable_preemption(); 364#endif 365 366 kprintf("ret from acpi_sleep_cpu hib=%d\n", did_hibernate); 367#endif 368 369#if CONFIG_SLEEP 370 /* Becase we don't save the bootstrap page, and we share it 371 * between sleep and mp slave init, we need to recreate it 372 * after coming back from sleep or hibernate */ 373 install_real_mode_bootstrap(slave_pstart); 374#endif 375} 376 377/* 378 * acpi_idle_kernel is called by the ACPI Platform kext to request the kernel 379 * to idle the boot processor in the deepest C-state for S0 sleep. All slave 380 * processors are expected already to have been offlined in the deepest C-state. 381 * 382 * The contract with ACPI is that although the kernel is called with interrupts 383 * disabled, interrupts may need to be re-enabled to dismiss any pending timer 384 * interrupt. However, the callback function will be called once this has 385 * occurred and interrupts are guaranteed to be disabled at that time, 386 * and to remain disabled during C-state entry, exit (wake) and return 387 * from acpi_idle_kernel. 388 */ 389void 390acpi_idle_kernel(acpi_sleep_callback func, void *refcon) 391{ 392 boolean_t istate = ml_get_interrupts_enabled(); 393 394 kprintf("acpi_idle_kernel, cpu=%d, interrupts %s\n", 395 cpu_number(), istate ? "enabled" : "disabled"); 396 397 assert(cpu_number() == master_cpu); 398 399 /* 400 * Effectively set the boot cpu offline. 401 * This will stop further deadlines being set. 402 */ 403 cpu_datap(master_cpu)->cpu_running = FALSE; 404 405 /* Cancel any pending deadline */ 406 setPop(0); 407 while (lapic_is_interrupting(LAPIC_TIMER_VECTOR)) { 408 (void) ml_set_interrupts_enabled(TRUE); 409 setPop(0); 410 ml_set_interrupts_enabled(FALSE); 411 } 412 413 /* 414 * Call back to caller to indicate that interrupts will remain 415 * disabled while we deep idle, wake and return. 416 */ 417 func(refcon); 418 419 acpi_idle_abstime = mach_absolute_time(); 420 421 KERNEL_DEBUG_CONSTANT( 422 MACHDBG_CODE(DBG_MACH_SCHED, MACH_DEEP_IDLE) | DBG_FUNC_START, 423 acpi_idle_abstime, deep_idle_rebase, 0, 0, 0); 424 425 /* 426 * Disable tracing during S0-sleep 427 * unless overridden by sysctl -w tsc.deep_idle_rebase=0 428 */ 429 if (deep_idle_rebase) { 430 save_kdebug_enable = kdebug_enable; 431 kdebug_enable = 0; 432 } 433 434 /* 435 * Call into power-management to enter the lowest C-state. 436 * Note when called on the boot processor this routine will 437 * return directly when awoken. 438 */ 439 pmCPUHalt(PM_HALT_SLEEP); 440 441 /* 442 * Get wakeup time relative to the TSC which has progressed. 443 * Then rebase nanotime to reflect time not progressing over sleep 444 * - unless overriden so that tracing can occur during deep_idle. 445 */ 446 acpi_wake_abstime = mach_absolute_time(); 447 if (deep_idle_rebase) { 448 rtc_sleep_wakeup(acpi_idle_abstime); 449 kdebug_enable = save_kdebug_enable; 450 } 451 452 cpu_datap(master_cpu)->cpu_running = TRUE; 453 454 KERNEL_DEBUG_CONSTANT( 455 MACHDBG_CODE(DBG_MACH_SCHED, MACH_DEEP_IDLE) | DBG_FUNC_END, 456 acpi_wake_abstime, acpi_wake_abstime - acpi_idle_abstime, 0, 0, 0); 457 458 /* Like S3 sleep, turn on tracing if trace_wake boot-arg is present */ 459 if (kdebug_enable == 0) { 460 if (wake_nkdbufs) 461 start_kern_tracing(wake_nkdbufs, TRUE); 462 } 463 464 IOCPURunPlatformActiveActions(); 465 466 /* Restart timer interrupts */ 467 rtc_timer_start(); 468} 469 470extern char real_mode_bootstrap_end[]; 471extern char real_mode_bootstrap_base[]; 472 473void 474install_real_mode_bootstrap(void *prot_entry) 475{ 476 /* 477 * Copy the boot entry code to the real-mode vector area REAL_MODE_BOOTSTRAP_OFFSET. 478 * This is in page 1 which has been reserved for this purpose by 479 * machine_startup() from the boot processor. 480 * The slave boot code is responsible for switching to protected 481 * mode and then jumping to the common startup, _start(). 482 */ 483 bcopy_phys(kvtophys((vm_offset_t) real_mode_bootstrap_base), 484 (addr64_t) REAL_MODE_BOOTSTRAP_OFFSET, 485 real_mode_bootstrap_end-real_mode_bootstrap_base); 486 487 /* 488 * Set the location at the base of the stack to point to the 489 * common startup entry. 490 */ 491 ml_phys_write_word( 492 PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, 493 (unsigned int)kvtophys((vm_offset_t)prot_entry)); 494 495 /* Flush caches */ 496 __asm__("wbinvd"); 497} 498 499