1/*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <i386/pmap.h>
30#include <i386/proc_reg.h>
31#include <i386/mp_desc.h>
32#include <i386/misc_protos.h>
33#include <i386/mp.h>
34#include <i386/cpu_data.h>
35#if CONFIG_MTRR
36#include <i386/mtrr.h>
37#endif
38#if CONFIG_VMX
39#include <i386/vmx/vmx_cpu.h>
40#endif
41#include <i386/ucode.h>
42#include <i386/acpi.h>
43#include <i386/fpu.h>
44#include <i386/lapic.h>
45#include <i386/mp.h>
46#include <i386/mp_desc.h>
47#include <i386/serial_io.h>
48#if CONFIG_MCA
49#include <i386/machine_check.h>
50#endif
51#include <i386/pmCPU.h>
52
53#include <i386/tsc.h>
54
55#include <kern/cpu_data.h>
56#include <kern/machine.h>
57#include <kern/timer_queue.h>
58#include <console/serial_protos.h>
59#include <machine/pal_routines.h>
60#include <vm/vm_page.h>
61
62#if HIBERNATION
63#include <IOKit/IOHibernatePrivate.h>
64#endif
65#include <IOKit/IOPlatformExpert.h>
66#include <sys/kdebug.h>
67
68#if CONFIG_SLEEP
69extern void	acpi_sleep_cpu(acpi_sleep_callback, void * refcon);
70extern void	acpi_wake_prot(void);
71#endif
72extern kern_return_t IOCPURunPlatformQuiesceActions(void);
73extern kern_return_t IOCPURunPlatformActiveActions(void);
74
75extern void 	fpinit(void);
76
77vm_offset_t
78acpi_install_wake_handler(void)
79{
80#if CONFIG_SLEEP
81	install_real_mode_bootstrap(acpi_wake_prot);
82	return REAL_MODE_BOOTSTRAP_OFFSET;
83#else
84	return 0;
85#endif
86}
87
88#if HIBERNATION
89struct acpi_hibernate_callback_data {
90	acpi_sleep_callback func;
91	void *refcon;
92};
93typedef struct acpi_hibernate_callback_data acpi_hibernate_callback_data_t;
94
95unsigned int		save_kdebug_enable = 0;
96static uint64_t		acpi_sleep_abstime;
97static uint64_t		acpi_idle_abstime;
98static uint64_t		acpi_wake_abstime, acpi_wake_postrebase_abstime;
99boolean_t		deep_idle_rebase = TRUE;
100
101#if CONFIG_SLEEP
102static void
103acpi_hibernate(void *refcon)
104{
105	uint32_t mode;
106
107	acpi_hibernate_callback_data_t *data =
108		(acpi_hibernate_callback_data_t *)refcon;
109
110	if (current_cpu_datap()->cpu_hibernate)
111	{
112		mode = hibernate_write_image();
113
114		if( mode == kIOHibernatePostWriteHalt )
115		{
116			// off
117			HIBLOG("power off\n");
118			if (PE_halt_restart) (*PE_halt_restart)(kPEHaltCPU);
119		}
120		else if( mode == kIOHibernatePostWriteRestart )
121		{
122			// restart
123			HIBLOG("restart\n");
124			if (PE_halt_restart) (*PE_halt_restart)(kPERestartCPU);
125		}
126		else
127		{
128			// sleep
129			HIBLOG("sleep\n");
130
131			// should we come back via regular wake, set the state in memory.
132			cpu_datap(0)->cpu_hibernate = 0;
133		}
134
135	}
136	kdebug_enable = 0;
137
138	IOCPURunPlatformQuiesceActions();
139
140	acpi_sleep_abstime = mach_absolute_time();
141
142	(data->func)(data->refcon);
143
144	/* should never get here! */
145}
146#endif /* CONFIG_SLEEP */
147#endif /* HIBERNATION */
148
149extern void			slave_pstart(void);
150extern void			hibernate_rebuild_vm_structs(void);
151
152extern	unsigned int		wake_nkdbufs;
153
154void
155acpi_sleep_kernel(acpi_sleep_callback func, void *refcon)
156{
157#if HIBERNATION
158	acpi_hibernate_callback_data_t data;
159#endif
160	boolean_t did_hibernate;
161	unsigned int	cpu;
162	kern_return_t	rc;
163	unsigned int	my_cpu;
164	uint64_t	start;
165	uint64_t	elapsed = 0;
166	uint64_t	elapsed_trace_start = 0;
167
168	kprintf("acpi_sleep_kernel hib=%d, cpu=%d\n",
169			current_cpu_datap()->cpu_hibernate, cpu_number());
170
171    	/* Get all CPUs to be in the "off" state */
172    	my_cpu = cpu_number();
173	for (cpu = 0; cpu < real_ncpus; cpu += 1) {
174	    	if (cpu == my_cpu)
175			continue;
176		rc = pmCPUExitHaltToOff(cpu);
177		if (rc != KERN_SUCCESS)
178			panic("Error %d trying to transition CPU %d to OFF",
179			      rc, cpu);
180	}
181
182	/* shutdown local APIC before passing control to firmware */
183	lapic_shutdown();
184
185#if HIBERNATION
186	data.func = func;
187	data.refcon = refcon;
188#endif
189
190	/* Save power management timer state */
191	pmTimerSave();
192
193#if CONFIG_VMX
194	/*
195	 * Turn off VT, otherwise switching to legacy mode will fail
196	 */
197	vmx_suspend();
198#endif
199
200	/*
201	 * Enable FPU/SIMD unit for potential hibernate acceleration
202	 */
203	clear_ts();
204
205	KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_START, 0, 0, 0, 0, 0);
206
207	save_kdebug_enable = kdebug_enable;
208	kdebug_enable = 0;
209
210	acpi_sleep_abstime = mach_absolute_time();
211
212#if CONFIG_SLEEP
213	/*
214	 * Save master CPU state and sleep platform.
215	 * Will not return until platform is woken up,
216	 * or if sleep failed.
217	 */
218	uint64_t old_cr3 = x86_64_pre_sleep();
219#if HIBERNATION
220	acpi_sleep_cpu(acpi_hibernate, &data);
221#else
222	acpi_sleep_cpu(func, refcon);
223#endif
224
225	start = mach_absolute_time();
226
227	x86_64_post_sleep(old_cr3);
228
229#endif /* CONFIG_SLEEP */
230
231	/* Reset UART if kprintf is enabled.
232	 * However kprintf should not be used before rtc_sleep_wakeup()
233	 * for compatibility with firewire kprintf.
234	 */
235
236	if (FALSE == disable_serial_output)
237		pal_serial_init();
238
239#if HIBERNATION
240	if (current_cpu_datap()->cpu_hibernate) {
241		did_hibernate = TRUE;
242
243	} else
244#endif
245	{
246		did_hibernate = FALSE;
247	}
248
249	/* Re-enable mode (including 64-bit if applicable) */
250	cpu_mode_init(current_cpu_datap());
251
252#if CONFIG_MCA
253	/* Re-enable machine check handling */
254	mca_cpu_init();
255#endif
256
257#if CONFIG_MTRR
258	/* restore MTRR settings */
259	mtrr_update_cpu();
260#endif
261
262	/* update CPU microcode */
263	ucode_update_wake();
264
265#if CONFIG_VMX
266	/*
267	 * Restore VT mode
268	 */
269	vmx_resume();
270#endif
271
272#if CONFIG_MTRR
273	/* set up PAT following boot processor power up */
274	pat_init();
275#endif
276
277	/*
278	 * Go through all of the CPUs and mark them as requiring
279	 * a full restart.
280	 */
281	pmMarkAllCPUsOff();
282
283
284	/* re-enable and re-init local apic (prior to starting timers) */
285	if (lapic_probe())
286		lapic_configure();
287
288	hibernate_rebuild_vm_structs();
289
290	elapsed += mach_absolute_time() - start;
291	acpi_wake_abstime = mach_absolute_time();
292
293	/* let the realtime clock reset */
294	rtc_sleep_wakeup(acpi_sleep_abstime);
295	acpi_wake_postrebase_abstime = mach_absolute_time();
296	kdebug_enable = save_kdebug_enable;
297
298	if (kdebug_enable == 0) {
299		if (wake_nkdbufs) {
300			start = mach_absolute_time();
301			start_kern_tracing(wake_nkdbufs, TRUE);
302			elapsed_trace_start += mach_absolute_time() - start;
303		}
304	}
305	start = mach_absolute_time();
306
307	/* Reconfigure FP/SIMD unit */
308	init_fpu();
309	clear_ts();
310
311	IOCPURunPlatformActiveActions();
312
313	if (did_hibernate) {
314		elapsed += mach_absolute_time() - start;
315
316		KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_START, elapsed, elapsed_trace_start, 0, 0, 0);
317		hibernate_machine_init();
318		KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_END, 0, 0, 0, 0, 0);
319
320		current_cpu_datap()->cpu_hibernate = 0;
321
322		KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0);
323	} else
324		KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 0) | DBG_FUNC_END, 0, 0, 0, 0, 0);
325
326	/* Restore power management register state */
327	pmCPUMarkRunning(current_cpu_datap());
328
329	/* Restore power management timer state */
330	pmTimerRestore();
331
332	/* Restart timer interrupts */
333	rtc_timer_start();
334
335#if HIBERNATION
336
337	kprintf("ret from acpi_sleep_cpu hib=%d\n", did_hibernate);
338#endif
339
340#if CONFIG_SLEEP
341	/* Becase we don't save the bootstrap page, and we share it
342	 * between sleep and mp slave init, we need to recreate it
343	 * after coming back from sleep or hibernate */
344	install_real_mode_bootstrap(slave_pstart);
345#endif
346}
347
348/*
349 * acpi_idle_kernel is called by the ACPI Platform kext to request the kernel
350 * to idle the boot processor in the deepest C-state for S0 sleep. All slave
351 * processors are expected already to have been offlined in the deepest C-state.
352 *
353 * The contract with ACPI is that although the kernel is called with interrupts
354 * disabled, interrupts may need to be re-enabled to dismiss any pending timer
355 * interrupt. However, the callback function will be called once this has
356 * occurred and interrupts are guaranteed to be disabled at that time,
357 * and to remain disabled during C-state entry, exit (wake) and return
358 * from acpi_idle_kernel.
359 */
360void
361acpi_idle_kernel(acpi_sleep_callback func, void *refcon)
362{
363	boolean_t	istate = ml_get_interrupts_enabled();
364
365	kprintf("acpi_idle_kernel, cpu=%d, interrupts %s\n",
366		cpu_number(), istate ? "enabled" : "disabled");
367
368	assert(cpu_number() == master_cpu);
369
370	/*
371	 * Effectively set the boot cpu offline.
372	 * This will stop further deadlines being set.
373	 */
374	cpu_datap(master_cpu)->cpu_running = FALSE;
375
376	/* Cancel any pending deadline */
377	setPop(0);
378	while (lapic_is_interrupting(LAPIC_TIMER_VECTOR)) {
379		(void) ml_set_interrupts_enabled(TRUE);
380		setPop(0);
381		ml_set_interrupts_enabled(FALSE);
382	}
383
384	/*
385	 * Call back to caller to indicate that interrupts will remain
386	 * disabled while we deep idle, wake and return.
387	 */
388	func(refcon);
389
390	acpi_idle_abstime = mach_absolute_time();
391
392	KERNEL_DEBUG_CONSTANT(
393		MACHDBG_CODE(DBG_MACH_SCHED, MACH_DEEP_IDLE) | DBG_FUNC_START,
394		acpi_idle_abstime, deep_idle_rebase, 0, 0, 0);
395
396	/*
397	 * Disable tracing during S0-sleep
398	 * unless overridden by sysctl -w tsc.deep_idle_rebase=0
399	 */
400	if (deep_idle_rebase) {
401		save_kdebug_enable = kdebug_enable;
402		kdebug_enable = 0;
403	}
404
405	/*
406	 * Call into power-management to enter the lowest C-state.
407	 * Note when called on the boot processor this routine will
408	 * return directly when awoken.
409	 */
410	pmCPUHalt(PM_HALT_SLEEP);
411
412	/*
413	 * Get wakeup time relative to the TSC which has progressed.
414	 * Then rebase nanotime to reflect time not progressing over sleep
415	 * - unless overriden so that tracing can occur during deep_idle.
416	 */
417	acpi_wake_abstime = mach_absolute_time();
418	if (deep_idle_rebase) {
419		rtc_sleep_wakeup(acpi_idle_abstime);
420		kdebug_enable = save_kdebug_enable;
421	}
422	acpi_wake_postrebase_abstime = mach_absolute_time();
423	cpu_datap(master_cpu)->cpu_running = TRUE;
424
425	KERNEL_DEBUG_CONSTANT(
426		MACHDBG_CODE(DBG_MACH_SCHED, MACH_DEEP_IDLE) | DBG_FUNC_END,
427		acpi_wake_abstime, acpi_wake_abstime - acpi_idle_abstime, 0, 0, 0);
428
429	/* Like S3 sleep, turn on tracing if trace_wake boot-arg is present */
430	if (kdebug_enable == 0) {
431		if (wake_nkdbufs)
432			start_kern_tracing(wake_nkdbufs, TRUE);
433	}
434
435	IOCPURunPlatformActiveActions();
436
437	/* Restart timer interrupts */
438	rtc_timer_start();
439}
440
441extern char real_mode_bootstrap_end[];
442extern char real_mode_bootstrap_base[];
443
444void
445install_real_mode_bootstrap(void *prot_entry)
446{
447	/*
448	 * Copy the boot entry code to the real-mode vector area REAL_MODE_BOOTSTRAP_OFFSET.
449	 * This is in page 1 which has been reserved for this purpose by
450	 * machine_startup() from the boot processor.
451	 * The slave boot code is responsible for switching to protected
452	 * mode and then jumping to the common startup, _start().
453	 */
454	bcopy_phys(kvtophys((vm_offset_t) real_mode_bootstrap_base),
455		   (addr64_t) REAL_MODE_BOOTSTRAP_OFFSET,
456		   real_mode_bootstrap_end-real_mode_bootstrap_base);
457
458	/*
459	 * Set the location at the base of the stack to point to the
460	 * common startup entry.
461	 */
462	ml_phys_write_word(
463		PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET,
464		(unsigned int)kvtophys((vm_offset_t)prot_entry));
465
466	/* Flush caches */
467	__asm__("wbinvd");
468}
469
470boolean_t
471ml_recent_wake(void) {
472	uint64_t ctime = mach_absolute_time();
473	assert(ctime > acpi_wake_postrebase_abstime);
474	return ((ctime - acpi_wake_postrebase_abstime) < 5 * NSEC_PER_SEC);
475}
476