1/*
2 * Copyright (c) 2004-2011 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * CPU-specific power management support.
31 *
32 * Implements the "wrappers" to the KEXT.
33 */
34#include <i386/asm.h>
35#include <i386/machine_cpu.h>
36#include <i386/mp.h>
37#include <i386/machine_routines.h>
38#include <i386/proc_reg.h>
39#include <i386/pmap.h>
40#include <i386/misc_protos.h>
41#include <kern/machine.h>
42#include <kern/pms.h>
43#include <kern/processor.h>
44#include <kern/etimer.h>
45#include <i386/cpu_threads.h>
46#include <i386/pmCPU.h>
47#include <i386/cpuid.h>
48#include <i386/rtclock_protos.h>
49#include <kern/sched_prim.h>
50#include <i386/lapic.h>
51#include <i386/pal_routines.h>
52#include <sys/kdebug.h>
53
54extern int disableConsoleOutput;
55
56#define DELAY_UNSET		0xFFFFFFFFFFFFFFFFULL
57
58uint64_t cpu_itime_bins[CPU_ITIME_BINS] = {16* NSEC_PER_USEC, 32* NSEC_PER_USEC, 64* NSEC_PER_USEC, 128* NSEC_PER_USEC, 256* NSEC_PER_USEC, 512* NSEC_PER_USEC, 1024* NSEC_PER_USEC, 2048* NSEC_PER_USEC, 4096* NSEC_PER_USEC, 8192* NSEC_PER_USEC, 16384* NSEC_PER_USEC, 32768* NSEC_PER_USEC};
59uint64_t *cpu_rtime_bins = &cpu_itime_bins[0];
60
61/*
62 * The following is set when the KEXT loads and initializes.
63 */
64pmDispatch_t	*pmDispatch	= NULL;
65
66uint32_t		pmInitDone		= 0;
67static boolean_t	earlyTopology		= FALSE;
68static uint64_t		earlyMaxBusDelay	= DELAY_UNSET;
69static uint64_t		earlyMaxIntDelay	= DELAY_UNSET;
70
71/*
72 * Initialize the Cstate change code.
73 */
74void
75power_management_init(void)
76{
77    if (pmDispatch != NULL && pmDispatch->cstateInit != NULL)
78	(*pmDispatch->cstateInit)();
79}
80
81static inline void machine_classify_interval(uint64_t interval, uint64_t *bins, uint64_t *binvals, uint32_t nbins) {
82	uint32_t i;
83 	for (i = 0; i < nbins; i++) {
84 		if (interval < binvals[i]) {
85 			bins[i]++;
86 			break;
87 		}
88 	}
89}
90
91/*
92 * Called when the CPU is idle.  It calls into the power management kext
93 * to determine the best way to idle the CPU.
94 */
95void
96machine_idle(void)
97{
98	cpu_data_t		*my_cpu		= current_cpu_datap();
99	uint64_t		ctime, rtime, itime;
100
101	if (my_cpu == NULL)
102		goto out;
103
104	ctime = mach_absolute_time();
105
106	my_cpu->lcpu.state = LCPU_IDLE;
107	DBGLOG(cpu_handle, cpu_number(), MP_IDLE);
108	MARK_CPU_IDLE(cpu_number());
109
110	rtime = ctime - my_cpu->cpu_ixtime;
111
112	my_cpu->cpu_rtime_total += rtime;
113	machine_classify_interval(rtime, &my_cpu->cpu_rtimes[0], &cpu_rtime_bins[0], CPU_RTIME_BINS);
114
115	if (pmInitDone) {
116		/*
117		 * Handle case where ml_set_maxbusdelay() or ml_set_maxintdelay()
118		 * were called prior to the CPU PM kext being registered.  We do
119		 * this here since we know at this point the values will be first
120		 * used since idle is where the decisions using these values is made.
121		 */
122		if (earlyMaxBusDelay != DELAY_UNSET)
123			ml_set_maxbusdelay((uint32_t)(earlyMaxBusDelay & 0xFFFFFFFF));
124
125		if (earlyMaxIntDelay != DELAY_UNSET)
126			ml_set_maxintdelay(earlyMaxIntDelay);
127	}
128
129	if (pmInitDone
130	    && pmDispatch != NULL
131	    && pmDispatch->MachineIdle != NULL)
132		(*pmDispatch->MachineIdle)(0x7FFFFFFFFFFFFFFFULL);
133	else {
134		/*
135		 * If no power management, re-enable interrupts and halt.
136		 * This will keep the CPU from spinning through the scheduler
137		 * and will allow at least some minimal power savings (but it
138		 * cause problems in some MP configurations w.r.t. the APIC
139		 * stopping during a GV3 transition).
140		 */
141		pal_hlt();
142
143		/* Once woken, re-disable interrupts. */
144		pal_cli();
145	}
146
147	/*
148	 * Mark the CPU as running again.
149	 */
150	MARK_CPU_ACTIVE(cpu_number());
151	DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE);
152
153	uint64_t ixtime = my_cpu->cpu_ixtime = mach_absolute_time();
154	my_cpu->cpu_idle_exits++;
155
156	itime = ixtime - ctime;
157
158	my_cpu->lcpu.state = LCPU_RUN;
159
160	machine_classify_interval(itime, &my_cpu->cpu_itimes[0], &cpu_itime_bins[0], CPU_ITIME_BINS);
161	my_cpu->cpu_itime_total += itime;
162
163
164	/*
165	 * Re-enable interrupts.
166	 */
167out:
168	pal_sti();
169}
170
171/*
172 * Called when the CPU is to be halted.  It will choose the best C-State
173 * to be in.
174 */
175void
176pmCPUHalt(uint32_t reason)
177{
178    cpu_data_t	*cpup	= current_cpu_datap();
179
180    switch (reason) {
181    case PM_HALT_DEBUG:
182	cpup->lcpu.state = LCPU_PAUSE;
183	pal_stop_cpu(FALSE);
184	break;
185
186    case PM_HALT_PANIC:
187	cpup->lcpu.state = LCPU_PAUSE;
188	pal_stop_cpu(TRUE);
189	break;
190
191    case PM_HALT_NORMAL:
192    case PM_HALT_SLEEP:
193    default:
194        pal_cli();
195
196	if (pmInitDone
197	    && pmDispatch != NULL
198	    && pmDispatch->pmCPUHalt != NULL) {
199	    /*
200	     * Halt the CPU (and put it in a low power state.
201	     */
202	    (*pmDispatch->pmCPUHalt)();
203
204	    /*
205	     * We've exited halt, so get the CPU schedulable again.
206	     * - by calling the fast init routine for a slave, or
207	     * - by returning if we're the master processor.
208	     */
209	    if (cpup->cpu_number != master_cpu) {
210		i386_init_slave_fast();
211		panic("init_slave_fast returned");
212	    }
213	} else
214	{
215	    /*
216	     * If no power managment and a processor is taken off-line,
217	     * then invalidate the cache and halt it (it will not be able
218	     * to be brought back on-line without resetting the CPU).
219	     */
220	    __asm__ volatile ("wbinvd");
221	    cpup->lcpu.state = LCPU_HALT;
222	    pal_stop_cpu(FALSE);
223
224	    panic("back from Halt");
225	}
226
227	break;
228    }
229}
230
231void
232pmMarkAllCPUsOff(void)
233{
234    if (pmInitDone
235	&& pmDispatch != NULL
236	&& pmDispatch->markAllCPUsOff != NULL)
237	(*pmDispatch->markAllCPUsOff)();
238}
239
240static void
241pmInitComplete(void)
242{
243    if (earlyTopology
244	&& pmDispatch != NULL
245	&& pmDispatch->pmCPUStateInit != NULL) {
246	(*pmDispatch->pmCPUStateInit)();
247	earlyTopology = FALSE;
248    }
249
250    pmInitDone = 1;
251}
252
253x86_lcpu_t *
254pmGetLogicalCPU(int cpu)
255{
256    return(cpu_to_lcpu(cpu));
257}
258
259x86_lcpu_t *
260pmGetMyLogicalCPU(void)
261{
262    cpu_data_t	*cpup	= current_cpu_datap();
263
264    return(&cpup->lcpu);
265}
266
267static x86_core_t *
268pmGetCore(int cpu)
269{
270    return(cpu_to_core(cpu));
271}
272
273static x86_core_t *
274pmGetMyCore(void)
275{
276    cpu_data_t	*cpup	= current_cpu_datap();
277
278    return(cpup->lcpu.core);
279}
280
281static x86_die_t *
282pmGetDie(int cpu)
283{
284    return(cpu_to_die(cpu));
285}
286
287static x86_die_t *
288pmGetMyDie(void)
289{
290    cpu_data_t	*cpup	= current_cpu_datap();
291
292    return(cpup->lcpu.die);
293}
294
295static x86_pkg_t *
296pmGetPackage(int cpu)
297{
298    return(cpu_to_package(cpu));
299}
300
301static x86_pkg_t *
302pmGetMyPackage(void)
303{
304    cpu_data_t	*cpup	= current_cpu_datap();
305
306    return(cpup->lcpu.package);
307}
308
309static void
310pmLockCPUTopology(int lock)
311{
312    if (lock) {
313	simple_lock(&x86_topo_lock);
314    } else {
315	simple_unlock(&x86_topo_lock);
316    }
317}
318
319/*
320 * Called to get the next deadline that has been set by the
321 * power management code.
322 * Note: a return of 0 from AICPM and this routine signifies
323 * that no deadline is set.
324 */
325uint64_t
326pmCPUGetDeadline(cpu_data_t *cpu)
327{
328    uint64_t	deadline	= 0;
329
330    if (pmInitDone
331	&& pmDispatch != NULL
332	&& pmDispatch->GetDeadline != NULL)
333	deadline = (*pmDispatch->GetDeadline)(&cpu->lcpu);
334
335    return(deadline);
336}
337
338/*
339 * Called to determine if the supplied deadline or the power management
340 * deadline is sooner.  Returns which ever one is first.
341 */
342uint64_t
343pmCPUSetDeadline(cpu_data_t *cpu, uint64_t deadline)
344{
345   if (pmInitDone
346	&& pmDispatch != NULL
347	&& pmDispatch->SetDeadline != NULL)
348	deadline = (*pmDispatch->SetDeadline)(&cpu->lcpu, deadline);
349
350    return(deadline);
351}
352
353/*
354 * Called when a power management deadline expires.
355 */
356void
357pmCPUDeadline(cpu_data_t *cpu)
358{
359    if (pmInitDone
360	&& pmDispatch != NULL
361	&& pmDispatch->Deadline != NULL)
362	(*pmDispatch->Deadline)(&cpu->lcpu);
363}
364
365/*
366 * Called to get a CPU out of idle.
367 */
368boolean_t
369pmCPUExitIdle(cpu_data_t *cpu)
370{
371    boolean_t		do_ipi;
372
373    if (pmInitDone
374	&& pmDispatch != NULL
375	&& pmDispatch->exitIdle != NULL)
376	do_ipi = (*pmDispatch->exitIdle)(&cpu->lcpu);
377    else
378	do_ipi = TRUE;
379
380    return(do_ipi);
381}
382
383kern_return_t
384pmCPUExitHalt(int cpu)
385{
386    kern_return_t	rc	= KERN_INVALID_ARGUMENT;
387
388    if (pmInitDone
389	&& pmDispatch != NULL
390	&& pmDispatch->exitHalt != NULL)
391	rc = pmDispatch->exitHalt(cpu_to_lcpu(cpu));
392
393    return(rc);
394}
395
396kern_return_t
397pmCPUExitHaltToOff(int cpu)
398{
399    kern_return_t	rc	= KERN_SUCCESS;
400
401    if (pmInitDone
402	&& pmDispatch != NULL
403	&& pmDispatch->exitHaltToOff != NULL)
404	rc = pmDispatch->exitHaltToOff(cpu_to_lcpu(cpu));
405
406    return(rc);
407}
408
409/*
410 * Called to initialize the power management structures for the CPUs.
411 */
412void
413pmCPUStateInit(void)
414{
415    if (pmDispatch != NULL && pmDispatch->pmCPUStateInit != NULL)
416	(*pmDispatch->pmCPUStateInit)();
417    else
418	earlyTopology = TRUE;
419}
420
421/*
422 * Called when a CPU is being restarted after being powered off (as in S3).
423 */
424void
425pmCPUMarkRunning(cpu_data_t *cpu)
426{
427    cpu_data_t	*cpup	= current_cpu_datap();
428
429    if (pmInitDone
430	&& pmDispatch != NULL
431	&& pmDispatch->markCPURunning != NULL)
432	(*pmDispatch->markCPURunning)(&cpu->lcpu);
433    else
434	cpup->lcpu.state = LCPU_RUN;
435}
436
437/*
438 * Called to get/set CPU power management state.
439 */
440int
441pmCPUControl(uint32_t cmd, void *datap)
442{
443    int		rc	= -1;
444
445    if (pmDispatch != NULL
446	&& pmDispatch->pmCPUControl != NULL)
447	rc = (*pmDispatch->pmCPUControl)(cmd, datap);
448
449    return(rc);
450}
451
452/*
453 * Called to save the timer state used by power management prior
454 * to "sleeping".
455 */
456void
457pmTimerSave(void)
458{
459    if (pmDispatch != NULL
460	&& pmDispatch->pmTimerStateSave != NULL)
461	(*pmDispatch->pmTimerStateSave)();
462}
463
464/*
465 * Called to restore the timer state used by power management after
466 * waking from "sleep".
467 */
468void
469pmTimerRestore(void)
470{
471    if (pmDispatch != NULL
472	&& pmDispatch->pmTimerStateRestore != NULL)
473	(*pmDispatch->pmTimerStateRestore)();
474}
475
476/*
477 * Set the worst-case time for the C4 to C2 transition.
478 * No longer does anything.
479 */
480void
481ml_set_maxsnoop(__unused uint32_t maxdelay)
482{
483}
484
485
486/*
487 * Get the worst-case time for the C4 to C2 transition.  Returns nanoseconds.
488 */
489unsigned
490ml_get_maxsnoop(void)
491{
492    uint64_t	max_snoop	= 0;
493
494    if (pmInitDone
495	&& pmDispatch != NULL
496	&& pmDispatch->getMaxSnoop != NULL)
497	max_snoop = pmDispatch->getMaxSnoop();
498
499    return((unsigned)(max_snoop & 0xffffffff));
500}
501
502
503uint32_t
504ml_get_maxbusdelay(void)
505{
506    uint64_t	max_delay	= 0;
507
508    if (pmInitDone
509	&& pmDispatch != NULL
510	&& pmDispatch->getMaxBusDelay != NULL)
511	max_delay = pmDispatch->getMaxBusDelay();
512
513    return((uint32_t)(max_delay & 0xffffffff));
514}
515
516/*
517 * Set the maximum delay time allowed for snoop on the bus.
518 *
519 * Note that this value will be compared to the amount of time that it takes
520 * to transition from a non-snooping power state (C4) to a snooping state (C2).
521 * If maxBusDelay is less than C4C2SnoopDelay,
522 * we will not enter the lowest power state.
523 */
524void
525ml_set_maxbusdelay(uint32_t mdelay)
526{
527    uint64_t	maxdelay	= mdelay;
528
529    if (pmDispatch != NULL
530	&& pmDispatch->setMaxBusDelay != NULL) {
531	earlyMaxBusDelay = DELAY_UNSET;
532	pmDispatch->setMaxBusDelay(maxdelay);
533    } else
534	earlyMaxBusDelay = maxdelay;
535}
536
537uint64_t
538ml_get_maxintdelay(void)
539{
540    uint64_t	max_delay	= 0;
541
542    if (pmDispatch != NULL
543	&& pmDispatch->getMaxIntDelay != NULL)
544	max_delay = pmDispatch->getMaxIntDelay();
545
546    return(max_delay);
547}
548
549/*
550 * Set the maximum delay allowed for an interrupt.
551 */
552void
553ml_set_maxintdelay(uint64_t mdelay)
554{
555    if (pmDispatch != NULL
556	&& pmDispatch->setMaxIntDelay != NULL) {
557	earlyMaxIntDelay = DELAY_UNSET;
558	pmDispatch->setMaxIntDelay(mdelay);
559    } else
560	earlyMaxIntDelay = mdelay;
561}
562
563boolean_t
564ml_get_interrupt_prewake_applicable()
565{
566    boolean_t applicable = FALSE;
567
568    if (pmInitDone
569	&& pmDispatch != NULL
570	&& pmDispatch->pmInterruptPrewakeApplicable != NULL)
571	applicable = pmDispatch->pmInterruptPrewakeApplicable();
572
573    return applicable;
574}
575
576/*
577 * Put a CPU into "safe" mode with respect to power.
578 *
579 * Some systems cannot operate at a continuous "normal" speed without
580 * exceeding the thermal design.  This is called per-CPU to place the
581 * CPUs into a "safe" operating mode.
582 */
583void
584pmSafeMode(x86_lcpu_t *lcpu, uint32_t flags)
585{
586    if (pmDispatch != NULL
587	&& pmDispatch->pmCPUSafeMode != NULL)
588	pmDispatch->pmCPUSafeMode(lcpu, flags);
589    else {
590	/*
591	 * Do something reasonable if the KEXT isn't present.
592	 *
593	 * We only look at the PAUSE and RESUME flags.  The other flag(s)
594	 * will not make any sense without the KEXT, so just ignore them.
595	 *
596	 * We set the CPU's state to indicate that it's halted.  If this
597	 * is the CPU we're currently running on, then spin until the
598	 * state becomes non-halted.
599	 */
600	if (flags & PM_SAFE_FL_PAUSE) {
601	    lcpu->state = LCPU_PAUSE;
602	    if (lcpu == x86_lcpu()) {
603		while (lcpu->state == LCPU_PAUSE)
604		    cpu_pause();
605	    }
606	}
607
608	/*
609	 * Clear the halted flag for the specified CPU, that will
610	 * get it out of it's spin loop.
611	 */
612	if (flags & PM_SAFE_FL_RESUME) {
613	    lcpu->state = LCPU_RUN;
614	}
615    }
616}
617
618static uint32_t		saved_run_count = 0;
619
620void
621machine_run_count(uint32_t count)
622{
623    if (pmDispatch != NULL
624	&& pmDispatch->pmSetRunCount != NULL)
625	pmDispatch->pmSetRunCount(count);
626    else
627	saved_run_count = count;
628}
629
630boolean_t
631machine_processor_is_inactive(processor_t processor)
632{
633    int		cpu = processor->cpu_id;
634
635    if (pmDispatch != NULL
636	&& pmDispatch->pmIsCPUUnAvailable != NULL)
637	return(pmDispatch->pmIsCPUUnAvailable(cpu_to_lcpu(cpu)));
638    else
639	return(FALSE);
640}
641
642processor_t
643machine_choose_processor(processor_set_t pset,
644			 processor_t preferred)
645{
646    int		startCPU;
647    int		endCPU;
648    int		preferredCPU;
649    int		chosenCPU;
650
651    if (!pmInitDone)
652	return(preferred);
653
654    if (pset == NULL) {
655	startCPU = -1;
656	endCPU = -1;
657    } else {
658	startCPU = pset->cpu_set_low;
659	endCPU = pset->cpu_set_hi;
660    }
661
662    if (preferred == NULL)
663	preferredCPU = -1;
664    else
665	preferredCPU = preferred->cpu_id;
666
667    if (pmDispatch != NULL
668	&& pmDispatch->pmChooseCPU != NULL) {
669	chosenCPU = pmDispatch->pmChooseCPU(startCPU, endCPU, preferredCPU);
670
671	if (chosenCPU == -1)
672	    return(NULL);
673	return(cpu_datap(chosenCPU)->cpu_processor);
674    }
675
676    return(preferred);
677}
678
679static int
680pmThreadGetUrgency(uint64_t *rt_period, uint64_t *rt_deadline)
681{
682
683    return(thread_get_urgency(rt_period, rt_deadline));
684}
685
686#if	DEBUG
687uint32_t	urgency_stats[64][THREAD_URGENCY_MAX];
688#endif
689
690#define		URGENCY_NOTIFICATION_ASSERT_NS (5 * 1000 * 1000)
691uint64_t	urgency_notification_assert_abstime_threshold, urgency_notification_max_recorded;
692
693void
694thread_tell_urgency(int urgency,
695    uint64_t rt_period,
696    uint64_t rt_deadline)
697{
698	uint64_t	urgency_notification_time_start, delta;
699	boolean_t	urgency_assert = (urgency_notification_assert_abstime_threshold != 0);
700	assert(get_preemption_level() > 0 || ml_get_interrupts_enabled() == FALSE);
701#if	DEBUG
702	urgency_stats[cpu_number() % 64][urgency]++;
703#endif
704	if (!pmInitDone
705	    || pmDispatch == NULL
706	    || pmDispatch->pmThreadTellUrgency == NULL)
707		return;
708
709	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_START, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0);
710
711	if (__improbable((urgency_assert == TRUE)))
712		urgency_notification_time_start = mach_absolute_time();
713
714	pmDispatch->pmThreadTellUrgency(urgency, rt_period, rt_deadline);
715
716	if (__improbable((urgency_assert == TRUE))) {
717		delta = mach_absolute_time() - urgency_notification_time_start;
718
719		if (__improbable(delta > urgency_notification_max_recorded)) {
720			/* This is not synchronized, but it doesn't matter
721			 * if we (rarely) miss an event, as it is statistically
722			 * unlikely that it will never recur.
723			 */
724			urgency_notification_max_recorded = delta;
725
726			if (__improbable((delta > urgency_notification_assert_abstime_threshold) && !machine_timeout_suspended()))
727				panic("Urgency notification callout %p exceeded threshold, 0x%llx abstime units", pmDispatch->pmThreadTellUrgency, delta);
728		}
729	}
730
731	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_END, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0);
732}
733
734void
735active_rt_threads(boolean_t active)
736{
737    if (!pmInitDone
738	|| pmDispatch == NULL
739	|| pmDispatch->pmActiveRTThreads == NULL)
740	return;
741
742    pmDispatch->pmActiveRTThreads(active);
743}
744
745static uint32_t
746pmGetSavedRunCount(void)
747{
748    return(saved_run_count);
749}
750
751/*
752 * Returns the root of the package tree.
753 */
754x86_pkg_t *
755pmGetPkgRoot(void)
756{
757    return(x86_pkgs);
758}
759
760static boolean_t
761pmCPUGetHibernate(int cpu)
762{
763    return(cpu_datap(cpu)->cpu_hibernate);
764}
765
766processor_t
767pmLCPUtoProcessor(int lcpu)
768{
769    return(cpu_datap(lcpu)->cpu_processor);
770}
771
772static void
773pmReSyncDeadlines(int cpu)
774{
775    static boolean_t	registered	= FALSE;
776
777    if (!registered) {
778	PM_interrupt_register(&etimer_resync_deadlines);
779	registered = TRUE;
780    }
781
782    if ((uint32_t)cpu == current_cpu_datap()->lcpu.cpu_num)
783	etimer_resync_deadlines();
784    else
785	cpu_PM_interrupt(cpu);
786}
787
788static void
789pmSendIPI(int cpu)
790{
791    lapic_send_ipi(cpu, LAPIC_PM_INTERRUPT);
792}
793
794static void
795pmGetNanotimeInfo(pm_rtc_nanotime_t *rtc_nanotime)
796{
797	/*
798	 * Make sure that nanotime didn't change while we were reading it.
799	 */
800	do {
801		rtc_nanotime->generation = pal_rtc_nanotime_info.generation; /* must be first */
802		rtc_nanotime->tsc_base = pal_rtc_nanotime_info.tsc_base;
803		rtc_nanotime->ns_base = pal_rtc_nanotime_info.ns_base;
804		rtc_nanotime->scale = pal_rtc_nanotime_info.scale;
805		rtc_nanotime->shift = pal_rtc_nanotime_info.shift;
806	} while(pal_rtc_nanotime_info.generation != 0
807		&& rtc_nanotime->generation != pal_rtc_nanotime_info.generation);
808}
809
810uint32_t
811pmTimerQueueMigrate(int target_cpu)
812{
813    /* Call the etimer code to do this. */
814    return (target_cpu != cpu_number())
815		? etimer_queue_migrate(target_cpu)
816		: 0;
817}
818
819
820/*
821 * Called by the power management kext to register itself and to get the
822 * callbacks it might need into other kernel functions.  This interface
823 * is versioned to allow for slight mis-matches between the kext and the
824 * kernel.
825 */
826void
827pmKextRegister(uint32_t version, pmDispatch_t *cpuFuncs,
828	       pmCallBacks_t *callbacks)
829{
830    if (callbacks != NULL && version == PM_DISPATCH_VERSION) {
831	callbacks->setRTCPop            = setPop;
832	callbacks->resyncDeadlines      = pmReSyncDeadlines;
833	callbacks->initComplete         = pmInitComplete;
834	callbacks->GetLCPU              = pmGetLogicalCPU;
835	callbacks->GetCore              = pmGetCore;
836	callbacks->GetDie               = pmGetDie;
837	callbacks->GetPackage           = pmGetPackage;
838	callbacks->GetMyLCPU            = pmGetMyLogicalCPU;
839	callbacks->GetMyCore            = pmGetMyCore;
840	callbacks->GetMyDie             = pmGetMyDie;
841	callbacks->GetMyPackage         = pmGetMyPackage;
842	callbacks->GetPkgRoot           = pmGetPkgRoot;
843	callbacks->LockCPUTopology      = pmLockCPUTopology;
844	callbacks->GetHibernate         = pmCPUGetHibernate;
845	callbacks->LCPUtoProcessor      = pmLCPUtoProcessor;
846	callbacks->ThreadBind           = thread_bind;
847	callbacks->GetSavedRunCount     = pmGetSavedRunCount;
848	callbacks->GetNanotimeInfo	= pmGetNanotimeInfo;
849	callbacks->ThreadGetUrgency	= pmThreadGetUrgency;
850	callbacks->RTCClockAdjust	= rtc_clock_adjust;
851	callbacks->timerQueueMigrate    = pmTimerQueueMigrate;
852	callbacks->topoParms            = &topoParms;
853	callbacks->pmSendIPI		= pmSendIPI;
854	callbacks->InterruptPending	= lapic_is_interrupt_pending;
855	callbacks->IsInterrupting	= lapic_is_interrupting;
856	callbacks->InterruptStats	= lapic_interrupt_counts;
857	callbacks->DisableApicTimer	= lapic_disable_timer;
858    } else {
859	panic("Version mis-match between Kernel and CPU PM");
860    }
861
862    if (cpuFuncs != NULL) {
863        if (pmDispatch) {
864            panic("Attempt to re-register power management interface--AICPM present in xcpm mode? %p->%p", pmDispatch, cpuFuncs);
865        }
866
867	pmDispatch = cpuFuncs;
868
869	if (earlyTopology
870	    && pmDispatch->pmCPUStateInit != NULL) {
871	    (*pmDispatch->pmCPUStateInit)();
872	    earlyTopology = FALSE;
873	}
874
875	if (pmDispatch->pmIPIHandler != NULL) {
876	    lapic_set_pm_func((i386_intr_func_t)pmDispatch->pmIPIHandler);
877	}
878    }
879}
880
881/*
882 * Unregisters the power management functions from the kext.
883 */
884void
885pmUnRegister(pmDispatch_t *cpuFuncs)
886{
887    if (cpuFuncs != NULL && pmDispatch == cpuFuncs) {
888	pmDispatch = NULL;
889    }
890}
891
892/******************************************************************************
893 *
894 * All of the following are deprecated interfaces and no longer used.
895 *
896 ******************************************************************************/
897kern_return_t
898pmsControl(__unused uint32_t request, __unused user_addr_t reqaddr,
899	   __unused uint32_t reqsize)
900{
901    return(KERN_SUCCESS);
902}
903
904void
905pmsInit(void)
906{
907}
908
909void
910pmsStart(void)
911{
912}
913
914void
915pmsPark(void)
916{
917}
918
919void
920pmsRun(__unused uint32_t nstep)
921{
922}
923
924kern_return_t
925pmsBuild(__unused pmsDef *pd, __unused uint32_t pdsize,
926	 __unused pmsSetFunc_t *functab,
927	 __unused uint32_t platformData, __unused pmsQueryFunc_t queryFunc)
928{
929    return(KERN_SUCCESS);
930}
931
932void machine_track_platform_idle(boolean_t entry) {
933	cpu_data_t		*my_cpu		= current_cpu_datap();
934
935	if (entry) {
936		(void)__sync_fetch_and_add(&my_cpu->lcpu.package->num_idle, 1);
937	}
938 	else {
939 		uint32_t nidle = __sync_fetch_and_sub(&my_cpu->lcpu.package->num_idle, 1);
940 		if (nidle == topoParms.nLThreadsPerPackage) {
941 			my_cpu->lcpu.package->package_idle_exits++;
942 		}
943 	}
944}
945