osfmk/pmc/pmc.c

/*
 * Copyright (c) 2009 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 */

#include <kern/kalloc.h>
#include <kern/kern_types.h>
#include <kern/locks.h>
#include <kern/misc_protos.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <machine/machine_cpu.h>

#include <pmc/pmc.h>

#include <libkern/OSAtomic.h>

#if defined(__i386__) || defined(__x86_64__)
#include <i386/mp.h>
#endif

#if CONFIG_COUNTERS

/* various debug logging enable */
#undef DEBUG_COUNTERS

typedef uint8_t pmc_state_event_t;

#define PMC_STATE_EVENT_START				0
#define PMC_STATE_EVENT_STOP				1
#define PMC_STATE_EVENT_FREE				2
#define PMC_STATE_EVENT_INTERRUPT			3
#define PMC_STATE_EVENT_END_OF_INTERRUPT	4
#define PMC_STATE_EVENT_CONTEXT_IN			5
#define PMC_STATE_EVENT_CONTEXT_OUT			6
#define PMC_STATE_EVENT_LOAD_FINISHED		7
#define PMC_STATE_EVENT_STORE_FINISHED		8

/* PMC spin timeouts */
#define PMC_SPIN_THRESHOLD	10	/* Number of spins to allow before checking mach_absolute_time() */
#define PMC_SPIN_TIMEOUT_US	10	/* Time in microseconds before the spin causes an assert */

uint64_t pmc_spin_timeout_count = 0;	/* Number of times where a PMC spin loop causes a timeout */

#ifdef DEBUG_COUNTERS
#	include <pexpert/pexpert.h>
#	define COUNTER_DEBUG(...) \
	do { \
		kprintf("[%s:%s][%u] ", __FILE__, __PRETTY_FUNCTION__, cpu_number()); \
		kprintf(__VA_ARGS__); \
	} while(0)

#	define PRINT_PERF_MON(x)	\
	do { \
		kprintf("perfmon: %p (obj: %p refCt: %u switchable: %u)\n", \
			x, x->object, x->useCount, \
			(x->methods.flags & PERFMON_FLAG_SUPPORTS_CONTEXT_SWITCHING) ? \
			1 : 0); \
	} while(0)

static const char const * pmc_state_state_name(pmc_state_t state) {
	switch (PMC_STATE_STATE(state)) {
		case PMC_STATE_STATE_INVALID:
			return "INVALID";
		case PMC_STATE_STATE_STOP:
			return "STOP";
		case PMC_STATE_STATE_CAN_RUN:
			return "CAN_RUN";
		case PMC_STATE_STATE_LOAD:
			return "LOAD";
		case PMC_STATE_STATE_RUN:
			return "RUN";
		case PMC_STATE_STATE_STORE:
			return "STORE";
		case PMC_STATE_STATE_INTERRUPT:
			return "INTERRUPT";
		case PMC_STATE_STATE_DEALLOC:
			return "DEALLOC";
		default:
			return "UNKNOWN";
	}
}

static const char const * pmc_state_event_name(pmc_state_event_t event) {
	switch (event) {
		case PMC_STATE_EVENT_START:
			return "START";
		case PMC_STATE_EVENT_STOP:
			return "STOP";
		case PMC_STATE_EVENT_FREE:
			return "FREE";
		case PMC_STATE_EVENT_INTERRUPT:
			return "INTERRUPT";
		case PMC_STATE_EVENT_END_OF_INTERRUPT:
			return "END OF INTERRUPT";
		case PMC_STATE_EVENT_CONTEXT_IN:
			return "CONTEXT IN";
		case PMC_STATE_EVENT_CONTEXT_OUT:
			return "CONTEXT OUT";
		case PMC_STATE_EVENT_LOAD_FINISHED:
			return "LOAD_FINISHED";
		case PMC_STATE_EVENT_STORE_FINISHED:
			return "STORE_FINISHED";
		default:
			return "UNKNOWN";
	}
}

#	define PMC_STATE_FORMAT	"<%s, %u, %s%s%s>"
#	define PMC_STATE_ARGS(x)	pmc_state_state_name(x), PMC_STATE_CONTEXT_COUNT(x), ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_INTERRUPTING) ? "I" : ""), \
					((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_STOPPING) ? "S" : ""), ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_DEALLOCING) ? "D" : "")
#else
#	define COUNTER_DEBUG(...)
#	define PRINT_PERF_MON(x)
#	define PMC_STATE_FORMAT
#	define PMC_STATE_ARGS(x)
#endif

/*!struct
 * pmc_config is the data behind a pmc_config_t.
 * @member object A pointer to an instance of IOPerformanceCounterConfiguration
 * @member method A pointer to a method to call to handle PMI.
 * @member interrupt_after_value Cause a PMI after the counter counts this many
 * events.
 * @member refCon Passed to the @method method as the refCon argument.
 */
struct pmc_config {
	pmc_config_object_t object;
	volatile pmc_interrupt_method_t method;
	uint64_t interrupt_after_value;
	void *refCon;
};

/*
 * Allocation Zones
 *
 * Two allocation zones - Perf zone small and Perf zone big.
 * Each zone has associated maximums, defined below.
 * The small zone is the max of the smallest allocation objects (all sizes on
 * K64):
 *	perf_monitor_t - 48 bytes
 *		perf_monitor_methods_t - 28 bytes
 *	pmc_reservation_t - 48 bytes
 *  pmc_config_t - 32 bytes
 * perf_small_zone unit size is (on K64) 48 bytes
 * perf_small_zone max count must be max number of perf monitors, plus (max
 * number of reservations * 2). The "*2" is because each reservation has a
 * pmc_config_t within.
 *
 * Big zone is max of the larger allocation units
 *	pmc_t - 144 bytes
 *		pmc_methods_t - 116 bytes
 * perf_big_zone unit size is (on K64) 144 bytes
 * perf_big_zone max count is the max number of PMCs we support.
 */

static zone_t perf_small_zone = NULL;
#define MAX_PERF_SMALLS		(256 + 8196 + 8196)
#define PERF_SMALL_UNIT_SZ	(MAX(MAX(sizeof(struct perf_monitor), \
	sizeof(struct pmc_reservation)), sizeof(struct pmc_config)))

static zone_t perf_big_zone = NULL;
#define MAX_PERF_BIGS		(1024)
#define PERF_BIG_UNIT_SZ	(sizeof(struct pmc))

/*
 * Locks and Lock groups
 */
static lck_grp_t *pmc_lock_grp = LCK_GRP_NULL;
static lck_grp_attr_t *pmc_lock_grp_attr;
static lck_attr_t *pmc_lock_attr;

/* PMC tracking queue locks */

static lck_mtx_t  cpu_monitor_queue_mutex;   /* protects per-cpu queues at initialisation time */
static lck_spin_t perf_monitor_queue_spin;   /* protects adding and removing from queue */
static lck_spin_t perf_counters_queue_spin;  /* protects adding and removing from queue */

/* Reservation tracking queues lock */
static lck_spin_t reservations_spin;

/*
 * Tracking queues
 *
 * Keeps track of registered perf monitors and perf counters
 */

static queue_head_t **cpu_monitor_queues = NULL;

static queue_head_t *perf_monitors_queue = NULL;
static volatile uint32_t perf_monitors_count = 0U;

static queue_head_t *perf_counters_queue = NULL;
static volatile uint32_t perf_counters_count = 0U;

/*
 * Reservation queues
 *
 * Keeps track of all system, task, and thread-level reservations (both active and
 * inactive).
 *
 * We track them all here (rather than in their respective task or thread only)
 * so that we can inspect our tracking data directly (rather than peeking at
 * every task and thread) to determine if/when a new reservation would
 * constitute a conflict.
 */

static queue_head_t *system_reservations = NULL;
static volatile uint32_t system_reservation_count = 0U;

static queue_head_t *task_reservations = NULL;
static volatile uint32_t task_reservation_count = 0U;

static queue_head_t *thread_reservations = NULL;
static volatile uint32_t thread_reservation_count = 0U;

#if XNU_KERNEL_PRIVATE

/*
 * init_pmc_locks creates and initializes all the locks and lock groups and lock
 * attributes required for the pmc sub-system.
 */
static void init_pmc_locks(void) {
	pmc_lock_attr = lck_attr_alloc_init();
	assert(pmc_lock_attr);

	pmc_lock_grp_attr = lck_grp_attr_alloc_init();
	assert(pmc_lock_grp_attr);

	pmc_lock_grp = lck_grp_alloc_init("pmc", pmc_lock_grp_attr);
	assert(pmc_lock_grp);

	lck_spin_init(&perf_monitor_queue_spin, pmc_lock_grp, pmc_lock_attr);
	lck_spin_init(&perf_counters_queue_spin, pmc_lock_grp, pmc_lock_attr);

	lck_spin_init(&reservations_spin, pmc_lock_grp, pmc_lock_attr);

	lck_mtx_init(&cpu_monitor_queue_mutex, pmc_lock_grp, pmc_lock_attr);
}

/*
 * init_pmc_zones initializes the allocation zones used by the pmc subsystem
 */
static void init_pmc_zones(void) {
	perf_small_zone = zinit(PERF_SMALL_UNIT_SZ,
		MAX_PERF_SMALLS * PERF_SMALL_UNIT_SZ, MAX_PERF_SMALLS,
		"pmc.small zone");

	assert(perf_small_zone);

	perf_big_zone = zinit(PERF_BIG_UNIT_SZ,
		MAX_PERF_BIGS * PERF_BIG_UNIT_SZ, MAX_PERF_BIGS,
		"pmc.big zone");

	assert(perf_big_zone);
}

/*
 * init_pmc_queues allocates and initializes the tracking queues for
 * registering and reserving individual pmcs and perf monitors.
 */
static void init_pmc_queues(void) {

	perf_monitors_queue = (queue_head_t*)kalloc(sizeof(queue_head_t));
	assert(perf_monitors_queue);

	queue_init(perf_monitors_queue);

	perf_counters_queue = (queue_head_t*)kalloc(sizeof(queue_head_t));
	assert(perf_counters_queue);

	queue_init(perf_counters_queue);

	system_reservations = (queue_head_t*)kalloc(sizeof(queue_t));
	assert(system_reservations);

	queue_init(system_reservations);

	task_reservations = (queue_head_t*)kalloc(sizeof(queue_head_t));
	assert(task_reservations);

	queue_init(task_reservations);

	thread_reservations = (queue_head_t*)kalloc(sizeof(queue_head_t));
	assert(thread_reservations);

	queue_init(thread_reservations);
}

/*
 * pmc_bootstrap brings up all the necessary infrastructure required to use the
 * pmc sub-system.
 */
__private_extern__
void pmc_bootstrap(void) {
	/* build our alloc zones */
	init_pmc_zones();

	/* build the locks */
	init_pmc_locks();

	/* build our tracking queues */
	init_pmc_queues();
}

#endif /* XNU_KERNEL_PRIVATE */

/*
 * Perf Monitor Internals
 */

static perf_monitor_t perf_monitor_alloc(void) {
	/* perf monitors come from the perf small zone */
	return (perf_monitor_t)zalloc(perf_small_zone);
}

static void perf_monitor_free(void *pm) {
	zfree(perf_small_zone, pm);
}

static void perf_monitor_init(perf_monitor_t pm, int cpu) {
	assert(pm);

	pm->object = NULL;

	bzero(&(pm->methods), sizeof(perf_monitor_methods_t));

	pm->useCount = 1;	/* initial retain count of 1, for caller */

	pm->reservedCounters = 0;

	pm->cpu = cpu;

	pm->link.next = pm->link.prev = (queue_entry_t)NULL;
	pm->cpu_link.next = pm->cpu_link.prev = (queue_entry_t)NULL;
}

/*
 * perf_monitor_dequeue removes the given perf_monitor_t from the
 * perf_monitor_queue, thereby unregistering it with the system.
 */
static void perf_monitor_dequeue(perf_monitor_t pm) {
	lck_spin_lock(&perf_monitor_queue_spin);

	if (pm->methods.flags & PERFMON_FLAG_REQUIRES_IDLE_NOTIFICATIONS) {
		/* If this flag is set, the monitor is already validated to be
		 * accessible from a single cpu only.
		 */
		queue_remove(cpu_monitor_queues[pm->cpu], pm, perf_monitor_t, cpu_link);
	}

	/*
	 * remove the @pm object from the @perf_monitor_queue queue (it is of type
	 * <perf_monitor_t> and has a field called @link that is the queue_link_t
	 */
	queue_remove(perf_monitors_queue, pm, perf_monitor_t, link);

	perf_monitors_count--;

	lck_spin_unlock(&perf_monitor_queue_spin);
}

/*
 * perf_monitor_enqueue adds the given perf_monitor_t to the perf_monitor_queue,
 * thereby registering it for use with the system.
 */
static void perf_monitor_enqueue(perf_monitor_t pm) {

	lck_mtx_lock(&cpu_monitor_queue_mutex);
	lck_spin_lock(&perf_monitor_queue_spin);

	if (pm->cpu >= 0) {
            	/* Deferred initialisation; saves memory and permits ml_get_max_cpus()
            	 * to block until cpu initialisation is complete.
            	 */
            	if (!cpu_monitor_queues) {
            		uint32_t max_cpus;
            		queue_head_t **queues;
            		uint32_t i;

            		lck_spin_unlock(&perf_monitor_queue_spin);

            		max_cpus = ml_get_max_cpus();

            		queues = (queue_head_t**)kalloc(sizeof(queue_head_t*) * max_cpus);
            		assert(queues);
            		for (i = 0; i < max_cpus; i++) {
            			queue_head_t *queue = (queue_head_t*)kalloc(sizeof(queue_head_t));
            			assert(queue);
            			queue_init(queue);
            			queues[i] = queue;
            		}

            		lck_spin_lock(&perf_monitor_queue_spin);

            		cpu_monitor_queues = queues;
            	}

		queue_enter(cpu_monitor_queues[pm->cpu], pm, perf_monitor_t, cpu_link);
	}

	queue_enter(perf_monitors_queue, pm, perf_monitor_t, link);
	perf_monitors_count++;

	lck_spin_unlock(&perf_monitor_queue_spin);
	lck_mtx_unlock(&cpu_monitor_queue_mutex);
}

/*
 * perf_monitor_reference increments the reference count for the given
 * perf_monitor_t.
 */
static void perf_monitor_reference(perf_monitor_t pm) {
	assert(pm);

	OSIncrementAtomic(&(pm->useCount));
}

/*
 * perf_monitor_deallocate decrements the reference count for the given
 * perf_monitor_t.  If the reference count hits 0, the object is released back
 * to the perf_small_zone via a call to perf_monitor_free().
 */
static void perf_monitor_deallocate(perf_monitor_t pm) {
	assert(pm);

	/* If we just removed the last reference count */
	if(1 == OSDecrementAtomic(&(pm->useCount))) {
		/* Free the object */
		perf_monitor_free(pm);
	}
}

/*
 * perf_monitor_find attempts to find a perf_monitor_t that corresponds to the
 * given C++ object pointer that was used when registering with the subsystem.
 *
 * If found, the method returns the perf_monitor_t with an extra reference
 * placed on the object (or NULL if not
 * found).
 *
 * NOTE: Caller must use perf_monitor_deallocate to remove the extra reference after
 * calling perf_monitor_find.
 */
static perf_monitor_t perf_monitor_find(perf_monitor_object_t monitor) {
	assert(monitor);
	perf_monitor_t element = NULL;
	perf_monitor_t found = NULL;

	lck_spin_lock(&perf_monitor_queue_spin);

	queue_iterate(perf_monitors_queue, element, perf_monitor_t, link) {
 		if(element->object == monitor) {
			perf_monitor_reference(element);
			found = element;
			break;
		}
	}

	lck_spin_unlock(&perf_monitor_queue_spin);

	return found;
}

/*
 * perf_monitor_add_pmc adds a newly registered PMC to the perf monitor it is
 * associated with.
 */

static void perf_monitor_add_pmc(perf_monitor_t pm, pmc_t pmc __unused) {
	assert(pm);
	assert(pmc);

	/* Today, we merely add a reference count now that a new pmc is attached */
	perf_monitor_reference(pm);
}

/*
 * perf_monitor_remove_pmc removes a newly *un*registered PMC from the perf
 * monitor it is associated with.
 */
static void perf_monitor_remove_pmc(perf_monitor_t pm, pmc_t pmc __unused) {
	assert(pm);
	assert(pmc);

	/* Today, we merely remove a reference count now that the pmc is detached */
	perf_monitor_deallocate(pm);
}

/*
 * Perf Counter internals
 */

static pmc_t pmc_alloc(void) {
	return (pmc_t)zalloc(perf_big_zone);
}

static void pmc_free(void *pmc) {
	zfree(perf_big_zone, pmc);
}

/*
 * pmc_init initializes a newly allocated pmc_t
 */
static void pmc_init(pmc_t pmc) {
	assert(pmc);

	pmc->object = NULL;
	pmc->monitor = NULL;

	bzero(&pmc->methods, sizeof(pmc_methods_t));

	/* One reference for the caller */
	pmc->useCount = 1;
}

/*
 * pmc_reference increments the reference count of the given pmc_t
 */
static void pmc_reference(pmc_t pmc) {
	assert(pmc);

	OSIncrementAtomic(&(pmc->useCount));
}

/*
 * pmc_deallocate decrements the reference count of the given pmc_t. If the
 * reference count hits zero, the given pmc_t is deallocated and released back
 * to the allocation zone.
 */
static void pmc_deallocate(pmc_t pmc) {
	assert(pmc);

	/* If we just removed the last reference count */
	if(1 == OSDecrementAtomic(&(pmc->useCount))) {
		/* Free the pmc */
		pmc_free(pmc);
	}
}

/*
 * pmc_dequeue removes the given, newly *un*registered pmc from the
 * perf_counters_queue.
 */
static void pmc_dequeue(pmc_t pmc) {
	lck_spin_lock(&perf_counters_queue_spin);

	queue_remove(perf_counters_queue, pmc, pmc_t, link);

	perf_counters_count--;

	lck_spin_unlock(&perf_counters_queue_spin);
}

/*
 * pmc_enqueue adds the given, newly registered pmc to the perf_counters_queue
 */
static void pmc_enqueue(pmc_t pmc) {
	lck_spin_lock(&perf_counters_queue_spin);

	queue_enter(perf_counters_queue, pmc, pmc_t, link);

	perf_counters_count++;

	lck_spin_unlock(&perf_counters_queue_spin);
}

/*
 * pmc_find attempts to locate a pmc_t that was registered with the given
 * pmc_object_t pointer.  If found, it returns the pmc_t with an extra reference
 * which must be dropped by the caller by calling pmc_deallocate().
 */
static pmc_t pmc_find(pmc_object_t object) {
	assert(object);

	lck_spin_lock(&perf_counters_queue_spin);

	pmc_t element = NULL;
	pmc_t found = NULL;

	queue_iterate(perf_counters_queue, element, pmc_t, link) {
		if(element->object == object) {
			pmc_reference(element);
			found = element;
			break;
		}
	}

	lck_spin_unlock(&perf_counters_queue_spin);

	return found;
}

/*
 * Config internals
 */

/* Allocate a pmc_config_t */
static pmc_config_t pmc_config_alloc(pmc_t pmc __unused) {
	return (pmc_config_t)zalloc(perf_small_zone);
}

/* Free a pmc_config_t, and underlying pmc_config_object_t (if needed) */
static void pmc_config_free(pmc_t pmc, pmc_config_t config) {
	assert(pmc);
	assert(config);

	if(config->object) {
		pmc->methods.free_config(pmc->object, config->object);
		config->object = NULL;
	}

	zfree(perf_small_zone, config);
}

static kern_return_t pmc_open(pmc_t pmc) {
	assert(pmc);
	assert(pmc->object);
	assert(pmc->open_object);

	return pmc->methods.open(pmc->object, pmc->open_object);
}

static kern_return_t pmc_close(pmc_t pmc) {
	assert(pmc);
	assert(pmc->object);
	assert(pmc->open_object);

	return pmc->methods.close(pmc->object, pmc->open_object);
}

/*
 * Reservation Internals
 */

static kern_return_t pmc_internal_reservation_set_pmc(pmc_reservation_t resv, pmc_t pmc);
static void pmc_internal_reservation_store(pmc_reservation_t reservation);
static void pmc_internal_reservation_load(pmc_reservation_t reservation);

static pmc_reservation_t reservation_alloc(void) {
	/* pmc reservations come from the perf small zone */
	return (pmc_reservation_t)zalloc(perf_small_zone);
}

/*
 * reservation_free deallocates and releases all resources associated with the
 * given pmc_reservation_t.  This includes freeing the config used to create the
 * reservation, decrementing the reference count for the pmc used to create the
 * reservation, and deallocating the reservation's memory.
 */
static void reservation_free(pmc_reservation_t resv) {
	/* Free config */
	if(resv->config) {
		assert(resv->pmc);

		pmc_free_config(resv->pmc, resv->config);

		resv->config = NULL;
	}

	/* release PMC */
	(void)pmc_internal_reservation_set_pmc(resv, NULL);

	/* Free reservation */
	zfree(perf_small_zone, resv);
}

/*
 * reservation_init initializes a newly created reservation.
 */
static void reservation_init(pmc_reservation_t resv) {
	assert(resv);

	resv->pmc = NULL;
	resv->config = NULL;
	resv->value = 0ULL;

	resv->flags = 0U;
	resv->state = PMC_STATE(PMC_STATE_STATE_STOP, 0, 0);
	resv->active_last_context_in = 0U;

	/*
	 * Since this member is a union, we only need to set either the task
	 * or thread to NULL.
	 */
	resv->task = TASK_NULL;
}

/*
 * pmc_internal_reservation_set_pmc sets the pmc associated with the reservation object. If
 * there was one set already, it is deallocated (reference is dropped) before
 * the new one is set.  This methods increases the reference count of the given
 * pmc_t.
 *
 * NOTE: It is okay to pass NULL as the pmc_t - this will have the effect of
 * dropping the reference on any previously set pmc, and setting the reservation
 * to having no pmc set.
 */
static kern_return_t pmc_internal_reservation_set_pmc(pmc_reservation_t resv, pmc_t pmc) {
	assert(resv);

	if(resv->pmc) {
		(void)pmc_close(resv->pmc);
		pmc_deallocate(resv->pmc);
		resv->pmc = NULL;
	}

	resv->pmc = pmc;

	if(resv->pmc) {
		pmc_reference(resv->pmc);
		if(KERN_SUCCESS != pmc_open(resv->pmc)) {
			pmc_deallocate(resv->pmc);
			resv->pmc = NULL;

			return KERN_FAILURE;
		}
	}

	return KERN_SUCCESS;
}

/*
 * Used to place reservation into one of the system, task, and thread queues
 * Assumes the queue's spin lock is already held.
 */
static void pmc_internal_reservation_enqueue(queue_t queue, pmc_reservation_t resv) {
	assert(queue);
	assert(resv);

	queue_enter(queue, resv, pmc_reservation_t, link);
}

static void pmc_internal_reservation_dequeue(queue_t queue, pmc_reservation_t resv) {
	assert(queue);
	assert(resv);

	queue_remove(queue, resv, pmc_reservation_t, link);
}

/* Returns TRUE if the reservation applies to the current execution context */
static boolean_t pmc_internal_reservation_matches_context(pmc_reservation_t resv) {
	boolean_t ret = FALSE;
	assert(resv);

	if(PMC_FLAG_IS_SYSTEM_SCOPE(resv->flags)) {
		ret = TRUE;
	} else if(PMC_FLAG_IS_TASK_SCOPE(resv->flags)) {
		if(current_task() == resv->task) {
			ret = TRUE;
		}
	} else if(PMC_FLAG_IS_THREAD_SCOPE(resv->flags)) {
		if(current_thread() == resv->thread) {
			ret = TRUE;
		}
	}

	return ret;
}

/*
 * pmc_accessible_core_count returns the number of logical cores that can access
 * a given @pmc.  0 means every core in the system.
 */
static uint32_t pmc_accessible_core_count(pmc_t pmc) {
	assert(pmc);

	uint32_t *cores = NULL;
	size_t coreCt = 0UL;

	if(KERN_SUCCESS != pmc->methods.accessible_cores(pmc->object,
		&cores, &coreCt)) {
		coreCt = 0U;
	}

	return (uint32_t)coreCt;
}

/* spin lock for the queue must already be held */
/*
 * This method will inspect the task/thread of the reservation to see if it
 * matches the new incoming one (for thread/task reservations only).  Will only
 * return TRUE if the task/thread matches.
 */
static boolean_t pmc_internal_reservation_queue_contains_pmc(queue_t queue, pmc_reservation_t resv) {
	assert(queue);
	assert(resv);

	boolean_t ret = FALSE;
	pmc_reservation_t tmp = NULL;

	queue_iterate(queue, tmp, pmc_reservation_t, link) {
		if(tmp->pmc == resv->pmc) {
			/* PMC matches - make sure scope matches first */
			switch(PMC_FLAG_SCOPE(tmp->flags)) {
				case PMC_FLAG_SCOPE_SYSTEM:
					/*
					 * Found a reservation in system queue with same pmc - always a
					 * conflict.
					 */
					ret = TRUE;
					break;
				case PMC_FLAG_SCOPE_THREAD:
					/*
					 * Found one in thread queue with the same PMC as the
					 * argument. Only a conflict if argument scope isn't
					 * thread or system, or the threads match.
					 */
					ret = (PMC_FLAG_SCOPE(resv->flags) != PMC_FLAG_SCOPE_THREAD) ||
						(tmp->thread == resv->thread);

					if(!ret) {
						/*
						 * so far, no conflict - check that the pmc that is
						 * being reserved isn't accessible from more than
						 * one core, if it is, we need to say it's already
						 * taken.
						 */
						if(1 != pmc_accessible_core_count(tmp->pmc)) {
							ret = TRUE;
						}
					}
					break;
				case PMC_FLAG_SCOPE_TASK:
					/*
					 * Follow similar semantics for task scope.
					 */

					ret = (PMC_FLAG_SCOPE(resv->flags) != PMC_FLAG_SCOPE_TASK) ||
						(tmp->task == resv->task);
					if(!ret) {
						/*
						 * so far, no conflict - check that the pmc that is
						 * being reserved isn't accessible from more than
						 * one core, if it is, we need to say it's already
						 * taken.
						 */
						if(1 != pmc_accessible_core_count(tmp->pmc)) {
							ret = TRUE;
						}
					}

					break;
			}

			if(ret) break;
		}
	}

	return ret;
}

/*
 * pmc_internal_reservation_validate_for_pmc returns TRUE if the given reservation can be
 * added to its target queue without creating conflicts (target queue is
 * determined by the reservation's scope flags). Further, this method returns
 * FALSE if any level contains a reservation for a PMC that can be accessed from
 * more than just 1 core, and the given reservation also wants the same PMC.
 */
static boolean_t pmc_internal_reservation_validate_for_pmc(pmc_reservation_t resv) {
	assert(resv);
	boolean_t ret = TRUE;

	if(pmc_internal_reservation_queue_contains_pmc(system_reservations, resv) ||
		pmc_internal_reservation_queue_contains_pmc(task_reservations, resv) ||
		pmc_internal_reservation_queue_contains_pmc(thread_reservations, resv)) {
		ret = FALSE;
	}

	return ret;
}

static void pmc_internal_update_thread_flag(thread_t thread, boolean_t newFlag) {
	assert(thread);

	/* See if this thread needs it's PMC flag set */
	pmc_reservation_t tmp = NULL;

	if(!newFlag) {
		/*
		 * If the parent task just dropped its reservation, iterate the thread
		 * reservations to see if we need to keep the pmc flag set for the given
		 * thread or not.
		 */
		lck_spin_lock(&reservations_spin);

		queue_iterate(thread_reservations, tmp, pmc_reservation_t, link) {
			if(tmp->thread == thread) {
				newFlag = TRUE;
				break;
			}
		}

		lck_spin_unlock(&reservations_spin);
	}

	if(newFlag) {
		OSBitOrAtomic(THREAD_PMC_FLAG, &thread->t_chud);
	} else {
		OSBitAndAtomic(~(THREAD_PMC_FLAG), &thread->t_chud);
	}
}

/*
 * This operation is (worst case) O(N*M) where N is number of threads in the
 * given task, and M is the number of thread reservations in our system.
 */
static void pmc_internal_update_task_flag(task_t task, boolean_t newFlag) {
	assert(task);
	thread_t thread = NULL;

	if(newFlag) {
		OSBitOrAtomic(TASK_PMC_FLAG, &task->t_chud);
	} else {
		OSBitAndAtomic(~(TASK_PMC_FLAG), &task->t_chud);
	}

	task_lock(task);

	queue_iterate(&task->threads, thread, thread_t, task_threads) {
		/* propagate the task's mask down to each thread  */
		pmc_internal_update_thread_flag(thread, newFlag);
	}

	task_unlock(task);
}

/*
 * pmc_internal_reservation_add adds a reservation to the global tracking queues after
 * ensuring there are no reservation conflicts.  To do this, it takes all the
 * spin locks for all the queue (to ensure no other core goes and adds a
 * reservation for the same pmc to a queue that has already been checked).
 */
static boolean_t pmc_internal_reservation_add(pmc_reservation_t resv) {
	assert(resv);

	boolean_t ret = FALSE;

	/* always lock all three in the same order */
	lck_spin_lock(&reservations_spin);

	/* Check if the reservation can be added without conflicts */
	if(pmc_internal_reservation_validate_for_pmc(resv)) {

		/* add reservation to appropriate scope */
		switch(PMC_FLAG_SCOPE(resv->flags)) {
		case PMC_FLAG_SCOPE_SYSTEM:
			/* Simply add it to the system queue */
			pmc_internal_reservation_enqueue(system_reservations, resv);
			system_reservation_count++;

			lck_spin_unlock(&reservations_spin);

			break;

		case PMC_FLAG_SCOPE_TASK:
			assert(resv->task);

			/* Not only do we enqueue it in our local queue for tracking */
			pmc_internal_reservation_enqueue(task_reservations, resv);
			task_reservation_count++;

			lck_spin_unlock(&reservations_spin);

			/* update the task mask, and propagate it to existing threads */
			pmc_internal_update_task_flag(resv->task, TRUE);
			break;

		/* Thread-switched counter */
		case PMC_FLAG_SCOPE_THREAD:
			assert(resv->thread);

			/*
			 * Works the same as a task-switched counter, only at
			 * thread-scope
			 */

			pmc_internal_reservation_enqueue(thread_reservations, resv);
			thread_reservation_count++;

			lck_spin_unlock(&reservations_spin);

			pmc_internal_update_thread_flag(resv->thread, TRUE);
			break;
		}

		ret = TRUE;
	} else {
		lck_spin_unlock(&reservations_spin);
	}

	return ret;
}

static void pmc_internal_reservation_broadcast(pmc_reservation_t reservation, void (*action_func)(void *)) {
	uint32_t * cores;
	size_t core_cnt;

	/* Get the list of accessible cores */
	if (KERN_SUCCESS == pmc_get_accessible_core_list(reservation->pmc, &cores, &core_cnt)) {
		boolean_t intrs_enabled = ml_set_interrupts_enabled(FALSE);

		/* Fast case: the PMC is only accessible from one core and we happen to be on it */
		if (core_cnt == 1 && cores[0] == (uint32_t)cpu_number()) {
			action_func(reservation);
		} else {
			/* Call action_func on every accessible core */
#if defined(__i386__) || defined(__x86_64__)
			size_t ii;
			cpumask_t mask = 0;

			/* Build a mask for the accessible cores */
			if (core_cnt > 0) {
				for (ii = 0; ii < core_cnt; ii++) {
					mask |= cpu_to_cpumask(cores[ii]);
				}
			} else {
				/* core_cnt = 0 really means all cpus */
				mask = CPUMASK_ALL;
			}
			mp_cpus_call(mask, ASYNC, action_func, reservation);
#else
#error pmc_reservation_interrupt needs an inter-processor method invocation mechanism for this architecture
#endif
		}

		ml_set_interrupts_enabled(intrs_enabled);
	}

}

/*
 * pmc_internal_reservation_remove removes the given reservation from the appropriate
 * reservation queue according to its scope.
 *
 * NOTE: The scope flag must have been set for this method to function.
 */
static void pmc_internal_reservation_remove(pmc_reservation_t resv) {
	assert(resv);

	/*
	 * Due to the way the macros are written, we can't just blindly queue-remove
	 * the reservation without knowing which queue it's in. We figure this out
	 * using the reservation's scope flags.
	 */

	/* Lock the global spin lock */
	lck_spin_lock(&reservations_spin);

	switch(PMC_FLAG_SCOPE(resv->flags)) {

		case PMC_FLAG_SCOPE_SYSTEM:
			pmc_internal_reservation_dequeue(system_reservations, resv);
			system_reservation_count--;

			lck_spin_unlock(&reservations_spin);

			break;

		case PMC_FLAG_SCOPE_TASK:
			/* remove from the global queue */
			pmc_internal_reservation_dequeue(task_reservations, resv);
			task_reservation_count--;

			/* unlock the global */
			lck_spin_unlock(&reservations_spin);

			/* Recalculate task's counter mask */
			pmc_internal_update_task_flag(resv->task, FALSE);

			break;

		case PMC_FLAG_SCOPE_THREAD:
			pmc_internal_reservation_dequeue(thread_reservations, resv);
			thread_reservation_count--;

			lck_spin_unlock(&reservations_spin);

			/* recalculate the thread's counter mask */
			pmc_internal_update_thread_flag(resv->thread, FALSE);

			break;
	}
}

/* Reservation State Machine
 *
 * The PMC subsystem uses a 3-tuple of state information packed into a 32-bit quantity and a
 * set of 9 events to provide MP-safe bookkeeping and control flow.  The 3-tuple is comprised
 * of a state, a count of active contexts, and a set of modifier flags.  A state machine defines
 * the possible transitions at each event point given the current 3-tuple.  Atomicity is handled
 * by reading the current 3-tuple, applying the transformations indicated by the state machine
 * and then attempting to OSCompareAndSwap the transformed value.  If the OSCompareAndSwap fails,
 * the process is repeated until either the OSCompareAndSwap succeeds or not valid transitions are
 * available.
 *
 * The state machine is described using tuple notation for the current state and a related notation
 * for describing the transformations.  For concisness, the flag and state names are abbreviated as
 * follows:
 *
 * states:
 * S = STOP
 * CR = CAN_RUN
 * L = LOAD
 * R = RUN
 * ST = STORE
 * I = INTERRUPT
 * D = DEALLOC
 *
 * flags:
 *
 * S = STOPPING
 * D = DEALLOCING
 * I = INTERRUPTING
 *
 * The tuple notation is formed from the following pattern:
 *
 * tuple = < state, active-context-count, flags >
 * state = S | CR | L | R | ST | I | D
 * active-context-count = 0 | >0 | 1 | >1
 * flags = flags flag | blank
 * flag = S | D | I
 *
 * The transform notation is similar, but only describes the modifications made to the current state.
 * The notation is formed from the following pattern:
 *
 * transform = < state, active-context-count, flags >
 * state = S | CR | L | R | ST | I | D
 * active-context-count = + | - | blank
 * flags = flags flag | flags !flag | blank
 * flag = S | D | I
 *
 * And now for the state machine:
 * State		Start		Stop		Free		Interrupt		End Interrupt		Context In		Context Out	Load Finished		Store Finished
 * <CR, 0, >				<S, , >		<D, , >			<L, +, >
 * <D, 0, >
 * <D, 1, D>									< , -, !D>
 * <D, >1, D>									< , -, >
 * <I, 0, D>									<D, , !D>
 * <I, 0, S>	< , , !S>				< , , !SD>		<S, , !S>
 * <I, 0, >					< , , S>	< , , D>	<CR, , >
 * <L, 1, D>									<ST, -, >
 * <L, 1, ID>									<ST, -, >
 * <L, 1, IS>							< , , !SD>	<ST, -, >
 * <L, 1, S>	< , , !S>				< , , !SD>		<ST, -, >
 * <L, 1, >					< , , S>	< , , D>	< , , IS>							< , +, >	<R, , >
 * <L, >1, D>									< , -, >		<R, -, >
 * <L, >1, ID>									< , -, >		<R, -, >
 * <L, >1, IS>							< , , !SD>	< , -, >		<R, -, >
 * <L, >1, S>	< , , !S>				< , , !SD>		< , -, >		<R, -, >
 * <L, >1, >				< , , S>	< , , D>	< , , IS>							< , +, >		< , -, >		<R, , >
 * <R, 1, D>									<ST, -, >
 * <R, 1, ID>									<ST, -, >
 * <R, 1, IS>							< , , !SD>	<ST, -, >
 * <R, 1, S>	< , , !S>				< , , !SD>		<ST, -, >
 * <R, 1, >					< , , S>	< , , D>	< , , IS>							< , +, >	<ST, -, >
 * <R, >1, D>									< , -, >
 * <R, >1, ID>									< , -, >
 * <R, >1, IS>							< , , !SD>	< , -, >
 * <R, >1, S>	< , , !S>				< , , !SD>		< , -, >
 * <R, >1, >				< , , S>	< , , D>	< , , IS>							< , +, >		< , -, >
 * <S, 0, >		<CR, , >				<D, , >
 * <S, 1, ID>									<I, -, !I>
 * <S, 1, IS>							< , , !SD>	<I, -, !I>
 * <S, 1, S>	< , , !S>				<D, , !SD>		< , -, !S>
 * <S, 1, >					< , , S>	<D, , D>	<L, +, >		<CR, -, >
 * <S, >1, ID>									< , -, >
 * <S, >1, IS>							< , , !SD>	< , -, >
 * <S, >1, S>	< , , !S>				<D, , !SD>		< , -, >
 * <S, >1, >				< , , S>	<D, , D>		<L, +, >		< , -, >
 * <ST, 0, D>									<D, , !D>
 * <ST, 0, ID>									<I, , !I>
 * <ST, 0, IS>							< , , !SD>	<I, , !I>
 * <ST, 0, S>	< , , !S>				< , , !SD>		<S, , !S>
 * <ST, 0, >				< , , S>	< , , D>	< , , IS>							< , +, >		<CR, , >
 * <ST, >0, D>									< , -, >							<D, , >
 * <ST, >0, ID>								< , -, >							<S, , >
 * <ST, >0, IS>							< , , !SD>										< , -, >			<S, , >
 * <ST, >0, S>	< , , !S>				< , , !SD>		< , -, >							<S, , >
 * <ST, >0, >				< , , S>	< , , D>	< , , IS>							< , +, >		< , -, >			<L, , >
 */

static uint32_t pmc_internal_reservation_next_state(uint32_t current_state, pmc_state_event_t event) {
	uint32_t new_state = PMC_STATE(PMC_STATE_STATE_INVALID, 0, 0);

	switch (event) {
		case PMC_STATE_EVENT_START:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
					new_state = PMC_STATE_MODIFY(current_state, 0, 0, PMC_STATE_FLAGS_STOPPING);
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
					}
					break;
			}
			break;
		case PMC_STATE_EVENT_STOP:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
					break;
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
					new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_STOPPING, 0);
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
						new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_STOPPING, 0);
					}
					break;
			}
			break;
		case PMC_STATE_EVENT_FREE:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
					break;
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
					new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_DEALLOCING, PMC_STATE_FLAGS_STOPPING);
					break;
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
					new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_DEALLOCING, 0);
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING, PMC_STATE_FLAGS_STOPPING);
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING, 0);
					} else {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
					}
					break;
			}
			break;
		case PMC_STATE_EVENT_INTERRUPT:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
					new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING, 0);
					break;
			}
			break;
		case PMC_STATE_EVENT_END_OF_INTERRUPT:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_DEALLOCING):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, PMC_STATE_FLAGS_DEALLOCING);
					break;
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, PMC_STATE_FLAGS_STOPPING);
					break;
				case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
					break;
			}
			break;
		case PMC_STATE_EVENT_CONTEXT_IN:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 1, 0, 0);
					break;
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
					new_state = PMC_STATE_MODIFY(current_state, 1, 0, 0);
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 1, 0, 0);
					}
					break;
			}
			break;
		case PMC_STATE_EVENT_CONTEXT_OUT:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, PMC_STATE_FLAGS_DEALLOCING);
					} else {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STORE, -1, 0, 0);
					} else {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_INTERRUPT, -1, 0, PMC_STATE_FLAGS_INTERRUPTING);
					} else {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, PMC_STATE_FLAGS_STOPPING);
					} else {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
						if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
							new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, -1, 0, 0);
						} else {
							new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
						}
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
						new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
					}
					break;
			}
			break;
		case PMC_STATE_EVENT_LOAD_FINISHED:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_RUN, -1, 0, 0);
					} else {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STORE, -1, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
					new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_RUN, 0, 0, 0);
					break;
			}
			break;
		case PMC_STATE_EVENT_STORE_FINISHED:
			switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_DEALLOCING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, PMC_STATE_FLAGS_DEALLOCING);
					} else {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_INTERRUPT, 0, 0, PMC_STATE_FLAGS_INTERRUPTING);
					} else {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, PMC_STATE_FLAGS_STOPPING);
					} else {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
					}
					break;
				case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
					if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
					} else {
						new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 0, 0, 0);
					}
					break;
			}
			break;
	}

	return new_state;
}

static uint32_t pmc_internal_reservation_move_for_event(pmc_reservation_t reservation, pmc_state_event_t event, pmc_state_t *old_state_out) {
	pmc_state_t oldState;
	pmc_state_t newState;

	assert(reservation);

	/* Determine what state change, if any, we need to do.  Keep trying until either we succeed doing a transition
	 * or the there is no valid move.
	 */
	do {
		oldState = reservation->state;
		newState = pmc_internal_reservation_next_state(oldState, event);
	} while (newState != PMC_STATE_INVALID && !OSCompareAndSwap(oldState, newState, &(reservation->state)));

	if (newState != PMC_STATE_INVALID) {
		COUNTER_DEBUG("Moved reservation %p from state "PMC_STATE_FORMAT" to state "PMC_STATE_FORMAT" for event %s\n", reservation, PMC_STATE_ARGS(oldState), PMC_STATE_ARGS(newState), pmc_state_event_name(event));
	} else {
		COUNTER_DEBUG("No valid moves for reservation %p in state "PMC_STATE_FORMAT" for event %s\n", reservation, PMC_STATE_ARGS(oldState), pmc_state_event_name(event));
	}

	if (old_state_out != NULL) {
		*old_state_out = oldState;
	}

	return newState;
}

static void pmc_internal_reservation_context_out(pmc_reservation_t reservation) {
	assert(reservation);
	pmc_state_t newState;
	pmc_state_t oldState;

	/* Clear that the this reservation was active when this cpu did its last context in */
	OSBitAndAtomic(~(1U << cpu_number()), &(reservation->active_last_context_in));

	/* Move the state machine */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_CONTEXT_OUT, &oldState))) {
		return;
	}

	/* Do any actions required based on the state change */
	if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_STORE && PMC_STATE_STATE(oldState) != PMC_STATE_STATE_STORE) {
		/* Just moved into STORE, so store the reservation. */
		pmc_internal_reservation_store(reservation);
	} else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
		/* Wakeup any thread blocking for this reservation to hit <DEALLOC, 0, > */
		thread_wakeup((event_t)reservation);
	}

}

static void pmc_internal_reservation_context_in(pmc_reservation_t reservation) {
	assert(reservation);
	pmc_state_t oldState;
	pmc_state_t newState;

	/* Move the state machine */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_CONTEXT_IN, &oldState))) {
		return;
	}

	/* Mark that the reservation was active when this cpu did its last context in */
	OSBitOrAtomic(1U << cpu_number(), &(reservation->active_last_context_in));

	/* Do any actions required based on the state change */
	if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_LOAD && PMC_STATE_STATE(oldState) != PMC_STATE_STATE_LOAD) {
		/* Just moved into LOAD, so load the reservation. */
		pmc_internal_reservation_load(reservation);
	}

}

static void pmc_internal_reservation_store(pmc_reservation_t reservation) {
	assert(reservation);
	assert(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_STORE);

	assert(reservation->pmc);
	assert(reservation->config);

	pmc_state_t newState;
	kern_return_t ret = KERN_SUCCESS;

	pmc_t store_pmc = reservation->pmc;
	pmc_object_t store_pmc_obj = store_pmc->object;
	perf_monitor_t store_pm = store_pmc->monitor;

	/*
	 * Instruct the Perf Monitor that contains this counter to turn
	 * off the global disable for this counter.
	 */
	ret = store_pm->methods.disable_counters(store_pm->object, &store_pmc_obj, 1);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG(" [error] disable_counters: 0x%x\n", ret);
		return;
	}

	/* Instruct the counter to disable itself */
	ret = store_pmc->methods.disable(store_pmc_obj);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG("  [error] disable: 0x%x\n", ret);
	}

	/* store the counter value into the reservation's stored count */
	ret = store_pmc->methods.get_count(store_pmc_obj, &reservation->value);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG("  [error] get_count: 0x%x\n", ret);
		return;
	}

	/* Advance the state machine now that the STORE is finished */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_STORE_FINISHED, NULL))) {
		return;
	}

	/* Do any actions required based on the state change */
	if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_LOAD) {
		/* Just moved into LOAD, so load the reservation. */
		pmc_internal_reservation_load(reservation);
	} else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
		/* Wakeup any thread blocking for this reservation to hit <DEALLOC, 0, > */
		thread_wakeup((event_t)reservation);
	}

}

static void pmc_internal_reservation_load(pmc_reservation_t reservation) {
	assert(reservation);
	assert(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_LOAD);

	pmc_state_t newState;
	kern_return_t ret = KERN_SUCCESS;

	assert(reservation->pmc);
	assert(reservation->config);

	pmc_t load_pmc = reservation->pmc;
	pmc_object_t load_pmc_obj = load_pmc->object;
	perf_monitor_t load_pm = load_pmc->monitor;

	/* Set the control register up with the stored configuration */
	ret = load_pmc->methods.set_config(load_pmc_obj, reservation->config->object);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG("  [error] set_config: 0x%x\n", ret);
		return;
	}

	/* load the counter value */
	ret = load_pmc->methods.set_count(load_pmc_obj, reservation->value);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG("  [error] set_count: 0x%x\n", ret);
		return;
	}

	/* Locally enable the counter */
	ret = load_pmc->methods.enable(load_pmc_obj);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG("  [error] enable: 0x%x\n", ret);
		return;
	}

	/*
	 * Instruct the Perf Monitor containing the pmc to enable the
	 * counter.
	 */
	ret = load_pm->methods.enable_counters(load_pm->object, &load_pmc_obj, 1);
	if(KERN_SUCCESS != ret) {
		COUNTER_DEBUG("  [error] enable_counters: 0x%x\n", ret);
		/* not on the hardware. */
		return;
	}

	/* Advance the state machine now that the STORE is finished */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_LOAD_FINISHED, NULL))) {
		return;
	}

	/* Do any actions required based on the state change */
	if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_STORE) {
		/* Just moved into STORE, so store the reservation. */
		pmc_internal_reservation_store(reservation);
	}

}

/*
 * pmc_accessible_from_core will return TRUE if the given @pmc is directly
 * (e.g., hardware) readable from the given logical core.
 *
 * NOTE: This method is interrupt safe.
 */
static inline boolean_t pmc_accessible_from_core(pmc_t pmc, uint32_t logicalCore) {
	boolean_t ret = FALSE;

	assert(pmc);

	ret = pmc->methods.accessible_from_core(pmc->object, logicalCore);

	return ret;
}

static void pmc_internal_reservation_start_cpu(void * arg) {
	pmc_reservation_t reservation = (pmc_reservation_t)arg;

	assert(reservation);


	if (pmc_internal_reservation_matches_context(reservation)) {
		/* We are in context, but the reservation may have already had the context_in method run.  Attempt
		 * to set this cpu's bit in the active_last_context_in mask.  If we set it, call context_in.
		 */
		uint32_t oldMask = OSBitOrAtomic(1U << cpu_number(), &(reservation->active_last_context_in));

		if ((oldMask & (1U << cpu_number())) == 0) {
			COUNTER_DEBUG("Starting already in-context reservation %p for cpu %d\n", reservation, cpu_number());

			pmc_internal_reservation_context_in(reservation);
		}
	}
}

static void pmc_internal_reservation_stop_cpu(void * arg) {
	pmc_reservation_t reservation = (pmc_reservation_t)arg;

	assert(reservation);


	if (pmc_internal_reservation_matches_context(reservation)) {
		COUNTER_DEBUG("Stopping in-context reservation %p for cpu %d\n", reservation, cpu_number());

		pmc_internal_reservation_context_out(reservation);
	}
}

/*!fn
 * pmc_reservation_interrupt is called when a PMC reservation which was setup
 * with an interrupt threshold counts the requested number of events. When the
 * underlying counter hits the threshold, an interrupt is generated, and this
 * method is called. This method marks the reservation as stopped, and passes
 * control off to the user-registered callback method, along with the
 * reservation (so that the user can, for example, write a 0 to the counter, and
 * restart the reservation).
 * This method assumes the reservation has a valid pmc_config_t within.
 *
 * @param target The pmc_reservation_t that caused the interrupt.
 * @param refCon User specified reference constant.
 */
static void pmc_reservation_interrupt(void *target, void *refCon) {
	pmc_reservation_t reservation = (pmc_reservation_t)target;
	pmc_state_t newState;
	uint64_t timeout;
	uint32_t spins;

	assert(reservation);

	/* Move the state machine */
	if (PMC_STATE_INVALID == pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_INTERRUPT, NULL)) {
		return;
	}

	/* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
	 * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_stop_cpu
	 * on every cpu that can access the PMC.
	 */
	pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);

	/* Spin waiting for the state to turn to INTERRUPT */
	nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
	timeout += mach_absolute_time();
	spins = 0;
	while (PMC_STATE_STATE(reservation->state) != PMC_STATE_STATE_INTERRUPT) {
		/* Assert if this takes longer than PMC_SPIN_TIMEOUT_US */
		if (++spins > PMC_SPIN_THRESHOLD) {
			if (mach_absolute_time() > timeout) {
				pmc_spin_timeout_count++;
				assert(0);
			}
		}

		cpu_pause();
	}

	assert(reservation->config);
	assert(reservation->config->method);

	/* Call the registered callback handler */
#if DEBUG_COUNTERS
	uint64_t start = mach_absolute_time();
#endif /* DEBUG */

	(void)reservation->config->method(reservation, refCon);

#if DEBUG_COUNTERS
	uint64_t end = mach_absolute_time();
	if((end - start) > 5000ULL) {
		kprintf("%s - user method %p took %llu ns\n", __FUNCTION__,
				reservation->config->method, (end - start));
	}
#endif

	/* Move the state machine */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_END_OF_INTERRUPT, NULL))) {
		return;
	}

	/* Do any post-move actions necessary */
	if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_CAN_RUN) {
		pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_start_cpu);
	} else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
		/* Wakeup any thread blocking for this reservation to hit <DEALLOC, 0, > */
		thread_wakeup((event_t)reservation);
	}
}

/*
 * Apple-private KPI for Apple kext's (IOProfileFamily) only
 */

#if 0
#pragma mark -
#pragma mark IOProfileFamily private KPI
#endif

/*
 * perf_monitor_register registers a new Performance Monitor, and its associated
 * callback methods.  The given perf_monitor_object_t is the first argument to
 * each callback when they are called.
 */
kern_return_t perf_monitor_register(perf_monitor_object_t monitor,
	perf_monitor_methods_t *methods) {
	int cpu = -1;

	COUNTER_DEBUG("registering perf monitor %p\n", monitor);

	if(!monitor || !methods) {
		return KERN_INVALID_ARGUMENT;
	}

	/* Protect against out-of-date driver kexts */
	if(MACH_PERFMON_METHODS_VERSION != methods->perf_monitor_methods_version) {
		return KERN_INVALID_ARGUMENT;
	}

	/* If the monitor requires idle notifications, ensure that it is
	 * accessible from a single core only.
	 */
	if (methods->flags & PERFMON_FLAG_REQUIRES_IDLE_NOTIFICATIONS) {
		uint32_t *cores;
		size_t core_cnt;

		if (KERN_SUCCESS == methods->accessible_cores(monitor, &cores, &core_cnt)) {
			/*
			 * Guard against disabled cores - monitors will always match and
			 * attempt registration, irrespective of 'cpus=x' boot-arg.
			 */
			if ((core_cnt == 1) && (cores[0] < (uint32_t)ml_get_max_cpus())) {
				cpu = cores[0];
			} else {
				return KERN_INVALID_ARGUMENT;
			}
		}
	}

	/* All methods are required */
	if(!methods->accessible_cores |
	   !methods->enable_counters || !methods->disable_counters ||
	   !methods->on_idle || !methods->on_idle_exit) {
		return KERN_INVALID_ARGUMENT;
	}

	/* prevent dupes. */
	perf_monitor_t dupe = perf_monitor_find(monitor);
	if(dupe) {
		COUNTER_DEBUG("Duplicate registration for %p\n", monitor);
		perf_monitor_deallocate(dupe);
		return KERN_FAILURE;
	}

	perf_monitor_t pm = perf_monitor_alloc();
	if(!pm) {
		return KERN_RESOURCE_SHORTAGE;
	}

	/* initialize the object */
	perf_monitor_init(pm, cpu);

	/* copy in the registration info */
	pm->object = monitor;
	memcpy(&(pm->methods), methods, sizeof(perf_monitor_methods_t));

	/* place it in the tracking queues */
	perf_monitor_enqueue(pm);

	/* debug it */
	PRINT_PERF_MON(pm);

	return KERN_SUCCESS;
}

/*
 * perf_monitor_unregister unregisters a previously registered Perf Monitor,
 * looking it up by reference pointer (the same that was used in
 * perf_monitor_register()).
 */
kern_return_t perf_monitor_unregister(perf_monitor_object_t monitor) {
	kern_return_t ret = KERN_FAILURE;

	COUNTER_DEBUG("unregistering perf monitor %p\n", monitor);

	if(!monitor) {
		return KERN_INVALID_ARGUMENT;
	}

	perf_monitor_t pm = perf_monitor_find(monitor);
	if(pm) {
		/* Remove it from the queues. */
		perf_monitor_dequeue(pm);

		/* drop extra retain from find */
		perf_monitor_deallocate(pm);

		/* and release the object */
		perf_monitor_deallocate(pm);

		ret = KERN_SUCCESS;
	} else {
		COUNTER_DEBUG("could not find a registered pm that matches!\n");
	}

	return ret;
}

/*
 * pmc_register registers a new PMC for use with the pmc subsystem. Each PMC is
 * associated with a Perf Monitor.  Perf Monitors are looked up by the reference
 * pointer that was used to previously register them.
 *
 * PMCs are registered with a reference pointer (@pmc_object), and a set of
 * callback methods.  When the given callback methods are called from xnu, the
 * first argument will always be the reference pointer used to register the PMC.
 *
 * NOTE: @monitor must have been successfully registered via
 * perf_monitor_register before this method will succeed.
 */
kern_return_t pmc_register(perf_monitor_object_t monitor, pmc_object_t pmc_object,
	pmc_methods_t *methods, void *object) {

	COUNTER_DEBUG("%p %p\n", monitor, pmc_object);

	if(!monitor || !pmc_object || !methods || !object) {
		return KERN_INVALID_ARGUMENT;
	}

	/* Prevent version mismatches */
	if(MACH_PMC_METHODS_VERSION != methods->pmc_methods_version) {
		COUNTER_DEBUG("version mismatch\n");
		return KERN_INVALID_ARGUMENT;
	}

	/* All methods are required. */
	if(!methods->create_config ||
		!methods->free_config ||
		!methods->config_set_value ||
		!methods->config_set_threshold ||
		!methods->config_set_handler ||
		!methods->set_config ||
		!methods->get_monitor ||
		!methods->get_name ||
		!methods->accessible_from_core ||
		!methods->accessible_cores ||
		!methods->get_count ||
		!methods->set_count ||
		!methods->disable ||
		!methods->enable ||
		!methods->open ||
		!methods->close) {
		return KERN_INVALID_ARGUMENT;
	}

	/* make sure this perf monitor object is already registered */
	/*
	 * NOTE: this adds a reference to the parent, so we'll have to drop it in
	 * any failure code paths from here on out.
	 */
	perf_monitor_t pm = perf_monitor_find(monitor);
	if(!pm) {
		COUNTER_DEBUG("Could not find perf monitor for %p\n", monitor);
		return KERN_INVALID_ARGUMENT;
	}

	/* make a new pmc */
	pmc_t pmc = pmc_alloc();
	if(!pmc) {
		/* drop the extra reference from perf_monitor_find() */
		perf_monitor_deallocate(pm);
		return KERN_RESOURCE_SHORTAGE;
	}

	/* init it */
	pmc_init(pmc);

	pmc->object = pmc_object;
	pmc->open_object = object;

	/* copy the callbacks in */
	memcpy(&(pmc->methods), methods, sizeof(pmc_methods_t));

	pmc->monitor = pm;

	perf_monitor_add_pmc(pmc->monitor, pmc);

	/* enqueue it in our tracking queue */
	pmc_enqueue(pmc);

	/* drop extra reference from perf_monitor_find() */
	perf_monitor_deallocate(pm);

	return KERN_SUCCESS;
}

/*
 * pmc_unregister unregisters a previously registered PMC, looking it up by
 * reference point to *both* the Perf Monitor it was created with, and the PMC's
 * reference pointer itself.
 */
kern_return_t pmc_unregister(perf_monitor_object_t monitor, pmc_object_t pmc_object) {
	COUNTER_DEBUG("%p %p\n", monitor, pmc_object);

	if(!monitor || !pmc_object) {
		return KERN_INVALID_ARGUMENT;
	}

	pmc_t pmc = pmc_find(pmc_object);
	if(!pmc) {
		COUNTER_DEBUG("Could not find a matching pmc.\n");
		return KERN_FAILURE;
	}

	/* remove it from the global queue */
	pmc_dequeue(pmc);

	perf_monitor_remove_pmc(pmc->monitor, pmc);

	/* remove extra reference count from pmc_find() */
	pmc_deallocate(pmc);

	/* dealloc the pmc */
	pmc_deallocate(pmc);

	return KERN_SUCCESS;
}

static void perf_monitor_reservation_add(perf_monitor_t monitor) {
    assert(monitor);
    OSIncrementAtomic(&(monitor->reservedCounters));
}

static void perf_monitor_reservation_remove(perf_monitor_t monitor) {
    assert(monitor);
    OSDecrementAtomic(&(monitor->reservedCounters));
}

#if 0
#pragma mark -
#pragma mark KPI
#endif

/*
 * Begin in-kernel and in-kext KPI methods
 */

/*
 * pmc_create_config creates a new configuration area from a given @pmc.
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_create_config(pmc_t pmc, pmc_config_t *config) {
	pmc_config_t tmp = NULL;

	if(!pmc || !config) {
		return KERN_INVALID_ARGUMENT;
	}

	pmc_reference(pmc);

	tmp = pmc_config_alloc(pmc);
	if(tmp) {
		tmp->object = pmc->methods.create_config(pmc->object);

		if(!tmp->object) {
			pmc_config_free(pmc, tmp);
			tmp = NULL;
		} else {
			tmp->interrupt_after_value = 0ULL;
			tmp->method = NULL;
			tmp->refCon = NULL;
		}
	}

	pmc_deallocate(pmc);

	if(!tmp) {
		return KERN_RESOURCE_SHORTAGE;
	}

	*config = tmp;

	return KERN_SUCCESS;
}

/*
 * pmc_free_config frees a configuration area created from a given @pmc
 *
 * NOTE: This method is not interrupt safe.
 */
void pmc_free_config(pmc_t pmc, pmc_config_t config) {
	assert(pmc);
	assert(config);

	pmc_reference(pmc);

	pmc_config_free(pmc, config);

	pmc_deallocate(pmc);
}

/*
 * pmc_config_set_value sets up configuration area key-value pairs.  These pairs
 * are to be either pre-known, or looked up via CoreProfile.framework.
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_config_set_value(pmc_t pmc, pmc_config_t config,
	uint8_t id, uint64_t value) {

	kern_return_t ret = KERN_INVALID_ARGUMENT;

	if(!pmc || !config) {
		return ret;
	}

	pmc_reference(pmc);

	ret = pmc->methods.config_set_value(config->object, id, value);

	pmc_deallocate(pmc);

	return ret;
}

/*
 * pmc_config_set_interrupt_threshold modifies a config object, instructing
 * the pmc that it should generate a call to the given pmc_interrupt_method_t
 * after the counter counts @threshold events.
 *
 * PMC Threshold handler methods will have the pmc_reservation_t that generated the interrupt
 * as the first argument when the interrupt handler is invoked, and the given
 * @refCon (which may be NULL) as the second.
 *
 * See pmc_interrupt_method_t.
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_config_set_interrupt_threshold(pmc_t pmc, pmc_config_t config,
	uint64_t threshold, pmc_interrupt_method_t method, void *refCon) {
	kern_return_t ret = KERN_INVALID_ARGUMENT;

	if(!config || !pmc) {
		return ret;
	}

	assert(config);
	assert(pmc);

	pmc_reference(pmc);

	do {
		/*
		 * We have a minor annoyance to side-step here. The driver layer expects
		 * the config to never change once a reservation has been taken out with
		 * it.  However, in order to have the PMI method have the reservation as
		 * the first argument (in order to allow the user-method to, for
		 * example, write a 0 to it, and restart it), we need to create the
		 * pmc_reservation_t before setting it up in the config object.
		 * We overcome this by caching the method in the pmc_config_t stand-in,
		 * and mutating the pmc_config_object_t just before returning a
		 * reservation (in pmc_reserve() and friends, below).
		 */

		/* might as well stash this away too. */
		config->interrupt_after_value = threshold;
		config->method = method;
		config->refCon = refCon;

		ret = KERN_SUCCESS;

	}while(0);

	pmc_deallocate(pmc);

	return ret;
}

/*
 * pmc_get_pmc_list returns an allocated list of pmc_t's, as well as the number
 * of pmc_t's returned. Callers should free this list with a call to
 * pmc_free_pmc_list().
 *
 * NOTE: This method is not interrupt safe.
 */
kern_return_t pmc_get_pmc_list(pmc_t **pmcs, size_t *pmcCount) {
	pmc_t *array = NULL;
	pmc_t pmc = NULL;
	size_t count = 0UL;

	do {
		/* Copy down (to the stack) the count of perf counters */
		vm_size_t size = perf_counters_count;

		/* Allocate that sized chunk */
		array = (pmc_t *)kalloc(sizeof(pmc_t) * size);
		if(!array) {
			return KERN_RESOURCE_SHORTAGE;
		}

		/* Take the spin lock */
		lck_spin_lock(&perf_counters_queue_spin);

		/* verify the size didn't change while we were allocating */
		if(size != perf_counters_count) {
			/*
			 * queue size has changed between alloc and now - go back and
			 * make another pass.
			 */

			/* drop the lock */
			lck_spin_unlock(&perf_counters_queue_spin);

			/* free the block */
			kfree(array, sizeof(pmc_t) * size);
			array = NULL;
		}

		/* if we get here, and array is NULL, we try again. */
	}while(!array);

	/* copy the bits out */
	queue_iterate(perf_counters_queue, pmc, pmc_t, link) {
		/* copy out the pointer */
		array[count++] = pmc;
	}

	lck_spin_unlock(&perf_counters_queue_spin);

	/* return the list and the size */
	*pmcs = array;
	*pmcCount = count;

	return KERN_SUCCESS;
}

/*
 * pmc_free_pmc_list frees an array of pmc_t that has been returned from
 * pmc_get_pmc_list.
 *
 * NOTE: This method is not interrupt safe.
 */
void pmc_free_pmc_list(pmc_t *pmcs, size_t pmcCount) {
	if(pmcs && pmcCount) {
		COUNTER_DEBUG("pmcs: %p pmcCount: %lu\n", pmcs, pmcCount);

		kfree(pmcs, pmcCount * sizeof(pmc_t));
	}
}

kern_return_t pmc_find_by_name(const char *name, pmc_t **pmcs, size_t *pmcCount) {
	kern_return_t ret = KERN_INVALID_ARGUMENT;

	if(!name || !pmcs || !pmcCount) {
		return ret;
	}

	pmc_t *list = NULL;
	size_t count = 0UL;

	if(KERN_SUCCESS == (ret = pmc_get_pmc_list(&list, &count))) {
		size_t matchCount = 0UL, ii = 0UL, swapPtr = 0UL;
		size_t len = strlen(name);

		for(ii = 0UL; ii < count; ii++) {
			const char *pmcName = pmc_get_name(list[ii]);

			if(strlen(pmcName) < len) {
				/*
				 * If the pmc name is shorter than the requested match, it's no
				 * match, as we're looking for the most specific match(es).
				 */
				continue;
			}

			if(0 == strncmp(name, pmcName, len)) {
				pmc_t temp = list[ii];

				// move matches to the head of the array.
				list[ii] = list[swapPtr];
				list[swapPtr] = temp;
				swapPtr++;

				// keep a count of the matches
				matchCount++;
			}
		}

		if(matchCount) {
			/*
			 * If we have matches, they are all at the head of the array, so
			 * just allocate enough space for @matchCount pmc_t's, and copy the
			 * head of the array to the new allocation.  Then free the old
			 * allocation.
			 */

			pmc_t *result = (pmc_t *)kalloc(sizeof(pmc_t) * matchCount);
			if(result) {
				// copy the matches
				memcpy(result, list, sizeof(pmc_t) * matchCount);

				ret = KERN_SUCCESS;
			}

			pmc_free_pmc_list(list, count);

			if(!result) {
				*pmcs = NULL;
				*pmcCount = 0UL;
				return KERN_RESOURCE_SHORTAGE;
			}

			*pmcs = result;
			*pmcCount = matchCount;
		} else {
			*pmcs = NULL;
			*pmcCount = 0UL;
		}
	}

	return ret;
}

/*
 * pmc_get_name returns a pointer (not copied) to the human-readable name of the
 * given pmc.
 *
 * NOTE: Driver authors must take care to not allocate during this method, as
 * this method *IS* interrupt safe.
 */
const char *pmc_get_name(pmc_t pmc) {
	assert(pmc);

	const char *name = pmc->methods.get_name(pmc->object);

	return name;
}

/*
 * pmc_get_accessible_core_list returns a pointer to an array of logical core
 * numbers (as well as the size of that array) that represent the local cores
 * (hardware threads) from which the given @pmc can be accessed directly.
 *
 * NOTE: This method is interrupt safe.
 */
kern_return_t pmc_get_accessible_core_list(pmc_t pmc, uint32_t **logicalCores,
	size_t *logicalCoreCt) {

	kern_return_t ret = KERN_INVALID_ARGUMENT;

	if(!pmc || !logicalCores || !logicalCoreCt) {
		return ret;
	}

	ret = pmc->methods.accessible_cores(pmc->object, logicalCores, logicalCoreCt);

	return ret;
}

static boolean_t pmc_reservation_setup_pmi(pmc_reservation_t resv, pmc_config_t config) {
	assert(resv);
	assert(resv->pmc);
	assert(config);
	assert(config->object);

	/* If there's no PMI to setup, return success */
	if(config->interrupt_after_value && config->method) {

		/* set the threshold */
		kern_return_t ret = resv->pmc->methods.config_set_threshold(config->object,
			config->interrupt_after_value);

		if(KERN_SUCCESS != ret) {
			/*
			 * This is the most useful error message here, as this only happens
			 * as a result of pmc_reserve*()
			 */
			COUNTER_DEBUG("Failed to set threshold for pmc %p\n", resv->pmc);
			return FALSE;
		}

		if(KERN_SUCCESS != resv->pmc->methods.config_set_handler(config->object,
			(void *)resv, &pmc_reservation_interrupt, config->refCon)) {

			COUNTER_DEBUG("Failed to set handler for pmc %p\n", resv->pmc);
			return FALSE;
		}
	}

	return TRUE;
}

/*
 * pmc_reserve will attempt to reserve the given @pmc, with a given
 * configuration object, for counting system-wide. This method will fail with
 * KERN_FAILURE if the given pmc is already reserved at any scope.
 *
 * This method consumes the given configuration object if it returns
 * KERN_SUCCESS. Any other return value indicates the caller
 * must free the config object via pmc_free_config().
 *
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reserve(pmc_t pmc, pmc_config_t config,
	pmc_reservation_t *reservation) {

	if(!pmc || !config || !reservation) {
		return KERN_INVALID_ARGUMENT;
	}

	pmc_reservation_t resv = reservation_alloc();
	if(!resv) {
		return KERN_RESOURCE_SHORTAGE;
	}

	reservation_init(resv);

	resv->flags |= PMC_FLAG_SCOPE_SYSTEM;
	resv->config = config;

	if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
		resv->config = NULL;
		return KERN_FAILURE;
	}

	/* enqueue reservation in proper place */
	if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
		/* Prevent free of config object */
		resv->config = NULL;

		reservation_free(resv);
		return KERN_FAILURE;
	}

	perf_monitor_reservation_add(pmc->monitor);

	*reservation = resv;

	return KERN_SUCCESS;
}

/*
 * pmc_reserve_task will attempt to reserve the given @pmc with a given
 * configuration object, for counting when the given @task is running on any
 * logical core that can directly access the given @pmc.  This method will fail
 * with KERN_FAILURE if the given pmc is already reserved at either system or
 * thread scope.
 *
 * This method consumes the given configuration object if it returns
 * KERN_SUCCESS. Any other return value indicates the caller
 * must free the config object via pmc_free_config().
 *
 * NOTE: You can reserve the same pmc for N different tasks concurrently.
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reserve_task(pmc_t pmc, pmc_config_t config,
	task_t task, pmc_reservation_t *reservation) {

	if(!pmc || !config || !reservation || !task) {
		return KERN_INVALID_ARGUMENT;
	}

	if (!(pmc->monitor->methods.flags & PERFMON_FLAG_SUPPORTS_CONTEXT_SWITCHING)) {
		COUNTER_DEBUG("pmc %p cannot be context switched!\n", pmc);
		return KERN_INVALID_ARGUMENT;
	}

	pmc_reservation_t resv = reservation_alloc();
	if(!resv) {
		return KERN_RESOURCE_SHORTAGE;
	}

	reservation_init(resv);

	resv->flags |= PMC_FLAG_SCOPE_TASK;
	resv->task = task;

	resv->config = config;

	if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
		resv->config = NULL;
		return KERN_FAILURE;
	}

	/* enqueue reservation in proper place */
	if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
		/* Prevent free of config object */
		resv->config = NULL;

		reservation_free(resv);
		return KERN_FAILURE;
	}

	perf_monitor_reservation_add(pmc->monitor);

	*reservation = resv;

	return KERN_SUCCESS;
}

/*
 * pmc_reserve_thread will attempt to reserve the given @pmc with a given
 * configuration object, for counting when the given @thread is running on any
 * logical core that can directly access the given @pmc.  This method will fail
 * with KERN_FAILURE if the given pmc is already reserved at either system or
 * task scope.
 *
 * This method consumes the given configuration object if it returns
 * KERN_SUCCESS. Any other return value indicates the caller
 * must free the config object via pmc_free_config().
 *
 * NOTE: You can reserve the same pmc for N different threads concurrently.
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reserve_thread(pmc_t pmc, pmc_config_t config,
	thread_t thread, pmc_reservation_t *reservation) {
	if(!pmc || !config || !reservation || !thread) {
		return KERN_INVALID_ARGUMENT;
	}

	if (!(pmc->monitor->methods.flags & PERFMON_FLAG_SUPPORTS_CONTEXT_SWITCHING)) {
		COUNTER_DEBUG("pmc %p cannot be context switched!\n", pmc);
		return KERN_INVALID_ARGUMENT;
	}

	pmc_reservation_t resv = reservation_alloc();
	if(!resv) {
		return KERN_RESOURCE_SHORTAGE;
	}

	reservation_init(resv);

	resv->flags |= PMC_FLAG_SCOPE_THREAD;
	resv->thread = thread;

	resv->config = config;

	if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
		resv->config = NULL;
		return KERN_FAILURE;
	}

	/* enqueue reservation in proper place */
	if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
		/* Prevent free of config object */
		resv->config = NULL;

		reservation_free(resv);
		return KERN_FAILURE;
	}

	perf_monitor_reservation_add(pmc->monitor);

	*reservation = resv;

	return KERN_SUCCESS;
}

/*
 * pmc_reservation_start instructs the given reservation to start counting as
 * soon as possible.
 *
 * NOTE: This method is interrupt safe.
 */
kern_return_t pmc_reservation_start(pmc_reservation_t reservation) {
	pmc_state_t newState;

	if(!reservation) {
		return KERN_INVALID_ARGUMENT;
	}

	/* Move the state machine */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_START, NULL))) {
		return KERN_FAILURE;
	}

	/* If we are currently in an interrupt, don't bother to broadcast since it won't do anything now and the interrupt will
	 * broadcast right before it leaves
	 */
	if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_INTERRUPT) {
		/* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
		 * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_start_cpu
		 * on every cpu that can access the PMC.
		 */
		pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_start_cpu);
	}

	return KERN_SUCCESS;
}

/*
 * pmc_reservation_stop instructs the given reservation to stop counting as
 * soon as possible.  When this method returns, the pmc will be marked as stopping
 * and subsequent calls to pmc_reservation_start will succeed.  This does not mean
 * that the pmc hardware has _actually_ stopped running.  Assuming no other changes
 * to the reservation state, the pmc hardware _will_ stop shortly.
 *
 */
kern_return_t pmc_reservation_stop(pmc_reservation_t reservation) {
	pmc_state_t newState;

	if(!reservation) {
		return KERN_INVALID_ARGUMENT;
	}

	/* Move the state machine */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_STOP, NULL))) {
		return KERN_FAILURE;
	}

	/* If we are currently in an interrupt, don't bother to broadcast since it won't do anything now and the interrupt will
	 * broadcast right before it leaves.  Similarly, if we just moved directly to STOP, don't bother broadcasting.
	 */
	if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_INTERRUPT && PMC_STATE_STATE(newState) != PMC_STATE_STATE_STOP) {
		/* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
			 * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_stop_cpu
		 * on every cpu that can access the PMC.
		 */

		pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
	}

	return KERN_SUCCESS;
}

/*
 * pmc_reservation_read will read the event count associated with a reservation.
 * If the caller is current executing in a context that both a) matches the
 * reservation's context, and b) can access the reservation's pmc directly, the
 * value will be read from hardware.  Otherwise, this returns the reservation's
 * stored value.
 *
 * NOTE: This method is interrupt safe.
 * NOTE: When not on the interrupt stack, this method may block.
 */
kern_return_t pmc_reservation_read(pmc_reservation_t reservation, uint64_t *value) {
	kern_return_t ret = KERN_FAILURE;
	uint64_t timeout;
	uint32_t spins;

	if(!reservation || !value) {
		return KERN_INVALID_ARGUMENT;
	}

	nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
	timeout += mach_absolute_time();
	spins = 0;
	do {
		uint32_t state = reservation->state;

		if((PMC_STATE_STATE(state) == PMC_STATE_STATE_RUN)) {
			/* Attempt read from hardware via drivers. */

			assert(reservation->pmc);

			ret = reservation->pmc->methods.get_count(reservation->pmc->object, value);

			break;
		} else if ((PMC_STATE_STATE(state) == PMC_STATE_STATE_STORE) ||
				   (PMC_STATE_STATE(state) == PMC_STATE_STATE_LOAD)) {
			/* Spin */
			/* Assert if this takes longer than PMC_SPIN_TIMEOUT_US */
			if (++spins > PMC_SPIN_THRESHOLD) {
				if (mach_absolute_time() > timeout) {
					pmc_spin_timeout_count++;
					assert(0);
				}
			}

			cpu_pause();
		} else {
			break;
		}
	} while (1);

	/* If the direct hardware read failed (for whatever reason) */
	if(KERN_SUCCESS != ret) {
		/* Read stored value */
		*value = reservation->value;
	}

	return KERN_SUCCESS;
}

/*
 * pmc_reservation_write will write the event count associated with a reservation.
 * If the caller is current executing in a context that both a) matches the
 * reservation's context, and b) can access the reservation's pmc directly, the
 * value will be written to hardware.  Otherwise, this writes the reservation's
 * stored value.
 *
 * NOTE: This method is interrupt safe.
 * NOTE: When not on the interrupt stack, this method may block.
 */
kern_return_t pmc_reservation_write(pmc_reservation_t reservation, uint64_t value) {
	kern_return_t ret = KERN_FAILURE;
	uint64_t timeout;
	uint32_t spins;

	if(!reservation) {
		return KERN_INVALID_ARGUMENT;
	}

	nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
	timeout += mach_absolute_time();
	spins = 0;
	do {
		uint32_t state = reservation->state;

		if((PMC_STATE_STATE(state) == PMC_STATE_STATE_RUN)) {
				/* Write to hardware via drivers. */
			assert(reservation->pmc);

			ret = reservation->pmc->methods.set_count(reservation->pmc->object, value);
			break;
		} else if ((PMC_STATE_STATE(state) == PMC_STATE_STATE_STORE) ||
				   (PMC_STATE_STATE(state) == PMC_STATE_STATE_LOAD)) {
			/* Spin */
			/* Assert if this takes longer than PMC_SPIN_TIMEOUT_US */
			if (++spins > PMC_SPIN_THRESHOLD) {
				if (mach_absolute_time() > timeout) {
					pmc_spin_timeout_count++;
					assert(0);
				}
			}

			cpu_pause();
		} else {
			break;
		}
	} while (1);

	if(KERN_SUCCESS != ret) {
		/* Write stored value */
		reservation->value = value;
	}

	return KERN_SUCCESS;
}

/*
 * pmc_reservation_free releases a reservation and all associated resources.
 *
 * NOTE: This method is NOT interrupt safe.
 */
kern_return_t pmc_reservation_free(pmc_reservation_t reservation) {
	pmc_state_t newState;

	if(!reservation) {
		return KERN_INVALID_ARGUMENT;
	}

	perf_monitor_reservation_remove(reservation->pmc->monitor);

	/* Move the state machine */
	if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_FREE, NULL))) {
		return KERN_FAILURE;
	}

	/* If we didn't move directly to DEALLOC, help things along */
	if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_DEALLOC) {
		/* A valid state move has been made, but won't be picked up until a context switch occurs.  To cause matching
		 * contexts that are currently running to update, we do an inter-processor message to run pmc_internal_reservation_stop_cpu
		 * on every cpu that can access the PMC.
		 */
		pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
	}

	/* Block until the reservation hits the <DEALLOC, 0, > state */
	while (!(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(reservation->state) == 0 && PMC_STATE_FLAGS(reservation->state) == 0)) {
		assert_wait((event_t)reservation, THREAD_UNINT);
		thread_block(THREAD_CONTINUE_NULL);
	}

	/* remove from queues */
	pmc_internal_reservation_remove(reservation);

	/* free reservation */
	reservation_free(reservation);

	return KERN_SUCCESS;
}

/*
 * pmc_idle notifies eligible monitors of impending per-CPU idle, and can be used to save state.
 */
boolean_t pmc_idle(void) {
	perf_monitor_t monitor = NULL;
	queue_head_t *cpu_queue;

	lck_spin_lock(&perf_monitor_queue_spin);

	if (cpu_monitor_queues) {
		cpu_queue = cpu_monitor_queues[cpu_number()];

		queue_iterate(cpu_queue, monitor, perf_monitor_t, cpu_link) {
			perf_monitor_methods_t *methods = &(monitor->methods);
			if ((methods->flags & PERFMON_FLAG_ALWAYS_ACTIVE) || (monitor->reservedCounters)) {
				methods->on_idle(monitor->object);
			}
		}
	}

	lck_spin_unlock(&perf_monitor_queue_spin);

	return TRUE;
}

/*
 * pmc_idle_exit notifies eligible monitors of wake from idle; it can be used to restore state.
 */
boolean_t pmc_idle_exit(void) {
	perf_monitor_t monitor = NULL;
	queue_head_t *cpu_queue;

	lck_spin_lock(&perf_monitor_queue_spin);

	if (cpu_monitor_queues) {
		cpu_queue = cpu_monitor_queues[cpu_number()];

		queue_iterate(cpu_queue, monitor, perf_monitor_t, cpu_link) {
			perf_monitor_methods_t *methods = &(monitor->methods);
			if ((methods->flags & PERFMON_FLAG_ALWAYS_ACTIVE) || (monitor->reservedCounters)) {
				methods->on_idle_exit(monitor->object);
			}
		}
	}

	lck_spin_unlock(&perf_monitor_queue_spin);

	return TRUE;
}

/*
 * pmc_context_switch performs all context switching necessary to save all pmc
 * state associated with @oldThread (and the task to which @oldThread belongs),
 * as well as to restore all pmc state associated with @newThread (and the task
 * to which @newThread belongs).
 *
 * NOTE: This method IS interrupt safe.
 */
boolean_t pmc_context_switch(thread_t oldThread, thread_t newThread) {
	pmc_reservation_t resv = NULL;
	uint32_t cpuNum = cpu_number();

	lck_spin_lock(&reservations_spin);

	/* Save pmc states */
	if (thread_reservation_count) {
 		queue_iterate(thread_reservations, resv, pmc_reservation_t, link) {
			if ((oldThread == resv->thread) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
				(void)pmc_internal_reservation_context_out(resv);
			}
		}
	}

	if (task_reservation_count) {
		queue_iterate(task_reservations, resv, pmc_reservation_t, link) {
			if ((resv->task == oldThread->task) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
    			(void)pmc_internal_reservation_context_out(resv);
			}
		}
	}

	/* Restore */
	if (thread_reservation_count) {
		queue_iterate(thread_reservations, resv, pmc_reservation_t, link) {
			if ((resv->thread == newThread) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
				(void)pmc_internal_reservation_context_in(resv);
			}
		}
	}

	if (task_reservation_count) {
		queue_iterate(task_reservations, resv, pmc_reservation_t, link) {
			if ((resv->task == newThread->task) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
				(void)pmc_internal_reservation_context_in(resv);
			}
		}
	}

	lck_spin_unlock(&reservations_spin);

	return TRUE;
}

#else /* !CONFIG_COUNTERS */

#if 0
#pragma mark -
#pragma mark Dummy functions
#endif

/*
 * In the case that someone has chosen not to include the PMC KPI in some
 * configuration, we still have exports for kexts, so we'll need to define stub
 * methods that return failures.
 */
kern_return_t perf_monitor_register(perf_monitor_object_t monitor __unused,
	perf_monitor_methods_t *methods __unused) {
	return KERN_FAILURE;
}

kern_return_t perf_monitor_unregister(perf_monitor_object_t monitor __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_register(perf_monitor_object_t monitor __unused,
	pmc_object_t pmc __unused, pmc_methods_t *methods __unused, void *object __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_unregister(perf_monitor_object_t monitor __unused,
	pmc_object_t pmc __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_create_config(pmc_t pmc __unused,
	pmc_config_t *config __unused) {
	return KERN_FAILURE;
}

void pmc_free_config(pmc_t pmc __unused, pmc_config_t config __unused) {
}

kern_return_t pmc_config_set_value(pmc_t pmc __unused,
	pmc_config_t config __unused, uint8_t id __unused,
	uint64_t value __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_config_set_interrupt_threshold(pmc_t pmc __unused,
	pmc_config_t config __unused, uint64_t threshold __unused,
	pmc_interrupt_method_t method __unused, void *refCon __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_get_pmc_list(pmc_t **pmcs __unused, size_t *pmcCount __unused) {
	return KERN_FAILURE;
}

void pmc_free_pmc_list(pmc_t *pmcs __unused, size_t pmcCount __unused) {
}

kern_return_t pmc_find_by_name(const char *name __unused, pmc_t **pmcs __unused,
	size_t *pmcCount __unused) {
	return KERN_FAILURE;
}

const char *pmc_get_name(pmc_t pmc __unused) {
	return "";
}

kern_return_t pmc_get_accessible_core_list(pmc_t pmc __unused,
	uint32_t **logicalCores __unused, size_t *logicalCoreCt __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reserve(pmc_t pmc __unused,
	pmc_config_t config __unused, pmc_reservation_t *reservation __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reserve_task(pmc_t pmc __unused,
	pmc_config_t config __unused, task_t task __unused,
	pmc_reservation_t *reservation __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reserve_thread(pmc_t pmc __unused,
	pmc_config_t config __unused, thread_t thread __unused,
	pmc_reservation_t *reservation __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reservation_start(pmc_reservation_t reservation __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reservation_stop(pmc_reservation_t reservation __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reservation_read(pmc_reservation_t reservation __unused,
	uint64_t *value __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reservation_write(pmc_reservation_t reservation __unused,
	uint64_t value __unused) {
	return KERN_FAILURE;
}

kern_return_t pmc_reservation_free(pmc_reservation_t reservation __unused) {
	return KERN_FAILURE;
}


#endif /* !CONFIG_COUNTERS */