osfmk/kern/sched_prim.c

274 					processor_set_t	pset,
280 					processor_set_t		pset);
283 					processor_set_t		pset) __attribute__((unused));
336 sched_traditional_pset_init(processor_set_t pset);
728 sched_traditional_pset_init(processor_set_t pset)
731 		run_queue_init(&pset->pset_runq);
733 	pset->pset_runq_bound_count = 0;
1721 /* Invoked with pset locked, returns with pset unlocked */
1786 	processor_set_t		pset = processor->processor_set;
1802 		pset_lock(pset);
1808 			 * Should this secondary SMT processor attempt to find work? For pset runqueue systems,
1815 			if (!SCHED(processor_bound_count)(processor) && !queue_empty(&pset->idle_queue) && !rt_runq.count) {
1831 		    (thread->affinity_set == AFFINITY_SET_NULL || thread->affinity_set->aset_pset == pset)) {
1850 				pset_unlock(pset);
1863 				pset_unlock(pset);
1875 				pset_unlock(pset);
1893 				pset_unlock(pset);
1907 			pset_unlock(pset);
1918 		new_thread = SCHED(steal_thread)(pset);
1930 		pset_lock(pset);
1942 				enqueue_head(&pset->idle_queue, (queue_entry_t)processor);
1945 				enqueue_head(&pset->idle_secondary_queue, (queue_entry_t)processor);
1949 		/* Invoked with pset locked, returns with pset unlocked */
1950 		sched_SMT_balance(processor, pset);
2126  *	Associated pset must be locked.  Returns THREAD_NULL
3102  *	Thread must be locked.  Associated pset must
3110 	processor_set_t		pset = processor->processor_set;
3116 	pset_unlock(pset);
3176  *	Thread must be locked.  Associated pset must
3184 	processor_set_t		pset = processor->processor_set;
3200 		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
3210 			if (!(pset->pending_AST_cpu_mask & (1U << processor->cpu_id))) {
3212 				pset->pending_AST_cpu_mask |= (1U << processor->cpu_id);
3216 		pset_unlock(pset);
3236 			enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
3246 				if (!(pset->pending_AST_cpu_mask & (1U << processor->cpu_id))) {
3248 					pset->pending_AST_cpu_mask |= (1U << processor->cpu_id);
3263 				if (!(pset->pending_AST_cpu_mask & (1U << processor->cpu_id))) {
3265 					pset->pending_AST_cpu_mask |= (1U << processor->cpu_id);
3274 	pset_unlock(pset);
3331  *	Thread must be locked.  Associated pset must
3340 	processor_set_t		pset = processor->processor_set;
3355 		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
3364 		if (!(pset->pending_AST_cpu_mask & (1U << processor->cpu_id))) {
3366 			pset->pending_AST_cpu_mask |= (1U << processor->cpu_id);
3370 		pset_unlock(pset);
3399 			enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
3432 			enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
3449 				if (csw_check_locked(processor, pset, AST_NONE) != AST_NONE)
3452 				if (!(pset->pending_AST_cpu_mask & (1U << processor->cpu_id))) {
3454 					pset->pending_AST_cpu_mask |= (1U << processor->cpu_id);
3461 				if (csw_check_locked(processor, pset, AST_NONE) != AST_NONE)
3464 				if (!(pset->pending_AST_cpu_mask & (1U << processor->cpu_id))) {
3466 					pset->pending_AST_cpu_mask |= (1U << processor->cpu_id);
3473 	pset_unlock(pset);
3494 	processor_set_t pset = processor->processor_set;
3498 	 * The pset runq contains the count of all runnable threads
3499 	 * for all processors in the pset. However, for threads that
3509 	count -= pset->pset_runq_bound_count;
3592  *	Return the next sibling pset containing
3595  *	Returns the original pset if none other is
3600 	processor_set_t		pset)
3602 	processor_set_t		nset = pset;
3606 	} while (nset->online_processor_count < 1 && nset != pset);
3615  *	the pset.  Accepts an optional processor hint in
3616  *	the pset.
3618  *	Returns a processor, possibly from a different pset.
3620  *	The thread must be locked.  The pset must be locked,
3621  *	and the resulting pset is locked on return.
3625 	processor_set_t		pset,
3629 	processor_set_t		nset, cset = pset;
3641 	 * Only consult platform layer if pset is active, which
3645 	if (pset->online_processor_count) {
3646 		if ((processor == PROCESSOR_NULL) || (processor->processor_set == pset && processor->state == PROCESSOR_IDLE)) {
3647 			processor_t mc_processor = machine_choose_processor(pset, processor);
3655 	 * an initial starting pset. If the hint is not in the pset, or
3660 		if (processor->processor_set != pset) {
3742 		 * Choose an idle processor, in pset traversal order
3825 			 * pset.
3844 			 * If all primary processor in this pset are running a higher
3845 			 * priority thread, move on to next pset. Only when we have
3855 		if (nset != pset) {
3861 	} while (nset != pset);
3867 	 * pset, it may have changed state.
3932 	processor_set_t		pset;
3954 			pset = thread->affinity_set->aset_pset;
3955 			pset_lock(pset);
3957 			processor = SCHED(choose_processor)(pset, PROCESSOR_NULL, thread);
3968 			pset = processor->processor_set;
3969 			pset_lock(pset);
3970 			processor = SCHED(choose_processor)(pset, processor, thread);
3984 			pset = task->pset_hint;
3985 			if (pset == PROCESSOR_SET_NULL)
3986 				pset = current_processor()->processor_set;
3988 			pset = choose_next_pset(pset);
3989 			pset_lock(pset);
3991 			processor = SCHED(choose_processor)(pset, PROCESSOR_NULL, thread);
4005 		pset = processor->processor_set;
4006 		pset_lock(pset);
4028 	processor_set_t		pset = task->pset_hint;
4030 	if (pset != PROCESSOR_SET_NULL)
4031 		pset = choose_next_pset(pset);
4033 	return (pset);
4044  *	Associated pset must be locked, and is
4051 	processor_set_t		pset = processor->processor_set;
4091 	pset_unlock(pset);
4115 	processor_set_t	pset = processor->processor_set;
4118 	pset_lock(pset);
4120 	/* If we were sent a remote AST and interrupted a running processor, acknowledge it here with pset lock held */
4121 	pset->pending_AST_cpu_mask &= ~(1U << processor->cpu_id);
4123 	result = csw_check_locked(processor, pset, check_reason);
4125 	pset_unlock(pset);
4132  * pset and thread locked
4137 	processor_set_t	pset __unused,
4346 		 * runq is set to NULL under the pset lock, not the thread
4395  *	Associated pset must be locked.  Returns THREAD_NULL
4441  *	at the pset.
4443  *	The pset must be locked, and is returned
4451 	processor_set_t		pset)
4453 	processor_set_t		nset, cset = pset;
4477 		if (nset != pset) {
4483 	} while (nset != pset);
4491 					processor_set_t		pset)
4493 	pset_unlock(pset);
4564 	processor_set_t		pset = processor->processor_set;
4582 		if (pset->pending_AST_cpu_mask & (1U << processor->cpu_id))
4621 	pset_lock(pset);
4623 	/* If we were sent a remote AST and came out of idle, acknowledge it here with pset lock held */
4624 	pset->pending_AST_cpu_mask &= ~(1U << processor->cpu_id);
4643 			pset_unlock(pset);
4657 		pset_unlock(pset);
4674 		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
4689 			pset_unlock(pset);
4703 	pset_unlock(pset);
5035 	processor_set_t		pset;
5045 			pset = processor->processor_set;
5048 			pset_lock(pset);
5052 			pset_unlock(pset);