osfmk/kern/wait_queue.c

82 			wait_queue_t		wq,
225 	wait_queue_t wq,
232 	wq->wq_fifo = ((policy & SYNC_POLICY_REVERSED) == 0);
233 	wq->wq_type = _WAIT_QUEUE_inited;
234 	queue_init(&wq->wq_queue);
235 	hw_lock_init(&wq->wq_interlock);
254 	wait_queue_t wq;
257 	wq = (wait_queue_t) zalloc(_wait_queue_zone);
258 	if (wq != WAIT_QUEUE_NULL) {
259 		ret = wait_queue_init(wq, policy);
261 			zfree(_wait_queue_zone, wq);
262 			wq = WAIT_QUEUE_NULL;
265 	return wq;
277 	wait_queue_t wq)
279 	if (!wait_queue_is_queue(wq))
281 	if (!queue_empty(&wq->wq_queue))
283 	zfree(_wait_queue_zone, wq);
419 #define WAIT_QUEUE_ELEMENT_CHECK(wq, wqe) \
420 	WQASSERT(((wqe)->wqe_queue == (wq) && \
422 	  "wait queue element list corruption: wq=%#x, wqe=%#x", \
423 	  (wq), (wqe))
447 #define WAIT_QUEUE_CHECK(wq) \
449 	queue_t q2 = &(wq)->wq_queue; \
452 		WAIT_QUEUE_ELEMENT_CHECK((wq), wqe2); \
471 #define WAIT_QUEUE_CHECK(wq)
486 	wait_queue_t wq,
492 	assert(wait_queue_held(wq));
495 	q = &wq->wq_queue;
499 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
523 	wait_queue_t wq,
533 	wait_queue_lock(wq);
534 	ret = wait_queue_member_locked(wq, wq_set);
535 	wait_queue_unlock(wq);
555 	wait_queue_t wq,
563 	if (!wait_queue_is_valid(wq) || !wait_queue_is_set(wq_set))
572 	wait_queue_lock(wq);
573 	q = &wq->wq_queue;
576 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
580 			wait_queue_unlock(wq);
598 	wql->wql_queue = wq;
600 	queue_enter(&wq->wq_queue, wql, wait_queue_link_t, wql_links);
605 	wait_queue_unlock(wq);
622 	wait_queue_t wq,
627 	return wait_queue_link_internal(wq, wq_set, wql);
641 	wait_queue_t wq,
652 	ret = wait_queue_link_internal(wq, wq_set, wql);
686 	wait_queue_t wq,
690 	assert(wait_queue_held(wq));
694 	queue_remove(&wq->wq_queue, wql, wait_queue_link_t, wql_links);
703 	WAIT_QUEUE_CHECK(wq);
718 	wait_queue_t wq,
727 	if (!wait_queue_is_valid(wq) || !wait_queue_is_set(wq_set)) {
731 	wait_queue_lock(wq);
733 	q = &wq->wq_queue;
736 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
745 				wait_queue_unlink_locked(wq, wq_set, wql);
747 				wait_queue_unlock(wq);
756 	wait_queue_unlock(wq);
771 	wait_queue_t wq,
779 	if (!wait_queue_is_valid(wq) || !wait_queue_is_set(wq_set)) {
783 	wait_queue_lock(wq);
785 	q = &wq->wq_queue;
788 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
799 				wait_queue_unlink_locked(wq, wq_set, wql);
801 				wait_queue_unlock(wq);
811 	wait_queue_unlock(wq);
828 	wait_queue_t wq,
837 	q = &wq->wq_queue;
842 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
851 			wait_queue_unlink_locked(wq, wq_set, wql);
871 	wait_queue_t wq,
876 	if (!wait_queue_is_valid(wq)) {
881 	wait_queue_lock(wq);
882 	wait_queue_unlink_all_nofree_locked(wq, links);
883 	wait_queue_unlock(wq);
900 	wait_queue_t wq,
909 	q = &wq->wq_queue;
915 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
924 			wait_queue_unlink_locked(wq, wq_set, wql);
947 	wait_queue_t wq)
954 	if (!wait_queue_is_valid(wq)) {
961 	wait_queue_lock(wq);
962 	wait_queue_unlink_all_locked(wq, links);
963 	wait_queue_unlock(wq);
999 	wait_queue_t wq;
1017 		wq = wql->wql_queue;
1018 		if (wait_queue_lock_try(wq)) {
1019 			wait_queue_unlink_locked(wq, wq_set, wql);
1020 			wait_queue_unlock(wq);
1055 	wait_queue_t wq;
1077 		wq = wql->wql_queue;
1078 		if (wait_queue_lock_try(wq)) {
1082 			wait_queue_unlink_locked(wq, wq_set, wql);
1083 			wait_queue_unlock(wq);
1114 	wait_queue_t wq;
1133 	wq = wql->wql_queue;
1134 	if (wait_queue_lock_try(wq)) {
1135 		wait_queue_unlink_locked(wq, wq_set, wql);
1136 		wait_queue_unlock(wq);
1164 	wait_queue_t wq,
1176 	if (wq->wq_type == _WAIT_QUEUE_SET_inited) {
1177 		wait_queue_set_t wqs = (wait_queue_set_t)wq;
1201 		if (!wq->wq_fifo
1204 			enqueue_head(&wq->wq_queue, (queue_entry_t) thread);
1206 			enqueue_tail(&wq->wq_queue, (queue_entry_t) thread);
1209 		thread->wait_queue = wq;
1235 	wait_queue_t wq,
1245 	if (!wait_queue_is_valid(wq))
1249 	wait_queue_lock(wq);
1251 	ret = wait_queue_assert_wait64_locked(wq, CAST_DOWN(event64_t,event),
1254 	wait_queue_unlock(wq);
1269 	wait_queue_t wq,
1279 	if (!wait_queue_is_valid(wq))
1283 	wait_queue_lock(wq);
1285 	ret = wait_queue_assert_wait64_locked(wq, event, interruptible, deadline, thread);
1287 	wait_queue_unlock(wq);
1307 	wait_queue_t wq,
1315 	q = &wq->wq_queue;
1319 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
1376  *		KERN_NOT_WAITING - No threads were waiting <wq,event> pair
1380 	wait_queue_t wq,
1389 //	assert(wait_queue_held(wq));
1390 //	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
1391 //		panic("wait_queue_wakeup64_all_locked: lock not held on %p\n", wq);	/* (BRINGUP) */
1401 	_wait_queue_select64_all(wq, event, q);
1403 		wait_queue_unlock(wq);
1428  *		KERN_NOT_WAITING - No threads were waiting <wq,event> pair
1432 	wait_queue_t wq,
1439 	if (!wait_queue_is_valid(wq)) {
1444 	wait_queue_lock(wq);
1445 //	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
1446 //		panic("wait_queue_wakeup_all: we did not get the lock on %p\n", wq);	/* (BRINGUP) */
1449 				wq, CAST_DOWN(event64_t,event),
1465  *		KERN_NOT_WAITING - No threads were waiting <wq,event> pair
1469 	wait_queue_t wq,
1476 	if (!wait_queue_is_valid(wq)) {
1481 	wait_queue_lock(wq);
1482 	ret = wait_queue_wakeup64_all_locked(wq, event, result, TRUE);
1505 	wait_queue_t wq,
1513 	q = &wq->wq_queue;
1517 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
1623 	wait_queue_t wq,
1630 	queue_t q = &wq->wq_queue;
1633 	if ((thread->wait_queue == wq) && (thread->wait_event == event)) {
1650 		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
1690 	wait_queue_t wq,
1698 	assert(wait_queue_held(wq));
1700 	thread = _wait_queue_select64_one(wq, event);
1702 		wait_queue_unlock(wq);
1724  *		KERN_NOT_WAITING - No thread was waiting <wq,event> pair
1728 	wait_queue_t wq,
1735 	assert(wait_queue_held(wq));
1737 	thread = _wait_queue_select64_one(wq, event);
1739 		wait_queue_unlock(wq);
1762  *		KERN_NOT_WAITING - No thread was waiting <wq,event> pair
1766 	wait_queue_t wq,
1774 	if (!wait_queue_is_valid(wq)) {
1779 	wait_queue_lock(wq);
1780 	thread = _wait_queue_select64_one(wq, CAST_DOWN(event64_t,event));
1781 	wait_queue_unlock(wq);
1814  *		KERN_NOT_WAITING - No thread was waiting <wq,event> pair
1818 	wait_queue_t wq,
1825 	if (!wait_queue_is_valid(wq)) {
1829 	wait_queue_lock(wq);
1830 	thread = _wait_queue_select64_one(wq, event);
1831 	wait_queue_unlock(wq);
1869 	wait_queue_t wq,
1877 	assert(wait_queue_held(wq));
1883 	res = _wait_queue_select64_thread(wq, event, thread);
1885 	    wait_queue_unlock(wq);
1917 	wait_queue_t wq,
1925 	if (!wait_queue_is_valid(wq)) {
1930 	wait_queue_lock(wq);
1931 	res = _wait_queue_select64_thread(wq, CAST_DOWN(event64_t,event), thread);
1932 	wait_queue_unlock(wq);
1966 	wait_queue_t wq,
1974 	if (!wait_queue_is_valid(wq)) {
1979 	wait_queue_lock(wq);
1980 	res = _wait_queue_select64_thread(wq, event, thread);
1981 	wait_queue_unlock(wq);