bsd/kern/pthread_support.c

483 kern_return_t ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int log, thread_continue_t, void * parameter);
484 kern_return_t ksyn_wakeup_thread(ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe);
491 int ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int firstfit);
493 void ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe);
590 	ksyn_waitq_element_t kwe = NULL;
609 			kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
610 			kwe->kwe_psynchretval = updatebits;
611 			kwe->kwe_kwqqueue = NULL;
614 			__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf1, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
617 			kret = ksyn_wakeup_thread(kwq, kwe);
631 				kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
634 				kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
635 				kwe->kwe_kwqqueue = NULL;
638 				__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
641 				kret = ksyn_wakeup_thread(kwq, kwe);
673 				kwe = ksyn_queue_find_seq(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (nextgen & PTHRW_COUNT_MASK), 1);
674 				if (kwe != NULL) {
677 					kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
678 					kwe->kwe_kwqqueue = NULL;
680 					__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
682 					kret = ksyn_wakeup_thread(kwq, kwe);
768 	ksyn_waitq_element_t kwe;
778 	kwe = &uth->uu_kwe;
779 	kwe->kwe_lockseq = uap->mgen;
780 	kwe->kwe_uth = uth;
781 	kwe->kwe_psynchretval = 0;
782 	kwe->kwe_kwqqueue = NULL;
813                 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
817 				*retval = kwe->kwe_psynchretval;
819 			__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, 0xfafafaf1, kwe->kwe_psynchretval, kwq->kw_pre_intrcount, 0);
837 			kwe->kwe_psynchretval = updatebits;
861 	error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], mgen, uth, kwe, ins_flags);
870 	kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, psynch_mtxcontinue, (void *)kwq);
893 	ksyn_waitq_element_t kwe;
895 	kwe = &uth->uu_kwe;
915 		if (kwe->kwe_kwqqueue != NULL)
916 			ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
919 		updatebits = kwe->kwe_psynchretval;
1042 	ksyn_waitq_element_t kwe = &uth->uu_kwe;
1044 	if (kwe->kwe_kwqqueue != ckwq ||
1045 	    is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) {
1049 	return kwe;
1065 	ksyn_waitq_element_t kwe, nkwe = NULL;
1119 		kwe = (th != NULL) ? ksyn_queue_find_threadseq(ckwq, kq, th, uptoseq) :
1121 		if (kwe != NULL) {
1122 			switch (kwe->kwe_flags) {
1130 				kwe->kwe_count += 1;
1134 				if (is_seqlower((kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq)) {
1149 					ksyn_queue_removeitem(ckwq, kq, kwe);
1150 					kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
1151 					kwe->kwe_kwqqueue = NULL;
1153 					__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
1155 					kret = ksyn_wakeup_thread(ckwq, kwe);
1246 	ksyn_waitq_element_t kwe, nkwe = NULL;
1313 	kwe = ksyn_queue_find_cvpreposeq(kq, lockseq);
1315 	if (kwe != NULL) {
1316 		switch (kwe->kwe_flags) {
1328 			if ((kwe->kwe_lockseq & PTHRW_COUNT_MASK) == lockseq) {
1330 				if (--kwe->kwe_count == 0) {
1331 					ksyn_queue_removeitem(ckwq, kq, kwe);
1333 					nkwe = kwe;
1342 				__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc2c2c2c2, kwe->kwe_lockseq, 0, 0);
1345 				ksyn_handle_cvbroad(ckwq, kwe->kwe_lockseq, &updatebits);
1360                                 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfefefefe, kwe->kwe_lockseq, 0, 0);
1385 	kwe = &uth->uu_kwe;
1386 	kwe->kwe_kwqqueue = ckwq;
1387 	kwe->kwe_flags = KWE_THREAD_INWAIT;
1388 	kwe->kwe_lockseq = lockseq;
1389 	kwe->kwe_count = 1;
1390 	kwe->kwe_uth = uth;
1391 	kwe->kwe_psynchretval = 0;
1397 	error = ksyn_queue_insert(ckwq, kq, cgen, uth, kwe, SEQFIT);
1404 	kret = ksyn_block_thread_locked(ckwq, abstime, kwe, 1, psynch_cvcontinue, (void *)ckwq);
1426 	ksyn_waitq_element_t kwe;
1445 	kwe = &uth->uu_kwe;
1450 		uth->uu_rval[0] = kwe->kwe_psynchretval;
1453 		if ((kwe->kwe_kwqqueue != NULL) && (kwe->kwe_kwqqueue != ckwq))
1459 		if (kwe->kwe_kwqqueue != NULL) {
1460 			ksyn_queue_removeitem(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
1461 			kwe->kwe_kwqqueue = NULL;
1463 		if ((kwe->kwe_psynchretval & PTH_RWL_MTX_WAIT) != 0) {
1493 		if ((kwe->kwe_psynchretval & PTH_RWS_CV_MBIT)  != 0)
1592 	ksyn_waitq_element_t kwe;
1601 	kwe = &uth->uu_kwe;
1602 	kwe->kwe_lockseq = lgen;
1603 	kwe->kwe_uth = uth;
1604 	kwe->kwe_psynchretval = 0;
1605 	kwe->kwe_kwqqueue = NULL;
1638 		kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
1659 			kwe->kwe_psynchretval = updatebits;
1700 	error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], lgen, uth, kwe, SEQFIT);
1705 	kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
1725 		if (kwe->kwe_kwqqueue != NULL)
1726 			ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwe);
1730 		*retval = kwe->kwe_psynchretval;
1731 		returnbits = kwe->kwe_psynchretval;
1759 	ksyn_waitq_element_t kwe;
1771 	kwe = &uth->uu_kwe;
1772 	kwe->kwe_lockseq = lgen;
1773 	kwe->kwe_uth = uth;
1774 	kwe->kwe_psynchretval = 0;
1775 	kwe->kwe_kwqqueue = NULL;
1806 		kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
1845 	error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], lgen, uth, kwe, SEQFIT);
1851 	kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
1870 		if (kwe->kwe_kwqqueue != NULL)
1871 			ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwe);
1875 		*retval = kwe->kwe_psynchretval;
1876 		returnbits = kwe->kwe_psynchretval;
1910 	ksyn_waitq_element_t kwe;
1917 	kwe = &uth->uu_kwe;
1918 	kwe->kwe_lockseq = lgen;
1919 	kwe->kwe_uth = uth;
1920 	kwe->kwe_psynchretval = 0;
1921 	kwe->kwe_kwqqueue = NULL;
1954 		kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
1996 	error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], lgen, uth, kwe, SEQFIT);
2002 	kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2022 		if (kwe->kwe_kwqqueue != NULL)
2023 			ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
2027 		*retval = kwe->kwe_psynchretval;
2028 		returnbits = kwe->kwe_psynchretval;
2062 	ksyn_waitq_element_t kwe;
2071 	kwe = &uth->uu_kwe;
2072 	kwe->kwe_lockseq = lgen;
2073 	kwe->kwe_uth = uth;
2074 	kwe->kwe_psynchretval = 0;
2075 	kwe->kwe_kwqqueue = NULL;
2106 		kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
2145 	error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], lgen, uth, kwe, SEQFIT);
2151 	kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2170 		if (kwe->kwe_kwqqueue != NULL)
2171 			ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwe);
2175 		*retval = kwe->kwe_psynchretval;
2176 		returnbits = kwe->kwe_psynchretval;
2323 	ksyn_waitq_element_t kwe;
2330 	kwe = &uth->uu_kwe;
2331 	kwe->kwe_lockseq = lgen;
2332 	kwe->kwe_uth = uth;
2333 	kwe->kwe_psynchretval = 0;
2334 	kwe->kwe_kwqqueue = NULL;
2365 		kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
2404 	error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], lgen, uth, kwe, SEQFIT);
2411 	kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2431 		if (kwe->kwe_kwqqueue != NULL)
2432 			ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwe);
2436 		*retval = kwe->kwe_psynchretval;
2713 	ksyn_waitq_element_t kwe;
2716 	kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
2717 	while (kwe != NULL) {
2718 		if (kwe->kwe_flags != KWE_THREAD_INWAIT) {
2719 			TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
2720 			zfree(kwe_zone, kwe);
2722 			TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
2724 		kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3084 ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int mylog, thread_continue_t continuation, void * parameter)
3086 ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, __unused int mylog, thread_continue_t continuation, void * parameter)
3095 	kwe->kwe_kwqqueue = (void *)kwq;
3096 	assert_wait_deadline(&kwe->kwe_psynchretval, THREAD_ABORTSAFE, abstime);
3127 ksyn_wakeup_thread(__unused ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe)
3134 	kret = thread_wakeup_one((caddr_t)&kwe->kwe_psynchretval);
3139 	uth = kwe->kwe_uth;
3278 	ksyn_waitq_element_t kwe = NULL;
3288 		while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwq)) != NULL) {
3289 			kwe->kwe_psynchretval = lbits;
3290 			kwe->kwe_kwqqueue = NULL;
3293 			kret = ksyn_wakeup_thread(kwq, kwe);
3302 		while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwq)) != NULL) {
3303 			kwe->kwe_psynchretval = lbits;
3304 			kwe->kwe_kwqqueue = NULL;
3306 			kret = ksyn_wakeup_thread(kwq, kwe);
3318 			kwe = ksyn_queue_removefirst(kq, kwq);
3319 			kwe->kwe_psynchretval = lbits;
3320 			kwe->kwe_kwqqueue = NULL;
3322 			kret = ksyn_wakeup_thread(kwq, kwe);
3349 	ksyn_waitq_element_t kwe;
3390 			kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwq);
3393 			kwe->kwe_psynchretval = updatebits;
3394 			kwe->kwe_kwqqueue = NULL;
3395 			kret = ksyn_wakeup_thread(kwq, kwe);
3401 				kwq->kw_pre_intrretbits = kwe->kwe_psynchretval;
3551 				kwe = &uth->uu_kwe;
3552 				kwe->kwe_psynchretval = updatebits;
3592 				kwe = &uth->uu_kwe;
3593 				kwe->kwe_psynchretval = updatebits;
3596 				kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
3603 				kwe->kwe_psynchretval = updatebits;
3604 				kwe->kwe_kwqqueue = NULL;
3606 				kret = ksyn_wakeup_thread(kwq, kwe);
3648 						kwe = &uth->uu_kwe;
3651 						kwe->kwe_psynchretval = updatebits;
3675 						kwe = &uth->uu_kwe;
3677 						kwe->kwe_psynchretval = updatebits;
3711 					kwe = &uth->uu_kwe;
3712 					kwe->kwe_psynchretval = updatebits;
3715 					kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwq);
3723 					kwe->kwe_psynchretval = updatebits;
3724 					kwe->kwe_kwqqueue = NULL;
3726 					kret = ksyn_wakeup_thread(kwq, kwe);
3856 ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int fit)
3864 		TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
3873 		TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
3883 		if (kwe->kwe_flags == KWE_THREAD_PREPOST) {
3887 				TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
3904 		TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
3910 		TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
3918 			TAILQ_INSERT_BEFORE(q_kwe, kwe, kwe_list);
3929 		kwe->kwe_uth = uth;
3940 	ksyn_waitq_element_t kwe = NULL;
3945 		kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
3946 		TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
3947 		curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
3969 	return(kwe);
3973 ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe)
3979 		TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
3992 		curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
4110 	ksyn_waitq_element_t kwe;
4120 	kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
4121 	while (kwe != NULL) {
4122 		if ((all == 0) && (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), tseq) != 0))
4124 		if (kwe->kwe_flags == KWE_THREAD_INWAIT) {
4132 			ut = (uthread_t)kwe->kwe_uth;
4137 			ksyn_queue_removeitem(ckwq, kq, kwe);
4139 			kwe->kwe_psynchretval = PTHRW_INC | (PTH_RWS_CV_MBIT | PTH_RWL_MTX_WAIT);
4140 			kwe->kwe_kwqqueue = NULL;
4142 		__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4144 			(void)ksyn_wakeup_thread(ckwq, kwe);
4146 			ksyn_queue_removeitem(ckwq, kq, kwe);
4147 			TAILQ_INSERT_TAIL(&kfreeq->ksynq_kwelist, kwe, kwe_list);
4151 		kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
4155 		kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
4156 		while (kwe != NULL) {
4157 			TAILQ_REMOVE(&kfreeq->ksynq_kwelist, kwe, kwe_list);
4158 			zfree(kwe_zone, kwe);
4159 			kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
4343 	ksyn_waitq_element_t kwe, newkwe;
4352 	TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
4353 		curval = (kwe->kwe_lockseq & PTHRW_COUNT_MASK);
4374 	ksyn_waitq_element_t kwe, newkwe;
4387 	TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
4389 		if (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto))	/* outside our range */
4393 		switch (kwe->kwe_flags) {
4396 			ut = (uthread_t)kwe->kwe_uth;
4399 			if (kwe->kwe_kwqqueue != ckwq ||
4404 			ksyn_queue_removeitem(ckwq, kq, kwe);
4405 			kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
4406 			kwe->kwe_kwqqueue = NULL;
4408 				__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4410 				kret = ksyn_wakeup_thread(ckwq, kwe);
4420 			ksyn_queue_removeitem(ckwq, kq, kwe);
4421 			TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, kwe, kwe_list);
4462 	kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
4463 	while (kwe != NULL) {
4464 		TAILQ_REMOVE(&kfreeq.ksynq_kwelist, kwe, kwe_list);
4465 		zfree(kwe_zone, kwe);
4466 		kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);