bsd/kern/pthread_support.c

478 void ksyn_wqrelease(ksyn_wait_queue_t mkwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype);
504 void ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t *updatep);
505 void ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release);
507 ksyn_waitq_element_t ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, thread_t th, uint32_t toseq);
974 	ksyn_wait_queue_t ckwq;
1003 	ckwq = NULL;
1005 	error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1015 	ksyn_wqlock(ckwq);
1018 	UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1021 	ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1024 	ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1025 	ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1026 	ksyn_wqunlock(ckwq);
1030 	ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
1039 ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, __unused ksyn_queue_t kq, thread_t th, uint32_t upto)
1044 	if (kwe->kwe_kwqqueue != ckwq ||
1064 	ksyn_wait_queue_t ckwq = NULL;
1100 	error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1108 	ksyn_wqlock(ckwq);
1111 	UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1113 	kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
1117 	if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
1119 		kwe = (th != NULL) ? ksyn_queue_find_threadseq(ckwq, kq, th, uptoseq) :
1120 			ksyn_queue_find_signalseq(ckwq, kq, uptoseq, signalseq);
1145 					__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc1c1c1c1, uptoseq, 0, 0);
1147 					ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1149 					ksyn_queue_removeitem(ckwq, kq, kwe);
1153 					__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
1155 					kret = ksyn_wakeup_thread(ckwq, kwe);
1163 				ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1177 			ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
1178 			ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
1180 			ksyn_wqunlock(ckwq);
1182 			ksyn_wqlock(ckwq);
1188 			nkwe->kwe_kwqqueue = ckwq;
1196 			__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfefe, uptoseq, 0, 0);
1199 			(void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], uptoseq, NULL, nkwe, SEQFIT);
1200 			ckwq->kw_fakecount++;
1205 		ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1208 	ksyn_wqunlock(ckwq);
1212 	ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
1240 	ksyn_wait_queue_t kwq, ckwq;
1274 	ckwq = kwq = NULL;
1275 	error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INWAIT, &ckwq);
1306 	ksyn_wqlock(ckwq);
1309 	UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
1312 	kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
1319 			ksyn_wqunlock(ckwq);
1331 					ksyn_queue_removeitem(ckwq, kq, kwe);
1332 					ckwq->kw_fakecount--;
1342 				__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc2c2c2c2, kwe->kwe_lockseq, 0, 0);
1345 				ksyn_handle_cvbroad(ckwq, kwe->kwe_lockseq, &updatebits);
1360                                 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfefefefe, kwe->kwe_lockseq, 0, 0);
1365 		ckwq->kw_sword += PTHRW_INC;
1368 		ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
1375 		ksyn_wqunlock(ckwq);
1386 	kwe->kwe_kwqqueue = ckwq;
1394 	__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, cgen, 0, 0);
1397 	error = ksyn_queue_insert(ckwq, kq, cgen, uth, kwe, SEQFIT);
1399 		ksyn_wqunlock(ckwq);
1404 	kret = ksyn_block_thread_locked(ckwq, abstime, kwe, 1, psynch_cvcontinue, (void *)ckwq);
1407 	psynch_cvcontinue(ckwq, kret);
1415 	ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
1425 	ksyn_wait_queue_t ckwq = (ksyn_wait_queue_t)parameter;
1448 		ksyn_wqlock(ckwq);
1453 		if ((kwe->kwe_kwqqueue != NULL) && (kwe->kwe_kwqqueue != ckwq))
1460 			ksyn_queue_removeitem(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
1469 			ksyn_wqunlock(ckwq);
1473 		ckwq->kw_sword += PTHRW_INC;
1476 		if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
1478 			if (ckwq->kw_inqueue != 0) {
1479 				(void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (ckwq->kw_lword & PTHRW_COUNT_MASK), &kfreeq, 1, 1);
1481 			ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
1482 			ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
1485 			if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
1489 		ksyn_wqunlock(ckwq);
1501 	__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)ckwq->kw_addr, 0xeeeeeeed, uth->uu_rval[0], local_error, 0);
1503 	ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
1522 	ksyn_wait_queue_t ckwq = NULL;
1531 		error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
1540 		ksyn_wqlock(ckwq);
1541 		(void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (pseq & PTHRW_COUNT_MASK), &kfreeq, 0, 1);
1542 		ksyn_wqunlock(ckwq);
1543 		ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP));
1546 		error = ksyn_wqfind(cond, cgen, cugen, 0, 0, flags, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP), &ckwq);
1555 		ksyn_wqlock(ckwq);
1556 		if (((flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT) != 0) && (ckwq->kw_pre_rwwc != 0)) {
1557 			if (is_seqlower_eq(ckwq->kw_pre_lockseq, cgen) != 0) {
1559 				ckwq->kw_pre_rwwc = 0;
1560 				ckwq->kw_pre_lockseq = 0;
1563 		ksyn_wqunlock(ckwq);
1564 		ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP));
2929 ksyn_wqrelease(ksyn_wait_queue_t kwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype)
2972 	if (ckwq != NULL) {
2973 		ckwq->kw_iocount--;
2977 		if ( ckwq->kw_iocount == 0) {
2983 			if ((ckwq->kw_pre_rwwc == 0) && (ckwq->kw_inqueue == 0) && (ckwq->kw_pre_intrcount == 0)) {
2986 					microuptime(&ckwq->kw_ts);
2987 					LIST_INSERT_HEAD(&pth_free_list, ckwq, kw_list);
2988 					ckwq->kw_pflags |= KSYN_WQ_FLIST;
2993 					ckwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
2994 					LIST_REMOVE(ckwq, kw_hash);
2995 					lck_mtx_destroy(&ckwq->kw_lock, pthread_lck_grp);
2998 					free_elem1 = ckwq;
4108 ksyn_queue_move_tofree(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t kfreeq, int all, int release)
4137 			ksyn_queue_removeitem(ckwq, kq, kwe);
4142 		__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4144 			(void)ksyn_wakeup_thread(ckwq, kwe);
4146 			ksyn_queue_removeitem(ckwq, kq, kwe);
4148 			ckwq->kw_fakecount--;
4370 ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t * updatep)
4384 	kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
4399 			if (kwe->kwe_kwqqueue != ckwq ||
4404 			ksyn_queue_removeitem(ckwq, kq, kwe);
4408 				__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
4410 				kret = ksyn_wakeup_thread(ckwq, kwe);
4420 			ksyn_queue_removeitem(ckwq, kq, kwe);
4422 			ckwq->kw_fakecount--;
4433 	if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
4437 			ksyn_wqunlock(ckwq);
4440 			ksyn_wqlock(ckwq);
4446 		newkwe->kwe_kwqqueue = ckwq;
4454 		__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, upto, 0, 0);
4457 		(void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], upto, NULL, newkwe, SEQFIT);
4458 		ckwq->kw_fakecount++;
4478 ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release)
4484 	if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
4486 		if (ckwq->kw_inqueue != 0) {
4488 			ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], ckwq->kw_lword, kfreeq, 0, release);
4490 			if (ckwq->kw_inqueue != 0)
4494 		ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
4495 		ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
4496 	} else if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {