Lines Matching refs:kwq
175 int num_total_kwq = 0; /* number of kwq in use currently */
176 int num_infreekwq = 0; /* number of kwq in free list */
177 int num_freekwq = 0; /* number of kwq actually freed from the free the list */
178 int num_reusekwq = 0; /* number of kwq pulled back for reuse from free list */
179 int num_addedfreekwq = 0; /* number of added free kwq from the last instance */
428 #define CLEAR_PREPOST_BITS(kwq) {\
429 kwq->kw_pre_lockseq = 0; \
430 kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
431 kwq->kw_pre_rwwc = 0; \
434 #define CLEAR_INITCOUNT_BITS(kwq) {\
435 kwq->kw_initcount = 0; \
436 kwq->kw_initrecv = 0; \
437 kwq->kw_initcountseq = 0; \
440 #define CLEAR_INTR_PREPOST_BITS(kwq) {\
441 kwq->kw_pre_intrcount = 0; \
442 kwq->kw_pre_intrseq = 0; \
443 kwq->kw_pre_intrretbits = 0; \
444 kwq->kw_pre_intrtype = 0; \
447 #define CLEAR_REINIT_BITS(kwq) {\
448 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) { \
449 if((kwq->kw_inqueue != 0) && (kwq->kw_inqueue != kwq->kw_fakecount)) \
450 panic("CV:entries in queue durinmg reinit %d:%d\n",kwq->kw_inqueue, kwq->kw_fakecount); \
452 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_RWLOCK) { \
453 kwq->kw_nextseqword = PTHRW_RWS_INIT; \
454 kwq->kw_overlapwatch = 0; \
456 kwq->kw_pre_lockseq = 0; \
457 kwq->kw_pre_rwwc = 0; \
458 kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
459 kwq->kw_lastunlockseq = PTHRW_RWL_INIT; \
460 kwq->kw_lastseqword = PTHRW_RWS_INIT; \
461 kwq->kw_pre_intrcount = 0; \
462 kwq->kw_pre_intrseq = 0; \
463 kwq->kw_pre_intrretbits = 0; \
464 kwq->kw_pre_intrtype = 0; \
465 kwq->kw_lword = 0; \
466 kwq->kw_uword = 0; \
467 kwq->kw_sword = PTHRW_RWS_INIT; \
474 void ksyn_wqlock(ksyn_wait_queue_t kwq);
475 void ksyn_wqunlock(ksyn_wait_queue_t kwq);
480 static void UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int wqtype);
483 kern_return_t ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int log, thread_continue_t, void * parameter);
484 kern_return_t ksyn_wakeup_thread(ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe);
487 uint32_t psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags);
491 int ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int firstfit);
492 ksyn_waitq_element_t ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq);
493 void ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe);
494 int ksyn_queue_move_tofree(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t freeq, int all, int reease);
495 void update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq);
496 uint32_t find_nextlowseq(ksyn_wait_queue_t kwq);
497 uint32_t find_nexthighseq(ksyn_wait_queue_t kwq);
499 int find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp);
506 ksyn_waitq_element_t ksyn_queue_find_signalseq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t toseq, uint32_t lockseq);
511 int ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp);
512 int kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * type, uint32_t lowest[]);
513 ksyn_waitq_element_t ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove);
514 int kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, int flags , int * blockp);
515 int kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, int flags, uint32_t premgen, int * blockp);
518 UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, __unused uint64_t tid, __unused int wqtype)
520 if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) {
521 if ((kwq->kw_kflags & KSYN_KWF_ZEROEDOUT) != 0) {
523 kwq->kw_lword = mgen;
524 kwq->kw_uword = ugen;
525 kwq->kw_sword = rw_wc;
526 kwq->kw_kflags &= ~KSYN_KWF_ZEROEDOUT;
528 if (is_seqhigher((mgen & PTHRW_COUNT_MASK), (kwq->kw_lword & PTHRW_COUNT_MASK)) != 0)
529 kwq->kw_lword = mgen;
530 if (is_seqhigher((ugen & PTHRW_COUNT_MASK), (kwq->kw_uword & PTHRW_COUNT_MASK)) != 0)
531 kwq->kw_uword = ugen;
533 if(is_seqlower(kwq->kw_cvkernelseq, (rw_wc & PTHRW_COUNT_MASK)) != 0) {
534 kwq->kw_cvkernelseq = (rw_wc & PTHRW_COUNT_MASK);
536 if (is_seqhigher((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_sword & PTHRW_COUNT_MASK)) != 0)
537 kwq->kw_sword = rw_wc;
571 ksyn_wqlock(ksyn_wait_queue_t kwq)
574 lck_mtx_lock(&kwq->kw_lock);
578 ksyn_wqunlock(ksyn_wait_queue_t kwq)
580 lck_mtx_unlock(&kwq->kw_lock);
586 psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags)
596 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_START, (uint32_t)kwq->kw_addr, lkseq, ugen, flags, 0);
599 ksyn_wqlock(kwq);
603 if (kwq->kw_inqueue != 0) {
604 updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_EBIT | PTH_RWL_KBIT);
605 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
609 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
614 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf1, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
617 kret = ksyn_wakeup_thread(kwq, kwe);
626 low_writer = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
631 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
638 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
641 kret = ksyn_wakeup_thread(kwq, kwe);
648 kwq->kw_pre_intrcount = 1;
649 kwq->kw_pre_intrseq = nextgen;
650 kwq->kw_pre_intrretbits = updatebits;
651 kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
653 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfafafaf1, nextgen, kwq->kw_pre_intrretbits, 0);
658 kwq->kw_pre_rwwc++;
660 if (kwq->kw_pre_rwwc > 1) {
665 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
667 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
673 kwe = ksyn_queue_find_seq(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (nextgen & PTHRW_COUNT_MASK), 1);
680 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
682 kret = ksyn_wakeup_thread(kwq, kwe);
691 kwq->kw_pre_rwwc++;
693 if (kwq->kw_pre_rwwc > 1) {
698 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
700 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
709 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
710 kwq->kw_pre_rwwc++;
712 if (kwq->kw_pre_rwwc > 1) {
717 kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
719 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
724 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_lastunlockseq, kwq->kw_pre_lockseq, 0);
726 kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
728 if ((kwq->kw_pre_rwwc == 0) || (is_seqlower(kwq->kw_pre_lockseq, lkseq) == 0))
729 kwq->kw_pre_lockseq = (lkseq & PTHRW_COUNT_MASK);
730 kwq->kw_pre_rwwc = 1;
732 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
741 ksyn_wqunlock(kwq);
744 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, 0, 0, 0);
746 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX));
762 ksyn_wait_queue_t kwq;
792 error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX), &kwq);
800 ksyn_wqlock(kwq);
809 if ((kwq->kw_pre_intrcount != 0) &&
810 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE)) &&
811 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
812 kwq->kw_pre_intrcount--;
813 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
814 if (kwq->kw_pre_intrcount==0)
815 CLEAR_INTR_PREPOST_BITS(kwq);
816 ksyn_wqunlock(kwq);
819 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, 0xfafafaf1, kwe->kwe_psynchretval, kwq->kw_pre_intrcount, 0);
824 if ((kwq->kw_pre_rwwc != 0) && ((ins_flags == FIRSTFIT) || ((lockseq & PTHRW_COUNT_MASK) == (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK) ))) {
826 kwq->kw_pre_rwwc--;
827 if (kwq->kw_pre_rwwc == 0) {
828 CLEAR_PREPOST_BITS(kwq);
829 kwq->kw_lastunlockseq = PTHRW_RWL_INIT;
830 if (kwq->kw_inqueue == 0) {
833 updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_KBIT | PTH_RWL_EBIT);
842 ksyn_wqunlock(kwq);
845 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
850 kwq->kw_pre_lockseq += PTHRW_INC; /* look for next one */
851 ksyn_wqunlock(kwq);
858 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfeedfeed, mgen, ins_flags, 0);
861 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], mgen, uth, kwe, ins_flags);
863 ksyn_wqunlock(kwq);
870 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, psynch_mtxcontinue, (void *)kwq);
872 psynch_mtxcontinue((void *)kwq, kret);
878 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
892 ksyn_wait_queue_t kwq = (ksyn_wait_queue_t)parameter;
910 ksyn_wqlock(kwq);
913 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, 0xdeadbeef, error, 0);
916 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
917 ksyn_wqunlock(kwq);
926 ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
928 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, updatebits, error, 0);
945 ksyn_wait_queue_t kwq;
949 error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
954 updateval = psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
955 /* drops the kwq reference */
1240 ksyn_wait_queue_t kwq, ckwq;
1274 ckwq = kwq = NULL;
1285 error = ksyn_wqfind(mutex, mgen, ugen, 0, 0, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
1294 (void)psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
1295 /* drops kwq reference */
1296 kwq = NULL;
1454 panic("cvwait waiting on some other kwq\n");
1588 ksyn_wait_queue_t kwq;
1610 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1618 ksyn_wqlock(kwq);
1622 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1624 CLEAR_REINIT_BITS(kwq);
1625 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1633 if ((kwq->kw_pre_intrcount != 0) &&
1634 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
1635 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1637 kwq->kw_pre_intrcount--;
1638 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
1639 if (kwq->kw_pre_intrcount==0)
1640 CLEAR_INTR_PREPOST_BITS(kwq);
1641 ksyn_wqunlock(kwq);
1648 if ((kwq->kw_overlapwatch != 0) && ((rw_wc & PTHRW_RWS_SAVEMASK) == 0) && ((lgen & PTH_RWL_WBIT) == 0)) {
1650 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 10, kwq->kw_nextseqword, kwq->kw_lastseqword, 0);
1652 error = kwq_handle_overlap(kwq, lgen, ugen, rw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block);
1663 ksyn_wqunlock(kwq);
1668 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1670 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
1672 kwq->kw_pre_rwwc--;
1673 if (kwq->kw_pre_rwwc == 0) {
1674 preseq = kwq->kw_pre_lockseq;
1675 prerw_wc = kwq->kw_pre_sseq;
1676 CLEAR_PREPOST_BITS(kwq);
1677 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1678 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1683 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1689 ksyn_wqunlock(kwq);
1700 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], lgen, uth, kwe, SEQFIT);
1705 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
1706 /* drops the kwq lock */
1724 ksyn_wqlock(kwq);
1726 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwe);
1727 ksyn_wqunlock(kwq);
1733 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
1762 ksyn_wait_queue_t kwq;
1778 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1786 ksyn_wqlock(kwq);
1790 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1792 CLEAR_REINIT_BITS(kwq);
1793 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1801 if ((kwq->kw_pre_intrcount != 0) &&
1802 (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD) &&
1803 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1805 kwq->kw_pre_intrcount--;
1806 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
1807 if (kwq->kw_pre_intrcount==0)
1808 CLEAR_INTR_PREPOST_BITS(kwq);
1809 ksyn_wqunlock(kwq);
1814 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1816 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
1818 kwq->kw_pre_rwwc--;
1819 if (kwq->kw_pre_rwwc == 0) {
1820 preseq = kwq->kw_pre_lockseq;
1821 prerw_wc = kwq->kw_pre_sseq;
1822 CLEAR_PREPOST_BITS(kwq);
1823 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1824 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1829 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_LREADLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1835 ksyn_wqunlock(kwq);
1845 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], lgen, uth, kwe, SEQFIT);
1851 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
1852 /* drops the kwq lock */
1869 ksyn_wqlock(kwq);
1871 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwe);
1872 ksyn_wqunlock(kwq);
1879 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
1904 ksyn_wait_queue_t kwq;
1924 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
1932 ksyn_wqlock(kwq);
1937 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
1939 CLEAR_REINIT_BITS(kwq);
1940 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
1949 if ((kwq->kw_pre_intrcount != 0) &&
1950 (kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE) &&
1951 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
1953 kwq->kw_pre_intrcount--;
1954 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
1955 if (kwq->kw_pre_intrcount==0)
1956 CLEAR_INTR_PREPOST_BITS(kwq);
1957 ksyn_wqunlock(kwq);
1962 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
1964 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
1966 kwq->kw_pre_rwwc--;
1967 if (kwq->kw_pre_rwwc == 0) {
1968 preseq = kwq->kw_pre_lockseq;
1969 prerw_wc = kwq->kw_pre_sseq;
1970 CLEAR_PREPOST_BITS(kwq);
1971 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
1972 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
1977 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_WRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
1983 ksyn_wqunlock(kwq);
1996 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], lgen, uth, kwe, SEQFIT);
2002 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2021 ksyn_wqlock(kwq);
2023 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
2024 ksyn_wqunlock(kwq);
2031 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
2057 ksyn_wait_queue_t kwq;
2078 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
2086 ksyn_wqlock(kwq);
2090 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
2092 CLEAR_REINIT_BITS(kwq);
2093 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2101 if ((kwq->kw_pre_intrcount != 0) &&
2102 (kwq->kw_pre_intrtype == PTH_RW_TYPE_YWRITE) &&
2103 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
2105 kwq->kw_pre_intrcount--;
2106 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
2107 if (kwq->kw_pre_intrcount==0)
2108 CLEAR_INTR_PREPOST_BITS(kwq);
2109 ksyn_wqunlock(kwq);
2113 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
2115 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
2117 kwq->kw_pre_rwwc--;
2118 if (kwq->kw_pre_rwwc == 0) {
2119 preseq = kwq->kw_pre_lockseq;
2120 prerw_wc = kwq->kw_pre_sseq;
2121 CLEAR_PREPOST_BITS(kwq);
2122 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
2123 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2128 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_YWRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
2134 ksyn_wqunlock(kwq);
2145 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], lgen, uth, kwe, SEQFIT);
2151 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2169 ksyn_wqlock(kwq);
2171 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwe);
2172 ksyn_wqunlock(kwq);
2179 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
2205 ksyn_wait_queue_t kwq;
2217 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
2225 ksyn_wqlock(kwq);
2229 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
2230 CLEAR_REINIT_BITS(kwq);
2231 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2240 if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
2255 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
2257 if (find_seq_till(kwq, curgen, diff, &count) == 0) {
2263 if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
2264 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2272 CLEAR_PREPOST_BITS(kwq);
2278 error = kwq_handle_downgrade(kwq, lgen, 0, 0, NULL);
2286 ksyn_wqunlock(kwq);
2290 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
2295 kwq->kw_pre_rwwc = (rw_wc - count);
2296 kwq->kw_pre_lockseq = lgen;
2298 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWDOWNGRADE | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
2318 ksyn_wait_queue_t kwq;
2337 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK), &kwq);
2345 ksyn_wqlock(kwq);
2349 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
2351 CLEAR_REINIT_BITS(kwq);
2352 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2360 if ((kwq->kw_pre_intrcount != 0) &&
2361 ((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
2362 (is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
2364 kwq->kw_pre_intrcount--;
2365 kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
2366 if (kwq->kw_pre_intrcount==0)
2367 CLEAR_INTR_PREPOST_BITS(kwq);
2368 ksyn_wqunlock(kwq);
2372 if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
2374 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
2376 kwq->kw_pre_rwwc--;
2377 if (kwq->kw_pre_rwwc == 0) {
2378 preseq = kwq->kw_pre_lockseq;
2379 prerw_wc = kwq->kw_pre_sseq;
2380 CLEAR_PREPOST_BITS(kwq);
2381 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
2382 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2387 error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_UPGRADE|KW_UNLOCK_PREPOST), &block, lgen);
2393 ksyn_wqunlock(kwq);
2404 error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], lgen, uth, kwe, SEQFIT);
2411 kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
2430 ksyn_wqlock(kwq);
2432 ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwe);
2433 ksyn_wqunlock(kwq);
2439 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
2475 ksyn_wait_queue_t kwq;
2487 error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
2497 ksyn_wqlock(kwq);
2501 if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
2502 CLEAR_REINIT_BITS(kwq);
2503 kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
2512 if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
2514 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, (uint32_t)0xeeeeeeee, rw_wc, kwq->kw_lastunlockseq, 0);
2524 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
2526 if (find_seq_till(kwq, curgen, diff, &count) == 0) {
2532 if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
2533 kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
2541 CLEAR_PREPOST_BITS(kwq);
2546 error = kwq_handle_unlock(kwq, lgen, rw_wc, &updatebits, 0, NULL, 0);
2558 ksyn_wqunlock(kwq);
2560 ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
2569 if ((is_rws_setseq(kwq->kw_pre_sseq) != 0) ||
2570 (is_seqhigher_eq((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_pre_sseq & PTHRW_COUNT_MASK)) != 0)) {
2571 kwq->kw_pre_rwwc = (diff - count);
2572 kwq->kw_pre_lockseq = curgen;
2573 kwq->kw_pre_sseq = rw_wc;
2576 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
2639 ksyn_wait_queue_t kwq;
2645 kwq = (&hashptr[object & pthhash])->lh_first;
2646 if (kwq != 0) {
2647 for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
2648 if ((kwq->kw_object == object) &&(kwq->kw_offset == objoffset)) {
2649 return (kwq);
2655 kwq = (&hashptr[mutex & pthhash])->lh_first;
2656 if (kwq != 0)
2657 for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
2658 if (kwq->kw_addr == mutex) {
2659 return (kwq);
2670 ksyn_wait_queue_t kwq;
2685 while ((kwq = LIST_FIRST(&hashptr[i])) != NULL) {
2686 if ((kwq->kw_pflags & KSYN_WQ_INHASH) != 0) {
2687 kwq->kw_pflags &= ~KSYN_WQ_INHASH;
2688 LIST_REMOVE(kwq, kw_hash);
2690 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2691 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
2692 LIST_REMOVE(kwq, kw_list);
2698 if (((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) && (kwq->kw_inqueue != 0))
2699 ksyn_freeallkwe(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER]);
2700 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
2701 zfree(kwq_zone, kwq);
2732 ksyn_wait_queue_t kwq;
2760 kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
2762 if (kwq != NULL) {
2763 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2764 LIST_REMOVE(kwq, kw_list);
2765 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
2769 if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
2770 if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
2771 if (kwq->kw_iocount == 0) {
2772 kwq->kw_addr = mutex;
2773 kwq->kw_flags = flags;
2774 kwq->kw_object = object;
2775 kwq->kw_offset = offset;
2776 kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
2777 CLEAR_REINIT_BITS(kwq);
2778 CLEAR_INTR_PREPOST_BITS(kwq);
2779 CLEAR_PREPOST_BITS(kwq);
2780 kwq->kw_lword = mgen;
2781 kwq->kw_uword = ugen;
2782 kwq->kw_sword = rw_wc;
2783 kwq->kw_owner = tid;
2784 } else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
2786 kwq->kw_pflags |= KSYN_WQ_WAITING;
2788 (void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
2802 kwq->kw_iocount++;
2804 kwq->kw_dropcount++;
2806 *kwqp = kwq;
2846 kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
2848 if (kwq != NULL) {
2849 if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
2850 LIST_REMOVE(kwq, kw_list);
2851 kwq->kw_pflags &= ~KSYN_WQ_FLIST;
2855 if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
2856 if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
2857 if (kwq->kw_iocount == 0) {
2858 kwq->kw_addr = mutex;
2859 kwq->kw_flags = flags;
2860 kwq->kw_object = object;
2861 kwq->kw_offset = offset;
2862 kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
2863 CLEAR_REINIT_BITS(kwq);
2864 CLEAR_INTR_PREPOST_BITS(kwq);
2865 CLEAR_PREPOST_BITS(kwq);
2866 kwq->kw_lword = mgen;
2867 kwq->kw_uword = ugen;
2868 kwq->kw_sword = rw_wc;
2869 kwq->kw_owner = tid;
2870 } else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
2871 kwq->kw_pflags |= KSYN_WQ_WAITING;
2873 (void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
2895 kwq->kw_iocount++;
2897 kwq->kw_dropcount++;
2899 *kwqp = kwq;
2905 kwq = nkwq;
2908 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, kwq->kw_lword, kwq->kw_uword, kwq->kw_sword, 0xffff, 0);
2912 kwq->kw_pflags |= KSYN_WQ_SHARED;
2913 LIST_INSERT_HEAD(&hashptr[kwq->kw_object & pthhash], kwq, kw_hash);
2915 LIST_INSERT_HEAD(&hashptr[mutex & pthhash], kwq, kw_hash);
2917 kwq->kw_pflags |= KSYN_WQ_INHASH;
2923 *kwqp = kwq;
2929 ksyn_wqrelease(ksyn_wait_queue_t kwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype)
2939 kwq->kw_iocount--;
2941 kwq->kw_dropcount--;
2943 if (kwq->kw_iocount == 0) {
2944 if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
2946 kwq->kw_pflags &= ~KSYN_WQ_WAITING;
2947 wakeup(&kwq->kw_pflags);
2950 if ((kwq->kw_pre_rwwc == 0) && (kwq->kw_inqueue == 0) && (kwq->kw_pre_intrcount == 0)) {
2952 microuptime(&kwq->kw_ts);
2953 LIST_INSERT_HEAD(&pth_free_list, kwq, kw_list);
2954 kwq->kw_pflags |= KSYN_WQ_FLIST;
2959 kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
2960 LIST_REMOVE(kwq, kw_hash);
2961 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
2964 free_elem = kwq;
2975 kwq->kw_dropcount--;
2978 if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
2980 kwq->kw_pflags &= ~KSYN_WQ_WAITING;
2981 wakeup(&kwq->kw_pflags);
3026 ksyn_wait_queue_t kwq;
3039 LIST_FOREACH(kwq, &pth_free_list, kw_list) {
3040 if ((kwq->kw_iocount != 0) || (kwq->kw_pre_rwwc != 0) || (kwq->kw_inqueue != 0) || (kwq->kw_pre_intrcount != 0)) {
3044 diff = t.tv_sec - kwq->kw_ts.tv_sec;
3049 kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
3052 LIST_REMOVE(kwq, kw_hash);
3053 LIST_REMOVE(kwq, kw_list);
3054 LIST_INSERT_HEAD(&freelist, kwq, kw_list);
3074 while ((kwq = LIST_FIRST(&freelist)) != NULL) {
3075 LIST_REMOVE(kwq, kw_list);
3076 lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
3077 zfree(kwq_zone, kwq);
3084 ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int mylog, thread_continue_t continuation, void * parameter)
3086 ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, __unused int mylog, thread_continue_t continuation, void * parameter)
3095 kwe->kwe_kwqqueue = (void *)kwq;
3097 ksyn_wqunlock(kwq);
3127 ksyn_wakeup_thread(__unused ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe)
3173 kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * typep, uint32_t lowest[])
3182 if ((kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
3185 if (kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) {
3186 kw_fr = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_firstnum;
3199 if ((kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0)) {
3202 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) {
3203 kw_flr = kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum;
3217 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0)) {
3220 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) {
3221 kw_fwr = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
3234 if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0)) {
3237 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) {
3238 kw_fywr = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
3276 ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp)
3288 while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwq)) != NULL) {
3293 kret = ksyn_wakeup_thread(kwq, kwe);
3302 while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwq)) != NULL) {
3306 kret = ksyn_wakeup_thread(kwq, kwe);
3316 kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
3318 kwe = ksyn_queue_removefirst(kq, kwq);
3322 kret = ksyn_wakeup_thread(kwq, kwe);
3341 kwq_handle_unlock(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int * blockp, uint32_t premgen)
3360 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_START, (uint32_t)kwq->kw_addr, mgen, premgen, rw_wc, 0);
3366 kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
3367 kwq->kw_lastseqword = rw_wc;
3368 kwq->kw_lastunlockseq = (rw_wc & PTHRW_COUNT_MASK);
3369 kwq->kw_overlapwatch = 0;
3376 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
3378 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
3389 if (kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE].ksynq_count > 0) {
3390 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwq);
3392 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3395 kret = ksyn_wakeup_thread(kwq, kwe);
3399 kwq->kw_pre_intrcount = 1; /* actually a count */
3400 kwq->kw_pre_intrseq = mgen;
3401 kwq->kw_pre_intrretbits = kwe->kwe_psynchretval;
3402 kwq->kw_pre_intrtype = PTH_RW_TYPE_UPGRADE;
3409 ksyn_wqunlock(kwq);
3413 error = kwq_find_rw_lowest(kwq, flags, premgen, &rwtype, lowest);
3420 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 1, rwtype, 0, 0);
3502 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3503 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count;
3505 kwq->kw_overlapwatch = 1;
3520 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3536 kwq->kw_overlapwatch = 1;
3537 numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3546 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3556 failed = ksyn_wakeupreaders(kwq, limitrdnum, longreadset, allreaders, updatebits, &woken);
3558 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3562 kwq->kw_pre_intrcount = failed; /* actually a count */
3563 kwq->kw_pre_intrseq = limitrdnum;
3564 kwq->kw_pre_intrretbits = updatebits;
3566 kwq->kw_pre_intrtype = PTH_RW_TYPE_LREAD;
3568 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3573 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) && ((updatebits & PTH_RWL_WBIT) == 0))
3586 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
3596 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
3599 if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) )
3606 kret = ksyn_wakeup_thread(kwq, kwe);
3608 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
3615 kwq->kw_pre_intrcount = 1; /* actually a count */
3616 kwq->kw_pre_intrseq = low_writer;
3617 kwq->kw_pre_intrretbits = updatebits;
3618 kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
3622 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3655 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3658 failed = ksyn_wakeupreaders(kwq, low_writer, 0, 0, updatebits, NULL);
3660 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3663 kwq->kw_pre_intrcount = failed; /* actually a count */
3664 kwq->kw_pre_intrseq = low_writer;
3665 kwq->kw_pre_intrretbits = updatebits;
3666 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3671 numneeded = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
3680 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3681 failed = ksyn_wakeupreaders(kwq, low_writer, 0, 1, updatebits, &woken);
3683 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
3686 kwq->kw_pre_intrcount = failed; /* actually a count */
3687 kwq->kw_pre_intrseq = kwq->kw_highseq;
3688 kwq->kw_pre_intrretbits = updatebits;
3689 kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
3706 else if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
3715 kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwq);
3720 else if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) )
3726 kret = ksyn_wakeup_thread(kwq, kwe);
3728 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
3735 kwq->kw_pre_intrcount = 1; /* actually a count */
3736 kwq->kw_pre_intrseq = low_ywriter;
3737 kwq->kw_pre_intrretbits = updatebits;
3738 kwq->kw_pre_intrtype = PTH_RW_TYPE_YWRITE;
3742 kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
3759 __PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0, updatebits, block, 0);
3765 kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, __unused uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, __unused int flags , int * blockp)
3767 uint32_t highword = kwq->kw_nextseqword & PTHRW_COUNT_MASK;
3768 uint32_t lowword = kwq->kw_lastseqword & PTHRW_COUNT_MASK;
3778 if ((kwq->kw_nextseqword & PTH_RWL_LBIT) == 0) {
3794 kwq->kw_nextseqword += PTHRW_INC;
3797 val |= ((kwq->kw_nextseqword & PTHRW_BIT_MASK) | PTH_RWL_MBIT);
3806 kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, __unused int flags, __unused uint32_t premgen, __unused int * blockp)
3816 if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count > 0) {
3817 lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
3818 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
3819 if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
3824 if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count > 0) {
3825 lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
3826 if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
3827 if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
3833 count = ksyn_wakeupreaders(kwq, lowriter, longreadset, allreaders, updatebits, NULL);
3835 kwq->kw_pre_limrd = count;
3836 kwq->kw_pre_limrdseq = lowriter;
3837 kwq->kw_pre_limrdbits = lowriter;
3856 ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, struct uthread * uth, ksyn_waitq_element_t kwe, int fit)
3884 q_kwe = ksyn_queue_find_seq(kwq, kq, lockseq, 0);
3931 kwq->kw_inqueue++;
3932 update_low_high(kwq, lockseq);
3938 ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq)
3949 kwq->kw_inqueue--;
3959 if (kwq->kw_inqueue == 0) {
3960 kwq->kw_lowseq = 0;
3961 kwq->kw_highseq = 0;
3963 if (kwq->kw_lowseq == curseq)
3964 kwq->kw_lowseq = find_nextlowseq(kwq);
3965 if (kwq->kw_highseq == curseq)
3966 kwq->kw_highseq = find_nexthighseq(kwq);
3973 ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe)
3991 kwq->kw_inqueue--;
3993 if (kwq->kw_inqueue == 0) {
3994 kwq->kw_lowseq = 0;
3995 kwq->kw_highseq = 0;
3997 if (kwq->kw_lowseq == curseq)
3998 kwq->kw_lowseq = find_nextlowseq(kwq);
3999 if (kwq->kw_highseq == curseq)
4000 kwq->kw_highseq = find_nexthighseq(kwq);
4007 ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove)
4016 ksyn_queue_removeitem(kwq, kq, q_kwe);
4055 ksyn_queue_find_signalseq(__unused ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t uptoseq, uint32_t signalseq)
4084 if (q_kwe->kwe_kwqqueue == kwq) {
4169 update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq)
4171 if (kwq->kw_inqueue == 1) {
4172 kwq->kw_lowseq = lockseq;
4173 kwq->kw_highseq = lockseq;
4175 if (is_seqlower(lockseq, kwq->kw_lowseq) != 0)
4176 kwq->kw_lowseq = lockseq;
4177 if (is_seqhigher(lockseq, kwq->kw_highseq) != 0)
4178 kwq->kw_highseq = lockseq;
4183 find_nextlowseq(ksyn_wait_queue_t kwq)
4190 if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
4191 numbers[count]= kwq->kw_ksynqueues[i].ksynq_firstnum;
4210 find_nexthighseq(ksyn_wait_queue_t kwq)
4217 if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
4218 numbers[count]= kwq->kw_ksynqueues[i].ksynq_lastnum;
4304 find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp)
4315 count += ksyn_queue_count_tolowest(&kwq->kw_ksynqueues[i], upto);