Lines Matching defs:uq
278 static void umtxq_insert_queue(struct umtx_q *uq, int q);
279 static void umtxq_remove_queue(struct umtx_q *uq, int q);
280 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *);
290 #define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
291 #define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
463 struct umtx_q *uq;
465 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
466 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
468 TAILQ_INIT(&uq->uq_spare_queue->head);
469 TAILQ_INIT(&uq->uq_pi_contested);
470 uq->uq_inherited_pri = PRI_MAX;
471 return (uq);
475 umtxq_free(struct umtx_q *uq)
478 MPASS(uq->uq_spare_queue != NULL);
479 free(uq->uq_spare_queue, M_UMTX);
480 free(uq, M_UMTX);
600 umtxq_insert_queue(struct umtx_q *uq, int q)
605 uc = umtxq_getchain(&uq->uq_key);
607 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
608 uh = umtxq_queue_lookup(&uq->uq_key, q);
610 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
612 uh = uq->uq_spare_queue;
613 uh->key = uq->uq_key;
624 uq->uq_spare_queue = NULL;
626 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
628 uq->uq_flags |= UQF_UMTXQ;
629 uq->uq_cur_queue = uh;
634 umtxq_remove_queue(struct umtx_q *uq, int q)
639 uc = umtxq_getchain(&uq->uq_key);
641 if (uq->uq_flags & UQF_UMTXQ) {
642 uh = uq->uq_cur_queue;
643 TAILQ_REMOVE(&uh->head, uq, uq_link);
645 uq->uq_flags &= ~UQF_UMTXQ;
658 uq->uq_spare_queue = uh;
659 uq->uq_cur_queue = NULL;
705 struct umtx_q *uq;
712 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
713 umtxq_remove_queue(uq, q);
714 wakeup(uq);
726 umtxq_signal_thread(struct umtx_q *uq)
729 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
730 umtxq_remove(uq);
731 wakeup(uq);
812 umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime)
822 uc = umtxq_getchain(&uq->uq_key);
825 if (!(uq->uq_flags & UQF_UMTXQ)) {
837 error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo);
839 umtxq_lock(&uq->uq_key);
848 umtxq_lock(&uq->uq_key);
919 struct umtx_q *uq;
924 uq = td->td_umtxq;
926 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
932 umtxq_lock(&uq->uq_key);
933 umtxq_insert(uq);
934 umtxq_unlock(&uq->uq_key);
946 umtxq_lock(&uq->uq_key);
949 error = umtxq_sleep(uq, "uwait", timeout == NULL ?
951 if ((uq->uq_flags & UQF_UMTXQ) == 0)
954 umtxq_remove(uq);
955 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
956 umtxq_remove(uq);
958 umtxq_unlock(&uq->uq_key);
959 umtx_key_release(&uq->uq_key);
992 struct umtx_q *uq;
997 uq = td->td_umtxq;
1104 GET_SHARE(flags), &uq->uq_key)) != 0)
1107 umtxq_lock(&uq->uq_key);
1108 umtxq_busy(&uq->uq_key);
1109 umtxq_insert(uq);
1110 umtxq_unlock(&uq->uq_key);
1123 umtxq_lock(&uq->uq_key);
1124 umtxq_remove(uq);
1125 umtxq_unbusy(&uq->uq_key);
1126 umtxq_unlock(&uq->uq_key);
1127 umtx_key_release(&uq->uq_key);
1143 umtxq_lock(&uq->uq_key);
1144 umtxq_unbusy(&uq->uq_key);
1146 error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
1148 umtxq_remove(uq);
1149 umtxq_unlock(&uq->uq_key);
1150 umtx_key_release(&uq->uq_key);
1404 struct umtx_q *uq, *uq1, *uq2;
1411 uq = td->td_umtxq;
1418 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1419 uq2 = TAILQ_NEXT(uq, uq_lockq);
1426 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1435 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1437 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1490 struct umtx_q *uq;
1496 uq = td->td_umtxq;
1497 pi = uq->uq_pi_blocked;
1523 uq = td->td_umtxq;
1524 pi = uq->uq_pi_blocked;
1539 struct umtx_q *uq, *uq_owner;
1552 uq = TAILQ_FIRST(&pi2->pi_blocked);
1553 if (uq != NULL) {
1554 if (pri > UPRI(uq->uq_thread))
1555 pri = UPRI(uq->uq_thread);
1602 struct umtx_q *uq;
1619 uq = TAILQ_FIRST(&pi->pi_blocked);
1620 if (uq != NULL) {
1621 pri = UPRI(uq->uq_thread);
1638 struct umtx_q *uq;
1641 uq = td->td_umtxq;
1646 pi = uq->uq_pi_blocked;
1658 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
1670 td = uq->uq_thread;
1672 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
1674 umtxq_insert(uq);
1694 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1696 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1698 uq->uq_pi_blocked = pi;
1704 umtxq_unbusy(&uq->uq_key);
1706 error = umtxq_sleep(uq, wmesg, timo);
1707 umtxq_remove(uq);
1710 uq->uq_pi_blocked = NULL;
1714 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1717 umtxq_unlock(&uq->uq_key);
1798 struct umtx_q *uq;
1804 uq = td->td_umtxq;
1808 &uq->uq_key)) != 0)
1814 umtxq_lock(&uq->uq_key);
1815 pi = umtx_pi_lookup(&uq->uq_key);
1819 umtxq_unlock(&uq->uq_key);
1821 umtxq_lock(&uq->uq_key);
1822 pi = umtx_pi_lookup(&uq->uq_key);
1829 new_pi->pi_key = uq->uq_key;
1835 umtxq_unlock(&uq->uq_key);
1899 umtxq_lock(&uq->uq_key);
1900 umtxq_busy(&uq->uq_key);
1902 umtxq_unbusy(&uq->uq_key);
1903 umtxq_unlock(&uq->uq_key);
1936 umtxq_lock(&uq->uq_key);
1937 umtxq_busy(&uq->uq_key);
1938 umtxq_unlock(&uq->uq_key);
1951 umtxq_unbusy_unlocked(&uq->uq_key);
1956 umtxq_unbusy_unlocked(&uq->uq_key);
1970 umtxq_lock(&uq->uq_key);
1974 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1985 umtxq_lock(&uq->uq_key);
1987 umtxq_unlock(&uq->uq_key);
1989 umtx_key_release(&uq->uq_key);
2134 struct umtx_q *uq, *uq2;
2141 uq = td->td_umtxq;
2144 &uq->uq_key)) != 0)
2152 old_inherited_pri = uq->uq_inherited_pri;
2153 umtxq_lock(&uq->uq_key);
2154 umtxq_busy(&uq->uq_key);
2155 umtxq_unlock(&uq->uq_key);
2174 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
2175 uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
2177 if (uq->uq_inherited_pri < UPRI(td))
2178 sched_lend_user_prio(td, uq->uq_inherited_pri);
2241 umtxq_lock(&uq->uq_key);
2242 umtxq_insert(uq);
2243 umtxq_unbusy(&uq->uq_key);
2244 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
2246 umtxq_remove(uq);
2247 umtxq_unlock(&uq->uq_key);
2250 uq->uq_inherited_pri = old_inherited_pri;
2252 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2259 if (pri > uq->uq_inherited_pri)
2260 pri = uq->uq_inherited_pri;
2269 uq->uq_inherited_pri = old_inherited_pri;
2271 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2278 if (pri > uq->uq_inherited_pri)
2279 pri = uq->uq_inherited_pri;
2287 umtxq_unbusy_unlocked(&uq->uq_key);
2288 umtx_key_release(&uq->uq_key);
2299 struct umtx_q *uq, *uq2;
2305 uq = td->td_umtxq;
2358 uq->uq_inherited_pri = new_inherited_pri;
2360 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2367 if (pri > uq->uq_inherited_pri)
2368 pri = uq->uq_inherited_pri;
2382 struct umtx_q *uq;
2394 uq = td->td_umtxq;
2397 &uq->uq_key)) != 0)
2400 umtxq_lock(&uq->uq_key);
2401 umtxq_busy(&uq->uq_key);
2402 umtxq_unlock(&uq->uq_key);
2451 umtxq_lock(&uq->uq_key);
2452 umtxq_insert(uq);
2453 umtxq_unbusy(&uq->uq_key);
2454 error = umtxq_sleep(uq, "umtxpp", NULL);
2455 umtxq_remove(uq);
2456 umtxq_unlock(&uq->uq_key);
2458 umtxq_lock(&uq->uq_key);
2460 umtxq_signal(&uq->uq_key, INT_MAX);
2461 umtxq_unbusy(&uq->uq_key);
2462 umtxq_unlock(&uq->uq_key);
2463 umtx_key_release(&uq->uq_key);
2539 struct umtx_q *uq;
2543 uq = td->td_umtxq;
2547 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2554 umtx_key_release(&uq->uq_key);
2560 umtx_key_release(&uq->uq_key);
2567 umtxq_lock(&uq->uq_key);
2568 umtxq_busy(&uq->uq_key);
2569 umtxq_insert(uq);
2570 umtxq_unlock(&uq->uq_key);
2580 umtxq_unbusy_unlocked(&uq->uq_key);
2588 umtxq_lock(&uq->uq_key);
2590 error = umtxq_sleep(uq, "ucond", timeout == NULL ?
2594 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2602 umtxq_busy(&uq->uq_key);
2603 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
2604 int oldlen = uq->uq_cur_queue->length;
2605 umtxq_remove(uq);
2607 umtxq_unlock(&uq->uq_key);
2609 umtxq_lock(&uq->uq_key);
2612 umtxq_unbusy(&uq->uq_key);
2617 umtxq_unlock(&uq->uq_key);
2618 umtx_key_release(&uq->uq_key);
2687 struct umtx_q *uq;
2693 uq = td->td_umtxq;
2697 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2711 umtx_key_release(&uq->uq_key);
2719 umtx_key_release(&uq->uq_key);
2725 umtx_key_release(&uq->uq_key);
2730 umtx_key_release(&uq->uq_key);
2743 umtxq_lock(&uq->uq_key);
2744 umtxq_busy(&uq->uq_key);
2745 umtxq_unlock(&uq->uq_key);
2774 umtxq_unbusy_unlocked(&uq->uq_key);
2780 umtxq_unbusy_unlocked(&uq->uq_key);
2795 umtxq_unbusy_unlocked(&uq->uq_key);
2802 umtxq_lock(&uq->uq_key);
2803 umtxq_insert(uq);
2804 umtxq_unbusy(&uq->uq_key);
2806 error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
2809 umtxq_busy(&uq->uq_key);
2810 umtxq_remove(uq);
2811 umtxq_unlock(&uq->uq_key);
2825 umtxq_unbusy_unlocked(&uq->uq_key);
2833 umtxq_unbusy_unlocked(&uq->uq_key);
2858 umtxq_unbusy_unlocked(&uq->uq_key);
2862 umtx_key_release(&uq->uq_key);
2872 struct umtx_q *uq;
2879 uq = td->td_umtxq;
2883 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2894 umtx_key_release(&uq->uq_key);
2902 umtx_key_release(&uq->uq_key);
2907 umtx_key_release(&uq->uq_key);
2920 umtxq_lock(&uq->uq_key);
2921 umtxq_busy(&uq->uq_key);
2922 umtxq_signal_queue(&uq->uq_key, INT_MAX,
2924 umtxq_unbusy(&uq->uq_key);
2925 umtxq_unlock(&uq->uq_key);
2932 umtxq_lock(&uq->uq_key);
2933 umtxq_busy(&uq->uq_key);
2934 umtxq_unlock(&uq->uq_key);
2963 umtxq_unbusy_unlocked(&uq->uq_key);
2969 umtxq_unbusy_unlocked(&uq->uq_key);
2979 umtxq_unbusy_unlocked(&uq->uq_key);
2987 umtxq_lock(&uq->uq_key);
2988 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2989 umtxq_unbusy(&uq->uq_key);
2991 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
2994 umtxq_busy(&uq->uq_key);
2995 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2996 umtxq_unlock(&uq->uq_key);
3009 umtxq_unbusy_unlocked(&uq->uq_key);
3017 umtxq_unbusy_unlocked(&uq->uq_key);
3048 umtxq_unbusy_unlocked(&uq->uq_key);
3055 umtxq_unbusy_unlocked(&uq->uq_key);
3058 umtx_key_release(&uq->uq_key);
3067 struct umtx_q *uq;
3072 uq = td->td_umtxq;
3076 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3151 umtxq_lock(&uq->uq_key);
3152 umtxq_busy(&uq->uq_key);
3153 umtxq_signal_queue(&uq->uq_key, count, q);
3154 umtxq_unbusy(&uq->uq_key);
3155 umtxq_unlock(&uq->uq_key);
3158 umtx_key_release(&uq->uq_key);
3167 struct umtx_q *uq;
3171 uq = td->td_umtxq;
3175 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3183 umtxq_lock(&uq->uq_key);
3184 umtxq_busy(&uq->uq_key);
3185 umtxq_insert(uq);
3186 umtxq_unlock(&uq->uq_key);
3192 umtxq_lock(&uq->uq_key);
3193 umtxq_unbusy(&uq->uq_key);
3194 umtxq_remove(uq);
3195 umtxq_unlock(&uq->uq_key);
3208 umtxq_lock(&uq->uq_key);
3209 umtxq_unbusy(&uq->uq_key);
3211 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3213 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3216 umtxq_remove(uq);
3222 umtxq_unlock(&uq->uq_key);
3224 umtx_key_release(&uq->uq_key);
3272 struct umtx_q *uq;
3276 uq = td->td_umtxq;
3282 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3285 umtxq_lock(&uq->uq_key);
3286 umtxq_busy(&uq->uq_key);
3287 umtxq_insert(uq);
3288 umtxq_unlock(&uq->uq_key);
3291 umtxq_lock(&uq->uq_key);
3292 umtxq_unbusy(&uq->uq_key);
3293 umtxq_remove(uq);
3294 umtxq_unlock(&uq->uq_key);
3295 umtx_key_release(&uq->uq_key);
3300 umtxq_lock(&uq->uq_key);
3301 umtxq_unbusy(&uq->uq_key);
3302 umtxq_remove(uq);
3303 umtxq_unlock(&uq->uq_key);
3304 umtx_key_release(&uq->uq_key);
3312 umtxq_lock(&uq->uq_key);
3313 umtxq_unbusy(&uq->uq_key);
3314 umtxq_remove(uq);
3315 umtxq_unlock(&uq->uq_key);
3316 umtx_key_release(&uq->uq_key);
3324 umtxq_lock(&uq->uq_key);
3325 umtxq_unbusy(&uq->uq_key);
3327 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3329 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3332 umtxq_remove(uq);
3344 umtxq_unlock(&uq->uq_key);
3345 umtx_key_release(&uq->uq_key);
4513 struct umtx_q *uq;
4515 uq = td->td_umtxq;
4516 uq->uq_inherited_pri = PRI_MAX;
4518 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
4519 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
4520 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
4521 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
4654 struct umtx_q *uq;
4662 uq = td->td_umtxq;
4663 if (uq != NULL) {
4664 if (uq->uq_inherited_pri != PRI_MAX ||
4665 !TAILQ_EMPTY(&uq->uq_pi_contested)) {
4667 uq->uq_inherited_pri = PRI_MAX;
4668 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
4670 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);