1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 43 unchanged lines hidden (view full) --- 52 * already has a turnstile, then it gives its turnstile to the lock's 53 * turnstile's free list. When a thread is woken up, it takes a turnstile from 54 * the free list if there are any other waiters. If it is the only thread 55 * blocked on the lock, then it reclaims the turnstile associated with the lock 56 * and removes it from the hash table. 57 */ 58 59#include <sys/cdefs.h> |
60__FBSDID("$FreeBSD: head/sys/kern/subr_turnstile.c 170295 2007-06-04 23:51:44Z jeff $"); |
61 62#include "opt_ddb.h" 63#include "opt_turnstile_profiling.h" 64 65#include <sys/param.h> 66#include <sys/systm.h> 67#include <sys/kernel.h> 68#include <sys/ktr.h> --- 42 unchanged lines hidden (view full) --- 111 * turnstile_signal() or turnstile_wait() that are waiting to be put on 112 * the run queue. 113 * 114 * Locking key: 115 * c - turnstile chain lock 116 * q - td_contested lock 117 */ 118struct turnstile { |
119 struct mtx ts_lock; /* Spin lock for self. */ |
120 struct threadqueue ts_blocked[2]; /* (c + q) Blocked threads. */ 121 struct threadqueue ts_pending; /* (c) Pending threads. */ 122 LIST_ENTRY(turnstile) ts_hash; /* (c) Chain and free list. */ 123 LIST_ENTRY(turnstile) ts_link; /* (q) Contested locks. */ 124 LIST_HEAD(, turnstile) ts_free; /* (c) Free turnstiles. */ 125 struct lock_object *ts_lockobj; /* (c) Lock we reference. */ 126 struct thread *ts_owner; /* (c + q) Who owns the lock. */ 127}; --- 30 unchanged lines hidden (view full) --- 158static int turnstile_adjust_thread(struct turnstile *ts, 159 struct thread *td); 160static struct thread *turnstile_first_waiter(struct turnstile *ts); 161static void turnstile_setowner(struct turnstile *ts, struct thread *owner); 162#ifdef INVARIANTS 163static void turnstile_dtor(void *mem, int size, void *arg); 164#endif 165static int turnstile_init(void *mem, int size, int flags); |
166static void turnstile_fini(void *mem, int size); |
167 168/* 169 * Walks the chain of turnstiles and their owners to propagate the priority 170 * of the thread being blocked to all the threads holding locks that have to 171 * release their locks before this thread can run again. 172 */ 173static void 174propagate_priority(struct thread *td) 175{ |
176 struct turnstile *ts; 177 int pri; 178 |
179 THREAD_LOCK_ASSERT(td, MA_OWNED); |
180 pri = td->td_priority; 181 ts = td->td_blocked; |
182 MPASS(td->td_lock == &ts->ts_lock); 183 /* 184 * Grab a recursive lock on this turnstile chain so it stays locked 185 * for the whole operation. The caller expects us to return with 186 * the original lock held. We only ever lock down the chain so 187 * the lock order is constant. 188 */ 189 mtx_lock_spin(&ts->ts_lock); |
190 for (;;) { 191 td = ts->ts_owner; 192 193 if (td == NULL) { 194 /* 195 * This might be a read lock with no owner. There's 196 * not much we can do, so just bail. 197 */ |
198 mtx_unlock_spin(&ts->ts_lock); |
199 return; 200 } 201 |
202 thread_lock_flags(td, MTX_DUPOK); 203 mtx_unlock_spin(&ts->ts_lock); |
204 MPASS(td->td_proc != NULL); 205 MPASS(td->td_proc->p_magic == P_MAGIC); 206 207 /* 208 * If the thread is asleep, then we are probably about 209 * to deadlock. To make debugging this easier, just 210 * panic and tell the user which thread misbehaved so 211 * they can hopefully get a stack trace from the truly --- 8 unchanged lines hidden (view full) --- 220#endif 221 panic("sleeping thread"); 222 } 223 224 /* 225 * If this thread already has higher priority than the 226 * thread that is being blocked, we are finished. 227 */ |
228 if (td->td_priority <= pri) { 229 thread_unlock(td); |
230 return; |
231 } |
232 233 /* 234 * Bump this thread's priority. 235 */ 236 sched_lend_prio(td, pri); 237 238 /* 239 * If lock holder is actually running or on the run queue 240 * then we are done. 241 */ 242 if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) { 243 MPASS(td->td_blocked == NULL); |
244 thread_unlock(td); |
245 return; 246 } 247 248#ifndef SMP 249 /* 250 * For UP, we check to see if td is curthread (this shouldn't 251 * ever happen however as it would mean we are in a deadlock.) 252 */ --- 8 unchanged lines hidden (view full) --- 261 td->td_tid, td->td_proc->p_comm, td->td_state, 262 ts->ts_lockobj->lo_name)); 263 264 /* 265 * Pick up the lock that td is blocked on. 266 */ 267 ts = td->td_blocked; 268 MPASS(ts != NULL); |
269 MPASS(td->td_lock == &ts->ts_lock); |
270 /* Resort td on the list if needed. */ 271 if (!turnstile_adjust_thread(ts, td)) { |
272 mtx_unlock_spin(&ts->ts_lock); |
273 return; 274 } |
275 /* The thread lock is released as ts lock above. */ |
276 } 277} 278 279/* 280 * Adjust the thread's position on a turnstile after its priority has been 281 * changed. 282 */ 283static int 284turnstile_adjust_thread(struct turnstile *ts, struct thread *td) 285{ |
286 struct thread *td1, *td2; 287 int queue; 288 |
289 THREAD_LOCK_ASSERT(td, MA_OWNED); |
290 MPASS(TD_ON_LOCK(td)); 291 292 /* 293 * This thread may not be blocked on this turnstile anymore 294 * but instead might already be woken up on another CPU |
295 * that is waiting on the thread lock in turnstile_unpend() to |
296 * finish waking this thread up. We can detect this case 297 * by checking to see if this thread has been given a 298 * turnstile by either turnstile_signal() or 299 * turnstile_broadcast(). In this case, treat the thread as 300 * if it was already running. 301 */ 302 if (td->td_turnstile != NULL) 303 return (0); 304 305 /* 306 * Check if the thread needs to be moved on the blocked chain. 307 * It needs to be moved if either its priority is lower than 308 * the previous thread or higher than the next thread. 309 */ |
310 MPASS(td->td_lock == &ts->ts_lock); |
311 td1 = TAILQ_PREV(td, threadqueue, td_lockq); 312 td2 = TAILQ_NEXT(td, td_lockq); 313 if ((td1 != NULL && td->td_priority < td1->td_priority) || 314 (td2 != NULL && td->td_priority > td2->td_priority)) { 315 316 /* 317 * Remove thread from blocked chain and determine where 318 * it should be moved to. --- 72 unchanged lines hidden (view full) --- 391#endif 392 393static void 394init_turnstile0(void *dummy) 395{ 396 397 turnstile_zone = uma_zcreate("TURNSTILE", sizeof(struct turnstile), 398#ifdef INVARIANTS |
399 NULL, turnstile_dtor, turnstile_init, turnstile_fini, 400 UMA_ALIGN_CACHE, 0); |
401#else |
402 NULL, NULL, turnstile_init, turnstile_fini, UMA_ALIGN_CACHE, 0); |
403#endif 404 thread0.td_turnstile = turnstile_alloc(); 405} 406SYSINIT(turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL); 407 408/* 409 * Update a thread on the turnstile list after it's priority has been changed. 410 * The old priority is passed in as an argument. 411 */ 412void 413turnstile_adjust(struct thread *td, u_char oldpri) 414{ |
415 struct turnstile *ts; 416 |
417 MPASS(TD_ON_LOCK(td)); 418 419 /* 420 * Pick up the lock that td is blocked on. 421 */ 422 ts = td->td_blocked; 423 MPASS(ts != NULL); |
424 MPASS(td->td_lock == &ts->ts_lock); 425 mtx_assert(&ts->ts_lock, MA_OWNED); |
426 427 /* Resort the turnstile on the list. */ |
428 if (!turnstile_adjust_thread(ts, td)) |
429 return; |
430 /* 431 * If our priority was lowered and we are at the head of the 432 * turnstile, then propagate our new priority up the chain. 433 * Note that we currently don't try to revoke lent priorities 434 * when our priority goes up. 435 */ 436 MPASS(td->td_tsqueue == TS_EXCLUSIVE_QUEUE || 437 td->td_tsqueue == TS_SHARED_QUEUE); 438 if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) && 439 td->td_priority < oldpri) { |
440 propagate_priority(td); |
441 } |
442} 443 444/* 445 * Set the owner of the lock this turnstile is attached to. 446 */ 447static void 448turnstile_setowner(struct turnstile *ts, struct thread *owner) 449{ --- 35 unchanged lines hidden (view full) --- 485 struct turnstile *ts; 486 487 bzero(mem, size); 488 ts = mem; 489 TAILQ_INIT(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]); 490 TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]); 491 TAILQ_INIT(&ts->ts_pending); 492 LIST_INIT(&ts->ts_free); |
493 mtx_init(&ts->ts_lock, "turnstile lock", NULL, MTX_SPIN | MTX_RECURSE); |
494 return (0); 495} 496 |
497static void 498turnstile_fini(void *mem, int size) 499{ 500 struct turnstile *ts; 501 502 ts = mem; 503 mtx_destroy(&ts->ts_lock); 504} 505 |
506/* 507 * Get a turnstile for a new thread. 508 */ 509struct turnstile * 510turnstile_alloc(void) 511{ 512 513 return (uma_zalloc(turnstile_zone, M_WAITOK)); --- 8 unchanged lines hidden (view full) --- 522 523 uma_zfree(turnstile_zone, ts); 524} 525 526/* 527 * Lock the turnstile chain associated with the specified lock. 528 */ 529void |
530turnstile_chain_lock(struct lock_object *lock) |
531{ 532 struct turnstile_chain *tc; 533 534 tc = TC_LOOKUP(lock); 535 mtx_lock_spin(&tc->tc_lock); 536} 537 |
538struct turnstile * 539turnstile_trywait(struct lock_object *lock) 540{ 541 struct turnstile_chain *tc; 542 struct turnstile *ts; 543 544 tc = TC_LOOKUP(lock); 545 mtx_lock_spin(&tc->tc_lock); 546 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash) 547 if (ts->ts_lockobj == lock) { 548 mtx_lock_spin(&ts->ts_lock); 549 return (ts); 550 } 551 552 ts = curthread->td_turnstile; 553 MPASS(ts != NULL); 554 mtx_lock_spin(&ts->ts_lock); 555 KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer")); 556 ts->ts_lockobj = lock; 557 558 return (ts); 559} 560 561void 562turnstile_cancel(struct turnstile *ts) 563{ 564 struct turnstile_chain *tc; 565 struct lock_object *lock; 566 567 mtx_assert(&ts->ts_lock, MA_OWNED); 568 569 mtx_unlock_spin(&ts->ts_lock); 570 lock = ts->ts_lockobj; 571 if (ts == curthread->td_turnstile) 572 ts->ts_lockobj = NULL; 573 tc = TC_LOOKUP(lock); 574 mtx_unlock_spin(&tc->tc_lock); 575} 576 |
577/* 578 * Look up the turnstile for a lock in the hash table locking the associated 579 * turnstile chain along the way. If no turnstile is found in the hash 580 * table, NULL is returned. 581 */ 582struct turnstile * 583turnstile_lookup(struct lock_object *lock) 584{ 585 struct turnstile_chain *tc; 586 struct turnstile *ts; 587 588 tc = TC_LOOKUP(lock); 589 mtx_assert(&tc->tc_lock, MA_OWNED); 590 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash) |
591 if (ts->ts_lockobj == lock) { 592 mtx_lock_spin(&ts->ts_lock); |
593 return (ts); |
594 } |
595 return (NULL); 596} 597 598/* 599 * Unlock the turnstile chain associated with a given lock. 600 */ 601void |
602turnstile_chain_unlock(struct lock_object *lock) |
603{ 604 struct turnstile_chain *tc; 605 606 tc = TC_LOOKUP(lock); 607 mtx_unlock_spin(&tc->tc_lock); 608} 609 610/* --- 12 unchanged lines hidden (view full) --- 623 return (xtd); 624} 625 626/* 627 * Take ownership of a turnstile and adjust the priority of the new 628 * owner appropriately. 629 */ 630void |
631turnstile_claim(struct turnstile *ts) |
632{ |
633 struct thread *td, *owner; |
634 struct turnstile_chain *tc; |
635 |
636 mtx_assert(&ts->ts_lock, MA_OWNED); 637 MPASS(ts != curthread->td_turnstile); |
638 639 owner = curthread; 640 mtx_lock_spin(&td_contested_lock); 641 turnstile_setowner(ts, owner); 642 mtx_unlock_spin(&td_contested_lock); 643 644 td = turnstile_first_waiter(ts); 645 MPASS(td != NULL); 646 MPASS(td->td_proc->p_magic == P_MAGIC); |
647 MPASS(td->td_lock == &ts->ts_lock); |
648 649 /* 650 * Update the priority of the new owner if needed. 651 */ |
652 thread_lock(owner); |
653 if (td->td_priority < owner->td_priority) 654 sched_lend_prio(owner, td->td_priority); |
655 thread_unlock(owner); 656 tc = TC_LOOKUP(ts->ts_lockobj); 657 mtx_unlock_spin(&ts->ts_lock); 658 mtx_unlock_spin(&tc->tc_lock); |
659} 660 661/* 662 * Block the current thread on the turnstile assicated with 'lock'. This 663 * function will context switch and not return until this thread has been 664 * woken back up. This function must be called with the appropriate 665 * turnstile chain locked and will return with it unlocked. 666 */ 667void |
668turnstile_wait(struct turnstile *ts, struct thread *owner, int queue) |
669{ 670 struct turnstile_chain *tc; |
671 struct thread *td, *td1; |
672 struct lock_object *lock; |
673 674 td = curthread; |
675 mtx_assert(&ts->ts_lock, MA_OWNED); |
676 if (queue == TS_SHARED_QUEUE) 677 MPASS(owner != NULL); 678 if (owner) 679 MPASS(owner->td_proc->p_magic == P_MAGIC); 680 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 681 |
682 /* 683 * If the lock does not already have a turnstile, use this thread's 684 * turnstile. Otherwise insert the current thread into the 685 * turnstile already in use by this lock. 686 */ |
687 tc = TC_LOOKUP(ts->ts_lockobj); 688 if (ts == td->td_turnstile) { 689 mtx_assert(&tc->tc_lock, MA_OWNED); |
690#ifdef TURNSTILE_PROFILING 691 tc->tc_depth++; 692 if (tc->tc_depth > tc->tc_max_depth) { 693 tc->tc_max_depth = tc->tc_depth; 694 if (tc->tc_max_depth > turnstile_max_depth) 695 turnstile_max_depth = tc->tc_max_depth; 696 } 697#endif |
698 tc = TC_LOOKUP(ts->ts_lockobj); |
699 LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash); 700 KASSERT(TAILQ_EMPTY(&ts->ts_pending), 701 ("thread's turnstile has pending threads")); 702 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]), 703 ("thread's turnstile has exclusive waiters")); 704 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]), 705 ("thread's turnstile has shared waiters")); 706 KASSERT(LIST_EMPTY(&ts->ts_free), 707 ("thread's turnstile has a non-empty free list")); |
708 MPASS(ts->ts_lockobj != NULL); |
709 mtx_lock_spin(&td_contested_lock); 710 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq); 711 turnstile_setowner(ts, owner); 712 mtx_unlock_spin(&td_contested_lock); 713 } else { 714 TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq) 715 if (td1->td_priority > td->td_priority) 716 break; 717 mtx_lock_spin(&td_contested_lock); 718 if (td1 != NULL) 719 TAILQ_INSERT_BEFORE(td1, td, td_lockq); 720 else 721 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq); 722 MPASS(owner == ts->ts_owner); 723 mtx_unlock_spin(&td_contested_lock); 724 MPASS(td->td_turnstile != NULL); 725 LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash); 726 } |
727 thread_lock(td); 728 thread_lock_set(td, &ts->ts_lock); |
729 td->td_turnstile = NULL; |
730 |
731 /* Save who we are blocked on and switch. */ |
732 lock = ts->ts_lockobj; |
733 td->td_tsqueue = queue; 734 td->td_blocked = ts; 735 td->td_lockname = lock->lo_name; 736 TD_SET_LOCK(td); |
737 mtx_unlock_spin(&tc->tc_lock); |
738 propagate_priority(td); |
739 740 if (LOCK_LOG_TEST(lock, 0)) 741 CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__, 742 td->td_tid, lock, lock->lo_name); 743 |
744 MPASS(td->td_lock == &ts->ts_lock); 745 SCHED_STAT_INC(switch_turnstile); |
746 mi_switch(SW_VOL, NULL); 747 748 if (LOCK_LOG_TEST(lock, 0)) 749 CTR4(KTR_LOCK, "%s: td %d free from blocked on [%p] %s", 750 __func__, td->td_tid, lock, lock->lo_name); |
751 thread_unlock(td); |
752} 753 754/* 755 * Pick the highest priority thread on this turnstile and put it on the 756 * pending list. This must be called with the turnstile chain locked. 757 */ 758int 759turnstile_signal(struct turnstile *ts, int queue) 760{ 761 struct turnstile_chain *tc; 762 struct thread *td; 763 int empty; 764 765 MPASS(ts != NULL); |
766 mtx_assert(&ts->ts_lock, MA_OWNED); |
767 MPASS(curthread->td_proc->p_magic == P_MAGIC); 768 MPASS(ts->ts_owner == curthread || 769 (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL)); |
770 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 771 772 /* 773 * Pick the highest priority thread blocked on this lock and 774 * move it to the pending list. 775 */ 776 td = TAILQ_FIRST(&ts->ts_blocked[queue]); 777 MPASS(td->td_proc->p_magic == P_MAGIC); --- 5 unchanged lines hidden (view full) --- 783 /* 784 * If the turnstile is now empty, remove it from its chain and 785 * give it to the about-to-be-woken thread. Otherwise take a 786 * turnstile from the free list and give it to the thread. 787 */ 788 empty = TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && 789 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]); 790 if (empty) { |
791 tc = TC_LOOKUP(ts->ts_lockobj); 792 mtx_assert(&tc->tc_lock, MA_OWNED); |
793 MPASS(LIST_EMPTY(&ts->ts_free)); 794#ifdef TURNSTILE_PROFILING 795 tc->tc_depth--; 796#endif 797 } else 798 ts = LIST_FIRST(&ts->ts_free); 799 MPASS(ts != NULL); 800 LIST_REMOVE(ts, ts_hash); --- 9 unchanged lines hidden (view full) --- 810void 811turnstile_broadcast(struct turnstile *ts, int queue) 812{ 813 struct turnstile_chain *tc; 814 struct turnstile *ts1; 815 struct thread *td; 816 817 MPASS(ts != NULL); |
818 mtx_assert(&ts->ts_lock, MA_OWNED); |
819 MPASS(curthread->td_proc->p_magic == P_MAGIC); 820 MPASS(ts->ts_owner == curthread || 821 (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL)); |
822 /* 823 * We must have the chain locked so that we can remove the empty 824 * turnstile from the hash queue. 825 */ |
826 tc = TC_LOOKUP(ts->ts_lockobj); 827 mtx_assert(&tc->tc_lock, MA_OWNED); 828 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 829 830 /* 831 * Transfer the blocked list to the pending list. 832 */ 833 mtx_lock_spin(&td_contested_lock); --- 23 unchanged lines hidden (view full) --- 857 * Wakeup all threads on the pending list and adjust the priority of the 858 * current thread appropriately. This must be called with the turnstile 859 * chain locked. 860 */ 861void 862turnstile_unpend(struct turnstile *ts, int owner_type) 863{ 864 TAILQ_HEAD( ,thread) pending_threads; |
865 struct turnstile *nts; |
866 struct thread *td; 867 u_char cp, pri; 868 869 MPASS(ts != NULL); |
870 mtx_assert(&ts->ts_lock, MA_OWNED); |
871 MPASS(ts->ts_owner == curthread || 872 (owner_type == TS_SHARED_LOCK && ts->ts_owner == NULL)); |
873 MPASS(!TAILQ_EMPTY(&ts->ts_pending)); 874 875 /* 876 * Move the list of pending threads out of the turnstile and 877 * into a local variable. 878 */ 879 TAILQ_INIT(&pending_threads); 880 TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq); 881#ifdef INVARIANTS 882 if (TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && 883 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])) 884 ts->ts_lockobj = NULL; 885#endif |
886 /* |
887 * Adjust the priority of curthread based on other contested 888 * locks it owns. Don't lower the priority below the base 889 * priority however. 890 */ 891 td = curthread; 892 pri = PRI_MAX; 893 thread_lock(td); 894 mtx_lock_spin(&td_contested_lock); 895 /* |
896 * Remove the turnstile from this thread's list of contested locks 897 * since this thread doesn't own it anymore. New threads will 898 * not be blocking on the turnstile until it is claimed by a new 899 * owner. There might not be a current owner if this is a shared 900 * lock. 901 */ 902 if (ts->ts_owner != NULL) { |
903 ts->ts_owner = NULL; 904 LIST_REMOVE(ts, ts_link); |
905 } |
906 LIST_FOREACH(nts, &td->td_contested, ts_link) { 907 cp = turnstile_first_waiter(nts)->td_priority; |
908 if (cp < pri) 909 pri = cp; 910 } 911 mtx_unlock_spin(&td_contested_lock); 912 sched_unlend_prio(td, pri); |
913 thread_unlock(td); |
914 /* 915 * Wake up all the pending threads. If a thread is not blocked 916 * on a lock, then it is currently executing on another CPU in 917 * turnstile_wait() or sitting on a run queue waiting to resume 918 * in turnstile_wait(). Set a flag to force it to try to acquire 919 * the lock again instead of blocking. 920 */ 921 while (!TAILQ_EMPTY(&pending_threads)) { 922 td = TAILQ_FIRST(&pending_threads); 923 TAILQ_REMOVE(&pending_threads, td, td_lockq); |
924 thread_lock(td); 925 MPASS(td->td_lock == &ts->ts_lock); |
926 MPASS(td->td_proc->p_magic == P_MAGIC); |
927 MPASS(TD_ON_LOCK(td)); 928 TD_CLR_LOCK(td); 929 MPASS(TD_CAN_RUN(td)); 930 td->td_blocked = NULL; 931 td->td_lockname = NULL; |
932#ifdef INVARIANTS |
933 td->td_tsqueue = 0xff; |
934#endif |
935 sched_add(td, SRQ_BORING); 936 thread_unlock(td); |
937 } |
938 mtx_unlock_spin(&ts->ts_lock); |
939} 940 941/* 942 * Give up ownership of a turnstile. This must be called with the 943 * turnstile chain locked. 944 */ 945void 946turnstile_disown(struct turnstile *ts) 947{ |
948 struct thread *td; 949 u_char cp, pri; 950 951 MPASS(ts != NULL); |
952 mtx_assert(&ts->ts_lock, MA_OWNED); |
953 MPASS(ts->ts_owner == curthread); |
954 MPASS(TAILQ_EMPTY(&ts->ts_pending)); 955 MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) || 956 !TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])); 957 958 /* 959 * Remove the turnstile from this thread's list of contested locks 960 * since this thread doesn't own it anymore. New threads will 961 * not be blocking on the turnstile until it is claimed by a new 962 * owner. 963 */ 964 mtx_lock_spin(&td_contested_lock); 965 ts->ts_owner = NULL; 966 LIST_REMOVE(ts, ts_link); 967 mtx_unlock_spin(&td_contested_lock); |
968 969 /* 970 * Adjust the priority of curthread based on other contested 971 * locks it owns. Don't lower the priority below the base 972 * priority however. 973 */ 974 td = curthread; 975 pri = PRI_MAX; |
976 thread_lock(td); 977 mtx_unlock_spin(&ts->ts_lock); |
978 mtx_lock_spin(&td_contested_lock); 979 LIST_FOREACH(ts, &td->td_contested, ts_link) { 980 cp = turnstile_first_waiter(ts)->td_priority; 981 if (cp < pri) 982 pri = cp; 983 } 984 mtx_unlock_spin(&td_contested_lock); 985 sched_unlend_prio(td, pri); |
986 thread_unlock(td); |
987} 988 989/* 990 * Return the first thread in a turnstile. 991 */ 992struct thread * 993turnstile_head(struct turnstile *ts, int queue) 994{ 995#ifdef INVARIANTS |
996 997 MPASS(ts != NULL); 998 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); |
999 mtx_assert(&ts->ts_lock, MA_OWNED); |
1000#endif 1001 return (TAILQ_FIRST(&ts->ts_blocked[queue])); 1002} 1003 1004/* 1005 * Returns true if a sub-queue of a turnstile is empty. 1006 */ 1007int 1008turnstile_empty(struct turnstile *ts, int queue) 1009{ 1010#ifdef INVARIANTS |
1011 1012 MPASS(ts != NULL); 1013 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); |
1014 mtx_assert(&ts->ts_lock, MA_OWNED); |
1015#endif 1016 return (TAILQ_EMPTY(&ts->ts_blocked[queue])); 1017} 1018 1019#ifdef DDB 1020static void 1021print_thread(struct thread *td, const char *prefix) 1022{ --- 280 unchanged lines hidden --- |