subr_turnstile.c (169666) | subr_turnstile.c (170295) |
---|---|
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 43 unchanged lines hidden (view full) --- 52 * already has a turnstile, then it gives its turnstile to the lock's 53 * turnstile's free list. When a thread is woken up, it takes a turnstile from 54 * the free list if there are any other waiters. If it is the only thread 55 * blocked on the lock, then it reclaims the turnstile associated with the lock 56 * and removes it from the hash table. 57 */ 58 59#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 43 unchanged lines hidden (view full) --- 52 * already has a turnstile, then it gives its turnstile to the lock's 53 * turnstile's free list. When a thread is woken up, it takes a turnstile from 54 * the free list if there are any other waiters. If it is the only thread 55 * blocked on the lock, then it reclaims the turnstile associated with the lock 56 * and removes it from the hash table. 57 */ 58 59#include <sys/cdefs.h> |
60__FBSDID("$FreeBSD: head/sys/kern/subr_turnstile.c 169666 2007-05-18 06:32:24Z jeff $"); | 60__FBSDID("$FreeBSD: head/sys/kern/subr_turnstile.c 170295 2007-06-04 23:51:44Z jeff $"); |
61 62#include "opt_ddb.h" 63#include "opt_turnstile_profiling.h" 64 65#include <sys/param.h> 66#include <sys/systm.h> 67#include <sys/kernel.h> 68#include <sys/ktr.h> --- 42 unchanged lines hidden (view full) --- 111 * turnstile_signal() or turnstile_wait() that are waiting to be put on 112 * the run queue. 113 * 114 * Locking key: 115 * c - turnstile chain lock 116 * q - td_contested lock 117 */ 118struct turnstile { | 61 62#include "opt_ddb.h" 63#include "opt_turnstile_profiling.h" 64 65#include <sys/param.h> 66#include <sys/systm.h> 67#include <sys/kernel.h> 68#include <sys/ktr.h> --- 42 unchanged lines hidden (view full) --- 111 * turnstile_signal() or turnstile_wait() that are waiting to be put on 112 * the run queue. 113 * 114 * Locking key: 115 * c - turnstile chain lock 116 * q - td_contested lock 117 */ 118struct turnstile { |
119 struct mtx ts_lock; /* Spin lock for self. */ |
|
119 struct threadqueue ts_blocked[2]; /* (c + q) Blocked threads. */ 120 struct threadqueue ts_pending; /* (c) Pending threads. */ 121 LIST_ENTRY(turnstile) ts_hash; /* (c) Chain and free list. */ 122 LIST_ENTRY(turnstile) ts_link; /* (q) Contested locks. */ 123 LIST_HEAD(, turnstile) ts_free; /* (c) Free turnstiles. */ 124 struct lock_object *ts_lockobj; /* (c) Lock we reference. */ 125 struct thread *ts_owner; /* (c + q) Who owns the lock. */ 126}; --- 30 unchanged lines hidden (view full) --- 157static int turnstile_adjust_thread(struct turnstile *ts, 158 struct thread *td); 159static struct thread *turnstile_first_waiter(struct turnstile *ts); 160static void turnstile_setowner(struct turnstile *ts, struct thread *owner); 161#ifdef INVARIANTS 162static void turnstile_dtor(void *mem, int size, void *arg); 163#endif 164static int turnstile_init(void *mem, int size, int flags); | 120 struct threadqueue ts_blocked[2]; /* (c + q) Blocked threads. */ 121 struct threadqueue ts_pending; /* (c) Pending threads. */ 122 LIST_ENTRY(turnstile) ts_hash; /* (c) Chain and free list. */ 123 LIST_ENTRY(turnstile) ts_link; /* (q) Contested locks. */ 124 LIST_HEAD(, turnstile) ts_free; /* (c) Free turnstiles. */ 125 struct lock_object *ts_lockobj; /* (c) Lock we reference. */ 126 struct thread *ts_owner; /* (c + q) Who owns the lock. */ 127}; --- 30 unchanged lines hidden (view full) --- 158static int turnstile_adjust_thread(struct turnstile *ts, 159 struct thread *td); 160static struct thread *turnstile_first_waiter(struct turnstile *ts); 161static void turnstile_setowner(struct turnstile *ts, struct thread *owner); 162#ifdef INVARIANTS 163static void turnstile_dtor(void *mem, int size, void *arg); 164#endif 165static int turnstile_init(void *mem, int size, int flags); |
166static void turnstile_fini(void *mem, int size); |
|
165 166/* 167 * Walks the chain of turnstiles and their owners to propagate the priority 168 * of the thread being blocked to all the threads holding locks that have to 169 * release their locks before this thread can run again. 170 */ 171static void 172propagate_priority(struct thread *td) 173{ | 167 168/* 169 * Walks the chain of turnstiles and their owners to propagate the priority 170 * of the thread being blocked to all the threads holding locks that have to 171 * release their locks before this thread can run again. 172 */ 173static void 174propagate_priority(struct thread *td) 175{ |
174 struct turnstile_chain *tc; | |
175 struct turnstile *ts; 176 int pri; 177 | 176 struct turnstile *ts; 177 int pri; 178 |
178 mtx_assert(&sched_lock, MA_OWNED); | 179 THREAD_LOCK_ASSERT(td, MA_OWNED); |
179 pri = td->td_priority; 180 ts = td->td_blocked; | 180 pri = td->td_priority; 181 ts = td->td_blocked; |
182 MPASS(td->td_lock == &ts->ts_lock); 183 /* 184 * Grab a recursive lock on this turnstile chain so it stays locked 185 * for the whole operation. The caller expects us to return with 186 * the original lock held. We only ever lock down the chain so 187 * the lock order is constant. 188 */ 189 mtx_lock_spin(&ts->ts_lock); |
|
181 for (;;) { 182 td = ts->ts_owner; 183 184 if (td == NULL) { 185 /* 186 * This might be a read lock with no owner. There's 187 * not much we can do, so just bail. 188 */ | 190 for (;;) { 191 td = ts->ts_owner; 192 193 if (td == NULL) { 194 /* 195 * This might be a read lock with no owner. There's 196 * not much we can do, so just bail. 197 */ |
198 mtx_unlock_spin(&ts->ts_lock); |
|
189 return; 190 } 191 | 199 return; 200 } 201 |
202 thread_lock_flags(td, MTX_DUPOK); 203 mtx_unlock_spin(&ts->ts_lock); |
|
192 MPASS(td->td_proc != NULL); 193 MPASS(td->td_proc->p_magic == P_MAGIC); 194 195 /* 196 * If the thread is asleep, then we are probably about 197 * to deadlock. To make debugging this easier, just 198 * panic and tell the user which thread misbehaved so 199 * they can hopefully get a stack trace from the truly --- 8 unchanged lines hidden (view full) --- 208#endif 209 panic("sleeping thread"); 210 } 211 212 /* 213 * If this thread already has higher priority than the 214 * thread that is being blocked, we are finished. 215 */ | 204 MPASS(td->td_proc != NULL); 205 MPASS(td->td_proc->p_magic == P_MAGIC); 206 207 /* 208 * If the thread is asleep, then we are probably about 209 * to deadlock. To make debugging this easier, just 210 * panic and tell the user which thread misbehaved so 211 * they can hopefully get a stack trace from the truly --- 8 unchanged lines hidden (view full) --- 220#endif 221 panic("sleeping thread"); 222 } 223 224 /* 225 * If this thread already has higher priority than the 226 * thread that is being blocked, we are finished. 227 */ |
216 if (td->td_priority <= pri) | 228 if (td->td_priority <= pri) { 229 thread_unlock(td); |
217 return; | 230 return; |
231 } |
|
218 219 /* 220 * Bump this thread's priority. 221 */ 222 sched_lend_prio(td, pri); 223 224 /* 225 * If lock holder is actually running or on the run queue 226 * then we are done. 227 */ 228 if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) { 229 MPASS(td->td_blocked == NULL); | 232 233 /* 234 * Bump this thread's priority. 235 */ 236 sched_lend_prio(td, pri); 237 238 /* 239 * If lock holder is actually running or on the run queue 240 * then we are done. 241 */ 242 if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) { 243 MPASS(td->td_blocked == NULL); |
244 thread_unlock(td); |
|
230 return; 231 } 232 233#ifndef SMP 234 /* 235 * For UP, we check to see if td is curthread (this shouldn't 236 * ever happen however as it would mean we are in a deadlock.) 237 */ --- 8 unchanged lines hidden (view full) --- 246 td->td_tid, td->td_proc->p_comm, td->td_state, 247 ts->ts_lockobj->lo_name)); 248 249 /* 250 * Pick up the lock that td is blocked on. 251 */ 252 ts = td->td_blocked; 253 MPASS(ts != NULL); | 245 return; 246 } 247 248#ifndef SMP 249 /* 250 * For UP, we check to see if td is curthread (this shouldn't 251 * ever happen however as it would mean we are in a deadlock.) 252 */ --- 8 unchanged lines hidden (view full) --- 261 td->td_tid, td->td_proc->p_comm, td->td_state, 262 ts->ts_lockobj->lo_name)); 263 264 /* 265 * Pick up the lock that td is blocked on. 266 */ 267 ts = td->td_blocked; 268 MPASS(ts != NULL); |
254 tc = TC_LOOKUP(ts->ts_lockobj); 255 mtx_lock_spin(&tc->tc_lock); 256 | 269 MPASS(td->td_lock == &ts->ts_lock); |
257 /* Resort td on the list if needed. */ 258 if (!turnstile_adjust_thread(ts, td)) { | 270 /* Resort td on the list if needed. */ 271 if (!turnstile_adjust_thread(ts, td)) { |
259 mtx_unlock_spin(&tc->tc_lock); | 272 mtx_unlock_spin(&ts->ts_lock); |
260 return; 261 } | 273 return; 274 } |
262 mtx_unlock_spin(&tc->tc_lock); | 275 /* The thread lock is released as ts lock above. */ |
263 } 264} 265 266/* 267 * Adjust the thread's position on a turnstile after its priority has been 268 * changed. 269 */ 270static int 271turnstile_adjust_thread(struct turnstile *ts, struct thread *td) 272{ | 276 } 277} 278 279/* 280 * Adjust the thread's position on a turnstile after its priority has been 281 * changed. 282 */ 283static int 284turnstile_adjust_thread(struct turnstile *ts, struct thread *td) 285{ |
273 struct turnstile_chain *tc; | |
274 struct thread *td1, *td2; 275 int queue; 276 | 286 struct thread *td1, *td2; 287 int queue; 288 |
277 mtx_assert(&sched_lock, MA_OWNED); | 289 THREAD_LOCK_ASSERT(td, MA_OWNED); |
278 MPASS(TD_ON_LOCK(td)); 279 280 /* 281 * This thread may not be blocked on this turnstile anymore 282 * but instead might already be woken up on another CPU | 290 MPASS(TD_ON_LOCK(td)); 291 292 /* 293 * This thread may not be blocked on this turnstile anymore 294 * but instead might already be woken up on another CPU |
283 * that is waiting on sched_lock in turnstile_unpend() to | 295 * that is waiting on the thread lock in turnstile_unpend() to |
284 * finish waking this thread up. We can detect this case 285 * by checking to see if this thread has been given a 286 * turnstile by either turnstile_signal() or 287 * turnstile_broadcast(). In this case, treat the thread as 288 * if it was already running. 289 */ 290 if (td->td_turnstile != NULL) 291 return (0); 292 293 /* 294 * Check if the thread needs to be moved on the blocked chain. 295 * It needs to be moved if either its priority is lower than 296 * the previous thread or higher than the next thread. 297 */ | 296 * finish waking this thread up. We can detect this case 297 * by checking to see if this thread has been given a 298 * turnstile by either turnstile_signal() or 299 * turnstile_broadcast(). In this case, treat the thread as 300 * if it was already running. 301 */ 302 if (td->td_turnstile != NULL) 303 return (0); 304 305 /* 306 * Check if the thread needs to be moved on the blocked chain. 307 * It needs to be moved if either its priority is lower than 308 * the previous thread or higher than the next thread. 309 */ |
298 tc = TC_LOOKUP(ts->ts_lockobj); 299 mtx_assert(&tc->tc_lock, MA_OWNED); | 310 MPASS(td->td_lock == &ts->ts_lock); |
300 td1 = TAILQ_PREV(td, threadqueue, td_lockq); 301 td2 = TAILQ_NEXT(td, td_lockq); 302 if ((td1 != NULL && td->td_priority < td1->td_priority) || 303 (td2 != NULL && td->td_priority > td2->td_priority)) { 304 305 /* 306 * Remove thread from blocked chain and determine where 307 * it should be moved to. --- 72 unchanged lines hidden (view full) --- 380#endif 381 382static void 383init_turnstile0(void *dummy) 384{ 385 386 turnstile_zone = uma_zcreate("TURNSTILE", sizeof(struct turnstile), 387#ifdef INVARIANTS | 311 td1 = TAILQ_PREV(td, threadqueue, td_lockq); 312 td2 = TAILQ_NEXT(td, td_lockq); 313 if ((td1 != NULL && td->td_priority < td1->td_priority) || 314 (td2 != NULL && td->td_priority > td2->td_priority)) { 315 316 /* 317 * Remove thread from blocked chain and determine where 318 * it should be moved to. --- 72 unchanged lines hidden (view full) --- 391#endif 392 393static void 394init_turnstile0(void *dummy) 395{ 396 397 turnstile_zone = uma_zcreate("TURNSTILE", sizeof(struct turnstile), 398#ifdef INVARIANTS |
388 NULL, turnstile_dtor, turnstile_init, NULL, UMA_ALIGN_CACHE, 0); | 399 NULL, turnstile_dtor, turnstile_init, turnstile_fini, 400 UMA_ALIGN_CACHE, 0); |
389#else | 401#else |
390 NULL, NULL, turnstile_init, NULL, UMA_ALIGN_CACHE, 0); | 402 NULL, NULL, turnstile_init, turnstile_fini, UMA_ALIGN_CACHE, 0); |
391#endif 392 thread0.td_turnstile = turnstile_alloc(); 393} 394SYSINIT(turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL); 395 396/* 397 * Update a thread on the turnstile list after it's priority has been changed. 398 * The old priority is passed in as an argument. 399 */ 400void 401turnstile_adjust(struct thread *td, u_char oldpri) 402{ | 403#endif 404 thread0.td_turnstile = turnstile_alloc(); 405} 406SYSINIT(turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL); 407 408/* 409 * Update a thread on the turnstile list after it's priority has been changed. 410 * The old priority is passed in as an argument. 411 */ 412void 413turnstile_adjust(struct thread *td, u_char oldpri) 414{ |
403 struct turnstile_chain *tc; | |
404 struct turnstile *ts; 405 | 415 struct turnstile *ts; 416 |
406 mtx_assert(&sched_lock, MA_OWNED); | |
407 MPASS(TD_ON_LOCK(td)); 408 409 /* 410 * Pick up the lock that td is blocked on. 411 */ 412 ts = td->td_blocked; 413 MPASS(ts != NULL); | 417 MPASS(TD_ON_LOCK(td)); 418 419 /* 420 * Pick up the lock that td is blocked on. 421 */ 422 ts = td->td_blocked; 423 MPASS(ts != NULL); |
414 tc = TC_LOOKUP(ts->ts_lockobj); 415 mtx_lock_spin(&tc->tc_lock); | 424 MPASS(td->td_lock == &ts->ts_lock); 425 mtx_assert(&ts->ts_lock, MA_OWNED); |
416 417 /* Resort the turnstile on the list. */ | 426 427 /* Resort the turnstile on the list. */ |
418 if (!turnstile_adjust_thread(ts, td)) { 419 mtx_unlock_spin(&tc->tc_lock); | 428 if (!turnstile_adjust_thread(ts, td)) |
420 return; | 429 return; |
421 } 422 | |
423 /* 424 * If our priority was lowered and we are at the head of the 425 * turnstile, then propagate our new priority up the chain. 426 * Note that we currently don't try to revoke lent priorities 427 * when our priority goes up. 428 */ 429 MPASS(td->td_tsqueue == TS_EXCLUSIVE_QUEUE || 430 td->td_tsqueue == TS_SHARED_QUEUE); 431 if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) && 432 td->td_priority < oldpri) { | 430 /* 431 * If our priority was lowered and we are at the head of the 432 * turnstile, then propagate our new priority up the chain. 433 * Note that we currently don't try to revoke lent priorities 434 * when our priority goes up. 435 */ 436 MPASS(td->td_tsqueue == TS_EXCLUSIVE_QUEUE || 437 td->td_tsqueue == TS_SHARED_QUEUE); 438 if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) && 439 td->td_priority < oldpri) { |
433 mtx_unlock_spin(&tc->tc_lock); 434 critical_enter(); | |
435 propagate_priority(td); | 440 propagate_priority(td); |
436 critical_exit(); 437 } else 438 mtx_unlock_spin(&tc->tc_lock); | 441 } |
439} 440 441/* 442 * Set the owner of the lock this turnstile is attached to. 443 */ 444static void 445turnstile_setowner(struct turnstile *ts, struct thread *owner) 446{ --- 35 unchanged lines hidden (view full) --- 482 struct turnstile *ts; 483 484 bzero(mem, size); 485 ts = mem; 486 TAILQ_INIT(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]); 487 TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]); 488 TAILQ_INIT(&ts->ts_pending); 489 LIST_INIT(&ts->ts_free); | 442} 443 444/* 445 * Set the owner of the lock this turnstile is attached to. 446 */ 447static void 448turnstile_setowner(struct turnstile *ts, struct thread *owner) 449{ --- 35 unchanged lines hidden (view full) --- 485 struct turnstile *ts; 486 487 bzero(mem, size); 488 ts = mem; 489 TAILQ_INIT(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]); 490 TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]); 491 TAILQ_INIT(&ts->ts_pending); 492 LIST_INIT(&ts->ts_free); |
493 mtx_init(&ts->ts_lock, "turnstile lock", NULL, MTX_SPIN | MTX_RECURSE); |
|
490 return (0); 491} 492 | 494 return (0); 495} 496 |
497static void 498turnstile_fini(void *mem, int size) 499{ 500 struct turnstile *ts; 501 502 ts = mem; 503 mtx_destroy(&ts->ts_lock); 504} 505 |
|
493/* 494 * Get a turnstile for a new thread. 495 */ 496struct turnstile * 497turnstile_alloc(void) 498{ 499 500 return (uma_zalloc(turnstile_zone, M_WAITOK)); --- 8 unchanged lines hidden (view full) --- 509 510 uma_zfree(turnstile_zone, ts); 511} 512 513/* 514 * Lock the turnstile chain associated with the specified lock. 515 */ 516void | 506/* 507 * Get a turnstile for a new thread. 508 */ 509struct turnstile * 510turnstile_alloc(void) 511{ 512 513 return (uma_zalloc(turnstile_zone, M_WAITOK)); --- 8 unchanged lines hidden (view full) --- 522 523 uma_zfree(turnstile_zone, ts); 524} 525 526/* 527 * Lock the turnstile chain associated with the specified lock. 528 */ 529void |
517turnstile_lock(struct lock_object *lock) | 530turnstile_chain_lock(struct lock_object *lock) |
518{ 519 struct turnstile_chain *tc; 520 521 tc = TC_LOOKUP(lock); 522 mtx_lock_spin(&tc->tc_lock); 523} 524 | 531{ 532 struct turnstile_chain *tc; 533 534 tc = TC_LOOKUP(lock); 535 mtx_lock_spin(&tc->tc_lock); 536} 537 |
538struct turnstile * 539turnstile_trywait(struct lock_object *lock) 540{ 541 struct turnstile_chain *tc; 542 struct turnstile *ts; 543 544 tc = TC_LOOKUP(lock); 545 mtx_lock_spin(&tc->tc_lock); 546 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash) 547 if (ts->ts_lockobj == lock) { 548 mtx_lock_spin(&ts->ts_lock); 549 return (ts); 550 } 551 552 ts = curthread->td_turnstile; 553 MPASS(ts != NULL); 554 mtx_lock_spin(&ts->ts_lock); 555 KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer")); 556 ts->ts_lockobj = lock; 557 558 return (ts); 559} 560 561void 562turnstile_cancel(struct turnstile *ts) 563{ 564 struct turnstile_chain *tc; 565 struct lock_object *lock; 566 567 mtx_assert(&ts->ts_lock, MA_OWNED); 568 569 mtx_unlock_spin(&ts->ts_lock); 570 lock = ts->ts_lockobj; 571 if (ts == curthread->td_turnstile) 572 ts->ts_lockobj = NULL; 573 tc = TC_LOOKUP(lock); 574 mtx_unlock_spin(&tc->tc_lock); 575} 576 |
|
525/* 526 * Look up the turnstile for a lock in the hash table locking the associated 527 * turnstile chain along the way. If no turnstile is found in the hash 528 * table, NULL is returned. 529 */ 530struct turnstile * 531turnstile_lookup(struct lock_object *lock) 532{ 533 struct turnstile_chain *tc; 534 struct turnstile *ts; 535 536 tc = TC_LOOKUP(lock); 537 mtx_assert(&tc->tc_lock, MA_OWNED); 538 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash) | 577/* 578 * Look up the turnstile for a lock in the hash table locking the associated 579 * turnstile chain along the way. If no turnstile is found in the hash 580 * table, NULL is returned. 581 */ 582struct turnstile * 583turnstile_lookup(struct lock_object *lock) 584{ 585 struct turnstile_chain *tc; 586 struct turnstile *ts; 587 588 tc = TC_LOOKUP(lock); 589 mtx_assert(&tc->tc_lock, MA_OWNED); 590 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash) |
539 if (ts->ts_lockobj == lock) | 591 if (ts->ts_lockobj == lock) { 592 mtx_lock_spin(&ts->ts_lock); |
540 return (ts); | 593 return (ts); |
594 } |
|
541 return (NULL); 542} 543 544/* 545 * Unlock the turnstile chain associated with a given lock. 546 */ 547void | 595 return (NULL); 596} 597 598/* 599 * Unlock the turnstile chain associated with a given lock. 600 */ 601void |
548turnstile_release(struct lock_object *lock) | 602turnstile_chain_unlock(struct lock_object *lock) |
549{ 550 struct turnstile_chain *tc; 551 552 tc = TC_LOOKUP(lock); 553 mtx_unlock_spin(&tc->tc_lock); 554} 555 556/* --- 12 unchanged lines hidden (view full) --- 569 return (xtd); 570} 571 572/* 573 * Take ownership of a turnstile and adjust the priority of the new 574 * owner appropriately. 575 */ 576void | 603{ 604 struct turnstile_chain *tc; 605 606 tc = TC_LOOKUP(lock); 607 mtx_unlock_spin(&tc->tc_lock); 608} 609 610/* --- 12 unchanged lines hidden (view full) --- 623 return (xtd); 624} 625 626/* 627 * Take ownership of a turnstile and adjust the priority of the new 628 * owner appropriately. 629 */ 630void |
577turnstile_claim(struct lock_object *lock) | 631turnstile_claim(struct turnstile *ts) |
578{ | 632{ |
579 struct turnstile_chain *tc; 580 struct turnstile *ts; | |
581 struct thread *td, *owner; | 633 struct thread *td, *owner; |
634 struct turnstile_chain *tc; |
|
582 | 635 |
583 tc = TC_LOOKUP(lock); 584 mtx_assert(&tc->tc_lock, MA_OWNED); 585 ts = turnstile_lookup(lock); 586 MPASS(ts != NULL); | 636 mtx_assert(&ts->ts_lock, MA_OWNED); 637 MPASS(ts != curthread->td_turnstile); |
587 588 owner = curthread; 589 mtx_lock_spin(&td_contested_lock); 590 turnstile_setowner(ts, owner); 591 mtx_unlock_spin(&td_contested_lock); 592 593 td = turnstile_first_waiter(ts); 594 MPASS(td != NULL); 595 MPASS(td->td_proc->p_magic == P_MAGIC); | 638 639 owner = curthread; 640 mtx_lock_spin(&td_contested_lock); 641 turnstile_setowner(ts, owner); 642 mtx_unlock_spin(&td_contested_lock); 643 644 td = turnstile_first_waiter(ts); 645 MPASS(td != NULL); 646 MPASS(td->td_proc->p_magic == P_MAGIC); |
596 mtx_unlock_spin(&tc->tc_lock); | 647 MPASS(td->td_lock == &ts->ts_lock); |
597 598 /* 599 * Update the priority of the new owner if needed. 600 */ | 648 649 /* 650 * Update the priority of the new owner if needed. 651 */ |
601 mtx_lock_spin(&sched_lock); | 652 thread_lock(owner); |
602 if (td->td_priority < owner->td_priority) 603 sched_lend_prio(owner, td->td_priority); | 653 if (td->td_priority < owner->td_priority) 654 sched_lend_prio(owner, td->td_priority); |
604 mtx_unlock_spin(&sched_lock); | 655 thread_unlock(owner); 656 tc = TC_LOOKUP(ts->ts_lockobj); 657 mtx_unlock_spin(&ts->ts_lock); 658 mtx_unlock_spin(&tc->tc_lock); |
605} 606 607/* 608 * Block the current thread on the turnstile assicated with 'lock'. This 609 * function will context switch and not return until this thread has been 610 * woken back up. This function must be called with the appropriate 611 * turnstile chain locked and will return with it unlocked. 612 */ 613void | 659} 660 661/* 662 * Block the current thread on the turnstile assicated with 'lock'. This 663 * function will context switch and not return until this thread has been 664 * woken back up. This function must be called with the appropriate 665 * turnstile chain locked and will return with it unlocked. 666 */ 667void |
614turnstile_wait(struct lock_object *lock, struct thread *owner, int queue) | 668turnstile_wait(struct turnstile *ts, struct thread *owner, int queue) |
615{ 616 struct turnstile_chain *tc; | 669{ 670 struct turnstile_chain *tc; |
617 struct turnstile *ts; | |
618 struct thread *td, *td1; | 671 struct thread *td, *td1; |
672 struct lock_object *lock; |
|
619 620 td = curthread; | 673 674 td = curthread; |
621 tc = TC_LOOKUP(lock); 622 mtx_assert(&tc->tc_lock, MA_OWNED); 623 MPASS(td->td_turnstile != NULL); | 675 mtx_assert(&ts->ts_lock, MA_OWNED); |
624 if (queue == TS_SHARED_QUEUE) 625 MPASS(owner != NULL); 626 if (owner) 627 MPASS(owner->td_proc->p_magic == P_MAGIC); 628 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 629 | 676 if (queue == TS_SHARED_QUEUE) 677 MPASS(owner != NULL); 678 if (owner) 679 MPASS(owner->td_proc->p_magic == P_MAGIC); 680 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 681 |
630 /* Look up the turnstile associated with the lock 'lock'. */ 631 ts = turnstile_lookup(lock); 632 | |
633 /* 634 * If the lock does not already have a turnstile, use this thread's 635 * turnstile. Otherwise insert the current thread into the 636 * turnstile already in use by this lock. 637 */ | 682 /* 683 * If the lock does not already have a turnstile, use this thread's 684 * turnstile. Otherwise insert the current thread into the 685 * turnstile already in use by this lock. 686 */ |
638 if (ts == NULL) { | 687 tc = TC_LOOKUP(ts->ts_lockobj); 688 if (ts == td->td_turnstile) { 689 mtx_assert(&tc->tc_lock, MA_OWNED); |
639#ifdef TURNSTILE_PROFILING 640 tc->tc_depth++; 641 if (tc->tc_depth > tc->tc_max_depth) { 642 tc->tc_max_depth = tc->tc_depth; 643 if (tc->tc_max_depth > turnstile_max_depth) 644 turnstile_max_depth = tc->tc_max_depth; 645 } 646#endif | 690#ifdef TURNSTILE_PROFILING 691 tc->tc_depth++; 692 if (tc->tc_depth > tc->tc_max_depth) { 693 tc->tc_max_depth = tc->tc_depth; 694 if (tc->tc_max_depth > turnstile_max_depth) 695 turnstile_max_depth = tc->tc_max_depth; 696 } 697#endif |
647 ts = td->td_turnstile; | 698 tc = TC_LOOKUP(ts->ts_lockobj); |
648 LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash); 649 KASSERT(TAILQ_EMPTY(&ts->ts_pending), 650 ("thread's turnstile has pending threads")); 651 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]), 652 ("thread's turnstile has exclusive waiters")); 653 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]), 654 ("thread's turnstile has shared waiters")); 655 KASSERT(LIST_EMPTY(&ts->ts_free), 656 ("thread's turnstile has a non-empty free list")); | 699 LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash); 700 KASSERT(TAILQ_EMPTY(&ts->ts_pending), 701 ("thread's turnstile has pending threads")); 702 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]), 703 ("thread's turnstile has exclusive waiters")); 704 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]), 705 ("thread's turnstile has shared waiters")); 706 KASSERT(LIST_EMPTY(&ts->ts_free), 707 ("thread's turnstile has a non-empty free list")); |
657 KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer")); 658 ts->ts_lockobj = lock; | 708 MPASS(ts->ts_lockobj != NULL); |
659 mtx_lock_spin(&td_contested_lock); 660 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq); 661 turnstile_setowner(ts, owner); 662 mtx_unlock_spin(&td_contested_lock); 663 } else { 664 TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq) 665 if (td1->td_priority > td->td_priority) 666 break; 667 mtx_lock_spin(&td_contested_lock); 668 if (td1 != NULL) 669 TAILQ_INSERT_BEFORE(td1, td, td_lockq); 670 else 671 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq); 672 MPASS(owner == ts->ts_owner); 673 mtx_unlock_spin(&td_contested_lock); 674 MPASS(td->td_turnstile != NULL); 675 LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash); 676 } | 709 mtx_lock_spin(&td_contested_lock); 710 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq); 711 turnstile_setowner(ts, owner); 712 mtx_unlock_spin(&td_contested_lock); 713 } else { 714 TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq) 715 if (td1->td_priority > td->td_priority) 716 break; 717 mtx_lock_spin(&td_contested_lock); 718 if (td1 != NULL) 719 TAILQ_INSERT_BEFORE(td1, td, td_lockq); 720 else 721 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq); 722 MPASS(owner == ts->ts_owner); 723 mtx_unlock_spin(&td_contested_lock); 724 MPASS(td->td_turnstile != NULL); 725 LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash); 726 } |
727 thread_lock(td); 728 thread_lock_set(td, &ts->ts_lock); |
|
677 td->td_turnstile = NULL; | 729 td->td_turnstile = NULL; |
678 mtx_unlock_spin(&tc->tc_lock); | |
679 | 730 |
680 mtx_lock_spin(&sched_lock); 681 /* 682 * Handle race condition where a thread on another CPU that owns 683 * lock 'lock' could have woken us in between us dropping the 684 * turnstile chain lock and acquiring the sched_lock. 685 */ 686 if (td->td_flags & TDF_TSNOBLOCK) { 687 td->td_flags &= ~TDF_TSNOBLOCK; 688 mtx_unlock_spin(&sched_lock); 689 return; 690 } 691 692#ifdef notyet 693 /* 694 * If we're borrowing an interrupted thread's VM context, we 695 * must clean up before going to sleep. 696 */ 697 if (td->td_ithd != NULL) { 698 struct ithd *it = td->td_ithd; 699 700 if (it->it_interrupted) { 701 if (LOCK_LOG_TEST(lock, 0)) 702 CTR3(KTR_LOCK, "%s: %p interrupted %p", 703 __func__, it, it->it_interrupted); 704 intr_thd_fixup(it); 705 } 706 } 707#endif 708 | |
709 /* Save who we are blocked on and switch. */ | 731 /* Save who we are blocked on and switch. */ |
732 lock = ts->ts_lockobj; |
|
710 td->td_tsqueue = queue; 711 td->td_blocked = ts; 712 td->td_lockname = lock->lo_name; 713 TD_SET_LOCK(td); | 733 td->td_tsqueue = queue; 734 td->td_blocked = ts; 735 td->td_lockname = lock->lo_name; 736 TD_SET_LOCK(td); |
714 critical_enter(); | 737 mtx_unlock_spin(&tc->tc_lock); |
715 propagate_priority(td); | 738 propagate_priority(td); |
716 critical_exit(); | |
717 718 if (LOCK_LOG_TEST(lock, 0)) 719 CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__, 720 td->td_tid, lock, lock->lo_name); 721 | 739 740 if (LOCK_LOG_TEST(lock, 0)) 741 CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__, 742 td->td_tid, lock, lock->lo_name); 743 |
744 MPASS(td->td_lock == &ts->ts_lock); 745 SCHED_STAT_INC(switch_turnstile); |
|
722 mi_switch(SW_VOL, NULL); 723 724 if (LOCK_LOG_TEST(lock, 0)) 725 CTR4(KTR_LOCK, "%s: td %d free from blocked on [%p] %s", 726 __func__, td->td_tid, lock, lock->lo_name); | 746 mi_switch(SW_VOL, NULL); 747 748 if (LOCK_LOG_TEST(lock, 0)) 749 CTR4(KTR_LOCK, "%s: td %d free from blocked on [%p] %s", 750 __func__, td->td_tid, lock, lock->lo_name); |
727 728 mtx_unlock_spin(&sched_lock); | 751 thread_unlock(td); |
729} 730 731/* 732 * Pick the highest priority thread on this turnstile and put it on the 733 * pending list. This must be called with the turnstile chain locked. 734 */ 735int 736turnstile_signal(struct turnstile *ts, int queue) 737{ 738 struct turnstile_chain *tc; 739 struct thread *td; 740 int empty; 741 742 MPASS(ts != NULL); | 752} 753 754/* 755 * Pick the highest priority thread on this turnstile and put it on the 756 * pending list. This must be called with the turnstile chain locked. 757 */ 758int 759turnstile_signal(struct turnstile *ts, int queue) 760{ 761 struct turnstile_chain *tc; 762 struct thread *td; 763 int empty; 764 765 MPASS(ts != NULL); |
766 mtx_assert(&ts->ts_lock, MA_OWNED); |
|
743 MPASS(curthread->td_proc->p_magic == P_MAGIC); 744 MPASS(ts->ts_owner == curthread || 745 (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL)); | 767 MPASS(curthread->td_proc->p_magic == P_MAGIC); 768 MPASS(ts->ts_owner == curthread || 769 (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL)); |
746 tc = TC_LOOKUP(ts->ts_lockobj); 747 mtx_assert(&tc->tc_lock, MA_OWNED); | |
748 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 749 750 /* 751 * Pick the highest priority thread blocked on this lock and 752 * move it to the pending list. 753 */ 754 td = TAILQ_FIRST(&ts->ts_blocked[queue]); 755 MPASS(td->td_proc->p_magic == P_MAGIC); --- 5 unchanged lines hidden (view full) --- 761 /* 762 * If the turnstile is now empty, remove it from its chain and 763 * give it to the about-to-be-woken thread. Otherwise take a 764 * turnstile from the free list and give it to the thread. 765 */ 766 empty = TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && 767 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]); 768 if (empty) { | 770 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 771 772 /* 773 * Pick the highest priority thread blocked on this lock and 774 * move it to the pending list. 775 */ 776 td = TAILQ_FIRST(&ts->ts_blocked[queue]); 777 MPASS(td->td_proc->p_magic == P_MAGIC); --- 5 unchanged lines hidden (view full) --- 783 /* 784 * If the turnstile is now empty, remove it from its chain and 785 * give it to the about-to-be-woken thread. Otherwise take a 786 * turnstile from the free list and give it to the thread. 787 */ 788 empty = TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && 789 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]); 790 if (empty) { |
791 tc = TC_LOOKUP(ts->ts_lockobj); 792 mtx_assert(&tc->tc_lock, MA_OWNED); |
|
769 MPASS(LIST_EMPTY(&ts->ts_free)); 770#ifdef TURNSTILE_PROFILING 771 tc->tc_depth--; 772#endif 773 } else 774 ts = LIST_FIRST(&ts->ts_free); 775 MPASS(ts != NULL); 776 LIST_REMOVE(ts, ts_hash); --- 9 unchanged lines hidden (view full) --- 786void 787turnstile_broadcast(struct turnstile *ts, int queue) 788{ 789 struct turnstile_chain *tc; 790 struct turnstile *ts1; 791 struct thread *td; 792 793 MPASS(ts != NULL); | 793 MPASS(LIST_EMPTY(&ts->ts_free)); 794#ifdef TURNSTILE_PROFILING 795 tc->tc_depth--; 796#endif 797 } else 798 ts = LIST_FIRST(&ts->ts_free); 799 MPASS(ts != NULL); 800 LIST_REMOVE(ts, ts_hash); --- 9 unchanged lines hidden (view full) --- 810void 811turnstile_broadcast(struct turnstile *ts, int queue) 812{ 813 struct turnstile_chain *tc; 814 struct turnstile *ts1; 815 struct thread *td; 816 817 MPASS(ts != NULL); |
818 mtx_assert(&ts->ts_lock, MA_OWNED); |
|
794 MPASS(curthread->td_proc->p_magic == P_MAGIC); 795 MPASS(ts->ts_owner == curthread || 796 (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL)); | 819 MPASS(curthread->td_proc->p_magic == P_MAGIC); 820 MPASS(ts->ts_owner == curthread || 821 (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL)); |
822 /* 823 * We must have the chain locked so that we can remove the empty 824 * turnstile from the hash queue. 825 */ |
|
797 tc = TC_LOOKUP(ts->ts_lockobj); 798 mtx_assert(&tc->tc_lock, MA_OWNED); 799 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 800 801 /* 802 * Transfer the blocked list to the pending list. 803 */ 804 mtx_lock_spin(&td_contested_lock); --- 23 unchanged lines hidden (view full) --- 828 * Wakeup all threads on the pending list and adjust the priority of the 829 * current thread appropriately. This must be called with the turnstile 830 * chain locked. 831 */ 832void 833turnstile_unpend(struct turnstile *ts, int owner_type) 834{ 835 TAILQ_HEAD( ,thread) pending_threads; | 826 tc = TC_LOOKUP(ts->ts_lockobj); 827 mtx_assert(&tc->tc_lock, MA_OWNED); 828 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); 829 830 /* 831 * Transfer the blocked list to the pending list. 832 */ 833 mtx_lock_spin(&td_contested_lock); --- 23 unchanged lines hidden (view full) --- 857 * Wakeup all threads on the pending list and adjust the priority of the 858 * current thread appropriately. This must be called with the turnstile 859 * chain locked. 860 */ 861void 862turnstile_unpend(struct turnstile *ts, int owner_type) 863{ 864 TAILQ_HEAD( ,thread) pending_threads; |
836 struct turnstile_chain *tc; | 865 struct turnstile *nts; |
837 struct thread *td; 838 u_char cp, pri; 839 840 MPASS(ts != NULL); | 866 struct thread *td; 867 u_char cp, pri; 868 869 MPASS(ts != NULL); |
870 mtx_assert(&ts->ts_lock, MA_OWNED); |
|
841 MPASS(ts->ts_owner == curthread || 842 (owner_type == TS_SHARED_LOCK && ts->ts_owner == NULL)); | 871 MPASS(ts->ts_owner == curthread || 872 (owner_type == TS_SHARED_LOCK && ts->ts_owner == NULL)); |
843 tc = TC_LOOKUP(ts->ts_lockobj); 844 mtx_assert(&tc->tc_lock, MA_OWNED); | |
845 MPASS(!TAILQ_EMPTY(&ts->ts_pending)); 846 847 /* 848 * Move the list of pending threads out of the turnstile and 849 * into a local variable. 850 */ 851 TAILQ_INIT(&pending_threads); 852 TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq); 853#ifdef INVARIANTS 854 if (TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && 855 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])) 856 ts->ts_lockobj = NULL; 857#endif | 873 MPASS(!TAILQ_EMPTY(&ts->ts_pending)); 874 875 /* 876 * Move the list of pending threads out of the turnstile and 877 * into a local variable. 878 */ 879 TAILQ_INIT(&pending_threads); 880 TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq); 881#ifdef INVARIANTS 882 if (TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && 883 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])) 884 ts->ts_lockobj = NULL; 885#endif |
858 | |
859 /* | 886 /* |
887 * Adjust the priority of curthread based on other contested 888 * locks it owns. Don't lower the priority below the base 889 * priority however. 890 */ 891 td = curthread; 892 pri = PRI_MAX; 893 thread_lock(td); 894 mtx_lock_spin(&td_contested_lock); 895 /* |
|
860 * Remove the turnstile from this thread's list of contested locks 861 * since this thread doesn't own it anymore. New threads will 862 * not be blocking on the turnstile until it is claimed by a new 863 * owner. There might not be a current owner if this is a shared 864 * lock. 865 */ 866 if (ts->ts_owner != NULL) { | 896 * Remove the turnstile from this thread's list of contested locks 897 * since this thread doesn't own it anymore. New threads will 898 * not be blocking on the turnstile until it is claimed by a new 899 * owner. There might not be a current owner if this is a shared 900 * lock. 901 */ 902 if (ts->ts_owner != NULL) { |
867 mtx_lock_spin(&td_contested_lock); | |
868 ts->ts_owner = NULL; 869 LIST_REMOVE(ts, ts_link); | 903 ts->ts_owner = NULL; 904 LIST_REMOVE(ts, ts_link); |
870 mtx_unlock_spin(&td_contested_lock); | |
871 } | 905 } |
872 critical_enter(); 873 mtx_unlock_spin(&tc->tc_lock); 874 875 /* 876 * Adjust the priority of curthread based on other contested 877 * locks it owns. Don't lower the priority below the base 878 * priority however. 879 */ 880 td = curthread; 881 pri = PRI_MAX; 882 mtx_lock_spin(&sched_lock); 883 mtx_lock_spin(&td_contested_lock); 884 LIST_FOREACH(ts, &td->td_contested, ts_link) { 885 cp = turnstile_first_waiter(ts)->td_priority; | 906 LIST_FOREACH(nts, &td->td_contested, ts_link) { 907 cp = turnstile_first_waiter(nts)->td_priority; |
886 if (cp < pri) 887 pri = cp; 888 } 889 mtx_unlock_spin(&td_contested_lock); 890 sched_unlend_prio(td, pri); | 908 if (cp < pri) 909 pri = cp; 910 } 911 mtx_unlock_spin(&td_contested_lock); 912 sched_unlend_prio(td, pri); |
891 | 913 thread_unlock(td); |
892 /* 893 * Wake up all the pending threads. If a thread is not blocked 894 * on a lock, then it is currently executing on another CPU in 895 * turnstile_wait() or sitting on a run queue waiting to resume 896 * in turnstile_wait(). Set a flag to force it to try to acquire 897 * the lock again instead of blocking. 898 */ 899 while (!TAILQ_EMPTY(&pending_threads)) { 900 td = TAILQ_FIRST(&pending_threads); 901 TAILQ_REMOVE(&pending_threads, td, td_lockq); | 914 /* 915 * Wake up all the pending threads. If a thread is not blocked 916 * on a lock, then it is currently executing on another CPU in 917 * turnstile_wait() or sitting on a run queue waiting to resume 918 * in turnstile_wait(). Set a flag to force it to try to acquire 919 * the lock again instead of blocking. 920 */ 921 while (!TAILQ_EMPTY(&pending_threads)) { 922 td = TAILQ_FIRST(&pending_threads); 923 TAILQ_REMOVE(&pending_threads, td, td_lockq); |
924 thread_lock(td); 925 MPASS(td->td_lock == &ts->ts_lock); |
|
902 MPASS(td->td_proc->p_magic == P_MAGIC); | 926 MPASS(td->td_proc->p_magic == P_MAGIC); |
903 if (TD_ON_LOCK(td)) { 904 td->td_blocked = NULL; 905 td->td_lockname = NULL; | 927 MPASS(TD_ON_LOCK(td)); 928 TD_CLR_LOCK(td); 929 MPASS(TD_CAN_RUN(td)); 930 td->td_blocked = NULL; 931 td->td_lockname = NULL; |
906#ifdef INVARIANTS | 932#ifdef INVARIANTS |
907 td->td_tsqueue = 0xff; | 933 td->td_tsqueue = 0xff; |
908#endif | 934#endif |
909 TD_CLR_LOCK(td); 910 MPASS(TD_CAN_RUN(td)); 911 sched_add(td, SRQ_BORING); 912 } else { 913 td->td_flags |= TDF_TSNOBLOCK; 914 MPASS(TD_IS_RUNNING(td) || TD_ON_RUNQ(td)); 915 } | 935 sched_add(td, SRQ_BORING); 936 thread_unlock(td); |
916 } | 937 } |
917 critical_exit(); 918 mtx_unlock_spin(&sched_lock); | 938 mtx_unlock_spin(&ts->ts_lock); |
919} 920 921/* 922 * Give up ownership of a turnstile. This must be called with the 923 * turnstile chain locked. 924 */ 925void 926turnstile_disown(struct turnstile *ts) 927{ | 939} 940 941/* 942 * Give up ownership of a turnstile. This must be called with the 943 * turnstile chain locked. 944 */ 945void 946turnstile_disown(struct turnstile *ts) 947{ |
928 struct turnstile_chain *tc; | |
929 struct thread *td; 930 u_char cp, pri; 931 932 MPASS(ts != NULL); | 948 struct thread *td; 949 u_char cp, pri; 950 951 MPASS(ts != NULL); |
952 mtx_assert(&ts->ts_lock, MA_OWNED); |
|
933 MPASS(ts->ts_owner == curthread); | 953 MPASS(ts->ts_owner == curthread); |
934 tc = TC_LOOKUP(ts->ts_lockobj); 935 mtx_assert(&tc->tc_lock, MA_OWNED); | |
936 MPASS(TAILQ_EMPTY(&ts->ts_pending)); 937 MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) || 938 !TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])); 939 940 /* 941 * Remove the turnstile from this thread's list of contested locks 942 * since this thread doesn't own it anymore. New threads will 943 * not be blocking on the turnstile until it is claimed by a new 944 * owner. 945 */ 946 mtx_lock_spin(&td_contested_lock); 947 ts->ts_owner = NULL; 948 LIST_REMOVE(ts, ts_link); 949 mtx_unlock_spin(&td_contested_lock); | 954 MPASS(TAILQ_EMPTY(&ts->ts_pending)); 955 MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) || 956 !TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])); 957 958 /* 959 * Remove the turnstile from this thread's list of contested locks 960 * since this thread doesn't own it anymore. New threads will 961 * not be blocking on the turnstile until it is claimed by a new 962 * owner. 963 */ 964 mtx_lock_spin(&td_contested_lock); 965 ts->ts_owner = NULL; 966 LIST_REMOVE(ts, ts_link); 967 mtx_unlock_spin(&td_contested_lock); |
950 mtx_unlock_spin(&tc->tc_lock); | |
951 952 /* 953 * Adjust the priority of curthread based on other contested 954 * locks it owns. Don't lower the priority below the base 955 * priority however. 956 */ 957 td = curthread; 958 pri = PRI_MAX; | 968 969 /* 970 * Adjust the priority of curthread based on other contested 971 * locks it owns. Don't lower the priority below the base 972 * priority however. 973 */ 974 td = curthread; 975 pri = PRI_MAX; |
959 mtx_lock_spin(&sched_lock); | 976 thread_lock(td); 977 mtx_unlock_spin(&ts->ts_lock); |
960 mtx_lock_spin(&td_contested_lock); 961 LIST_FOREACH(ts, &td->td_contested, ts_link) { 962 cp = turnstile_first_waiter(ts)->td_priority; 963 if (cp < pri) 964 pri = cp; 965 } 966 mtx_unlock_spin(&td_contested_lock); 967 sched_unlend_prio(td, pri); | 978 mtx_lock_spin(&td_contested_lock); 979 LIST_FOREACH(ts, &td->td_contested, ts_link) { 980 cp = turnstile_first_waiter(ts)->td_priority; 981 if (cp < pri) 982 pri = cp; 983 } 984 mtx_unlock_spin(&td_contested_lock); 985 sched_unlend_prio(td, pri); |
968 mtx_unlock_spin(&sched_lock); | 986 thread_unlock(td); |
969} 970 971/* 972 * Return the first thread in a turnstile. 973 */ 974struct thread * 975turnstile_head(struct turnstile *ts, int queue) 976{ 977#ifdef INVARIANTS | 987} 988 989/* 990 * Return the first thread in a turnstile. 991 */ 992struct thread * 993turnstile_head(struct turnstile *ts, int queue) 994{ 995#ifdef INVARIANTS |
978 struct turnstile_chain *tc; | |
979 980 MPASS(ts != NULL); 981 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); | 996 997 MPASS(ts != NULL); 998 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); |
982 tc = TC_LOOKUP(ts->ts_lockobj); 983 mtx_assert(&tc->tc_lock, MA_OWNED); | 999 mtx_assert(&ts->ts_lock, MA_OWNED); |
984#endif 985 return (TAILQ_FIRST(&ts->ts_blocked[queue])); 986} 987 988/* 989 * Returns true if a sub-queue of a turnstile is empty. 990 */ 991int 992turnstile_empty(struct turnstile *ts, int queue) 993{ 994#ifdef INVARIANTS | 1000#endif 1001 return (TAILQ_FIRST(&ts->ts_blocked[queue])); 1002} 1003 1004/* 1005 * Returns true if a sub-queue of a turnstile is empty. 1006 */ 1007int 1008turnstile_empty(struct turnstile *ts, int queue) 1009{ 1010#ifdef INVARIANTS |
995 struct turnstile_chain *tc; | |
996 997 MPASS(ts != NULL); 998 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); | 1011 1012 MPASS(ts != NULL); 1013 MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE); |
999 tc = TC_LOOKUP(ts->ts_lockobj); 1000 mtx_assert(&tc->tc_lock, MA_OWNED); | 1014 mtx_assert(&ts->ts_lock, MA_OWNED); |
1001#endif 1002 return (TAILQ_EMPTY(&ts->ts_blocked[queue])); 1003} 1004 1005#ifdef DDB 1006static void 1007print_thread(struct thread *td, const char *prefix) 1008{ --- 280 unchanged lines hidden --- | 1015#endif 1016 return (TAILQ_EMPTY(&ts->ts_blocked[queue])); 1017} 1018 1019#ifdef DDB 1020static void 1021print_thread(struct thread *td, const char *prefix) 1022{ --- 280 unchanged lines hidden --- |