kern_rwlock.c (227588) | kern_rwlock.c (228424) |
---|---|
1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 18 unchanged lines hidden (view full) --- 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Machine independent bits of reader/writer lock implementation. 32 */ 33 34#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 18 unchanged lines hidden (view full) --- 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Machine independent bits of reader/writer lock implementation. 32 */ 33 34#include <sys/cdefs.h> |
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 227588 2011-11-16 21:51:17Z pjd $"); | 35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 228424 2011-12-11 21:02:01Z avg $"); |
36 37#include "opt_ddb.h" 38#include "opt_kdtrace.h" 39#include "opt_no_adaptive_rwlocks.h" 40 41#include <sys/param.h> 42#include <sys/ktr.h> 43#include <sys/kernel.h> --- 184 unchanged lines hidden (view full) --- 228 229 return (rw_wowner(rw) == curthread); 230} 231 232void 233_rw_wlock(struct rwlock *rw, const char *file, int line) 234{ 235 | 36 37#include "opt_ddb.h" 38#include "opt_kdtrace.h" 39#include "opt_no_adaptive_rwlocks.h" 40 41#include <sys/param.h> 42#include <sys/ktr.h> 43#include <sys/kernel.h> --- 184 unchanged lines hidden (view full) --- 228 229 return (rw_wowner(rw) == curthread); 230} 231 232void 233_rw_wlock(struct rwlock *rw, const char *file, int line) 234{ 235 |
236 if (SCHEDULER_STOPPED()) 237 return; |
|
236 MPASS(curthread != NULL); 237 KASSERT(rw->rw_lock != RW_DESTROYED, 238 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 239 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 240 line, NULL); 241 __rw_wlock(rw, curthread, file, line); 242 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 243 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 244 curthread->td_locks++; 245} 246 247int 248_rw_try_wlock(struct rwlock *rw, const char *file, int line) 249{ 250 int rval; 251 | 238 MPASS(curthread != NULL); 239 KASSERT(rw->rw_lock != RW_DESTROYED, 240 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 241 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 242 line, NULL); 243 __rw_wlock(rw, curthread, file, line); 244 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 245 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 246 curthread->td_locks++; 247} 248 249int 250_rw_try_wlock(struct rwlock *rw, const char *file, int line) 251{ 252 int rval; 253 |
254 if (SCHEDULER_STOPPED()) 255 return (1); 256 |
|
252 KASSERT(rw->rw_lock != RW_DESTROYED, 253 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 254 255 if (rw_wlocked(rw) && 256 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { 257 rw->rw_recurse++; 258 rval = 1; 259 } else --- 8 unchanged lines hidden (view full) --- 268 } 269 return (rval); 270} 271 272void 273_rw_wunlock(struct rwlock *rw, const char *file, int line) 274{ 275 | 257 KASSERT(rw->rw_lock != RW_DESTROYED, 258 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 259 260 if (rw_wlocked(rw) && 261 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { 262 rw->rw_recurse++; 263 rval = 1; 264 } else --- 8 unchanged lines hidden (view full) --- 273 } 274 return (rval); 275} 276 277void 278_rw_wunlock(struct rwlock *rw, const char *file, int line) 279{ 280 |
281 if (SCHEDULER_STOPPED()) 282 return; |
|
276 MPASS(curthread != NULL); 277 KASSERT(rw->rw_lock != RW_DESTROYED, 278 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 279 _rw_assert(rw, RA_WLOCKED, file, line); 280 curthread->td_locks--; 281 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 282 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 283 line); --- 28 unchanged lines hidden (view full) --- 312#endif 313 uintptr_t v; 314#ifdef KDTRACE_HOOKS 315 uint64_t spin_cnt = 0; 316 uint64_t sleep_cnt = 0; 317 int64_t sleep_time = 0; 318#endif 319 | 283 MPASS(curthread != NULL); 284 KASSERT(rw->rw_lock != RW_DESTROYED, 285 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 286 _rw_assert(rw, RA_WLOCKED, file, line); 287 curthread->td_locks--; 288 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 289 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 290 line); --- 28 unchanged lines hidden (view full) --- 319#endif 320 uintptr_t v; 321#ifdef KDTRACE_HOOKS 322 uint64_t spin_cnt = 0; 323 uint64_t sleep_cnt = 0; 324 int64_t sleep_time = 0; 325#endif 326 |
327 if (SCHEDULER_STOPPED()) 328 return; 329 |
|
320 KASSERT(rw->rw_lock != RW_DESTROYED, 321 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 322 KASSERT(rw_wowner(rw) != curthread, 323 ("%s (%s): wlock already held @ %s:%d", __func__, 324 rw->lock_object.lo_name, file, line)); 325 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 326 327 for (;;) { --- 166 unchanged lines hidden (view full) --- 494#endif 495} 496 497int 498_rw_try_rlock(struct rwlock *rw, const char *file, int line) 499{ 500 uintptr_t x; 501 | 330 KASSERT(rw->rw_lock != RW_DESTROYED, 331 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 332 KASSERT(rw_wowner(rw) != curthread, 333 ("%s (%s): wlock already held @ %s:%d", __func__, 334 rw->lock_object.lo_name, file, line)); 335 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 336 337 for (;;) { --- 166 unchanged lines hidden (view full) --- 504#endif 505} 506 507int 508_rw_try_rlock(struct rwlock *rw, const char *file, int line) 509{ 510 uintptr_t x; 511 |
512 if (SCHEDULER_STOPPED()) 513 return (1); 514 |
|
502 for (;;) { 503 x = rw->rw_lock; 504 KASSERT(rw->rw_lock != RW_DESTROYED, 505 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 506 if (!(x & RW_LOCK_READ)) 507 break; 508 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 509 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, --- 10 unchanged lines hidden (view full) --- 520} 521 522void 523_rw_runlock(struct rwlock *rw, const char *file, int line) 524{ 525 struct turnstile *ts; 526 uintptr_t x, v, queue; 527 | 515 for (;;) { 516 x = rw->rw_lock; 517 KASSERT(rw->rw_lock != RW_DESTROYED, 518 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 519 if (!(x & RW_LOCK_READ)) 520 break; 521 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 522 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, --- 10 unchanged lines hidden (view full) --- 533} 534 535void 536_rw_runlock(struct rwlock *rw, const char *file, int line) 537{ 538 struct turnstile *ts; 539 uintptr_t x, v, queue; 540 |
541 if (SCHEDULER_STOPPED()) 542 return; 543 |
|
528 KASSERT(rw->rw_lock != RW_DESTROYED, 529 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 530 _rw_assert(rw, RA_RLOCKED, file, line); 531 curthread->td_locks--; 532 curthread->td_rw_rlocks--; 533 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 534 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 535 --- 109 unchanged lines hidden (view full) --- 645 int contested = 0; 646#endif 647#ifdef KDTRACE_HOOKS 648 uint64_t spin_cnt = 0; 649 uint64_t sleep_cnt = 0; 650 int64_t sleep_time = 0; 651#endif 652 | 544 KASSERT(rw->rw_lock != RW_DESTROYED, 545 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 546 _rw_assert(rw, RA_RLOCKED, file, line); 547 curthread->td_locks--; 548 curthread->td_rw_rlocks--; 549 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 550 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 551 --- 109 unchanged lines hidden (view full) --- 661 int contested = 0; 662#endif 663#ifdef KDTRACE_HOOKS 664 uint64_t spin_cnt = 0; 665 uint64_t sleep_cnt = 0; 666 int64_t sleep_time = 0; 667#endif 668 |
669 if (SCHEDULER_STOPPED()) 670 return; 671 |
|
653 if (rw_wlocked(rw)) { 654 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 655 ("%s: recursing but non-recursive rw %s @ %s:%d\n", 656 __func__, rw->lock_object.lo_name, file, line)); 657 rw->rw_recurse++; 658 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 659 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 660 return; --- 148 unchanged lines hidden (view full) --- 809 */ 810void 811_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 812{ 813 struct turnstile *ts; 814 uintptr_t v; 815 int queue; 816 | 672 if (rw_wlocked(rw)) { 673 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 674 ("%s: recursing but non-recursive rw %s @ %s:%d\n", 675 __func__, rw->lock_object.lo_name, file, line)); 676 rw->rw_recurse++; 677 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 678 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 679 return; --- 148 unchanged lines hidden (view full) --- 828 */ 829void 830_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 831{ 832 struct turnstile *ts; 833 uintptr_t v; 834 int queue; 835 |
836 if (SCHEDULER_STOPPED()) 837 return; 838 |
|
817 if (rw_wlocked(rw) && rw_recursed(rw)) { 818 rw->rw_recurse--; 819 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 820 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 821 return; 822 } 823 824 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), --- 46 unchanged lines hidden (view full) --- 871 */ 872int 873_rw_try_upgrade(struct rwlock *rw, const char *file, int line) 874{ 875 uintptr_t v, x, tid; 876 struct turnstile *ts; 877 int success; 878 | 839 if (rw_wlocked(rw) && rw_recursed(rw)) { 840 rw->rw_recurse--; 841 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 842 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 843 return; 844 } 845 846 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), --- 46 unchanged lines hidden (view full) --- 893 */ 894int 895_rw_try_upgrade(struct rwlock *rw, const char *file, int line) 896{ 897 uintptr_t v, x, tid; 898 struct turnstile *ts; 899 int success; 900 |
901 if (SCHEDULER_STOPPED()) 902 return (1); 903 |
|
879 KASSERT(rw->rw_lock != RW_DESTROYED, 880 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 881 _rw_assert(rw, RA_RLOCKED, file, line); 882 883 /* 884 * Attempt to switch from one reader to a writer. If there 885 * are any write waiters, then we will have to lock the 886 * turnstile first to prevent races with another writer --- 54 unchanged lines hidden (view full) --- 941 */ 942void 943_rw_downgrade(struct rwlock *rw, const char *file, int line) 944{ 945 struct turnstile *ts; 946 uintptr_t tid, v; 947 int rwait, wwait; 948 | 904 KASSERT(rw->rw_lock != RW_DESTROYED, 905 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 906 _rw_assert(rw, RA_RLOCKED, file, line); 907 908 /* 909 * Attempt to switch from one reader to a writer. If there 910 * are any write waiters, then we will have to lock the 911 * turnstile first to prevent races with another writer --- 54 unchanged lines hidden (view full) --- 966 */ 967void 968_rw_downgrade(struct rwlock *rw, const char *file, int line) 969{ 970 struct turnstile *ts; 971 uintptr_t tid, v; 972 int rwait, wwait; 973 |
974 if (SCHEDULER_STOPPED()) 975 return; 976 |
|
949 KASSERT(rw->rw_lock != RW_DESTROYED, 950 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 951 _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line); 952#ifndef INVARIANTS 953 if (rw_recursed(rw)) 954 panic("downgrade of a recursed lock"); 955#endif 956 --- 171 unchanged lines hidden --- | 977 KASSERT(rw->rw_lock != RW_DESTROYED, 978 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 979 _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line); 980#ifndef INVARIANTS 981 if (rw_recursed(rw)) 982 panic("downgrade of a recursed lock"); 983#endif 984 --- 171 unchanged lines hidden --- |