kern_rwlock.c revision 315382
1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: stable/11/sys/kern/kern_rwlock.c 315382 2017-03-16 07:06:06Z mjg $"); 33 34#include "opt_ddb.h" 35#include "opt_hwpmc_hooks.h" 36#include "opt_no_adaptive_rwlocks.h" 37 38#include <sys/param.h> 39#include <sys/kdb.h> 40#include <sys/ktr.h> 41#include <sys/kernel.h> 42#include <sys/lock.h> 43#include <sys/mutex.h> 44#include <sys/proc.h> 45#include <sys/rwlock.h> 46#include <sys/sched.h> 47#include <sys/smp.h> 48#include <sys/sysctl.h> 49#include <sys/systm.h> 50#include <sys/turnstile.h> 51 52#include <machine/cpu.h> 53 54#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS) 55#define ADAPTIVE_RWLOCKS 56#endif 57 58#ifdef HWPMC_HOOKS 59#include <sys/pmckern.h> 60PMC_SOFT_DECLARE( , , lock, failed); 61#endif 62 63/* 64 * Return the rwlock address when the lock cookie address is provided. 65 * This functionality assumes that struct rwlock* have a member named rw_lock. 66 */ 67#define rwlock2rw(c) (__containerof(c, struct rwlock, rw_lock)) 68 69#ifdef DDB 70#include <ddb/ddb.h> 71 72static void db_show_rwlock(const struct lock_object *lock); 73#endif 74static void assert_rw(const struct lock_object *lock, int what); 75static void lock_rw(struct lock_object *lock, uintptr_t how); 76#ifdef KDTRACE_HOOKS 77static int owner_rw(const struct lock_object *lock, struct thread **owner); 78#endif 79static uintptr_t unlock_rw(struct lock_object *lock); 80 81struct lock_class lock_class_rw = { 82 .lc_name = "rw", 83 .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, 84 .lc_assert = assert_rw, 85#ifdef DDB 86 .lc_ddb_show = db_show_rwlock, 87#endif 88 .lc_lock = lock_rw, 89 .lc_unlock = unlock_rw, 90#ifdef KDTRACE_HOOKS 91 .lc_owner = owner_rw, 92#endif 93}; 94 95#ifdef ADAPTIVE_RWLOCKS 96static int rowner_retries = 10; 97static int rowner_loops = 10000; 98static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, 99 "rwlock debugging"); 100SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); 101SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); 102 103static struct lock_delay_config __read_mostly rw_delay; 104 105SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_base, CTLFLAG_RW, &rw_delay.base, 106 0, ""); 107SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max, 108 0, ""); 109 110LOCK_DELAY_SYSINIT_DEFAULT(rw_delay); 111#endif 112 113/* 114 * Return a pointer to the owning thread if the lock is write-locked or 115 * NULL if the lock is unlocked or read-locked. 116 */ 117 118#define lv_rw_wowner(v) \ 119 ((v) & RW_LOCK_READ ? NULL : \ 120 (struct thread *)RW_OWNER((v))) 121 122#define rw_wowner(rw) lv_rw_wowner(RW_READ_VALUE(rw)) 123 124/* 125 * Returns if a write owner is recursed. Write ownership is not assured 126 * here and should be previously checked. 127 */ 128#define rw_recursed(rw) ((rw)->rw_recurse != 0) 129 130/* 131 * Return true if curthread helds the lock. 132 */ 133#define rw_wlocked(rw) (rw_wowner((rw)) == curthread) 134 135/* 136 * Return a pointer to the owning thread for this lock who should receive 137 * any priority lent by threads that block on this lock. Currently this 138 * is identical to rw_wowner(). 139 */ 140#define rw_owner(rw) rw_wowner(rw) 141 142#ifndef INVARIANTS 143#define __rw_assert(c, what, file, line) 144#endif 145 146void 147assert_rw(const struct lock_object *lock, int what) 148{ 149 150 rw_assert((const struct rwlock *)lock, what); 151} 152 153void 154lock_rw(struct lock_object *lock, uintptr_t how) 155{ 156 struct rwlock *rw; 157 158 rw = (struct rwlock *)lock; 159 if (how) 160 rw_rlock(rw); 161 else 162 rw_wlock(rw); 163} 164 165uintptr_t 166unlock_rw(struct lock_object *lock) 167{ 168 struct rwlock *rw; 169 170 rw = (struct rwlock *)lock; 171 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); 172 if (rw->rw_lock & RW_LOCK_READ) { 173 rw_runlock(rw); 174 return (1); 175 } else { 176 rw_wunlock(rw); 177 return (0); 178 } 179} 180 181#ifdef KDTRACE_HOOKS 182int 183owner_rw(const struct lock_object *lock, struct thread **owner) 184{ 185 const struct rwlock *rw = (const struct rwlock *)lock; 186 uintptr_t x = rw->rw_lock; 187 188 *owner = rw_wowner(rw); 189 return ((x & RW_LOCK_READ) != 0 ? (RW_READERS(x) != 0) : 190 (*owner != NULL)); 191} 192#endif 193 194void 195_rw_init_flags(volatile uintptr_t *c, const char *name, int opts) 196{ 197 struct rwlock *rw; 198 int flags; 199 200 rw = rwlock2rw(c); 201 202 MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET | 203 RW_RECURSE | RW_NEW)) == 0); 204 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, 205 ("%s: rw_lock not aligned for %s: %p", __func__, name, 206 &rw->rw_lock)); 207 208 flags = LO_UPGRADABLE; 209 if (opts & RW_DUPOK) 210 flags |= LO_DUPOK; 211 if (opts & RW_NOPROFILE) 212 flags |= LO_NOPROFILE; 213 if (!(opts & RW_NOWITNESS)) 214 flags |= LO_WITNESS; 215 if (opts & RW_RECURSE) 216 flags |= LO_RECURSABLE; 217 if (opts & RW_QUIET) 218 flags |= LO_QUIET; 219 if (opts & RW_NEW) 220 flags |= LO_NEW; 221 222 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); 223 rw->rw_lock = RW_UNLOCKED; 224 rw->rw_recurse = 0; 225} 226 227void 228_rw_destroy(volatile uintptr_t *c) 229{ 230 struct rwlock *rw; 231 232 rw = rwlock2rw(c); 233 234 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); 235 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); 236 rw->rw_lock = RW_DESTROYED; 237 lock_destroy(&rw->lock_object); 238} 239 240void 241rw_sysinit(void *arg) 242{ 243 struct rw_args *args = arg; 244 245 rw_init((struct rwlock *)args->ra_rw, args->ra_desc); 246} 247 248void 249rw_sysinit_flags(void *arg) 250{ 251 struct rw_args_flags *args = arg; 252 253 rw_init_flags((struct rwlock *)args->ra_rw, args->ra_desc, 254 args->ra_flags); 255} 256 257int 258_rw_wowned(const volatile uintptr_t *c) 259{ 260 261 return (rw_wowner(rwlock2rw(c)) == curthread); 262} 263 264void 265_rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) 266{ 267 struct rwlock *rw; 268 uintptr_t tid, v; 269 270 if (SCHEDULER_STOPPED()) 271 return; 272 273 rw = rwlock2rw(c); 274 275 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 276 ("rw_wlock() by idle thread %p on rwlock %s @ %s:%d", 277 curthread, rw->lock_object.lo_name, file, line)); 278 KASSERT(rw->rw_lock != RW_DESTROYED, 279 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 280 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 281 line, NULL); 282 tid = (uintptr_t)curthread; 283 v = RW_UNLOCKED; 284 if (!_rw_write_lock_fetch(rw, &v, tid)) 285 _rw_wlock_hard(rw, v, tid, file, line); 286 else 287 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, 288 0, 0, file, line, LOCKSTAT_WRITER); 289 290 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 291 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 292 TD_LOCKS_INC(curthread); 293} 294 295int 296__rw_try_wlock(volatile uintptr_t *c, const char *file, int line) 297{ 298 struct rwlock *rw; 299 int rval; 300 301 if (SCHEDULER_STOPPED()) 302 return (1); 303 304 rw = rwlock2rw(c); 305 306 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 307 ("rw_try_wlock() by idle thread %p on rwlock %s @ %s:%d", 308 curthread, rw->lock_object.lo_name, file, line)); 309 KASSERT(rw->rw_lock != RW_DESTROYED, 310 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 311 312 if (rw_wlocked(rw) && 313 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { 314 rw->rw_recurse++; 315 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); 316 rval = 1; 317 } else 318 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, 319 (uintptr_t)curthread); 320 321 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 322 if (rval) { 323 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 324 file, line); 325 if (!rw_recursed(rw)) 326 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, 327 rw, 0, 0, file, line, LOCKSTAT_WRITER); 328 TD_LOCKS_INC(curthread); 329 } 330 return (rval); 331} 332 333void 334_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) 335{ 336 struct rwlock *rw; 337 338 if (SCHEDULER_STOPPED()) 339 return; 340 341 rw = rwlock2rw(c); 342 343 KASSERT(rw->rw_lock != RW_DESTROYED, 344 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 345 __rw_assert(c, RA_WLOCKED, file, line); 346 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 347 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 348 line); 349 350 _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); 351 352 TD_LOCKS_DEC(curthread); 353} 354 355/* 356 * Determines whether a new reader can acquire a lock. Succeeds if the 357 * reader already owns a read lock and the lock is locked for read to 358 * prevent deadlock from reader recursion. Also succeeds if the lock 359 * is unlocked and has no writer waiters or spinners. Failing otherwise 360 * prioritizes writers before readers. 361 */ 362#define RW_CAN_READ(td, _rw) \ 363 (((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) & \ 364 (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) == \ 365 RW_LOCK_READ) 366 367static bool __always_inline 368__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, 369 const char *file, int line) 370{ 371 372 /* 373 * Handle the easy case. If no other thread has a write 374 * lock, then try to bump up the count of read locks. Note 375 * that we have to preserve the current state of the 376 * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 377 * read lock, then rw_lock must have changed, so restart 378 * the loop. Note that this handles the case of a 379 * completely unlocked rwlock since such a lock is encoded 380 * as a read lock with no waiters. 381 */ 382 while (RW_CAN_READ(td, *vp)) { 383 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp, 384 *vp + RW_ONE_READER)) { 385 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 386 CTR4(KTR_LOCK, 387 "%s: %p succeed %p -> %p", __func__, 388 rw, (void *)*vp, 389 (void *)(*vp + RW_ONE_READER)); 390 td->td_rw_rlocks++; 391 return (true); 392 } 393 } 394 return (false); 395} 396 397static void __noinline 398__rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v, 399 const char *file, int line) 400{ 401 struct rwlock *rw; 402 struct turnstile *ts; 403#ifdef ADAPTIVE_RWLOCKS 404 volatile struct thread *owner; 405 int spintries = 0; 406 int i; 407#endif 408#ifdef LOCK_PROFILING 409 uint64_t waittime = 0; 410 int contested = 0; 411#endif 412#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) 413 struct lock_delay_arg lda; 414#endif 415#ifdef KDTRACE_HOOKS 416 uintptr_t state; 417 u_int sleep_cnt = 0; 418 int64_t sleep_time = 0; 419 int64_t all_time = 0; 420#endif 421 422 if (SCHEDULER_STOPPED()) 423 return; 424 425#if defined(ADAPTIVE_RWLOCKS) 426 lock_delay_arg_init(&lda, &rw_delay); 427#elif defined(KDTRACE_HOOKS) 428 lock_delay_arg_init(&lda, NULL); 429#endif 430 rw = rwlock2rw(c); 431 432#ifdef KDTRACE_HOOKS 433 all_time -= lockstat_nsecs(&rw->lock_object); 434#endif 435#ifdef KDTRACE_HOOKS 436 state = v; 437#endif 438 for (;;) { 439 if (__rw_rlock_try(rw, td, &v, file, line)) 440 break; 441#ifdef KDTRACE_HOOKS 442 lda.spin_cnt++; 443#endif 444#ifdef HWPMC_HOOKS 445 PMC_SOFT_CALL( , , lock, failed); 446#endif 447 lock_profile_obtain_lock_failed(&rw->lock_object, 448 &contested, &waittime); 449 450#ifdef ADAPTIVE_RWLOCKS 451 /* 452 * If the owner is running on another CPU, spin until 453 * the owner stops running or the state of the lock 454 * changes. 455 */ 456 if ((v & RW_LOCK_READ) == 0) { 457 owner = (struct thread *)RW_OWNER(v); 458 if (TD_IS_RUNNING(owner)) { 459 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 460 CTR3(KTR_LOCK, 461 "%s: spinning on %p held by %p", 462 __func__, rw, owner); 463 KTR_STATE1(KTR_SCHED, "thread", 464 sched_tdname(curthread), "spinning", 465 "lockname:\"%s\"", rw->lock_object.lo_name); 466 do { 467 lock_delay(&lda); 468 v = RW_READ_VALUE(rw); 469 owner = lv_rw_wowner(v); 470 } while (owner != NULL && TD_IS_RUNNING(owner)); 471 KTR_STATE0(KTR_SCHED, "thread", 472 sched_tdname(curthread), "running"); 473 continue; 474 } 475 } else if (spintries < rowner_retries) { 476 spintries++; 477 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 478 "spinning", "lockname:\"%s\"", 479 rw->lock_object.lo_name); 480 for (i = 0; i < rowner_loops; i++) { 481 v = RW_READ_VALUE(rw); 482 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, v)) 483 break; 484 cpu_spinwait(); 485 } 486 v = RW_READ_VALUE(rw); 487#ifdef KDTRACE_HOOKS 488 lda.spin_cnt += rowner_loops - i; 489#endif 490 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 491 "running"); 492 if (i != rowner_loops) 493 continue; 494 } 495#endif 496 497 /* 498 * Okay, now it's the hard case. Some other thread already 499 * has a write lock or there are write waiters present, 500 * acquire the turnstile lock so we can begin the process 501 * of blocking. 502 */ 503 ts = turnstile_trywait(&rw->lock_object); 504 505 /* 506 * The lock might have been released while we spun, so 507 * recheck its state and restart the loop if needed. 508 */ 509 v = RW_READ_VALUE(rw); 510 if (RW_CAN_READ(td, v)) { 511 turnstile_cancel(ts); 512 continue; 513 } 514 515#ifdef ADAPTIVE_RWLOCKS 516 /* 517 * The current lock owner might have started executing 518 * on another CPU (or the lock could have changed 519 * owners) while we were waiting on the turnstile 520 * chain lock. If so, drop the turnstile lock and try 521 * again. 522 */ 523 if ((v & RW_LOCK_READ) == 0) { 524 owner = (struct thread *)RW_OWNER(v); 525 if (TD_IS_RUNNING(owner)) { 526 turnstile_cancel(ts); 527 continue; 528 } 529 } 530#endif 531 532 /* 533 * The lock is held in write mode or it already has waiters. 534 */ 535 MPASS(!RW_CAN_READ(td, v)); 536 537 /* 538 * If the RW_LOCK_READ_WAITERS flag is already set, then 539 * we can go ahead and block. If it is not set then try 540 * to set it. If we fail to set it drop the turnstile 541 * lock and restart the loop. 542 */ 543 if (!(v & RW_LOCK_READ_WAITERS)) { 544 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 545 v | RW_LOCK_READ_WAITERS)) { 546 turnstile_cancel(ts); 547 v = RW_READ_VALUE(rw); 548 continue; 549 } 550 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 551 CTR2(KTR_LOCK, "%s: %p set read waiters flag", 552 __func__, rw); 553 } 554 555 /* 556 * We were unable to acquire the lock and the read waiters 557 * flag is set, so we must block on the turnstile. 558 */ 559 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 560 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 561 rw); 562#ifdef KDTRACE_HOOKS 563 sleep_time -= lockstat_nsecs(&rw->lock_object); 564#endif 565 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 566#ifdef KDTRACE_HOOKS 567 sleep_time += lockstat_nsecs(&rw->lock_object); 568 sleep_cnt++; 569#endif 570 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 571 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 572 __func__, rw); 573 v = RW_READ_VALUE(rw); 574 } 575#ifdef KDTRACE_HOOKS 576 all_time += lockstat_nsecs(&rw->lock_object); 577 if (sleep_time) 578 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, 579 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 580 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 581 582 /* Record only the loops spinning and not sleeping. */ 583 if (lda.spin_cnt > sleep_cnt) 584 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, 585 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 586 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 587#endif 588 /* 589 * TODO: acquire "owner of record" here. Here be turnstile dragons 590 * however. turnstiles don't like owners changing between calls to 591 * turnstile_wait() currently. 592 */ 593 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, 594 waittime, file, line, LOCKSTAT_READER); 595} 596 597void 598__rw_rlock(volatile uintptr_t *c, const char *file, int line) 599{ 600 struct rwlock *rw; 601 struct thread *td; 602 uintptr_t v; 603 604 td = curthread; 605 rw = rwlock2rw(c); 606 607 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td), 608 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", 609 td, rw->lock_object.lo_name, file, line)); 610 KASSERT(rw->rw_lock != RW_DESTROYED, 611 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 612 KASSERT(rw_wowner(rw) != td, 613 ("rw_rlock: wlock already held for %s @ %s:%d", 614 rw->lock_object.lo_name, file, line)); 615 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 616 617 v = RW_READ_VALUE(rw); 618 if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) || 619 !__rw_rlock_try(rw, td, &v, file, line))) 620 __rw_rlock_hard(c, td, v, file, line); 621 622 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 623 WITNESS_LOCK(&rw->lock_object, 0, file, line); 624 TD_LOCKS_INC(curthread); 625} 626 627int 628__rw_try_rlock(volatile uintptr_t *c, const char *file, int line) 629{ 630 struct rwlock *rw; 631 uintptr_t x; 632 633 if (SCHEDULER_STOPPED()) 634 return (1); 635 636 rw = rwlock2rw(c); 637 638 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 639 ("rw_try_rlock() by idle thread %p on rwlock %s @ %s:%d", 640 curthread, rw->lock_object.lo_name, file, line)); 641 642 for (;;) { 643 x = rw->rw_lock; 644 KASSERT(rw->rw_lock != RW_DESTROYED, 645 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 646 if (!(x & RW_LOCK_READ)) 647 break; 648 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 649 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 650 line); 651 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); 652 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, 653 rw, 0, 0, file, line, LOCKSTAT_READER); 654 TD_LOCKS_INC(curthread); 655 curthread->td_rw_rlocks++; 656 return (1); 657 } 658 } 659 660 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 661 return (0); 662} 663 664static bool __always_inline 665__rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp) 666{ 667 668 for (;;) { 669 /* 670 * See if there is more than one read lock held. If so, 671 * just drop one and return. 672 */ 673 if (RW_READERS(*vp) > 1) { 674 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp, 675 *vp - RW_ONE_READER)) { 676 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 677 CTR4(KTR_LOCK, 678 "%s: %p succeeded %p -> %p", 679 __func__, rw, (void *)*vp, 680 (void *)(*vp - RW_ONE_READER)); 681 td->td_rw_rlocks--; 682 return (true); 683 } 684 continue; 685 } 686 /* 687 * If there aren't any waiters for a write lock, then try 688 * to drop it quickly. 689 */ 690 if (!(*vp & RW_LOCK_WAITERS)) { 691 MPASS((*vp & ~RW_LOCK_WRITE_SPINNER) == 692 RW_READERS_LOCK(1)); 693 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp, 694 RW_UNLOCKED)) { 695 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 696 CTR2(KTR_LOCK, "%s: %p last succeeded", 697 __func__, rw); 698 td->td_rw_rlocks--; 699 return (true); 700 } 701 continue; 702 } 703 break; 704 } 705 return (false); 706} 707 708static void __noinline 709__rw_runlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v, 710 const char *file, int line) 711{ 712 struct rwlock *rw; 713 struct turnstile *ts; 714 uintptr_t x, queue; 715 716 if (SCHEDULER_STOPPED()) 717 return; 718 719 rw = rwlock2rw(c); 720 721 for (;;) { 722 if (__rw_runlock_try(rw, td, &v)) 723 break; 724 725 /* 726 * Ok, we know we have waiters and we think we are the 727 * last reader, so grab the turnstile lock. 728 */ 729 turnstile_chain_lock(&rw->lock_object); 730 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 731 MPASS(v & RW_LOCK_WAITERS); 732 733 /* 734 * Try to drop our lock leaving the lock in a unlocked 735 * state. 736 * 737 * If you wanted to do explicit lock handoff you'd have to 738 * do it here. You'd also want to use turnstile_signal() 739 * and you'd have to handle the race where a higher 740 * priority thread blocks on the write lock before the 741 * thread you wakeup actually runs and have the new thread 742 * "steal" the lock. For now it's a lot simpler to just 743 * wakeup all of the waiters. 744 * 745 * As above, if we fail, then another thread might have 746 * acquired a read lock, so drop the turnstile lock and 747 * restart. 748 */ 749 x = RW_UNLOCKED; 750 if (v & RW_LOCK_WRITE_WAITERS) { 751 queue = TS_EXCLUSIVE_QUEUE; 752 x |= (v & RW_LOCK_READ_WAITERS); 753 } else 754 queue = TS_SHARED_QUEUE; 755 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, 756 x)) { 757 turnstile_chain_unlock(&rw->lock_object); 758 x = RW_READ_VALUE(rw); 759 continue; 760 } 761 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 762 CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 763 __func__, rw); 764 765 /* 766 * Ok. The lock is released and all that's left is to 767 * wake up the waiters. Note that the lock might not be 768 * free anymore, but in that case the writers will just 769 * block again if they run before the new lock holder(s) 770 * release the lock. 771 */ 772 ts = turnstile_lookup(&rw->lock_object); 773 MPASS(ts != NULL); 774 turnstile_broadcast(ts, queue); 775 turnstile_unpend(ts, TS_SHARED_LOCK); 776 turnstile_chain_unlock(&rw->lock_object); 777 td->td_rw_rlocks--; 778 break; 779 } 780 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER); 781} 782 783void 784_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) 785{ 786 struct rwlock *rw; 787 struct thread *td; 788 uintptr_t v; 789 790 rw = rwlock2rw(c); 791 792 KASSERT(rw->rw_lock != RW_DESTROYED, 793 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 794 __rw_assert(c, RA_RLOCKED, file, line); 795 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 796 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 797 798 td = curthread; 799 v = RW_READ_VALUE(rw); 800 801 if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__release) || 802 !__rw_runlock_try(rw, td, &v))) 803 __rw_runlock_hard(c, td, v, file, line); 804 805 TD_LOCKS_DEC(curthread); 806} 807 808 809/* 810 * This function is called when we are unable to obtain a write lock on the 811 * first try. This means that at least one other thread holds either a 812 * read or write lock. 813 */ 814void 815__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, 816 const char *file, int line) 817{ 818 struct rwlock *rw; 819 struct turnstile *ts; 820#ifdef ADAPTIVE_RWLOCKS 821 volatile struct thread *owner; 822 int spintries = 0; 823 int i; 824#endif 825 uintptr_t x; 826#ifdef LOCK_PROFILING 827 uint64_t waittime = 0; 828 int contested = 0; 829#endif 830#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) 831 struct lock_delay_arg lda; 832#endif 833#ifdef KDTRACE_HOOKS 834 uintptr_t state; 835 u_int sleep_cnt = 0; 836 int64_t sleep_time = 0; 837 int64_t all_time = 0; 838#endif 839 840 if (SCHEDULER_STOPPED()) 841 return; 842 843#if defined(ADAPTIVE_RWLOCKS) 844 lock_delay_arg_init(&lda, &rw_delay); 845#elif defined(KDTRACE_HOOKS) 846 lock_delay_arg_init(&lda, NULL); 847#endif 848 rw = rwlock2rw(c); 849 if (__predict_false(v == RW_UNLOCKED)) 850 v = RW_READ_VALUE(rw); 851 852 if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) { 853 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 854 ("%s: recursing but non-recursive rw %s @ %s:%d\n", 855 __func__, rw->lock_object.lo_name, file, line)); 856 rw->rw_recurse++; 857 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); 858 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 859 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 860 return; 861 } 862 863 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 864 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 865 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 866 867#ifdef KDTRACE_HOOKS 868 all_time -= lockstat_nsecs(&rw->lock_object); 869 state = v; 870#endif 871 for (;;) { 872 if (v == RW_UNLOCKED) { 873 if (_rw_write_lock_fetch(rw, &v, tid)) 874 break; 875 continue; 876 } 877#ifdef KDTRACE_HOOKS 878 lda.spin_cnt++; 879#endif 880#ifdef HWPMC_HOOKS 881 PMC_SOFT_CALL( , , lock, failed); 882#endif 883 lock_profile_obtain_lock_failed(&rw->lock_object, 884 &contested, &waittime); 885#ifdef ADAPTIVE_RWLOCKS 886 /* 887 * If the lock is write locked and the owner is 888 * running on another CPU, spin until the owner stops 889 * running or the state of the lock changes. 890 */ 891 owner = lv_rw_wowner(v); 892 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 893 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 894 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 895 __func__, rw, owner); 896 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 897 "spinning", "lockname:\"%s\"", 898 rw->lock_object.lo_name); 899 do { 900 lock_delay(&lda); 901 v = RW_READ_VALUE(rw); 902 owner = lv_rw_wowner(v); 903 } while (owner != NULL && TD_IS_RUNNING(owner)); 904 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 905 "running"); 906 continue; 907 } 908 if ((v & RW_LOCK_READ) && RW_READERS(v) && 909 spintries < rowner_retries) { 910 if (!(v & RW_LOCK_WRITE_SPINNER)) { 911 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 912 v | RW_LOCK_WRITE_SPINNER)) { 913 v = RW_READ_VALUE(rw); 914 continue; 915 } 916 } 917 spintries++; 918 KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), 919 "spinning", "lockname:\"%s\"", 920 rw->lock_object.lo_name); 921 for (i = 0; i < rowner_loops; i++) { 922 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0) 923 break; 924 cpu_spinwait(); 925 } 926 KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), 927 "running"); 928 v = RW_READ_VALUE(rw); 929#ifdef KDTRACE_HOOKS 930 lda.spin_cnt += rowner_loops - i; 931#endif 932 if (i != rowner_loops) 933 continue; 934 } 935#endif 936 ts = turnstile_trywait(&rw->lock_object); 937 v = RW_READ_VALUE(rw); 938 939#ifdef ADAPTIVE_RWLOCKS 940 /* 941 * The current lock owner might have started executing 942 * on another CPU (or the lock could have changed 943 * owners) while we were waiting on the turnstile 944 * chain lock. If so, drop the turnstile lock and try 945 * again. 946 */ 947 if (!(v & RW_LOCK_READ)) { 948 owner = (struct thread *)RW_OWNER(v); 949 if (TD_IS_RUNNING(owner)) { 950 turnstile_cancel(ts); 951 continue; 952 } 953 } 954#endif 955 /* 956 * Check for the waiters flags about this rwlock. 957 * If the lock was released, without maintain any pending 958 * waiters queue, simply try to acquire it. 959 * If a pending waiters queue is present, claim the lock 960 * ownership and maintain the pending queue. 961 */ 962 x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 963 if ((v & ~x) == RW_UNLOCKED) { 964 x &= ~RW_LOCK_WRITE_SPINNER; 965 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) { 966 if (x) 967 turnstile_claim(ts); 968 else 969 turnstile_cancel(ts); 970 break; 971 } 972 turnstile_cancel(ts); 973 v = RW_READ_VALUE(rw); 974 continue; 975 } 976 /* 977 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 978 * set it. If we fail to set it, then loop back and try 979 * again. 980 */ 981 if (!(v & RW_LOCK_WRITE_WAITERS)) { 982 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 983 v | RW_LOCK_WRITE_WAITERS)) { 984 turnstile_cancel(ts); 985 v = RW_READ_VALUE(rw); 986 continue; 987 } 988 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 989 CTR2(KTR_LOCK, "%s: %p set write waiters flag", 990 __func__, rw); 991 } 992 /* 993 * We were unable to acquire the lock and the write waiters 994 * flag is set, so we must block on the turnstile. 995 */ 996 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 997 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 998 rw); 999#ifdef KDTRACE_HOOKS 1000 sleep_time -= lockstat_nsecs(&rw->lock_object); 1001#endif 1002 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 1003#ifdef KDTRACE_HOOKS 1004 sleep_time += lockstat_nsecs(&rw->lock_object); 1005 sleep_cnt++; 1006#endif 1007 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1008 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 1009 __func__, rw); 1010#ifdef ADAPTIVE_RWLOCKS 1011 spintries = 0; 1012#endif 1013 v = RW_READ_VALUE(rw); 1014 } 1015#ifdef KDTRACE_HOOKS 1016 all_time += lockstat_nsecs(&rw->lock_object); 1017 if (sleep_time) 1018 LOCKSTAT_RECORD4(rw__block, rw, sleep_time, 1019 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 1020 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 1021 1022 /* Record only the loops spinning and not sleeping. */ 1023 if (lda.spin_cnt > sleep_cnt) 1024 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, 1025 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 1026 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 1027#endif 1028 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested, 1029 waittime, file, line, LOCKSTAT_WRITER); 1030} 1031 1032/* 1033 * This function is called if lockstat is active or the first try at releasing 1034 * a write lock failed. The latter means that the lock is recursed or one of 1035 * the 2 waiter bits must be set indicating that at least one thread is waiting 1036 * on this lock. 1037 */ 1038void 1039__rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, 1040 int line) 1041{ 1042 struct rwlock *rw; 1043 struct turnstile *ts; 1044 uintptr_t v; 1045 int queue; 1046 1047 if (SCHEDULER_STOPPED()) 1048 return; 1049 1050 rw = rwlock2rw(c); 1051 v = RW_READ_VALUE(rw); 1052 if (v & RW_LOCK_WRITER_RECURSED) { 1053 if (--(rw->rw_recurse) == 0) 1054 atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); 1055 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1056 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 1057 return; 1058 } 1059 1060 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_WRITER); 1061 if (v == tid && _rw_write_unlock(rw, tid)) 1062 return; 1063 1064 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 1065 ("%s: neither of the waiter flags are set", __func__)); 1066 1067 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1068 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 1069 1070 turnstile_chain_lock(&rw->lock_object); 1071 ts = turnstile_lookup(&rw->lock_object); 1072 MPASS(ts != NULL); 1073 1074 /* 1075 * Use the same algo as sx locks for now. Prefer waking up shared 1076 * waiters if we have any over writers. This is probably not ideal. 1077 * 1078 * 'v' is the value we are going to write back to rw_lock. If we 1079 * have waiters on both queues, we need to preserve the state of 1080 * the waiter flag for the queue we don't wake up. For now this is 1081 * hardcoded for the algorithm mentioned above. 1082 * 1083 * In the case of both readers and writers waiting we wakeup the 1084 * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 1085 * new writer comes in before a reader it will claim the lock up 1086 * above. There is probably a potential priority inversion in 1087 * there that could be worked around either by waking both queues 1088 * of waiters or doing some complicated lock handoff gymnastics. 1089 */ 1090 v = RW_UNLOCKED; 1091 if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) { 1092 queue = TS_EXCLUSIVE_QUEUE; 1093 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS); 1094 } else 1095 queue = TS_SHARED_QUEUE; 1096 1097 /* Wake up all waiters for the specific queue. */ 1098 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 1099 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 1100 queue == TS_SHARED_QUEUE ? "read" : "write"); 1101 turnstile_broadcast(ts, queue); 1102 atomic_store_rel_ptr(&rw->rw_lock, v); 1103 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1104 turnstile_chain_unlock(&rw->lock_object); 1105} 1106 1107/* 1108 * Attempt to do a non-blocking upgrade from a read lock to a write 1109 * lock. This will only succeed if this thread holds a single read 1110 * lock. Returns true if the upgrade succeeded and false otherwise. 1111 */ 1112int 1113__rw_try_upgrade(volatile uintptr_t *c, const char *file, int line) 1114{ 1115 struct rwlock *rw; 1116 uintptr_t v, x, tid; 1117 struct turnstile *ts; 1118 int success; 1119 1120 if (SCHEDULER_STOPPED()) 1121 return (1); 1122 1123 rw = rwlock2rw(c); 1124 1125 KASSERT(rw->rw_lock != RW_DESTROYED, 1126 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 1127 __rw_assert(c, RA_RLOCKED, file, line); 1128 1129 /* 1130 * Attempt to switch from one reader to a writer. If there 1131 * are any write waiters, then we will have to lock the 1132 * turnstile first to prevent races with another writer 1133 * calling turnstile_wait() before we have claimed this 1134 * turnstile. So, do the simple case of no waiters first. 1135 */ 1136 tid = (uintptr_t)curthread; 1137 success = 0; 1138 for (;;) { 1139 v = rw->rw_lock; 1140 if (RW_READERS(v) > 1) 1141 break; 1142 if (!(v & RW_LOCK_WAITERS)) { 1143 success = atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid); 1144 if (!success) 1145 continue; 1146 break; 1147 } 1148 1149 /* 1150 * Ok, we think we have waiters, so lock the turnstile. 1151 */ 1152 ts = turnstile_trywait(&rw->lock_object); 1153 v = rw->rw_lock; 1154 if (RW_READERS(v) > 1) { 1155 turnstile_cancel(ts); 1156 break; 1157 } 1158 /* 1159 * Try to switch from one reader to a writer again. This time 1160 * we honor the current state of the waiters flags. 1161 * If we obtain the lock with the flags set, then claim 1162 * ownership of the turnstile. 1163 */ 1164 x = rw->rw_lock & RW_LOCK_WAITERS; 1165 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x); 1166 if (success) { 1167 if (x) 1168 turnstile_claim(ts); 1169 else 1170 turnstile_cancel(ts); 1171 break; 1172 } 1173 turnstile_cancel(ts); 1174 } 1175 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); 1176 if (success) { 1177 curthread->td_rw_rlocks--; 1178 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 1179 file, line); 1180 LOCKSTAT_RECORD0(rw__upgrade, rw); 1181 } 1182 return (success); 1183} 1184 1185/* 1186 * Downgrade a write lock into a single read lock. 1187 */ 1188void 1189__rw_downgrade(volatile uintptr_t *c, const char *file, int line) 1190{ 1191 struct rwlock *rw; 1192 struct turnstile *ts; 1193 uintptr_t tid, v; 1194 int rwait, wwait; 1195 1196 if (SCHEDULER_STOPPED()) 1197 return; 1198 1199 rw = rwlock2rw(c); 1200 1201 KASSERT(rw->rw_lock != RW_DESTROYED, 1202 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 1203 __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line); 1204#ifndef INVARIANTS 1205 if (rw_recursed(rw)) 1206 panic("downgrade of a recursed lock"); 1207#endif 1208 1209 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); 1210 1211 /* 1212 * Convert from a writer to a single reader. First we handle 1213 * the easy case with no waiters. If there are any waiters, we 1214 * lock the turnstile and "disown" the lock. 1215 */ 1216 tid = (uintptr_t)curthread; 1217 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) 1218 goto out; 1219 1220 /* 1221 * Ok, we think we have waiters, so lock the turnstile so we can 1222 * read the waiter flags without any races. 1223 */ 1224 turnstile_chain_lock(&rw->lock_object); 1225 v = rw->rw_lock & RW_LOCK_WAITERS; 1226 rwait = v & RW_LOCK_READ_WAITERS; 1227 wwait = v & RW_LOCK_WRITE_WAITERS; 1228 MPASS(rwait | wwait); 1229 1230 /* 1231 * Downgrade from a write lock while preserving waiters flag 1232 * and give up ownership of the turnstile. 1233 */ 1234 ts = turnstile_lookup(&rw->lock_object); 1235 MPASS(ts != NULL); 1236 if (!wwait) 1237 v &= ~RW_LOCK_READ_WAITERS; 1238 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); 1239 /* 1240 * Wake other readers if there are no writers pending. Otherwise they 1241 * won't be able to acquire the lock anyway. 1242 */ 1243 if (rwait && !wwait) { 1244 turnstile_broadcast(ts, TS_SHARED_QUEUE); 1245 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1246 } else 1247 turnstile_disown(ts); 1248 turnstile_chain_unlock(&rw->lock_object); 1249out: 1250 curthread->td_rw_rlocks++; 1251 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); 1252 LOCKSTAT_RECORD0(rw__downgrade, rw); 1253} 1254 1255#ifdef INVARIANT_SUPPORT 1256#ifndef INVARIANTS 1257#undef __rw_assert 1258#endif 1259 1260/* 1261 * In the non-WITNESS case, rw_assert() can only detect that at least 1262 * *some* thread owns an rlock, but it cannot guarantee that *this* 1263 * thread owns an rlock. 1264 */ 1265void 1266__rw_assert(const volatile uintptr_t *c, int what, const char *file, int line) 1267{ 1268 const struct rwlock *rw; 1269 1270 if (panicstr != NULL) 1271 return; 1272 1273 rw = rwlock2rw(c); 1274 1275 switch (what) { 1276 case RA_LOCKED: 1277 case RA_LOCKED | RA_RECURSED: 1278 case RA_LOCKED | RA_NOTRECURSED: 1279 case RA_RLOCKED: 1280 case RA_RLOCKED | RA_RECURSED: 1281 case RA_RLOCKED | RA_NOTRECURSED: 1282#ifdef WITNESS 1283 witness_assert(&rw->lock_object, what, file, line); 1284#else 1285 /* 1286 * If some other thread has a write lock or we have one 1287 * and are asserting a read lock, fail. Also, if no one 1288 * has a lock at all, fail. 1289 */ 1290 if (rw->rw_lock == RW_UNLOCKED || 1291 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED || 1292 rw_wowner(rw) != curthread))) 1293 panic("Lock %s not %slocked @ %s:%d\n", 1294 rw->lock_object.lo_name, (what & RA_RLOCKED) ? 1295 "read " : "", file, line); 1296 1297 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) { 1298 if (rw_recursed(rw)) { 1299 if (what & RA_NOTRECURSED) 1300 panic("Lock %s recursed @ %s:%d\n", 1301 rw->lock_object.lo_name, file, 1302 line); 1303 } else if (what & RA_RECURSED) 1304 panic("Lock %s not recursed @ %s:%d\n", 1305 rw->lock_object.lo_name, file, line); 1306 } 1307#endif 1308 break; 1309 case RA_WLOCKED: 1310 case RA_WLOCKED | RA_RECURSED: 1311 case RA_WLOCKED | RA_NOTRECURSED: 1312 if (rw_wowner(rw) != curthread) 1313 panic("Lock %s not exclusively locked @ %s:%d\n", 1314 rw->lock_object.lo_name, file, line); 1315 if (rw_recursed(rw)) { 1316 if (what & RA_NOTRECURSED) 1317 panic("Lock %s recursed @ %s:%d\n", 1318 rw->lock_object.lo_name, file, line); 1319 } else if (what & RA_RECURSED) 1320 panic("Lock %s not recursed @ %s:%d\n", 1321 rw->lock_object.lo_name, file, line); 1322 break; 1323 case RA_UNLOCKED: 1324#ifdef WITNESS 1325 witness_assert(&rw->lock_object, what, file, line); 1326#else 1327 /* 1328 * If we hold a write lock fail. We can't reliably check 1329 * to see if we hold a read lock or not. 1330 */ 1331 if (rw_wowner(rw) == curthread) 1332 panic("Lock %s exclusively locked @ %s:%d\n", 1333 rw->lock_object.lo_name, file, line); 1334#endif 1335 break; 1336 default: 1337 panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 1338 line); 1339 } 1340} 1341#endif /* INVARIANT_SUPPORT */ 1342 1343#ifdef DDB 1344void 1345db_show_rwlock(const struct lock_object *lock) 1346{ 1347 const struct rwlock *rw; 1348 struct thread *td; 1349 1350 rw = (const struct rwlock *)lock; 1351 1352 db_printf(" state: "); 1353 if (rw->rw_lock == RW_UNLOCKED) 1354 db_printf("UNLOCKED\n"); 1355 else if (rw->rw_lock == RW_DESTROYED) { 1356 db_printf("DESTROYED\n"); 1357 return; 1358 } else if (rw->rw_lock & RW_LOCK_READ) 1359 db_printf("RLOCK: %ju locks\n", 1360 (uintmax_t)(RW_READERS(rw->rw_lock))); 1361 else { 1362 td = rw_wowner(rw); 1363 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1364 td->td_tid, td->td_proc->p_pid, td->td_name); 1365 if (rw_recursed(rw)) 1366 db_printf(" recursed: %u\n", rw->rw_recurse); 1367 } 1368 db_printf(" waiters: "); 1369 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 1370 case RW_LOCK_READ_WAITERS: 1371 db_printf("readers\n"); 1372 break; 1373 case RW_LOCK_WRITE_WAITERS: 1374 db_printf("writers\n"); 1375 break; 1376 case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 1377 db_printf("readers and writers\n"); 1378 break; 1379 default: 1380 db_printf("none\n"); 1381 break; 1382 } 1383} 1384 1385#endif 1386