kern_rwlock.c revision 233628
1154941Sjhb/*- 2154941Sjhb * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3154941Sjhb * All rights reserved. 4154941Sjhb * 5154941Sjhb * Redistribution and use in source and binary forms, with or without 6154941Sjhb * modification, are permitted provided that the following conditions 7154941Sjhb * are met: 8154941Sjhb * 1. Redistributions of source code must retain the above copyright 9154941Sjhb * notice, this list of conditions and the following disclaimer. 10154941Sjhb * 2. Redistributions in binary form must reproduce the above copyright 11154941Sjhb * notice, this list of conditions and the following disclaimer in the 12154941Sjhb * documentation and/or other materials provided with the distribution. 13154941Sjhb * 3. Neither the name of the author nor the names of any co-contributors 14154941Sjhb * may be used to endorse or promote products derived from this software 15154941Sjhb * without specific prior written permission. 16154941Sjhb * 17154941Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18154941Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19154941Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20154941Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21154941Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22154941Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23154941Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24154941Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25154941Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26154941Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27154941Sjhb * SUCH DAMAGE. 28154941Sjhb */ 29154941Sjhb 30154941Sjhb/* 31154941Sjhb * Machine independent bits of reader/writer lock implementation. 32154941Sjhb */ 33154941Sjhb 34154941Sjhb#include <sys/cdefs.h> 35154941Sjhb__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 233628 2012-03-28 20:58:30Z fabient $"); 36154941Sjhb 37154941Sjhb#include "opt_ddb.h" 38233628Sfabient#include "opt_hwpmc_hooks.h" 39192853Ssson#include "opt_kdtrace.h" 40167801Sjhb#include "opt_no_adaptive_rwlocks.h" 41154941Sjhb 42154941Sjhb#include <sys/param.h> 43154941Sjhb#include <sys/ktr.h> 44177912Sjeff#include <sys/kernel.h> 45154941Sjhb#include <sys/lock.h> 46154941Sjhb#include <sys/mutex.h> 47154941Sjhb#include <sys/proc.h> 48154941Sjhb#include <sys/rwlock.h> 49177912Sjeff#include <sys/sysctl.h> 50154941Sjhb#include <sys/systm.h> 51154941Sjhb#include <sys/turnstile.h> 52171516Sattilio 53154941Sjhb#include <machine/cpu.h> 54154941Sjhb 55167801Sjhb#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS) 56167801Sjhb#define ADAPTIVE_RWLOCKS 57167801Sjhb#endif 58167801Sjhb 59233628Sfabient#ifdef HWPMC_HOOKS 60233628Sfabient#include <sys/pmckern.h> 61233628SfabientPMC_SOFT_DECLARE( , , lock, failed); 62233628Sfabient#endif 63233628Sfabient 64177912Sjeff#ifdef ADAPTIVE_RWLOCKS 65177912Sjeffstatic int rowner_retries = 10; 66177912Sjeffstatic int rowner_loops = 10000; 67227309Sedstatic SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, 68227309Sed "rwlock debugging"); 69177912SjeffSYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); 70177912SjeffSYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); 71177912Sjeff#endif 72177912Sjeff 73154941Sjhb#ifdef DDB 74154941Sjhb#include <ddb/ddb.h> 75154941Sjhb 76227588Spjdstatic void db_show_rwlock(const struct lock_object *lock); 77154941Sjhb#endif 78227588Spjdstatic void assert_rw(const struct lock_object *lock, int what); 79167368Sjhbstatic void lock_rw(struct lock_object *lock, int how); 80192853Ssson#ifdef KDTRACE_HOOKS 81227588Spjdstatic int owner_rw(const struct lock_object *lock, struct thread **owner); 82192853Ssson#endif 83167368Sjhbstatic int unlock_rw(struct lock_object *lock); 84154941Sjhb 85154941Sjhbstruct lock_class lock_class_rw = { 86167365Sjhb .lc_name = "rw", 87167365Sjhb .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, 88173733Sattilio .lc_assert = assert_rw, 89154941Sjhb#ifdef DDB 90167365Sjhb .lc_ddb_show = db_show_rwlock, 91154941Sjhb#endif 92167368Sjhb .lc_lock = lock_rw, 93167368Sjhb .lc_unlock = unlock_rw, 94192853Ssson#ifdef KDTRACE_HOOKS 95192853Ssson .lc_owner = owner_rw, 96192853Ssson#endif 97154941Sjhb}; 98154941Sjhb 99157826Sjhb/* 100157826Sjhb * Return a pointer to the owning thread if the lock is write-locked or 101157826Sjhb * NULL if the lock is unlocked or read-locked. 102157826Sjhb */ 103157826Sjhb#define rw_wowner(rw) \ 104154941Sjhb ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ 105154941Sjhb (struct thread *)RW_OWNER((rw)->rw_lock)) 106154941Sjhb 107157826Sjhb/* 108171052Sattilio * Returns if a write owner is recursed. Write ownership is not assured 109171052Sattilio * here and should be previously checked. 110171052Sattilio */ 111171052Sattilio#define rw_recursed(rw) ((rw)->rw_recurse != 0) 112171052Sattilio 113171052Sattilio/* 114171052Sattilio * Return true if curthread helds the lock. 115171052Sattilio */ 116171052Sattilio#define rw_wlocked(rw) (rw_wowner((rw)) == curthread) 117171052Sattilio 118171052Sattilio/* 119157826Sjhb * Return a pointer to the owning thread for this lock who should receive 120157826Sjhb * any priority lent by threads that block on this lock. Currently this 121157826Sjhb * is identical to rw_wowner(). 122157826Sjhb */ 123157826Sjhb#define rw_owner(rw) rw_wowner(rw) 124157826Sjhb 125154941Sjhb#ifndef INVARIANTS 126154941Sjhb#define _rw_assert(rw, what, file, line) 127154941Sjhb#endif 128154941Sjhb 129154941Sjhbvoid 130227588Spjdassert_rw(const struct lock_object *lock, int what) 131173733Sattilio{ 132173733Sattilio 133227588Spjd rw_assert((const struct rwlock *)lock, what); 134173733Sattilio} 135173733Sattilio 136173733Sattiliovoid 137167368Sjhblock_rw(struct lock_object *lock, int how) 138167368Sjhb{ 139167368Sjhb struct rwlock *rw; 140167368Sjhb 141167368Sjhb rw = (struct rwlock *)lock; 142167368Sjhb if (how) 143167368Sjhb rw_wlock(rw); 144167368Sjhb else 145167368Sjhb rw_rlock(rw); 146167368Sjhb} 147167368Sjhb 148167368Sjhbint 149167368Sjhbunlock_rw(struct lock_object *lock) 150167368Sjhb{ 151167368Sjhb struct rwlock *rw; 152167368Sjhb 153167368Sjhb rw = (struct rwlock *)lock; 154167368Sjhb rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); 155167368Sjhb if (rw->rw_lock & RW_LOCK_READ) { 156167368Sjhb rw_runlock(rw); 157167368Sjhb return (0); 158167368Sjhb } else { 159167368Sjhb rw_wunlock(rw); 160167368Sjhb return (1); 161167368Sjhb } 162167368Sjhb} 163167368Sjhb 164192853Ssson#ifdef KDTRACE_HOOKS 165192853Sssonint 166227588Spjdowner_rw(const struct lock_object *lock, struct thread **owner) 167192853Ssson{ 168227588Spjd const struct rwlock *rw = (const struct rwlock *)lock; 169192853Ssson uintptr_t x = rw->rw_lock; 170192853Ssson 171192853Ssson *owner = rw_wowner(rw); 172192853Ssson return ((x & RW_LOCK_READ) != 0 ? (RW_READERS(x) != 0) : 173192853Ssson (*owner != NULL)); 174192853Ssson} 175192853Ssson#endif 176192853Ssson 177167368Sjhbvoid 178171052Sattiliorw_init_flags(struct rwlock *rw, const char *name, int opts) 179154941Sjhb{ 180171052Sattilio int flags; 181154941Sjhb 182171052Sattilio MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET | 183171052Sattilio RW_RECURSE)) == 0); 184196334Sattilio ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, 185196334Sattilio ("%s: rw_lock not aligned for %s: %p", __func__, name, 186196334Sattilio &rw->rw_lock)); 187171052Sattilio 188193307Sattilio flags = LO_UPGRADABLE; 189171052Sattilio if (opts & RW_DUPOK) 190171052Sattilio flags |= LO_DUPOK; 191171052Sattilio if (opts & RW_NOPROFILE) 192171052Sattilio flags |= LO_NOPROFILE; 193171052Sattilio if (!(opts & RW_NOWITNESS)) 194171052Sattilio flags |= LO_WITNESS; 195193307Sattilio if (opts & RW_RECURSE) 196193307Sattilio flags |= LO_RECURSABLE; 197171052Sattilio if (opts & RW_QUIET) 198171052Sattilio flags |= LO_QUIET; 199171052Sattilio 200154941Sjhb rw->rw_lock = RW_UNLOCKED; 201171052Sattilio rw->rw_recurse = 0; 202171052Sattilio lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); 203154941Sjhb} 204154941Sjhb 205154941Sjhbvoid 206154941Sjhbrw_destroy(struct rwlock *rw) 207154941Sjhb{ 208154941Sjhb 209205626Sbz KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); 210205626Sbz KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); 211169394Sjhb rw->rw_lock = RW_DESTROYED; 212167787Sjhb lock_destroy(&rw->lock_object); 213154941Sjhb} 214154941Sjhb 215154941Sjhbvoid 216154941Sjhbrw_sysinit(void *arg) 217154941Sjhb{ 218154941Sjhb struct rw_args *args = arg; 219154941Sjhb 220154941Sjhb rw_init(args->ra_rw, args->ra_desc); 221154941Sjhb} 222154941Sjhb 223185778Skmacyvoid 224185778Skmacyrw_sysinit_flags(void *arg) 225185778Skmacy{ 226185778Skmacy struct rw_args_flags *args = arg; 227185778Skmacy 228185778Skmacy rw_init_flags(args->ra_rw, args->ra_desc, args->ra_flags); 229185778Skmacy} 230185778Skmacy 231167024Srwatsonint 232227588Spjdrw_wowned(const struct rwlock *rw) 233167024Srwatson{ 234167024Srwatson 235167024Srwatson return (rw_wowner(rw) == curthread); 236167024Srwatson} 237167024Srwatson 238154941Sjhbvoid 239154941Sjhb_rw_wlock(struct rwlock *rw, const char *file, int line) 240154941Sjhb{ 241154941Sjhb 242228424Savg if (SCHEDULER_STOPPED()) 243228424Savg return; 244154941Sjhb MPASS(curthread != NULL); 245169394Sjhb KASSERT(rw->rw_lock != RW_DESTROYED, 246169394Sjhb ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); 247167787Sjhb WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 248182914Sjhb line, NULL); 249154941Sjhb __rw_wlock(rw, curthread, file, line); 250171052Sattilio LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); 251167787Sjhb WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 252160771Sjhb curthread->td_locks++; 253154941Sjhb} 254154941Sjhb 255177843Sattilioint 256177843Sattilio_rw_try_wlock(struct rwlock *rw, const char *file, int line) 257177843Sattilio{ 258177843Sattilio int rval; 259177843Sattilio 260228424Savg if (SCHEDULER_STOPPED()) 261228424Savg return (1); 262228424Savg 263177843Sattilio KASSERT(rw->rw_lock != RW_DESTROYED, 264177843Sattilio ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line)); 265177843Sattilio 266193307Sattilio if (rw_wlocked(rw) && 267193307Sattilio (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { 268177843Sattilio rw->rw_recurse++; 269177843Sattilio rval = 1; 270177843Sattilio } else 271177843Sattilio rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, 272177843Sattilio (uintptr_t)curthread); 273177843Sattilio 274177843Sattilio LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 275177843Sattilio if (rval) { 276177843Sattilio WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 277177843Sattilio file, line); 278177843Sattilio curthread->td_locks++; 279177843Sattilio } 280177843Sattilio return (rval); 281177843Sattilio} 282177843Sattilio 283154941Sjhbvoid 284154941Sjhb_rw_wunlock(struct rwlock *rw, const char *file, int line) 285154941Sjhb{ 286154941Sjhb 287228424Savg if (SCHEDULER_STOPPED()) 288228424Savg return; 289154941Sjhb MPASS(curthread != NULL); 290169394Sjhb KASSERT(rw->rw_lock != RW_DESTROYED, 291169394Sjhb ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line)); 292154941Sjhb _rw_assert(rw, RA_WLOCKED, file, line); 293160771Sjhb curthread->td_locks--; 294167787Sjhb WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); 295171052Sattilio LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, 296171052Sattilio line); 297171052Sattilio if (!rw_recursed(rw)) 298192853Ssson LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_WUNLOCK_RELEASE, rw); 299154941Sjhb __rw_wunlock(rw, curthread, file, line); 300154941Sjhb} 301176017Sjeff/* 302176017Sjeff * Determines whether a new reader can acquire a lock. Succeeds if the 303176017Sjeff * reader already owns a read lock and the lock is locked for read to 304176017Sjeff * prevent deadlock from reader recursion. Also succeeds if the lock 305176017Sjeff * is unlocked and has no writer waiters or spinners. Failing otherwise 306176017Sjeff * prioritizes writers before readers. 307176017Sjeff */ 308176017Sjeff#define RW_CAN_READ(_rw) \ 309176017Sjeff ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) & \ 310176017Sjeff (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) == \ 311176017Sjeff RW_LOCK_READ) 312154941Sjhb 313154941Sjhbvoid 314154941Sjhb_rw_rlock(struct rwlock *rw, const char *file, int line) 315154941Sjhb{ 316170295Sjeff struct turnstile *ts; 317167801Sjhb#ifdef ADAPTIVE_RWLOCKS 318157846Sjhb volatile struct thread *owner; 319177912Sjeff int spintries = 0; 320177912Sjeff int i; 321157851Swkoszek#endif 322189846Sjeff#ifdef LOCK_PROFILING 323167307Sjhb uint64_t waittime = 0; 324167054Skmacy int contested = 0; 325189846Sjeff#endif 326176017Sjeff uintptr_t v; 327192853Ssson#ifdef KDTRACE_HOOKS 328192853Ssson uint64_t spin_cnt = 0; 329192853Ssson uint64_t sleep_cnt = 0; 330192853Ssson int64_t sleep_time = 0; 331192853Ssson#endif 332154941Sjhb 333228424Savg if (SCHEDULER_STOPPED()) 334228424Savg return; 335228424Savg 336169394Sjhb KASSERT(rw->rw_lock != RW_DESTROYED, 337169394Sjhb ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 338157826Sjhb KASSERT(rw_wowner(rw) != curthread, 339154941Sjhb ("%s (%s): wlock already held @ %s:%d", __func__, 340167787Sjhb rw->lock_object.lo_name, file, line)); 341182914Sjhb WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 342154941Sjhb 343154941Sjhb for (;;) { 344192853Ssson#ifdef KDTRACE_HOOKS 345192853Ssson spin_cnt++; 346192853Ssson#endif 347154941Sjhb /* 348154941Sjhb * Handle the easy case. If no other thread has a write 349154941Sjhb * lock, then try to bump up the count of read locks. Note 350154941Sjhb * that we have to preserve the current state of the 351154941Sjhb * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 352154941Sjhb * read lock, then rw_lock must have changed, so restart 353154941Sjhb * the loop. Note that this handles the case of a 354154941Sjhb * completely unlocked rwlock since such a lock is encoded 355154941Sjhb * as a read lock with no waiters. 356154941Sjhb */ 357176017Sjeff v = rw->rw_lock; 358176017Sjeff if (RW_CAN_READ(v)) { 359154941Sjhb /* 360154941Sjhb * The RW_LOCK_READ_WAITERS flag should only be set 361176017Sjeff * if the lock has been unlocked and write waiters 362176017Sjeff * were present. 363154941Sjhb */ 364176017Sjeff if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, 365176017Sjeff v + RW_ONE_READER)) { 366167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 367154941Sjhb CTR4(KTR_LOCK, 368154941Sjhb "%s: %p succeed %p -> %p", __func__, 369176017Sjeff rw, (void *)v, 370176017Sjeff (void *)(v + RW_ONE_READER)); 371154941Sjhb break; 372154941Sjhb } 373154941Sjhb continue; 374154941Sjhb } 375233628Sfabient#ifdef HWPMC_HOOKS 376233628Sfabient PMC_SOFT_CALL( , , lock, failed); 377233628Sfabient#endif 378174629Sjeff lock_profile_obtain_lock_failed(&rw->lock_object, 379174629Sjeff &contested, &waittime); 380154941Sjhb 381173960Sattilio#ifdef ADAPTIVE_RWLOCKS 382154941Sjhb /* 383173960Sattilio * If the owner is running on another CPU, spin until 384173960Sattilio * the owner stops running or the state of the lock 385173960Sattilio * changes. 386173960Sattilio */ 387176017Sjeff if ((v & RW_LOCK_READ) == 0) { 388176017Sjeff owner = (struct thread *)RW_OWNER(v); 389176017Sjeff if (TD_IS_RUNNING(owner)) { 390176017Sjeff if (LOCK_LOG_TEST(&rw->lock_object, 0)) 391176017Sjeff CTR3(KTR_LOCK, 392176017Sjeff "%s: spinning on %p held by %p", 393176017Sjeff __func__, rw, owner); 394176017Sjeff while ((struct thread*)RW_OWNER(rw->rw_lock) == 395192853Ssson owner && TD_IS_RUNNING(owner)) { 396176017Sjeff cpu_spinwait(); 397192853Ssson#ifdef KDTRACE_HOOKS 398192853Ssson spin_cnt++; 399192853Ssson#endif 400192853Ssson } 401176017Sjeff continue; 402176017Sjeff } 403177912Sjeff } else if (spintries < rowner_retries) { 404177912Sjeff spintries++; 405177912Sjeff for (i = 0; i < rowner_loops; i++) { 406177912Sjeff v = rw->rw_lock; 407177912Sjeff if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v)) 408177912Sjeff break; 409177912Sjeff cpu_spinwait(); 410177912Sjeff } 411177912Sjeff if (i != rowner_loops) 412177912Sjeff continue; 413173960Sattilio } 414173960Sattilio#endif 415173960Sattilio 416173960Sattilio /* 417154941Sjhb * Okay, now it's the hard case. Some other thread already 418176017Sjeff * has a write lock or there are write waiters present, 419176017Sjeff * acquire the turnstile lock so we can begin the process 420176017Sjeff * of blocking. 421154941Sjhb */ 422170295Sjeff ts = turnstile_trywait(&rw->lock_object); 423154941Sjhb 424154941Sjhb /* 425154941Sjhb * The lock might have been released while we spun, so 426176017Sjeff * recheck its state and restart the loop if needed. 427154941Sjhb */ 428176017Sjeff v = rw->rw_lock; 429176017Sjeff if (RW_CAN_READ(v)) { 430170295Sjeff turnstile_cancel(ts); 431154941Sjhb continue; 432154941Sjhb } 433154941Sjhb 434173960Sattilio#ifdef ADAPTIVE_RWLOCKS 435154941Sjhb /* 436193035Sjhb * The current lock owner might have started executing 437193035Sjhb * on another CPU (or the lock could have changed 438193035Sjhb * owners) while we were waiting on the turnstile 439193035Sjhb * chain lock. If so, drop the turnstile lock and try 440193035Sjhb * again. 441173960Sattilio */ 442176017Sjeff if ((v & RW_LOCK_READ) == 0) { 443176017Sjeff owner = (struct thread *)RW_OWNER(v); 444176017Sjeff if (TD_IS_RUNNING(owner)) { 445176017Sjeff turnstile_cancel(ts); 446176017Sjeff continue; 447176017Sjeff } 448173960Sattilio } 449173960Sattilio#endif 450173960Sattilio 451173960Sattilio /* 452176017Sjeff * The lock is held in write mode or it already has waiters. 453154941Sjhb */ 454176017Sjeff MPASS(!RW_CAN_READ(v)); 455176017Sjeff 456176017Sjeff /* 457176017Sjeff * If the RW_LOCK_READ_WAITERS flag is already set, then 458176017Sjeff * we can go ahead and block. If it is not set then try 459176017Sjeff * to set it. If we fail to set it drop the turnstile 460176017Sjeff * lock and restart the loop. 461176017Sjeff */ 462176017Sjeff if (!(v & RW_LOCK_READ_WAITERS)) { 463176017Sjeff if (!atomic_cmpset_ptr(&rw->rw_lock, v, 464176017Sjeff v | RW_LOCK_READ_WAITERS)) { 465170295Sjeff turnstile_cancel(ts); 466157826Sjhb continue; 467157826Sjhb } 468167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 469157826Sjhb CTR2(KTR_LOCK, "%s: %p set read waiters flag", 470157826Sjhb __func__, rw); 471154941Sjhb } 472154941Sjhb 473154941Sjhb /* 474154941Sjhb * We were unable to acquire the lock and the read waiters 475154941Sjhb * flag is set, so we must block on the turnstile. 476154941Sjhb */ 477167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 478154941Sjhb CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 479154941Sjhb rw); 480192853Ssson#ifdef KDTRACE_HOOKS 481192853Ssson sleep_time -= lockstat_nsecs(); 482192853Ssson#endif 483170295Sjeff turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 484192853Ssson#ifdef KDTRACE_HOOKS 485192853Ssson sleep_time += lockstat_nsecs(); 486192853Ssson sleep_cnt++; 487192853Ssson#endif 488167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 489154941Sjhb CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 490154941Sjhb __func__, rw); 491154941Sjhb } 492154941Sjhb 493154941Sjhb /* 494154941Sjhb * TODO: acquire "owner of record" here. Here be turnstile dragons 495154941Sjhb * however. turnstiles don't like owners changing between calls to 496154941Sjhb * turnstile_wait() currently. 497154941Sjhb */ 498192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, 499174629Sjeff waittime, file, line); 500167787Sjhb LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 501167787Sjhb WITNESS_LOCK(&rw->lock_object, 0, file, line); 502160771Sjhb curthread->td_locks++; 503176017Sjeff curthread->td_rw_rlocks++; 504192853Ssson#ifdef KDTRACE_HOOKS 505192853Ssson if (sleep_time) 506192853Ssson LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time); 507192853Ssson 508192853Ssson /* 509192853Ssson * Record only the loops spinning and not sleeping. 510192853Ssson */ 511192853Ssson if (spin_cnt > sleep_cnt) 512192853Ssson LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); 513192853Ssson#endif 514154941Sjhb} 515154941Sjhb 516177843Sattilioint 517177843Sattilio_rw_try_rlock(struct rwlock *rw, const char *file, int line) 518177843Sattilio{ 519177843Sattilio uintptr_t x; 520177843Sattilio 521228424Savg if (SCHEDULER_STOPPED()) 522228424Savg return (1); 523228424Savg 524177843Sattilio for (;;) { 525177843Sattilio x = rw->rw_lock; 526177843Sattilio KASSERT(rw->rw_lock != RW_DESTROYED, 527177843Sattilio ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 528177843Sattilio if (!(x & RW_LOCK_READ)) 529177843Sattilio break; 530177843Sattilio if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 531177843Sattilio LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 532177843Sattilio line); 533177843Sattilio WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); 534177843Sattilio curthread->td_locks++; 535177843Sattilio curthread->td_rw_rlocks++; 536177843Sattilio return (1); 537177843Sattilio } 538177843Sattilio } 539177843Sattilio 540177843Sattilio LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 541177843Sattilio return (0); 542177843Sattilio} 543177843Sattilio 544154941Sjhbvoid 545154941Sjhb_rw_runlock(struct rwlock *rw, const char *file, int line) 546154941Sjhb{ 547154941Sjhb struct turnstile *ts; 548176017Sjeff uintptr_t x, v, queue; 549154941Sjhb 550228424Savg if (SCHEDULER_STOPPED()) 551228424Savg return; 552228424Savg 553169394Sjhb KASSERT(rw->rw_lock != RW_DESTROYED, 554169394Sjhb ("rw_runlock() of destroyed rwlock @ %s:%d", file, line)); 555154941Sjhb _rw_assert(rw, RA_RLOCKED, file, line); 556160771Sjhb curthread->td_locks--; 557176017Sjeff curthread->td_rw_rlocks--; 558167787Sjhb WITNESS_UNLOCK(&rw->lock_object, 0, file, line); 559167787Sjhb LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); 560154941Sjhb 561154941Sjhb /* TODO: drop "owner of record" here. */ 562154941Sjhb 563154941Sjhb for (;;) { 564154941Sjhb /* 565154941Sjhb * See if there is more than one read lock held. If so, 566154941Sjhb * just drop one and return. 567154941Sjhb */ 568154941Sjhb x = rw->rw_lock; 569154941Sjhb if (RW_READERS(x) > 1) { 570197643Sattilio if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, 571154941Sjhb x - RW_ONE_READER)) { 572167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 573154941Sjhb CTR4(KTR_LOCK, 574154941Sjhb "%s: %p succeeded %p -> %p", 575154941Sjhb __func__, rw, (void *)x, 576154941Sjhb (void *)(x - RW_ONE_READER)); 577154941Sjhb break; 578154941Sjhb } 579154941Sjhb continue; 580167307Sjhb } 581154941Sjhb /* 582154941Sjhb * If there aren't any waiters for a write lock, then try 583154941Sjhb * to drop it quickly. 584154941Sjhb */ 585176017Sjeff if (!(x & RW_LOCK_WAITERS)) { 586176017Sjeff MPASS((x & ~RW_LOCK_WRITE_SPINNER) == 587176017Sjeff RW_READERS_LOCK(1)); 588197643Sattilio if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, 589197643Sattilio RW_UNLOCKED)) { 590167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 591154941Sjhb CTR2(KTR_LOCK, "%s: %p last succeeded", 592154941Sjhb __func__, rw); 593154941Sjhb break; 594154941Sjhb } 595154941Sjhb continue; 596154941Sjhb } 597154941Sjhb /* 598176017Sjeff * Ok, we know we have waiters and we think we are the 599176017Sjeff * last reader, so grab the turnstile lock. 600154941Sjhb */ 601170295Sjeff turnstile_chain_lock(&rw->lock_object); 602176017Sjeff v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 603176017Sjeff MPASS(v & RW_LOCK_WAITERS); 604154941Sjhb 605154941Sjhb /* 606154941Sjhb * Try to drop our lock leaving the lock in a unlocked 607154941Sjhb * state. 608154941Sjhb * 609154941Sjhb * If you wanted to do explicit lock handoff you'd have to 610154941Sjhb * do it here. You'd also want to use turnstile_signal() 611154941Sjhb * and you'd have to handle the race where a higher 612154941Sjhb * priority thread blocks on the write lock before the 613154941Sjhb * thread you wakeup actually runs and have the new thread 614154941Sjhb * "steal" the lock. For now it's a lot simpler to just 615154941Sjhb * wakeup all of the waiters. 616154941Sjhb * 617154941Sjhb * As above, if we fail, then another thread might have 618154941Sjhb * acquired a read lock, so drop the turnstile lock and 619154941Sjhb * restart. 620154941Sjhb */ 621176017Sjeff x = RW_UNLOCKED; 622176017Sjeff if (v & RW_LOCK_WRITE_WAITERS) { 623176017Sjeff queue = TS_EXCLUSIVE_QUEUE; 624176017Sjeff x |= (v & RW_LOCK_READ_WAITERS); 625176017Sjeff } else 626176017Sjeff queue = TS_SHARED_QUEUE; 627197643Sattilio if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, 628176017Sjeff x)) { 629170295Sjeff turnstile_chain_unlock(&rw->lock_object); 630154941Sjhb continue; 631154941Sjhb } 632167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 633154941Sjhb CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 634154941Sjhb __func__, rw); 635154941Sjhb 636154941Sjhb /* 637154941Sjhb * Ok. The lock is released and all that's left is to 638154941Sjhb * wake up the waiters. Note that the lock might not be 639154941Sjhb * free anymore, but in that case the writers will just 640154941Sjhb * block again if they run before the new lock holder(s) 641154941Sjhb * release the lock. 642154941Sjhb */ 643167787Sjhb ts = turnstile_lookup(&rw->lock_object); 644157846Sjhb MPASS(ts != NULL); 645176017Sjeff turnstile_broadcast(ts, queue); 646154941Sjhb turnstile_unpend(ts, TS_SHARED_LOCK); 647170295Sjeff turnstile_chain_unlock(&rw->lock_object); 648154941Sjhb break; 649154941Sjhb } 650192853Ssson LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_RUNLOCK_RELEASE, rw); 651154941Sjhb} 652154941Sjhb 653154941Sjhb/* 654154941Sjhb * This function is called when we are unable to obtain a write lock on the 655154941Sjhb * first try. This means that at least one other thread holds either a 656154941Sjhb * read or write lock. 657154941Sjhb */ 658154941Sjhbvoid 659154941Sjhb_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 660154941Sjhb{ 661170295Sjeff struct turnstile *ts; 662167801Sjhb#ifdef ADAPTIVE_RWLOCKS 663157846Sjhb volatile struct thread *owner; 664176017Sjeff int spintries = 0; 665176017Sjeff int i; 666157851Swkoszek#endif 667189846Sjeff uintptr_t v, x; 668189846Sjeff#ifdef LOCK_PROFILING 669171516Sattilio uint64_t waittime = 0; 670171516Sattilio int contested = 0; 671189846Sjeff#endif 672192853Ssson#ifdef KDTRACE_HOOKS 673192853Ssson uint64_t spin_cnt = 0; 674192853Ssson uint64_t sleep_cnt = 0; 675192853Ssson int64_t sleep_time = 0; 676192853Ssson#endif 677154941Sjhb 678228424Savg if (SCHEDULER_STOPPED()) 679228424Savg return; 680228424Savg 681171052Sattilio if (rw_wlocked(rw)) { 682193307Sattilio KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, 683171052Sattilio ("%s: recursing but non-recursive rw %s @ %s:%d\n", 684171052Sattilio __func__, rw->lock_object.lo_name, file, line)); 685171052Sattilio rw->rw_recurse++; 686171052Sattilio if (LOCK_LOG_TEST(&rw->lock_object, 0)) 687171052Sattilio CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 688171052Sattilio return; 689171052Sattilio } 690171052Sattilio 691167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 692154941Sjhb CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 693167787Sjhb rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 694154941Sjhb 695154941Sjhb while (!_rw_write_lock(rw, tid)) { 696192853Ssson#ifdef KDTRACE_HOOKS 697192853Ssson spin_cnt++; 698192853Ssson#endif 699233628Sfabient#ifdef HWPMC_HOOKS 700233628Sfabient PMC_SOFT_CALL( , , lock, failed); 701233628Sfabient#endif 702174629Sjeff lock_profile_obtain_lock_failed(&rw->lock_object, 703174629Sjeff &contested, &waittime); 704173960Sattilio#ifdef ADAPTIVE_RWLOCKS 705173960Sattilio /* 706173960Sattilio * If the lock is write locked and the owner is 707173960Sattilio * running on another CPU, spin until the owner stops 708173960Sattilio * running or the state of the lock changes. 709173960Sattilio */ 710173960Sattilio v = rw->rw_lock; 711173960Sattilio owner = (struct thread *)RW_OWNER(v); 712173960Sattilio if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 713173960Sattilio if (LOCK_LOG_TEST(&rw->lock_object, 0)) 714173960Sattilio CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 715173960Sattilio __func__, rw, owner); 716173960Sattilio while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 717192853Ssson TD_IS_RUNNING(owner)) { 718173960Sattilio cpu_spinwait(); 719192853Ssson#ifdef KDTRACE_HOOKS 720192853Ssson spin_cnt++; 721192853Ssson#endif 722192853Ssson } 723173960Sattilio continue; 724173960Sattilio } 725177912Sjeff if ((v & RW_LOCK_READ) && RW_READERS(v) && 726177912Sjeff spintries < rowner_retries) { 727176017Sjeff if (!(v & RW_LOCK_WRITE_SPINNER)) { 728176017Sjeff if (!atomic_cmpset_ptr(&rw->rw_lock, v, 729176017Sjeff v | RW_LOCK_WRITE_SPINNER)) { 730176017Sjeff continue; 731176017Sjeff } 732176017Sjeff } 733176017Sjeff spintries++; 734177912Sjeff for (i = 0; i < rowner_loops; i++) { 735176017Sjeff if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0) 736176017Sjeff break; 737176017Sjeff cpu_spinwait(); 738176017Sjeff } 739192853Ssson#ifdef KDTRACE_HOOKS 740192853Ssson spin_cnt += rowner_loops - i; 741192853Ssson#endif 742177912Sjeff if (i != rowner_loops) 743176017Sjeff continue; 744176017Sjeff } 745173960Sattilio#endif 746170295Sjeff ts = turnstile_trywait(&rw->lock_object); 747154941Sjhb v = rw->rw_lock; 748154941Sjhb 749173960Sattilio#ifdef ADAPTIVE_RWLOCKS 750154941Sjhb /* 751193035Sjhb * The current lock owner might have started executing 752193035Sjhb * on another CPU (or the lock could have changed 753193035Sjhb * owners) while we were waiting on the turnstile 754193035Sjhb * chain lock. If so, drop the turnstile lock and try 755193035Sjhb * again. 756173960Sattilio */ 757173960Sattilio if (!(v & RW_LOCK_READ)) { 758173960Sattilio owner = (struct thread *)RW_OWNER(v); 759173960Sattilio if (TD_IS_RUNNING(owner)) { 760173960Sattilio turnstile_cancel(ts); 761173960Sattilio continue; 762173960Sattilio } 763173960Sattilio } 764173960Sattilio#endif 765173960Sattilio /* 766179334Sattilio * Check for the waiters flags about this rwlock. 767179334Sattilio * If the lock was released, without maintain any pending 768179334Sattilio * waiters queue, simply try to acquire it. 769179334Sattilio * If a pending waiters queue is present, claim the lock 770179334Sattilio * ownership and maintain the pending queue. 771154941Sjhb */ 772176017Sjeff x = v & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); 773176017Sjeff if ((v & ~x) == RW_UNLOCKED) { 774176017Sjeff x &= ~RW_LOCK_WRITE_SPINNER; 775176017Sjeff if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) { 776176017Sjeff if (x) 777176017Sjeff turnstile_claim(ts); 778176017Sjeff else 779176017Sjeff turnstile_cancel(ts); 780154941Sjhb break; 781154941Sjhb } 782170295Sjeff turnstile_cancel(ts); 783154941Sjhb continue; 784154941Sjhb } 785154941Sjhb /* 786154941Sjhb * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 787154941Sjhb * set it. If we fail to set it, then loop back and try 788154941Sjhb * again. 789154941Sjhb */ 790157826Sjhb if (!(v & RW_LOCK_WRITE_WAITERS)) { 791157826Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, v, 792157826Sjhb v | RW_LOCK_WRITE_WAITERS)) { 793170295Sjeff turnstile_cancel(ts); 794157826Sjhb continue; 795157826Sjhb } 796167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 797157826Sjhb CTR2(KTR_LOCK, "%s: %p set write waiters flag", 798157826Sjhb __func__, rw); 799154941Sjhb } 800157846Sjhb /* 801154941Sjhb * We were unable to acquire the lock and the write waiters 802154941Sjhb * flag is set, so we must block on the turnstile. 803154941Sjhb */ 804167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 805154941Sjhb CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 806154941Sjhb rw); 807192853Ssson#ifdef KDTRACE_HOOKS 808192853Ssson sleep_time -= lockstat_nsecs(); 809192853Ssson#endif 810170295Sjeff turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 811192853Ssson#ifdef KDTRACE_HOOKS 812192853Ssson sleep_time += lockstat_nsecs(); 813192853Ssson sleep_cnt++; 814192853Ssson#endif 815167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 816154941Sjhb CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 817154941Sjhb __func__, rw); 818176017Sjeff#ifdef ADAPTIVE_RWLOCKS 819176017Sjeff spintries = 0; 820176017Sjeff#endif 821154941Sjhb } 822192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, 823192853Ssson waittime, file, line); 824192853Ssson#ifdef KDTRACE_HOOKS 825192853Ssson if (sleep_time) 826192853Ssson LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time); 827192853Ssson 828192853Ssson /* 829192853Ssson * Record only the loops spinning and not sleeping. 830192853Ssson */ 831192853Ssson if (spin_cnt > sleep_cnt) 832192853Ssson LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); 833192853Ssson#endif 834154941Sjhb} 835154941Sjhb 836154941Sjhb/* 837154941Sjhb * This function is called if the first try at releasing a write lock failed. 838154941Sjhb * This means that one of the 2 waiter bits must be set indicating that at 839154941Sjhb * least one thread is waiting on this lock. 840154941Sjhb */ 841154941Sjhbvoid 842154941Sjhb_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 843154941Sjhb{ 844154941Sjhb struct turnstile *ts; 845154941Sjhb uintptr_t v; 846154941Sjhb int queue; 847154941Sjhb 848228424Savg if (SCHEDULER_STOPPED()) 849228424Savg return; 850228424Savg 851171052Sattilio if (rw_wlocked(rw) && rw_recursed(rw)) { 852176017Sjeff rw->rw_recurse--; 853171052Sattilio if (LOCK_LOG_TEST(&rw->lock_object, 0)) 854171052Sattilio CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); 855171052Sattilio return; 856171052Sattilio } 857171052Sattilio 858154941Sjhb KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 859154941Sjhb ("%s: neither of the waiter flags are set", __func__)); 860154941Sjhb 861167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 862154941Sjhb CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 863154941Sjhb 864170295Sjeff turnstile_chain_lock(&rw->lock_object); 865167787Sjhb ts = turnstile_lookup(&rw->lock_object); 866154941Sjhb MPASS(ts != NULL); 867154941Sjhb 868154941Sjhb /* 869154941Sjhb * Use the same algo as sx locks for now. Prefer waking up shared 870154941Sjhb * waiters if we have any over writers. This is probably not ideal. 871154941Sjhb * 872154941Sjhb * 'v' is the value we are going to write back to rw_lock. If we 873154941Sjhb * have waiters on both queues, we need to preserve the state of 874154941Sjhb * the waiter flag for the queue we don't wake up. For now this is 875154941Sjhb * hardcoded for the algorithm mentioned above. 876154941Sjhb * 877154941Sjhb * In the case of both readers and writers waiting we wakeup the 878154941Sjhb * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 879154941Sjhb * new writer comes in before a reader it will claim the lock up 880154941Sjhb * above. There is probably a potential priority inversion in 881154941Sjhb * there that could be worked around either by waking both queues 882154941Sjhb * of waiters or doing some complicated lock handoff gymnastics. 883154941Sjhb */ 884157846Sjhb v = RW_UNLOCKED; 885176076Sjeff if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) { 886176076Sjeff queue = TS_EXCLUSIVE_QUEUE; 887176076Sjeff v |= (rw->rw_lock & RW_LOCK_READ_WAITERS); 888176076Sjeff } else 889154941Sjhb queue = TS_SHARED_QUEUE; 890157846Sjhb 891157846Sjhb /* Wake up all waiters for the specific queue. */ 892167787Sjhb if (LOCK_LOG_TEST(&rw->lock_object, 0)) 893154941Sjhb CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 894154941Sjhb queue == TS_SHARED_QUEUE ? "read" : "write"); 895154941Sjhb turnstile_broadcast(ts, queue); 896154941Sjhb atomic_store_rel_ptr(&rw->rw_lock, v); 897154941Sjhb turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 898170295Sjeff turnstile_chain_unlock(&rw->lock_object); 899154941Sjhb} 900154941Sjhb 901157882Sjhb/* 902157882Sjhb * Attempt to do a non-blocking upgrade from a read lock to a write 903157882Sjhb * lock. This will only succeed if this thread holds a single read 904157882Sjhb * lock. Returns true if the upgrade succeeded and false otherwise. 905157882Sjhb */ 906157882Sjhbint 907157882Sjhb_rw_try_upgrade(struct rwlock *rw, const char *file, int line) 908157882Sjhb{ 909176017Sjeff uintptr_t v, x, tid; 910170295Sjeff struct turnstile *ts; 911157882Sjhb int success; 912157882Sjhb 913228424Savg if (SCHEDULER_STOPPED()) 914228424Savg return (1); 915228424Savg 916169394Sjhb KASSERT(rw->rw_lock != RW_DESTROYED, 917169394Sjhb ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line)); 918157882Sjhb _rw_assert(rw, RA_RLOCKED, file, line); 919157882Sjhb 920157882Sjhb /* 921157882Sjhb * Attempt to switch from one reader to a writer. If there 922157882Sjhb * are any write waiters, then we will have to lock the 923157882Sjhb * turnstile first to prevent races with another writer 924157882Sjhb * calling turnstile_wait() before we have claimed this 925157882Sjhb * turnstile. So, do the simple case of no waiters first. 926157882Sjhb */ 927157882Sjhb tid = (uintptr_t)curthread; 928176017Sjeff success = 0; 929176017Sjeff for (;;) { 930176017Sjeff v = rw->rw_lock; 931176017Sjeff if (RW_READERS(v) > 1) 932176017Sjeff break; 933176017Sjeff if (!(v & RW_LOCK_WAITERS)) { 934176017Sjeff success = atomic_cmpset_ptr(&rw->rw_lock, v, tid); 935176017Sjeff if (!success) 936176017Sjeff continue; 937176017Sjeff break; 938176017Sjeff } 939157882Sjhb 940176017Sjeff /* 941176017Sjeff * Ok, we think we have waiters, so lock the turnstile. 942176017Sjeff */ 943176017Sjeff ts = turnstile_trywait(&rw->lock_object); 944176017Sjeff v = rw->rw_lock; 945176017Sjeff if (RW_READERS(v) > 1) { 946176017Sjeff turnstile_cancel(ts); 947176017Sjeff break; 948176017Sjeff } 949176017Sjeff /* 950176017Sjeff * Try to switch from one reader to a writer again. This time 951176017Sjeff * we honor the current state of the waiters flags. 952176017Sjeff * If we obtain the lock with the flags set, then claim 953176017Sjeff * ownership of the turnstile. 954176017Sjeff */ 955176017Sjeff x = rw->rw_lock & RW_LOCK_WAITERS; 956176017Sjeff success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x); 957176017Sjeff if (success) { 958176017Sjeff if (x) 959176017Sjeff turnstile_claim(ts); 960176017Sjeff else 961176017Sjeff turnstile_cancel(ts); 962176017Sjeff break; 963176017Sjeff } 964170295Sjeff turnstile_cancel(ts); 965176017Sjeff } 966167787Sjhb LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); 967176017Sjeff if (success) { 968176017Sjeff curthread->td_rw_rlocks--; 969167787Sjhb WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 970157882Sjhb file, line); 971192853Ssson LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, rw); 972176017Sjeff } 973157882Sjhb return (success); 974157882Sjhb} 975157882Sjhb 976157882Sjhb/* 977157882Sjhb * Downgrade a write lock into a single read lock. 978157882Sjhb */ 979157882Sjhbvoid 980157882Sjhb_rw_downgrade(struct rwlock *rw, const char *file, int line) 981157882Sjhb{ 982157882Sjhb struct turnstile *ts; 983157882Sjhb uintptr_t tid, v; 984176017Sjeff int rwait, wwait; 985157882Sjhb 986228424Savg if (SCHEDULER_STOPPED()) 987228424Savg return; 988228424Savg 989169394Sjhb KASSERT(rw->rw_lock != RW_DESTROYED, 990169394Sjhb ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); 991171052Sattilio _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line); 992171052Sattilio#ifndef INVARIANTS 993171052Sattilio if (rw_recursed(rw)) 994171052Sattilio panic("downgrade of a recursed lock"); 995171052Sattilio#endif 996157882Sjhb 997167787Sjhb WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); 998157882Sjhb 999157882Sjhb /* 1000157882Sjhb * Convert from a writer to a single reader. First we handle 1001157882Sjhb * the easy case with no waiters. If there are any waiters, we 1002176017Sjeff * lock the turnstile and "disown" the lock. 1003157882Sjhb */ 1004157882Sjhb tid = (uintptr_t)curthread; 1005157882Sjhb if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) 1006157882Sjhb goto out; 1007157882Sjhb 1008157882Sjhb /* 1009157882Sjhb * Ok, we think we have waiters, so lock the turnstile so we can 1010157882Sjhb * read the waiter flags without any races. 1011157882Sjhb */ 1012170295Sjeff turnstile_chain_lock(&rw->lock_object); 1013176017Sjeff v = rw->rw_lock & RW_LOCK_WAITERS; 1014176017Sjeff rwait = v & RW_LOCK_READ_WAITERS; 1015176017Sjeff wwait = v & RW_LOCK_WRITE_WAITERS; 1016176017Sjeff MPASS(rwait | wwait); 1017157882Sjhb 1018157882Sjhb /* 1019176017Sjeff * Downgrade from a write lock while preserving waiters flag 1020176017Sjeff * and give up ownership of the turnstile. 1021157882Sjhb */ 1022167787Sjhb ts = turnstile_lookup(&rw->lock_object); 1023157882Sjhb MPASS(ts != NULL); 1024176017Sjeff if (!wwait) 1025176017Sjeff v &= ~RW_LOCK_READ_WAITERS; 1026176017Sjeff atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); 1027176017Sjeff /* 1028176017Sjeff * Wake other readers if there are no writers pending. Otherwise they 1029176017Sjeff * won't be able to acquire the lock anyway. 1030176017Sjeff */ 1031176017Sjeff if (rwait && !wwait) { 1032157882Sjhb turnstile_broadcast(ts, TS_SHARED_QUEUE); 1033157882Sjhb turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1034176017Sjeff } else 1035157882Sjhb turnstile_disown(ts); 1036170295Sjeff turnstile_chain_unlock(&rw->lock_object); 1037157882Sjhbout: 1038176017Sjeff curthread->td_rw_rlocks++; 1039167787Sjhb LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); 1040192853Ssson LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, rw); 1041157882Sjhb} 1042157882Sjhb 1043154941Sjhb#ifdef INVARIANT_SUPPORT 1044155162Sscottl#ifndef INVARIANTS 1045154941Sjhb#undef _rw_assert 1046154941Sjhb#endif 1047154941Sjhb 1048154941Sjhb/* 1049154941Sjhb * In the non-WITNESS case, rw_assert() can only detect that at least 1050154941Sjhb * *some* thread owns an rlock, but it cannot guarantee that *this* 1051154941Sjhb * thread owns an rlock. 1052154941Sjhb */ 1053154941Sjhbvoid 1054227588Spjd_rw_assert(const struct rwlock *rw, int what, const char *file, int line) 1055154941Sjhb{ 1056154941Sjhb 1057154941Sjhb if (panicstr != NULL) 1058154941Sjhb return; 1059154941Sjhb switch (what) { 1060154941Sjhb case RA_LOCKED: 1061171052Sattilio case RA_LOCKED | RA_RECURSED: 1062171052Sattilio case RA_LOCKED | RA_NOTRECURSED: 1063154941Sjhb case RA_RLOCKED: 1064154941Sjhb#ifdef WITNESS 1065167787Sjhb witness_assert(&rw->lock_object, what, file, line); 1066154941Sjhb#else 1067154941Sjhb /* 1068154941Sjhb * If some other thread has a write lock or we have one 1069154941Sjhb * and are asserting a read lock, fail. Also, if no one 1070154941Sjhb * has a lock at all, fail. 1071154941Sjhb */ 1072155061Sscottl if (rw->rw_lock == RW_UNLOCKED || 1073155061Sscottl (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED || 1074157826Sjhb rw_wowner(rw) != curthread))) 1075154941Sjhb panic("Lock %s not %slocked @ %s:%d\n", 1076167787Sjhb rw->lock_object.lo_name, (what == RA_RLOCKED) ? 1077154941Sjhb "read " : "", file, line); 1078171052Sattilio 1079171052Sattilio if (!(rw->rw_lock & RW_LOCK_READ)) { 1080171052Sattilio if (rw_recursed(rw)) { 1081171052Sattilio if (what & RA_NOTRECURSED) 1082171052Sattilio panic("Lock %s recursed @ %s:%d\n", 1083171052Sattilio rw->lock_object.lo_name, file, 1084171052Sattilio line); 1085171052Sattilio } else if (what & RA_RECURSED) 1086171052Sattilio panic("Lock %s not recursed @ %s:%d\n", 1087171052Sattilio rw->lock_object.lo_name, file, line); 1088171052Sattilio } 1089154941Sjhb#endif 1090154941Sjhb break; 1091154941Sjhb case RA_WLOCKED: 1092171052Sattilio case RA_WLOCKED | RA_RECURSED: 1093171052Sattilio case RA_WLOCKED | RA_NOTRECURSED: 1094157826Sjhb if (rw_wowner(rw) != curthread) 1095154941Sjhb panic("Lock %s not exclusively locked @ %s:%d\n", 1096167787Sjhb rw->lock_object.lo_name, file, line); 1097171052Sattilio if (rw_recursed(rw)) { 1098171052Sattilio if (what & RA_NOTRECURSED) 1099171052Sattilio panic("Lock %s recursed @ %s:%d\n", 1100171052Sattilio rw->lock_object.lo_name, file, line); 1101171052Sattilio } else if (what & RA_RECURSED) 1102171052Sattilio panic("Lock %s not recursed @ %s:%d\n", 1103171052Sattilio rw->lock_object.lo_name, file, line); 1104154941Sjhb break; 1105154941Sjhb case RA_UNLOCKED: 1106154941Sjhb#ifdef WITNESS 1107167787Sjhb witness_assert(&rw->lock_object, what, file, line); 1108154941Sjhb#else 1109154941Sjhb /* 1110154941Sjhb * If we hold a write lock fail. We can't reliably check 1111154941Sjhb * to see if we hold a read lock or not. 1112154941Sjhb */ 1113157826Sjhb if (rw_wowner(rw) == curthread) 1114154941Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 1115167787Sjhb rw->lock_object.lo_name, file, line); 1116154941Sjhb#endif 1117154941Sjhb break; 1118154941Sjhb default: 1119154941Sjhb panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 1120154941Sjhb line); 1121154941Sjhb } 1122154941Sjhb} 1123154941Sjhb#endif /* INVARIANT_SUPPORT */ 1124154941Sjhb 1125154941Sjhb#ifdef DDB 1126154941Sjhbvoid 1127227588Spjddb_show_rwlock(const struct lock_object *lock) 1128154941Sjhb{ 1129227588Spjd const struct rwlock *rw; 1130154941Sjhb struct thread *td; 1131154941Sjhb 1132227588Spjd rw = (const struct rwlock *)lock; 1133154941Sjhb 1134154941Sjhb db_printf(" state: "); 1135154941Sjhb if (rw->rw_lock == RW_UNLOCKED) 1136154941Sjhb db_printf("UNLOCKED\n"); 1137169394Sjhb else if (rw->rw_lock == RW_DESTROYED) { 1138169394Sjhb db_printf("DESTROYED\n"); 1139169394Sjhb return; 1140169394Sjhb } else if (rw->rw_lock & RW_LOCK_READ) 1141167504Sjhb db_printf("RLOCK: %ju locks\n", 1142167504Sjhb (uintmax_t)(RW_READERS(rw->rw_lock))); 1143154941Sjhb else { 1144157826Sjhb td = rw_wowner(rw); 1145154941Sjhb db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1146173600Sjulian td->td_tid, td->td_proc->p_pid, td->td_name); 1147171052Sattilio if (rw_recursed(rw)) 1148171052Sattilio db_printf(" recursed: %u\n", rw->rw_recurse); 1149154941Sjhb } 1150154941Sjhb db_printf(" waiters: "); 1151154941Sjhb switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 1152154941Sjhb case RW_LOCK_READ_WAITERS: 1153154941Sjhb db_printf("readers\n"); 1154154941Sjhb break; 1155154941Sjhb case RW_LOCK_WRITE_WAITERS: 1156154941Sjhb db_printf("writers\n"); 1157154941Sjhb break; 1158154941Sjhb case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 1159167492Sjhb db_printf("readers and writers\n"); 1160154941Sjhb break; 1161154941Sjhb default: 1162154941Sjhb db_printf("none\n"); 1163154941Sjhb break; 1164154941Sjhb } 1165154941Sjhb} 1166154941Sjhb 1167154941Sjhb#endif 1168