kern_rwlock.c revision 157846
1154941Sjhb/*- 2154941Sjhb * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3154941Sjhb * All rights reserved. 4154941Sjhb * 5154941Sjhb * Redistribution and use in source and binary forms, with or without 6154941Sjhb * modification, are permitted provided that the following conditions 7154941Sjhb * are met: 8154941Sjhb * 1. Redistributions of source code must retain the above copyright 9154941Sjhb * notice, this list of conditions and the following disclaimer. 10154941Sjhb * 2. Redistributions in binary form must reproduce the above copyright 11154941Sjhb * notice, this list of conditions and the following disclaimer in the 12154941Sjhb * documentation and/or other materials provided with the distribution. 13154941Sjhb * 3. Neither the name of the author nor the names of any co-contributors 14154941Sjhb * may be used to endorse or promote products derived from this software 15154941Sjhb * without specific prior written permission. 16154941Sjhb * 17154941Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18154941Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19154941Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20154941Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21154941Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22154941Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23154941Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24154941Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25154941Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26154941Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27154941Sjhb * SUCH DAMAGE. 28154941Sjhb */ 29154941Sjhb 30154941Sjhb/* 31154941Sjhb * Machine independent bits of reader/writer lock implementation. 32154941Sjhb */ 33154941Sjhb 34154941Sjhb#include <sys/cdefs.h> 35154941Sjhb__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 157846 2006-04-18 18:27:54Z jhb $"); 36154941Sjhb 37154941Sjhb#include "opt_ddb.h" 38154941Sjhb 39154941Sjhb#include <sys/param.h> 40154941Sjhb#include <sys/ktr.h> 41154941Sjhb#include <sys/lock.h> 42154941Sjhb#include <sys/mutex.h> 43154941Sjhb#include <sys/proc.h> 44154941Sjhb#include <sys/rwlock.h> 45154941Sjhb#include <sys/systm.h> 46154941Sjhb#include <sys/turnstile.h> 47154941Sjhb 48154941Sjhb#include <machine/cpu.h> 49154941Sjhb 50154941Sjhb#ifdef DDB 51154941Sjhb#include <ddb/ddb.h> 52154941Sjhb 53154941Sjhbstatic void db_show_rwlock(struct lock_object *lock); 54154941Sjhb#endif 55154941Sjhb 56154941Sjhbstruct lock_class lock_class_rw = { 57154941Sjhb "rw", 58154941Sjhb LC_SLEEPLOCK | LC_RECURSABLE /* | LC_UPGRADABLE */, 59154941Sjhb#ifdef DDB 60154941Sjhb db_show_rwlock 61154941Sjhb#endif 62154941Sjhb}; 63154941Sjhb 64157826Sjhb/* 65157826Sjhb * Return a pointer to the owning thread if the lock is write-locked or 66157826Sjhb * NULL if the lock is unlocked or read-locked. 67157826Sjhb */ 68157826Sjhb#define rw_wowner(rw) \ 69154941Sjhb ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ 70154941Sjhb (struct thread *)RW_OWNER((rw)->rw_lock)) 71154941Sjhb 72157826Sjhb/* 73157826Sjhb * Return a pointer to the owning thread for this lock who should receive 74157826Sjhb * any priority lent by threads that block on this lock. Currently this 75157826Sjhb * is identical to rw_wowner(). 76157826Sjhb */ 77157826Sjhb#define rw_owner(rw) rw_wowner(rw) 78157826Sjhb 79154941Sjhb#ifndef INVARIANTS 80154941Sjhb#define _rw_assert(rw, what, file, line) 81154941Sjhb#endif 82154941Sjhb 83154941Sjhbvoid 84154941Sjhbrw_init(struct rwlock *rw, const char *name) 85154941Sjhb{ 86154941Sjhb 87154941Sjhb rw->rw_lock = RW_UNLOCKED; 88154941Sjhb 89154941Sjhb lock_init(&rw->rw_object, &lock_class_rw, name, NULL, LO_WITNESS | 90154941Sjhb LO_RECURSABLE /* | LO_UPGRADABLE */); 91154941Sjhb} 92154941Sjhb 93154941Sjhbvoid 94154941Sjhbrw_destroy(struct rwlock *rw) 95154941Sjhb{ 96154941Sjhb 97154941Sjhb KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked")); 98154941Sjhb lock_destroy(&rw->rw_object); 99154941Sjhb} 100154941Sjhb 101154941Sjhbvoid 102154941Sjhbrw_sysinit(void *arg) 103154941Sjhb{ 104154941Sjhb struct rw_args *args = arg; 105154941Sjhb 106154941Sjhb rw_init(args->ra_rw, args->ra_desc); 107154941Sjhb} 108154941Sjhb 109154941Sjhbvoid 110154941Sjhb_rw_wlock(struct rwlock *rw, const char *file, int line) 111154941Sjhb{ 112154941Sjhb 113154941Sjhb MPASS(curthread != NULL); 114157826Sjhb KASSERT(rw_wowner(rw) != curthread, 115154941Sjhb ("%s (%s): wlock already held @ %s:%d", __func__, 116154941Sjhb rw->rw_object.lo_name, file, line)); 117154941Sjhb WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 118154941Sjhb line); 119154941Sjhb __rw_wlock(rw, curthread, file, line); 120154941Sjhb LOCK_LOG_LOCK("WLOCK", &rw->rw_object, 0, 0, file, line); 121154941Sjhb WITNESS_LOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); 122154941Sjhb} 123154941Sjhb 124154941Sjhbvoid 125154941Sjhb_rw_wunlock(struct rwlock *rw, const char *file, int line) 126154941Sjhb{ 127154941Sjhb 128154941Sjhb MPASS(curthread != NULL); 129154941Sjhb _rw_assert(rw, RA_WLOCKED, file, line); 130154941Sjhb WITNESS_UNLOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); 131154941Sjhb LOCK_LOG_LOCK("WUNLOCK", &rw->rw_object, 0, 0, file, line); 132154941Sjhb __rw_wunlock(rw, curthread, file, line); 133154941Sjhb} 134154941Sjhb 135154941Sjhbvoid 136154941Sjhb_rw_rlock(struct rwlock *rw, const char *file, int line) 137154941Sjhb{ 138157846Sjhb volatile struct thread *owner; 139154941Sjhb uintptr_t x; 140154941Sjhb 141157826Sjhb KASSERT(rw_wowner(rw) != curthread, 142154941Sjhb ("%s (%s): wlock already held @ %s:%d", __func__, 143154941Sjhb rw->rw_object.lo_name, file, line)); 144154941Sjhb WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER, file, line); 145154941Sjhb 146154941Sjhb /* 147154941Sjhb * Note that we don't make any attempt to try to block read 148154941Sjhb * locks once a writer has blocked on the lock. The reason is 149154941Sjhb * that we currently allow for read locks to recurse and we 150154941Sjhb * don't keep track of all the holders of read locks. Thus, if 151154941Sjhb * we were to block readers once a writer blocked and a reader 152154941Sjhb * tried to recurse on their reader lock after a writer had 153154941Sjhb * blocked we would end up in a deadlock since the reader would 154154941Sjhb * be blocked on the writer, and the writer would be blocked 155154941Sjhb * waiting for the reader to release its original read lock. 156154941Sjhb */ 157154941Sjhb for (;;) { 158154941Sjhb /* 159154941Sjhb * Handle the easy case. If no other thread has a write 160154941Sjhb * lock, then try to bump up the count of read locks. Note 161154941Sjhb * that we have to preserve the current state of the 162154941Sjhb * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 163154941Sjhb * read lock, then rw_lock must have changed, so restart 164154941Sjhb * the loop. Note that this handles the case of a 165154941Sjhb * completely unlocked rwlock since such a lock is encoded 166154941Sjhb * as a read lock with no waiters. 167154941Sjhb */ 168154941Sjhb x = rw->rw_lock; 169154941Sjhb if (x & RW_LOCK_READ) { 170154941Sjhb 171154941Sjhb /* 172154941Sjhb * The RW_LOCK_READ_WAITERS flag should only be set 173154941Sjhb * if another thread currently holds a write lock, 174154941Sjhb * and in that case RW_LOCK_READ should be clear. 175154941Sjhb */ 176154941Sjhb MPASS((x & RW_LOCK_READ_WAITERS) == 0); 177154941Sjhb if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, 178154941Sjhb x + RW_ONE_READER)) { 179154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 180154941Sjhb CTR4(KTR_LOCK, 181154941Sjhb "%s: %p succeed %p -> %p", __func__, 182154941Sjhb rw, (void *)x, 183154941Sjhb (void *)(x + RW_ONE_READER)); 184154941Sjhb break; 185154941Sjhb } 186157846Sjhb cpu_spinwait(); 187154941Sjhb continue; 188154941Sjhb } 189154941Sjhb 190154941Sjhb /* 191154941Sjhb * Okay, now it's the hard case. Some other thread already 192154941Sjhb * has a write lock, so acquire the turnstile lock so we can 193154941Sjhb * begin the process of blocking. 194154941Sjhb */ 195154941Sjhb turnstile_lock(&rw->rw_object); 196154941Sjhb 197154941Sjhb /* 198154941Sjhb * The lock might have been released while we spun, so 199154941Sjhb * recheck its state and restart the loop if there is no 200154941Sjhb * longer a write lock. 201154941Sjhb */ 202154941Sjhb x = rw->rw_lock; 203154941Sjhb if (x & RW_LOCK_READ) { 204154941Sjhb turnstile_release(&rw->rw_object); 205157846Sjhb cpu_spinwait(); 206154941Sjhb continue; 207154941Sjhb } 208154941Sjhb 209154941Sjhb /* 210154941Sjhb * Ok, it's still a write lock. If the RW_LOCK_READ_WAITERS 211154941Sjhb * flag is already set, then we can go ahead and block. If 212154941Sjhb * it is not set then try to set it. If we fail to set it 213154941Sjhb * drop the turnstile lock and restart the loop. 214154941Sjhb */ 215157826Sjhb if (!(x & RW_LOCK_READ_WAITERS)) { 216157826Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, x, 217157826Sjhb x | RW_LOCK_READ_WAITERS)) { 218157826Sjhb turnstile_release(&rw->rw_object); 219157826Sjhb cpu_spinwait(); 220157826Sjhb continue; 221157826Sjhb } 222157826Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 223157826Sjhb CTR2(KTR_LOCK, "%s: %p set read waiters flag", 224157826Sjhb __func__, rw); 225154941Sjhb } 226154941Sjhb 227157846Sjhb#ifdef SMP 228154941Sjhb /* 229157846Sjhb * If the owner is running on another CPU, spin until 230157846Sjhb * the owner stops running or the state of the lock 231157846Sjhb * changes. 232157846Sjhb */ 233157846Sjhb owner = (struct thread *)RW_OWNER(x); 234157846Sjhb if (TD_IS_RUNNING(owner)) { 235157846Sjhb turnstile_release(&rw->rw_object); 236157846Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 237157846Sjhb CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 238157846Sjhb __func__, rw, owner); 239157846Sjhb while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && 240157846Sjhb TD_IS_RUNNING(owner)) 241157846Sjhb cpu_spinwait(); 242157846Sjhb continue; 243157846Sjhb } 244157846Sjhb#endif 245157846Sjhb 246157846Sjhb /* 247154941Sjhb * We were unable to acquire the lock and the read waiters 248154941Sjhb * flag is set, so we must block on the turnstile. 249154941Sjhb */ 250154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 251154941Sjhb CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 252154941Sjhb rw); 253154941Sjhb turnstile_wait(&rw->rw_object, rw_owner(rw), TS_SHARED_QUEUE); 254154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 255154941Sjhb CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 256154941Sjhb __func__, rw); 257154941Sjhb } 258154941Sjhb 259154941Sjhb /* 260154941Sjhb * TODO: acquire "owner of record" here. Here be turnstile dragons 261154941Sjhb * however. turnstiles don't like owners changing between calls to 262154941Sjhb * turnstile_wait() currently. 263154941Sjhb */ 264154941Sjhb 265154941Sjhb LOCK_LOG_LOCK("RLOCK", &rw->rw_object, 0, 0, file, line); 266154941Sjhb WITNESS_LOCK(&rw->rw_object, 0, file, line); 267154941Sjhb} 268154941Sjhb 269154941Sjhbvoid 270154941Sjhb_rw_runlock(struct rwlock *rw, const char *file, int line) 271154941Sjhb{ 272154941Sjhb struct turnstile *ts; 273154941Sjhb uintptr_t x; 274154941Sjhb 275154941Sjhb _rw_assert(rw, RA_RLOCKED, file, line); 276154941Sjhb WITNESS_UNLOCK(&rw->rw_object, 0, file, line); 277154941Sjhb LOCK_LOG_LOCK("RUNLOCK", &rw->rw_object, 0, 0, file, line); 278154941Sjhb 279154941Sjhb /* TODO: drop "owner of record" here. */ 280154941Sjhb 281154941Sjhb for (;;) { 282154941Sjhb /* 283154941Sjhb * See if there is more than one read lock held. If so, 284154941Sjhb * just drop one and return. 285154941Sjhb */ 286154941Sjhb x = rw->rw_lock; 287154941Sjhb if (RW_READERS(x) > 1) { 288154941Sjhb if (atomic_cmpset_ptr(&rw->rw_lock, x, 289154941Sjhb x - RW_ONE_READER)) { 290154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 291154941Sjhb CTR4(KTR_LOCK, 292154941Sjhb "%s: %p succeeded %p -> %p", 293154941Sjhb __func__, rw, (void *)x, 294154941Sjhb (void *)(x - RW_ONE_READER)); 295154941Sjhb break; 296154941Sjhb } 297154941Sjhb continue; 298154941Sjhb } 299154941Sjhb 300154941Sjhb /* 301154941Sjhb * We should never have read waiters while at least one 302154941Sjhb * thread holds a read lock. (See note above) 303154941Sjhb */ 304154941Sjhb KASSERT(!(x & RW_LOCK_READ_WAITERS), 305154941Sjhb ("%s: waiting readers", __func__)); 306154941Sjhb 307154941Sjhb /* 308154941Sjhb * If there aren't any waiters for a write lock, then try 309154941Sjhb * to drop it quickly. 310154941Sjhb */ 311154941Sjhb if (!(x & RW_LOCK_WRITE_WAITERS)) { 312154941Sjhb 313154941Sjhb /* 314154941Sjhb * There shouldn't be any flags set and we should 315154941Sjhb * be the only read lock. If we fail to release 316154941Sjhb * the single read lock, then another thread might 317154941Sjhb * have just acquired a read lock, so go back up 318154941Sjhb * to the multiple read locks case. 319154941Sjhb */ 320154941Sjhb MPASS(x == RW_READERS_LOCK(1)); 321154941Sjhb if (atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1), 322154941Sjhb RW_UNLOCKED)) { 323154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 324154941Sjhb CTR2(KTR_LOCK, "%s: %p last succeeded", 325154941Sjhb __func__, rw); 326154941Sjhb break; 327154941Sjhb } 328154941Sjhb continue; 329154941Sjhb } 330154941Sjhb 331154941Sjhb /* 332154941Sjhb * There should just be one reader with one or more 333154941Sjhb * writers waiting. 334154941Sjhb */ 335154941Sjhb MPASS(x == (RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS)); 336154941Sjhb 337154941Sjhb /* 338154941Sjhb * Ok, we know we have a waiting writer and we think we 339154941Sjhb * are the last reader, so grab the turnstile lock. 340154941Sjhb */ 341154941Sjhb turnstile_lock(&rw->rw_object); 342154941Sjhb 343154941Sjhb /* 344154941Sjhb * Try to drop our lock leaving the lock in a unlocked 345154941Sjhb * state. 346154941Sjhb * 347154941Sjhb * If you wanted to do explicit lock handoff you'd have to 348154941Sjhb * do it here. You'd also want to use turnstile_signal() 349154941Sjhb * and you'd have to handle the race where a higher 350154941Sjhb * priority thread blocks on the write lock before the 351154941Sjhb * thread you wakeup actually runs and have the new thread 352154941Sjhb * "steal" the lock. For now it's a lot simpler to just 353154941Sjhb * wakeup all of the waiters. 354154941Sjhb * 355154941Sjhb * As above, if we fail, then another thread might have 356154941Sjhb * acquired a read lock, so drop the turnstile lock and 357154941Sjhb * restart. 358154941Sjhb */ 359154941Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, 360154941Sjhb RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) { 361154941Sjhb turnstile_release(&rw->rw_object); 362154941Sjhb continue; 363154941Sjhb } 364154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 365154941Sjhb CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 366154941Sjhb __func__, rw); 367154941Sjhb 368154941Sjhb /* 369154941Sjhb * Ok. The lock is released and all that's left is to 370154941Sjhb * wake up the waiters. Note that the lock might not be 371154941Sjhb * free anymore, but in that case the writers will just 372154941Sjhb * block again if they run before the new lock holder(s) 373154941Sjhb * release the lock. 374154941Sjhb */ 375154941Sjhb ts = turnstile_lookup(&rw->rw_object); 376157846Sjhb MPASS(ts != NULL); 377154941Sjhb turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 378154941Sjhb turnstile_unpend(ts, TS_SHARED_LOCK); 379154941Sjhb break; 380154941Sjhb } 381154941Sjhb} 382154941Sjhb 383154941Sjhb/* 384154941Sjhb * This function is called when we are unable to obtain a write lock on the 385154941Sjhb * first try. This means that at least one other thread holds either a 386154941Sjhb * read or write lock. 387154941Sjhb */ 388154941Sjhbvoid 389154941Sjhb_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 390154941Sjhb{ 391157846Sjhb volatile struct thread *owner; 392154941Sjhb uintptr_t v; 393154941Sjhb 394154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 395154941Sjhb CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 396154941Sjhb rw->rw_object.lo_name, (void *)rw->rw_lock, file, line); 397154941Sjhb 398154941Sjhb while (!_rw_write_lock(rw, tid)) { 399154941Sjhb turnstile_lock(&rw->rw_object); 400154941Sjhb v = rw->rw_lock; 401154941Sjhb 402154941Sjhb /* 403154941Sjhb * If the lock was released while spinning on the 404154941Sjhb * turnstile chain lock, try again. 405154941Sjhb */ 406154941Sjhb if (v == RW_UNLOCKED) { 407154941Sjhb turnstile_release(&rw->rw_object); 408154941Sjhb cpu_spinwait(); 409154941Sjhb continue; 410154941Sjhb } 411154941Sjhb 412154941Sjhb /* 413154941Sjhb * If the lock was released by a writer with both readers 414154941Sjhb * and writers waiting and a reader hasn't woken up and 415154941Sjhb * acquired the lock yet, rw_lock will be set to the 416154941Sjhb * value RW_UNLOCKED | RW_LOCK_WRITE_WAITERS. If we see 417154941Sjhb * that value, try to acquire it once. Note that we have 418154941Sjhb * to preserve the RW_LOCK_WRITE_WAITERS flag as there are 419154941Sjhb * other writers waiting still. If we fail, restart the 420154941Sjhb * loop. 421154941Sjhb */ 422154941Sjhb if (v == (RW_UNLOCKED | RW_LOCK_WRITE_WAITERS)) { 423154941Sjhb if (atomic_cmpset_acq_ptr(&rw->rw_lock, 424154941Sjhb RW_UNLOCKED | RW_LOCK_WRITE_WAITERS, 425154941Sjhb tid | RW_LOCK_WRITE_WAITERS)) { 426154941Sjhb turnstile_claim(&rw->rw_object); 427154941Sjhb CTR2(KTR_LOCK, "%s: %p claimed by new writer", 428154941Sjhb __func__, rw); 429154941Sjhb break; 430154941Sjhb } 431154941Sjhb turnstile_release(&rw->rw_object); 432154941Sjhb cpu_spinwait(); 433154941Sjhb continue; 434154941Sjhb } 435154941Sjhb 436154941Sjhb /* 437154941Sjhb * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 438154941Sjhb * set it. If we fail to set it, then loop back and try 439154941Sjhb * again. 440154941Sjhb */ 441157826Sjhb if (!(v & RW_LOCK_WRITE_WAITERS)) { 442157826Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, v, 443157826Sjhb v | RW_LOCK_WRITE_WAITERS)) { 444157826Sjhb turnstile_release(&rw->rw_object); 445157826Sjhb cpu_spinwait(); 446157826Sjhb continue; 447157826Sjhb } 448157826Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 449157826Sjhb CTR2(KTR_LOCK, "%s: %p set write waiters flag", 450157826Sjhb __func__, rw); 451154941Sjhb } 452154941Sjhb 453157846Sjhb#ifdef SMP 454157846Sjhb /* 455157846Sjhb * If the lock is write locked and the owner is 456157846Sjhb * running on another CPU, spin until the owner stops 457157846Sjhb * running or the state of the lock changes. 458157846Sjhb */ 459157846Sjhb owner = (struct thread *)RW_OWNER(v); 460157846Sjhb if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 461157846Sjhb turnstile_release(&rw->rw_object); 462157846Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 463157846Sjhb CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 464157846Sjhb __func__, rw, owner); 465157846Sjhb while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && 466157846Sjhb TD_IS_RUNNING(owner)) 467157846Sjhb cpu_spinwait(); 468157846Sjhb continue; 469157846Sjhb } 470157846Sjhb#endif 471154941Sjhb 472154941Sjhb /* 473154941Sjhb * We were unable to acquire the lock and the write waiters 474154941Sjhb * flag is set, so we must block on the turnstile. 475154941Sjhb */ 476154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 477154941Sjhb CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 478154941Sjhb rw); 479154941Sjhb turnstile_wait(&rw->rw_object, rw_owner(rw), 480154941Sjhb TS_EXCLUSIVE_QUEUE); 481154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 482154941Sjhb CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 483154941Sjhb __func__, rw); 484154941Sjhb } 485154941Sjhb} 486154941Sjhb 487154941Sjhb/* 488154941Sjhb * This function is called if the first try at releasing a write lock failed. 489154941Sjhb * This means that one of the 2 waiter bits must be set indicating that at 490154941Sjhb * least one thread is waiting on this lock. 491154941Sjhb */ 492154941Sjhbvoid 493154941Sjhb_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 494154941Sjhb{ 495154941Sjhb struct turnstile *ts; 496154941Sjhb uintptr_t v; 497154941Sjhb int queue; 498154941Sjhb 499154941Sjhb KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 500154941Sjhb ("%s: neither of the waiter flags are set", __func__)); 501154941Sjhb 502154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 503154941Sjhb CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 504154941Sjhb 505154941Sjhb turnstile_lock(&rw->rw_object); 506154941Sjhb ts = turnstile_lookup(&rw->rw_object); 507154941Sjhb 508157846Sjhb#ifdef SMP 509157846Sjhb /* 510157846Sjhb * There might not be a turnstile for this lock if all of 511157846Sjhb * the waiters are adaptively spinning. In that case, just 512157846Sjhb * reset the lock to the unlocked state and return. 513157846Sjhb */ 514157846Sjhb if (ts == NULL) { 515157846Sjhb atomic_store_rel_ptr(&rw->rw_lock, RW_UNLOCKED); 516157846Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 517157846Sjhb CTR2(KTR_LOCK, "%s: %p no sleepers", __func__, rw); 518157846Sjhb turnstile_release(&rw->rw_object); 519157846Sjhb return; 520157846Sjhb } 521157846Sjhb#else 522154941Sjhb MPASS(ts != NULL); 523157846Sjhb#endif 524154941Sjhb 525154941Sjhb /* 526154941Sjhb * Use the same algo as sx locks for now. Prefer waking up shared 527154941Sjhb * waiters if we have any over writers. This is probably not ideal. 528154941Sjhb * 529154941Sjhb * 'v' is the value we are going to write back to rw_lock. If we 530154941Sjhb * have waiters on both queues, we need to preserve the state of 531154941Sjhb * the waiter flag for the queue we don't wake up. For now this is 532154941Sjhb * hardcoded for the algorithm mentioned above. 533154941Sjhb * 534154941Sjhb * In the case of both readers and writers waiting we wakeup the 535154941Sjhb * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 536154941Sjhb * new writer comes in before a reader it will claim the lock up 537154941Sjhb * above. There is probably a potential priority inversion in 538154941Sjhb * there that could be worked around either by waking both queues 539154941Sjhb * of waiters or doing some complicated lock handoff gymnastics. 540157846Sjhb * 541157846Sjhb * Note that in the SMP case, if both flags are set, there might 542157846Sjhb * not be any actual writers on the turnstile as they might all 543157846Sjhb * be spinning. In that case, we don't want to preserve the 544157846Sjhb * RW_LOCK_WRITE_WAITERS flag as the turnstile is going to go 545157846Sjhb * away once we wakeup all the readers. 546154941Sjhb */ 547157846Sjhb v = RW_UNLOCKED; 548154941Sjhb if (rw->rw_lock & RW_LOCK_READ_WAITERS) { 549154941Sjhb queue = TS_SHARED_QUEUE; 550157846Sjhb#ifdef SMP 551157846Sjhb if (rw->rw_lock & RW_LOCK_WRITE_WAITERS && 552157846Sjhb !turnstile_empty(ts, TS_EXCLUSIVE_QUEUE)) 553157846Sjhb v |= RW_LOCK_WRITE_WAITERS; 554157846Sjhb#else 555157846Sjhb v |= (rw->rw_lock & RW_LOCK_WRITE_WAITERS); 556157846Sjhb#endif 557157846Sjhb } else 558154941Sjhb queue = TS_EXCLUSIVE_QUEUE; 559157846Sjhb 560157846Sjhb#ifdef SMP 561157846Sjhb /* 562157846Sjhb * We have to make sure that we actually have waiters to 563157846Sjhb * wakeup. If they are all spinning, then we just need to 564157846Sjhb * disown the turnstile and return. 565157846Sjhb */ 566157846Sjhb if (turnstile_empty(ts, queue)) { 567157846Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 568157846Sjhb CTR2(KTR_LOCK, "%s: %p no sleepers 2", __func__, rw); 569157846Sjhb atomic_store_rel_ptr(&rw->rw_lock, v); 570157846Sjhb turnstile_disown(ts); 571157846Sjhb return; 572154941Sjhb } 573157846Sjhb#endif 574157846Sjhb 575157846Sjhb /* Wake up all waiters for the specific queue. */ 576154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 577154941Sjhb CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 578154941Sjhb queue == TS_SHARED_QUEUE ? "read" : "write"); 579154941Sjhb turnstile_broadcast(ts, queue); 580154941Sjhb atomic_store_rel_ptr(&rw->rw_lock, v); 581154941Sjhb turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 582154941Sjhb} 583154941Sjhb 584154941Sjhb#ifdef INVARIANT_SUPPORT 585155162Sscottl#ifndef INVARIANTS 586154941Sjhb#undef _rw_assert 587154941Sjhb#endif 588154941Sjhb 589154941Sjhb/* 590154941Sjhb * In the non-WITNESS case, rw_assert() can only detect that at least 591154941Sjhb * *some* thread owns an rlock, but it cannot guarantee that *this* 592154941Sjhb * thread owns an rlock. 593154941Sjhb */ 594154941Sjhbvoid 595154941Sjhb_rw_assert(struct rwlock *rw, int what, const char *file, int line) 596154941Sjhb{ 597154941Sjhb 598154941Sjhb if (panicstr != NULL) 599154941Sjhb return; 600154941Sjhb switch (what) { 601154941Sjhb case RA_LOCKED: 602154941Sjhb case RA_RLOCKED: 603154941Sjhb#ifdef WITNESS 604154941Sjhb witness_assert(&rw->rw_object, what, file, line); 605154941Sjhb#else 606154941Sjhb /* 607154941Sjhb * If some other thread has a write lock or we have one 608154941Sjhb * and are asserting a read lock, fail. Also, if no one 609154941Sjhb * has a lock at all, fail. 610154941Sjhb */ 611155061Sscottl if (rw->rw_lock == RW_UNLOCKED || 612155061Sscottl (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED || 613157826Sjhb rw_wowner(rw) != curthread))) 614154941Sjhb panic("Lock %s not %slocked @ %s:%d\n", 615155012Sscottl rw->rw_object.lo_name, (what == RA_RLOCKED) ? 616154941Sjhb "read " : "", file, line); 617154941Sjhb#endif 618154941Sjhb break; 619154941Sjhb case RA_WLOCKED: 620157826Sjhb if (rw_wowner(rw) != curthread) 621154941Sjhb panic("Lock %s not exclusively locked @ %s:%d\n", 622154941Sjhb rw->rw_object.lo_name, file, line); 623154941Sjhb break; 624154941Sjhb case RA_UNLOCKED: 625154941Sjhb#ifdef WITNESS 626154941Sjhb witness_assert(&rw->rw_object, what, file, line); 627154941Sjhb#else 628154941Sjhb /* 629154941Sjhb * If we hold a write lock fail. We can't reliably check 630154941Sjhb * to see if we hold a read lock or not. 631154941Sjhb */ 632157826Sjhb if (rw_wowner(rw) == curthread) 633154941Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 634154941Sjhb rw->rw_object.lo_name, file, line); 635154941Sjhb#endif 636154941Sjhb break; 637154941Sjhb default: 638154941Sjhb panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 639154941Sjhb line); 640154941Sjhb } 641154941Sjhb} 642154941Sjhb#endif /* INVARIANT_SUPPORT */ 643154941Sjhb 644154941Sjhb#ifdef DDB 645154941Sjhbvoid 646154941Sjhbdb_show_rwlock(struct lock_object *lock) 647154941Sjhb{ 648154941Sjhb struct rwlock *rw; 649154941Sjhb struct thread *td; 650154941Sjhb 651154941Sjhb rw = (struct rwlock *)lock; 652154941Sjhb 653154941Sjhb db_printf(" state: "); 654154941Sjhb if (rw->rw_lock == RW_UNLOCKED) 655154941Sjhb db_printf("UNLOCKED\n"); 656154941Sjhb else if (rw->rw_lock & RW_LOCK_READ) 657154973Smlaier db_printf("RLOCK: %jd locks\n", 658154973Smlaier (intmax_t)(RW_READERS(rw->rw_lock))); 659154941Sjhb else { 660157826Sjhb td = rw_wowner(rw); 661154941Sjhb db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 662154941Sjhb td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 663154941Sjhb } 664154941Sjhb db_printf(" waiters: "); 665154941Sjhb switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 666154941Sjhb case RW_LOCK_READ_WAITERS: 667154941Sjhb db_printf("readers\n"); 668154941Sjhb break; 669154941Sjhb case RW_LOCK_WRITE_WAITERS: 670154941Sjhb db_printf("writers\n"); 671154941Sjhb break; 672154941Sjhb case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 673154941Sjhb db_printf("readers and waiters\n"); 674154941Sjhb break; 675154941Sjhb default: 676154941Sjhb db_printf("none\n"); 677154941Sjhb break; 678154941Sjhb } 679154941Sjhb} 680154941Sjhb 681154941Sjhb#endif 682