kern_rwlock.c revision 157826
1154941Sjhb/*- 2154941Sjhb * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3154941Sjhb * All rights reserved. 4154941Sjhb * 5154941Sjhb * Redistribution and use in source and binary forms, with or without 6154941Sjhb * modification, are permitted provided that the following conditions 7154941Sjhb * are met: 8154941Sjhb * 1. Redistributions of source code must retain the above copyright 9154941Sjhb * notice, this list of conditions and the following disclaimer. 10154941Sjhb * 2. Redistributions in binary form must reproduce the above copyright 11154941Sjhb * notice, this list of conditions and the following disclaimer in the 12154941Sjhb * documentation and/or other materials provided with the distribution. 13154941Sjhb * 3. Neither the name of the author nor the names of any co-contributors 14154941Sjhb * may be used to endorse or promote products derived from this software 15154941Sjhb * without specific prior written permission. 16154941Sjhb * 17154941Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18154941Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19154941Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20154941Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21154941Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22154941Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23154941Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24154941Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25154941Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26154941Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27154941Sjhb * SUCH DAMAGE. 28154941Sjhb */ 29154941Sjhb 30154941Sjhb/* 31154941Sjhb * Machine independent bits of reader/writer lock implementation. 32154941Sjhb */ 33154941Sjhb 34154941Sjhb#include <sys/cdefs.h> 35154941Sjhb__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 157826 2006-04-17 21:11:01Z jhb $"); 36154941Sjhb 37154941Sjhb#include "opt_ddb.h" 38154941Sjhb 39154941Sjhb#include <sys/param.h> 40154941Sjhb#include <sys/ktr.h> 41154941Sjhb#include <sys/lock.h> 42154941Sjhb#include <sys/mutex.h> 43154941Sjhb#include <sys/proc.h> 44154941Sjhb#include <sys/rwlock.h> 45154941Sjhb#include <sys/systm.h> 46154941Sjhb#include <sys/turnstile.h> 47154941Sjhb 48154941Sjhb#include <machine/cpu.h> 49154941Sjhb 50154941Sjhb#ifdef DDB 51154941Sjhb#include <ddb/ddb.h> 52154941Sjhb 53154941Sjhbstatic void db_show_rwlock(struct lock_object *lock); 54154941Sjhb#endif 55154941Sjhb 56154941Sjhbstruct lock_class lock_class_rw = { 57154941Sjhb "rw", 58154941Sjhb LC_SLEEPLOCK | LC_RECURSABLE /* | LC_UPGRADABLE */, 59154941Sjhb#ifdef DDB 60154941Sjhb db_show_rwlock 61154941Sjhb#endif 62154941Sjhb}; 63154941Sjhb 64157826Sjhb/* 65157826Sjhb * Return a pointer to the owning thread if the lock is write-locked or 66157826Sjhb * NULL if the lock is unlocked or read-locked. 67157826Sjhb */ 68157826Sjhb#define rw_wowner(rw) \ 69154941Sjhb ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ 70154941Sjhb (struct thread *)RW_OWNER((rw)->rw_lock)) 71154941Sjhb 72157826Sjhb/* 73157826Sjhb * Return a pointer to the owning thread for this lock who should receive 74157826Sjhb * any priority lent by threads that block on this lock. Currently this 75157826Sjhb * is identical to rw_wowner(). 76157826Sjhb */ 77157826Sjhb#define rw_owner(rw) rw_wowner(rw) 78157826Sjhb 79154941Sjhb#ifndef INVARIANTS 80154941Sjhb#define _rw_assert(rw, what, file, line) 81154941Sjhb#endif 82154941Sjhb 83154941Sjhbvoid 84154941Sjhbrw_init(struct rwlock *rw, const char *name) 85154941Sjhb{ 86154941Sjhb 87154941Sjhb rw->rw_lock = RW_UNLOCKED; 88154941Sjhb 89154941Sjhb lock_init(&rw->rw_object, &lock_class_rw, name, NULL, LO_WITNESS | 90154941Sjhb LO_RECURSABLE /* | LO_UPGRADABLE */); 91154941Sjhb} 92154941Sjhb 93154941Sjhbvoid 94154941Sjhbrw_destroy(struct rwlock *rw) 95154941Sjhb{ 96154941Sjhb 97154941Sjhb KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked")); 98154941Sjhb lock_destroy(&rw->rw_object); 99154941Sjhb} 100154941Sjhb 101154941Sjhbvoid 102154941Sjhbrw_sysinit(void *arg) 103154941Sjhb{ 104154941Sjhb struct rw_args *args = arg; 105154941Sjhb 106154941Sjhb rw_init(args->ra_rw, args->ra_desc); 107154941Sjhb} 108154941Sjhb 109154941Sjhbvoid 110154941Sjhb_rw_wlock(struct rwlock *rw, const char *file, int line) 111154941Sjhb{ 112154941Sjhb 113154941Sjhb MPASS(curthread != NULL); 114157826Sjhb KASSERT(rw_wowner(rw) != curthread, 115154941Sjhb ("%s (%s): wlock already held @ %s:%d", __func__, 116154941Sjhb rw->rw_object.lo_name, file, line)); 117154941Sjhb WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 118154941Sjhb line); 119154941Sjhb __rw_wlock(rw, curthread, file, line); 120154941Sjhb LOCK_LOG_LOCK("WLOCK", &rw->rw_object, 0, 0, file, line); 121154941Sjhb WITNESS_LOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); 122154941Sjhb} 123154941Sjhb 124154941Sjhbvoid 125154941Sjhb_rw_wunlock(struct rwlock *rw, const char *file, int line) 126154941Sjhb{ 127154941Sjhb 128154941Sjhb MPASS(curthread != NULL); 129154941Sjhb _rw_assert(rw, RA_WLOCKED, file, line); 130154941Sjhb WITNESS_UNLOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); 131154941Sjhb LOCK_LOG_LOCK("WUNLOCK", &rw->rw_object, 0, 0, file, line); 132154941Sjhb __rw_wunlock(rw, curthread, file, line); 133154941Sjhb} 134154941Sjhb 135154941Sjhbvoid 136154941Sjhb_rw_rlock(struct rwlock *rw, const char *file, int line) 137154941Sjhb{ 138154941Sjhb uintptr_t x; 139154941Sjhb 140157826Sjhb KASSERT(rw_wowner(rw) != curthread, 141154941Sjhb ("%s (%s): wlock already held @ %s:%d", __func__, 142154941Sjhb rw->rw_object.lo_name, file, line)); 143154941Sjhb WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER, file, line); 144154941Sjhb 145154941Sjhb /* 146154941Sjhb * Note that we don't make any attempt to try to block read 147154941Sjhb * locks once a writer has blocked on the lock. The reason is 148154941Sjhb * that we currently allow for read locks to recurse and we 149154941Sjhb * don't keep track of all the holders of read locks. Thus, if 150154941Sjhb * we were to block readers once a writer blocked and a reader 151154941Sjhb * tried to recurse on their reader lock after a writer had 152154941Sjhb * blocked we would end up in a deadlock since the reader would 153154941Sjhb * be blocked on the writer, and the writer would be blocked 154154941Sjhb * waiting for the reader to release its original read lock. 155154941Sjhb */ 156154941Sjhb for (;;) { 157154941Sjhb /* 158154941Sjhb * Handle the easy case. If no other thread has a write 159154941Sjhb * lock, then try to bump up the count of read locks. Note 160154941Sjhb * that we have to preserve the current state of the 161154941Sjhb * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 162154941Sjhb * read lock, then rw_lock must have changed, so restart 163154941Sjhb * the loop. Note that this handles the case of a 164154941Sjhb * completely unlocked rwlock since such a lock is encoded 165154941Sjhb * as a read lock with no waiters. 166154941Sjhb */ 167154941Sjhb x = rw->rw_lock; 168154941Sjhb if (x & RW_LOCK_READ) { 169154941Sjhb 170154941Sjhb /* 171154941Sjhb * The RW_LOCK_READ_WAITERS flag should only be set 172154941Sjhb * if another thread currently holds a write lock, 173154941Sjhb * and in that case RW_LOCK_READ should be clear. 174154941Sjhb */ 175154941Sjhb MPASS((x & RW_LOCK_READ_WAITERS) == 0); 176154941Sjhb if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, 177154941Sjhb x + RW_ONE_READER)) { 178154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 179154941Sjhb CTR4(KTR_LOCK, 180154941Sjhb "%s: %p succeed %p -> %p", __func__, 181154941Sjhb rw, (void *)x, 182154941Sjhb (void *)(x + RW_ONE_READER)); 183154941Sjhb break; 184154941Sjhb } 185154941Sjhb continue; 186154941Sjhb } 187154941Sjhb 188154941Sjhb /* 189154941Sjhb * Okay, now it's the hard case. Some other thread already 190154941Sjhb * has a write lock, so acquire the turnstile lock so we can 191154941Sjhb * begin the process of blocking. 192154941Sjhb */ 193154941Sjhb turnstile_lock(&rw->rw_object); 194154941Sjhb 195154941Sjhb /* 196154941Sjhb * The lock might have been released while we spun, so 197154941Sjhb * recheck its state and restart the loop if there is no 198154941Sjhb * longer a write lock. 199154941Sjhb */ 200154941Sjhb x = rw->rw_lock; 201154941Sjhb if (x & RW_LOCK_READ) { 202154941Sjhb turnstile_release(&rw->rw_object); 203154941Sjhb continue; 204154941Sjhb } 205154941Sjhb 206154941Sjhb /* 207154941Sjhb * Ok, it's still a write lock. If the RW_LOCK_READ_WAITERS 208154941Sjhb * flag is already set, then we can go ahead and block. If 209154941Sjhb * it is not set then try to set it. If we fail to set it 210154941Sjhb * drop the turnstile lock and restart the loop. 211154941Sjhb */ 212157826Sjhb if (!(x & RW_LOCK_READ_WAITERS)) { 213157826Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, x, 214157826Sjhb x | RW_LOCK_READ_WAITERS)) { 215157826Sjhb turnstile_release(&rw->rw_object); 216157826Sjhb cpu_spinwait(); 217157826Sjhb continue; 218157826Sjhb } 219157826Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 220157826Sjhb CTR2(KTR_LOCK, "%s: %p set read waiters flag", 221157826Sjhb __func__, rw); 222154941Sjhb } 223154941Sjhb 224154941Sjhb /* 225154941Sjhb * We were unable to acquire the lock and the read waiters 226154941Sjhb * flag is set, so we must block on the turnstile. 227154941Sjhb */ 228154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 229154941Sjhb CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 230154941Sjhb rw); 231154941Sjhb turnstile_wait(&rw->rw_object, rw_owner(rw), TS_SHARED_QUEUE); 232154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 233154941Sjhb CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 234154941Sjhb __func__, rw); 235154941Sjhb } 236154941Sjhb 237154941Sjhb /* 238154941Sjhb * TODO: acquire "owner of record" here. Here be turnstile dragons 239154941Sjhb * however. turnstiles don't like owners changing between calls to 240154941Sjhb * turnstile_wait() currently. 241154941Sjhb */ 242154941Sjhb 243154941Sjhb LOCK_LOG_LOCK("RLOCK", &rw->rw_object, 0, 0, file, line); 244154941Sjhb WITNESS_LOCK(&rw->rw_object, 0, file, line); 245154941Sjhb} 246154941Sjhb 247154941Sjhbvoid 248154941Sjhb_rw_runlock(struct rwlock *rw, const char *file, int line) 249154941Sjhb{ 250154941Sjhb struct turnstile *ts; 251154941Sjhb uintptr_t x; 252154941Sjhb 253154941Sjhb _rw_assert(rw, RA_RLOCKED, file, line); 254154941Sjhb WITNESS_UNLOCK(&rw->rw_object, 0, file, line); 255154941Sjhb LOCK_LOG_LOCK("RUNLOCK", &rw->rw_object, 0, 0, file, line); 256154941Sjhb 257154941Sjhb /* TODO: drop "owner of record" here. */ 258154941Sjhb 259154941Sjhb for (;;) { 260154941Sjhb /* 261154941Sjhb * See if there is more than one read lock held. If so, 262154941Sjhb * just drop one and return. 263154941Sjhb */ 264154941Sjhb x = rw->rw_lock; 265154941Sjhb if (RW_READERS(x) > 1) { 266154941Sjhb if (atomic_cmpset_ptr(&rw->rw_lock, x, 267154941Sjhb x - RW_ONE_READER)) { 268154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 269154941Sjhb CTR4(KTR_LOCK, 270154941Sjhb "%s: %p succeeded %p -> %p", 271154941Sjhb __func__, rw, (void *)x, 272154941Sjhb (void *)(x - RW_ONE_READER)); 273154941Sjhb break; 274154941Sjhb } 275154941Sjhb continue; 276154941Sjhb } 277154941Sjhb 278154941Sjhb /* 279154941Sjhb * We should never have read waiters while at least one 280154941Sjhb * thread holds a read lock. (See note above) 281154941Sjhb */ 282154941Sjhb KASSERT(!(x & RW_LOCK_READ_WAITERS), 283154941Sjhb ("%s: waiting readers", __func__)); 284154941Sjhb 285154941Sjhb /* 286154941Sjhb * If there aren't any waiters for a write lock, then try 287154941Sjhb * to drop it quickly. 288154941Sjhb */ 289154941Sjhb if (!(x & RW_LOCK_WRITE_WAITERS)) { 290154941Sjhb 291154941Sjhb /* 292154941Sjhb * There shouldn't be any flags set and we should 293154941Sjhb * be the only read lock. If we fail to release 294154941Sjhb * the single read lock, then another thread might 295154941Sjhb * have just acquired a read lock, so go back up 296154941Sjhb * to the multiple read locks case. 297154941Sjhb */ 298154941Sjhb MPASS(x == RW_READERS_LOCK(1)); 299154941Sjhb if (atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1), 300154941Sjhb RW_UNLOCKED)) { 301154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 302154941Sjhb CTR2(KTR_LOCK, "%s: %p last succeeded", 303154941Sjhb __func__, rw); 304154941Sjhb break; 305154941Sjhb } 306154941Sjhb continue; 307154941Sjhb } 308154941Sjhb 309154941Sjhb /* 310154941Sjhb * There should just be one reader with one or more 311154941Sjhb * writers waiting. 312154941Sjhb */ 313154941Sjhb MPASS(x == (RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS)); 314154941Sjhb 315154941Sjhb /* 316154941Sjhb * Ok, we know we have a waiting writer and we think we 317154941Sjhb * are the last reader, so grab the turnstile lock. 318154941Sjhb */ 319154941Sjhb turnstile_lock(&rw->rw_object); 320154941Sjhb 321154941Sjhb /* 322154941Sjhb * Try to drop our lock leaving the lock in a unlocked 323154941Sjhb * state. 324154941Sjhb * 325154941Sjhb * If you wanted to do explicit lock handoff you'd have to 326154941Sjhb * do it here. You'd also want to use turnstile_signal() 327154941Sjhb * and you'd have to handle the race where a higher 328154941Sjhb * priority thread blocks on the write lock before the 329154941Sjhb * thread you wakeup actually runs and have the new thread 330154941Sjhb * "steal" the lock. For now it's a lot simpler to just 331154941Sjhb * wakeup all of the waiters. 332154941Sjhb * 333154941Sjhb * As above, if we fail, then another thread might have 334154941Sjhb * acquired a read lock, so drop the turnstile lock and 335154941Sjhb * restart. 336154941Sjhb */ 337154941Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, 338154941Sjhb RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) { 339154941Sjhb turnstile_release(&rw->rw_object); 340154941Sjhb continue; 341154941Sjhb } 342154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 343154941Sjhb CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", 344154941Sjhb __func__, rw); 345154941Sjhb 346154941Sjhb /* 347154941Sjhb * Ok. The lock is released and all that's left is to 348154941Sjhb * wake up the waiters. Note that the lock might not be 349154941Sjhb * free anymore, but in that case the writers will just 350154941Sjhb * block again if they run before the new lock holder(s) 351154941Sjhb * release the lock. 352154941Sjhb */ 353154941Sjhb ts = turnstile_lookup(&rw->rw_object); 354154941Sjhb turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 355154941Sjhb turnstile_unpend(ts, TS_SHARED_LOCK); 356154941Sjhb break; 357154941Sjhb } 358154941Sjhb} 359154941Sjhb 360154941Sjhb/* 361154941Sjhb * This function is called when we are unable to obtain a write lock on the 362154941Sjhb * first try. This means that at least one other thread holds either a 363154941Sjhb * read or write lock. 364154941Sjhb */ 365154941Sjhbvoid 366154941Sjhb_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 367154941Sjhb{ 368154941Sjhb uintptr_t v; 369154941Sjhb 370154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 371154941Sjhb CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 372154941Sjhb rw->rw_object.lo_name, (void *)rw->rw_lock, file, line); 373154941Sjhb 374154941Sjhb while (!_rw_write_lock(rw, tid)) { 375154941Sjhb turnstile_lock(&rw->rw_object); 376154941Sjhb v = rw->rw_lock; 377154941Sjhb 378154941Sjhb /* 379154941Sjhb * If the lock was released while spinning on the 380154941Sjhb * turnstile chain lock, try again. 381154941Sjhb */ 382154941Sjhb if (v == RW_UNLOCKED) { 383154941Sjhb turnstile_release(&rw->rw_object); 384154941Sjhb cpu_spinwait(); 385154941Sjhb continue; 386154941Sjhb } 387154941Sjhb 388154941Sjhb /* 389154941Sjhb * If the lock was released by a writer with both readers 390154941Sjhb * and writers waiting and a reader hasn't woken up and 391154941Sjhb * acquired the lock yet, rw_lock will be set to the 392154941Sjhb * value RW_UNLOCKED | RW_LOCK_WRITE_WAITERS. If we see 393154941Sjhb * that value, try to acquire it once. Note that we have 394154941Sjhb * to preserve the RW_LOCK_WRITE_WAITERS flag as there are 395154941Sjhb * other writers waiting still. If we fail, restart the 396154941Sjhb * loop. 397154941Sjhb */ 398154941Sjhb if (v == (RW_UNLOCKED | RW_LOCK_WRITE_WAITERS)) { 399154941Sjhb if (atomic_cmpset_acq_ptr(&rw->rw_lock, 400154941Sjhb RW_UNLOCKED | RW_LOCK_WRITE_WAITERS, 401154941Sjhb tid | RW_LOCK_WRITE_WAITERS)) { 402154941Sjhb turnstile_claim(&rw->rw_object); 403154941Sjhb CTR2(KTR_LOCK, "%s: %p claimed by new writer", 404154941Sjhb __func__, rw); 405154941Sjhb break; 406154941Sjhb } 407154941Sjhb turnstile_release(&rw->rw_object); 408154941Sjhb cpu_spinwait(); 409154941Sjhb continue; 410154941Sjhb } 411154941Sjhb 412154941Sjhb /* 413154941Sjhb * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 414154941Sjhb * set it. If we fail to set it, then loop back and try 415154941Sjhb * again. 416154941Sjhb */ 417157826Sjhb if (!(v & RW_LOCK_WRITE_WAITERS)) { 418157826Sjhb if (!atomic_cmpset_ptr(&rw->rw_lock, v, 419157826Sjhb v | RW_LOCK_WRITE_WAITERS)) { 420157826Sjhb turnstile_release(&rw->rw_object); 421157826Sjhb cpu_spinwait(); 422157826Sjhb continue; 423157826Sjhb } 424157826Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 425157826Sjhb CTR2(KTR_LOCK, "%s: %p set write waiters flag", 426157826Sjhb __func__, rw); 427154941Sjhb } 428154941Sjhb 429154941Sjhb /* XXX: Adaptively spin if current wlock owner on another CPU? */ 430154941Sjhb 431154941Sjhb /* 432154941Sjhb * We were unable to acquire the lock and the write waiters 433154941Sjhb * flag is set, so we must block on the turnstile. 434154941Sjhb */ 435154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 436154941Sjhb CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 437154941Sjhb rw); 438154941Sjhb turnstile_wait(&rw->rw_object, rw_owner(rw), 439154941Sjhb TS_EXCLUSIVE_QUEUE); 440154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 441154941Sjhb CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 442154941Sjhb __func__, rw); 443154941Sjhb } 444154941Sjhb} 445154941Sjhb 446154941Sjhb/* 447154941Sjhb * This function is called if the first try at releasing a write lock failed. 448154941Sjhb * This means that one of the 2 waiter bits must be set indicating that at 449154941Sjhb * least one thread is waiting on this lock. 450154941Sjhb */ 451154941Sjhbvoid 452154941Sjhb_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 453154941Sjhb{ 454154941Sjhb struct turnstile *ts; 455154941Sjhb uintptr_t v; 456154941Sjhb int queue; 457154941Sjhb 458154941Sjhb KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), 459154941Sjhb ("%s: neither of the waiter flags are set", __func__)); 460154941Sjhb 461154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 462154941Sjhb CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); 463154941Sjhb 464154941Sjhb turnstile_lock(&rw->rw_object); 465154941Sjhb ts = turnstile_lookup(&rw->rw_object); 466154941Sjhb 467154941Sjhb /* XXX: Adaptive fixup would be required here. */ 468154941Sjhb MPASS(ts != NULL); 469154941Sjhb 470154941Sjhb /* 471154941Sjhb * Use the same algo as sx locks for now. Prefer waking up shared 472154941Sjhb * waiters if we have any over writers. This is probably not ideal. 473154941Sjhb * 474154941Sjhb * 'v' is the value we are going to write back to rw_lock. If we 475154941Sjhb * have waiters on both queues, we need to preserve the state of 476154941Sjhb * the waiter flag for the queue we don't wake up. For now this is 477154941Sjhb * hardcoded for the algorithm mentioned above. 478154941Sjhb * 479154941Sjhb * In the case of both readers and writers waiting we wakeup the 480154941Sjhb * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a 481154941Sjhb * new writer comes in before a reader it will claim the lock up 482154941Sjhb * above. There is probably a potential priority inversion in 483154941Sjhb * there that could be worked around either by waking both queues 484154941Sjhb * of waiters or doing some complicated lock handoff gymnastics. 485154941Sjhb */ 486154941Sjhb if (rw->rw_lock & RW_LOCK_READ_WAITERS) { 487154941Sjhb queue = TS_SHARED_QUEUE; 488154941Sjhb v = RW_UNLOCKED | (rw->rw_lock & RW_LOCK_WRITE_WAITERS); 489154941Sjhb } else { 490154941Sjhb queue = TS_EXCLUSIVE_QUEUE; 491154941Sjhb v = RW_UNLOCKED; 492154941Sjhb } 493154941Sjhb if (LOCK_LOG_TEST(&rw->rw_object, 0)) 494154941Sjhb CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, 495154941Sjhb queue == TS_SHARED_QUEUE ? "read" : "write"); 496154941Sjhb 497154941Sjhb /* Wake up all waiters for the specific queue. */ 498154941Sjhb turnstile_broadcast(ts, queue); 499154941Sjhb atomic_store_rel_ptr(&rw->rw_lock, v); 500154941Sjhb turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 501154941Sjhb} 502154941Sjhb 503154941Sjhb#ifdef INVARIANT_SUPPORT 504155162Sscottl#ifndef INVARIANTS 505154941Sjhb#undef _rw_assert 506154941Sjhb#endif 507154941Sjhb 508154941Sjhb/* 509154941Sjhb * In the non-WITNESS case, rw_assert() can only detect that at least 510154941Sjhb * *some* thread owns an rlock, but it cannot guarantee that *this* 511154941Sjhb * thread owns an rlock. 512154941Sjhb */ 513154941Sjhbvoid 514154941Sjhb_rw_assert(struct rwlock *rw, int what, const char *file, int line) 515154941Sjhb{ 516154941Sjhb 517154941Sjhb if (panicstr != NULL) 518154941Sjhb return; 519154941Sjhb switch (what) { 520154941Sjhb case RA_LOCKED: 521154941Sjhb case RA_RLOCKED: 522154941Sjhb#ifdef WITNESS 523154941Sjhb witness_assert(&rw->rw_object, what, file, line); 524154941Sjhb#else 525154941Sjhb /* 526154941Sjhb * If some other thread has a write lock or we have one 527154941Sjhb * and are asserting a read lock, fail. Also, if no one 528154941Sjhb * has a lock at all, fail. 529154941Sjhb */ 530155061Sscottl if (rw->rw_lock == RW_UNLOCKED || 531155061Sscottl (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED || 532157826Sjhb rw_wowner(rw) != curthread))) 533154941Sjhb panic("Lock %s not %slocked @ %s:%d\n", 534155012Sscottl rw->rw_object.lo_name, (what == RA_RLOCKED) ? 535154941Sjhb "read " : "", file, line); 536154941Sjhb#endif 537154941Sjhb break; 538154941Sjhb case RA_WLOCKED: 539157826Sjhb if (rw_wowner(rw) != curthread) 540154941Sjhb panic("Lock %s not exclusively locked @ %s:%d\n", 541154941Sjhb rw->rw_object.lo_name, file, line); 542154941Sjhb break; 543154941Sjhb case RA_UNLOCKED: 544154941Sjhb#ifdef WITNESS 545154941Sjhb witness_assert(&rw->rw_object, what, file, line); 546154941Sjhb#else 547154941Sjhb /* 548154941Sjhb * If we hold a write lock fail. We can't reliably check 549154941Sjhb * to see if we hold a read lock or not. 550154941Sjhb */ 551157826Sjhb if (rw_wowner(rw) == curthread) 552154941Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 553154941Sjhb rw->rw_object.lo_name, file, line); 554154941Sjhb#endif 555154941Sjhb break; 556154941Sjhb default: 557154941Sjhb panic("Unknown rw lock assertion: %d @ %s:%d", what, file, 558154941Sjhb line); 559154941Sjhb } 560154941Sjhb} 561154941Sjhb#endif /* INVARIANT_SUPPORT */ 562154941Sjhb 563154941Sjhb#ifdef DDB 564154941Sjhbvoid 565154941Sjhbdb_show_rwlock(struct lock_object *lock) 566154941Sjhb{ 567154941Sjhb struct rwlock *rw; 568154941Sjhb struct thread *td; 569154941Sjhb 570154941Sjhb rw = (struct rwlock *)lock; 571154941Sjhb 572154941Sjhb db_printf(" state: "); 573154941Sjhb if (rw->rw_lock == RW_UNLOCKED) 574154941Sjhb db_printf("UNLOCKED\n"); 575154941Sjhb else if (rw->rw_lock & RW_LOCK_READ) 576154973Smlaier db_printf("RLOCK: %jd locks\n", 577154973Smlaier (intmax_t)(RW_READERS(rw->rw_lock))); 578154941Sjhb else { 579157826Sjhb td = rw_wowner(rw); 580154941Sjhb db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 581154941Sjhb td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 582154941Sjhb } 583154941Sjhb db_printf(" waiters: "); 584154941Sjhb switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { 585154941Sjhb case RW_LOCK_READ_WAITERS: 586154941Sjhb db_printf("readers\n"); 587154941Sjhb break; 588154941Sjhb case RW_LOCK_WRITE_WAITERS: 589154941Sjhb db_printf("writers\n"); 590154941Sjhb break; 591154941Sjhb case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: 592154941Sjhb db_printf("readers and waiters\n"); 593154941Sjhb break; 594154941Sjhb default: 595154941Sjhb db_printf("none\n"); 596154941Sjhb break; 597154941Sjhb } 598154941Sjhb} 599154941Sjhb 600154941Sjhb#endif 601