1234949Sbapt/*- 2234949Sbapt * SPDX-License-Identifier: BSD-2-Clause 3234949Sbapt * 4234949Sbapt * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 5234949Sbapt * 6234949Sbapt * Redistribution and use in source and binary forms, with or without 7234949Sbapt * modification, are permitted provided that the following conditions 8234949Sbapt * are met: 9234949Sbapt * 1. Redistributions of source code must retain the above copyright 10234949Sbapt * notice, this list of conditions and the following disclaimer. 11234949Sbapt * 2. Redistributions in binary form must reproduce the above copyright 12234949Sbapt * notice, this list of conditions and the following disclaimer in the 13234949Sbapt * documentation and/or other materials provided with the distribution. 14234949Sbapt * 15234949Sbapt * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16234949Sbapt * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17234949Sbapt * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18234949Sbapt * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19234949Sbapt * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20234949Sbapt * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21234949Sbapt * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22234949Sbapt * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23234949Sbapt * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24234949Sbapt * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25234949Sbapt * SUCH DAMAGE. 26234949Sbapt */ 27234949Sbapt 28234949Sbapt#ifndef _SYS_RWLOCK_H_ 29234949Sbapt#define _SYS_RWLOCK_H_ 30234949Sbapt 31234949Sbapt#include <sys/_lock.h> 32234949Sbapt#include <sys/_rwlock.h> 33234949Sbapt#include <sys/lock_profile.h> 34234949Sbapt#include <sys/lockstat.h> 35234949Sbapt 36234949Sbapt#ifdef _KERNEL 37234949Sbapt#include <sys/pcpu.h> 38234949Sbapt#include <machine/atomic.h> 39234949Sbapt#endif 40234949Sbapt 41234949Sbapt/* 42234949Sbapt * The rw_lock field consists of several fields. The low bit indicates 43234949Sbapt * if the lock is locked with a read (shared) or write (exclusive) lock. 44234949Sbapt * A value of 0 indicates a write lock, and a value of 1 indicates a read 45234949Sbapt * lock. Bit 1 is a boolean indicating if there are any threads waiting 46234949Sbapt * for a read lock. Bit 2 is a boolean indicating if there are any threads 47234949Sbapt * waiting for a write lock. The rest of the variable's definition is 48234949Sbapt * dependent on the value of the first bit. For a write lock, it is a 49234949Sbapt * pointer to the thread holding the lock, similar to the mtx_lock field of 50234949Sbapt * mutexes. For read locks, it is a count of read locks that are held. 51234949Sbapt * 52234949Sbapt * When the lock is not locked by any thread, it is encoded as a read lock 53234949Sbapt * with zero waiters. 54234949Sbapt */ 55234949Sbapt 56234949Sbapt#define RW_LOCK_READ 0x01 57234949Sbapt#define RW_LOCK_READ_WAITERS 0x02 58234949Sbapt#define RW_LOCK_WRITE_WAITERS 0x04 59234949Sbapt#define RW_LOCK_WRITE_SPINNER 0x08 60234949Sbapt#define RW_LOCK_WRITER_RECURSED 0x10 61234949Sbapt#define RW_LOCK_FLAGMASK \ 62234949Sbapt (RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS | \ 63234949Sbapt RW_LOCK_WRITE_SPINNER | RW_LOCK_WRITER_RECURSED) 64234949Sbapt#define RW_LOCK_WAITERS (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS) 65234949Sbapt 66234949Sbapt#define RW_OWNER(x) ((x) & ~RW_LOCK_FLAGMASK) 67234949Sbapt#define RW_READERS_SHIFT 5 68234949Sbapt#define RW_READERS(x) (RW_OWNER((x)) >> RW_READERS_SHIFT) 69234949Sbapt#define RW_READERS_LOCK(x) ((x) << RW_READERS_SHIFT | RW_LOCK_READ) 70234949Sbapt#define RW_ONE_READER (1 << RW_READERS_SHIFT) 71234949Sbapt 72234949Sbapt#define RW_UNLOCKED RW_READERS_LOCK(0) 73234949Sbapt#define RW_DESTROYED (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS) 74234949Sbapt 75234949Sbapt#ifdef _KERNEL 76234949Sbapt 77234949Sbapt#define rw_recurse lock_object.lo_data 78234949Sbapt 79234949Sbapt#define RW_READ_VALUE(x) ((x)->rw_lock) 80234949Sbapt 81234949Sbapt/* Very simple operations on rw_lock. */ 82234949Sbapt 83234949Sbapt/* Try to obtain a write lock once. */ 84234949Sbapt#define _rw_write_lock(rw, tid) \ 85234949Sbapt atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid)) 86234949Sbapt 87234949Sbapt#define _rw_write_lock_fetch(rw, vp, tid) \ 88234949Sbapt atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)) 89234949Sbapt 90234949Sbapt/* Release a write lock quickly if there are no waiters. */ 91234949Sbapt#define _rw_write_unlock(rw, tid) \ 92234949Sbapt atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED) 93234949Sbapt 94234949Sbapt#define _rw_write_unlock_fetch(rw, tid) \ 95234949Sbapt atomic_fcmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED) 96234949Sbapt 97234949Sbapt/* 98234949Sbapt * Full lock operations that are suitable to be inlined in non-debug 99234949Sbapt * kernels. If the lock cannot be acquired or released trivially then 100234949Sbapt * the work is deferred to another function. 101234949Sbapt */ 102234949Sbapt 103234949Sbapt/* Acquire a write lock. */ 104234949Sbapt#define __rw_wlock(rw, tid, file, line) __extension__ ({ \ 105234949Sbapt uintptr_t _tid = (uintptr_t)(tid); \ 106234949Sbapt uintptr_t _v = RW_UNLOCKED; \ 107234949Sbapt \ 108234949Sbapt if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \ 109234949Sbapt !_rw_write_lock_fetch((rw), &_v, _tid))) \ 110234949Sbapt _rw_wlock_hard((rw), _v, (file), (line)); \ 111234949Sbapt (void)0; /* ensure void type for expression */ \ 112234949Sbapt}) 113234949Sbapt 114234949Sbapt/* Release a write lock. */ 115234949Sbapt#define __rw_wunlock(rw, tid, file, line) __extension__ ({ \ 116234949Sbapt uintptr_t _v = (uintptr_t)(tid); \ 117234949Sbapt \ 118234949Sbapt if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) || \ 119234949Sbapt !_rw_write_unlock_fetch((rw), &_v))) \ 120234949Sbapt _rw_wunlock_hard((rw), _v, (file), (line)); \ 121234949Sbapt (void)0; /* ensure void type for expression */ \ 122234949Sbapt}) 123234949Sbapt 124234949Sbapt/* 125234949Sbapt * Function prototypes. Routines that start with _ are not part of the 126234949Sbapt * external API and should not be called directly. Wrapper macros should 127234949Sbapt * be used instead. 128234949Sbapt */ 129234949Sbaptvoid _rw_init_flags(volatile uintptr_t *c, const char *name, int opts); 130234949Sbaptvoid _rw_destroy(volatile uintptr_t *c); 131234949Sbaptvoid rw_sysinit(void *arg); 132234949Sbaptint _rw_wowned(const volatile uintptr_t *c); 133234949Sbaptvoid _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line); 134234949Sbaptint __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF); 135234949Sbaptint __rw_try_wlock(volatile uintptr_t *c, const char *file, int line); 136234949Sbaptvoid _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line); 137234949Sbaptvoid __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF); 138234949Sbaptvoid __rw_rlock(volatile uintptr_t *c, const char *file, int line); 139234949Sbaptint __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF); 140234949Sbaptint __rw_try_rlock(volatile uintptr_t *c, const char *file, int line); 141234949Sbaptvoid _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF); 142234949Sbaptvoid _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line); 143234949Sbaptvoid __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v 144234949Sbapt LOCK_FILE_LINE_ARG_DEF); 145234949Sbaptvoid __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v 146234949Sbapt LOCK_FILE_LINE_ARG_DEF); 147234949Sbaptint __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF); 148234949Sbaptint __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line); 149234949Sbaptvoid __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF); 150234949Sbaptvoid __rw_downgrade(volatile uintptr_t *c, const char *file, int line); 151234949Sbapt#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 152234949Sbaptvoid __rw_assert(const volatile uintptr_t *c, int what, const char *file, 153234949Sbapt int line); 154234949Sbapt#endif 155234949Sbapt 156234949Sbapt/* 157234949Sbapt * Top-level macros to provide lock cookie once the actual rwlock is passed. 158234949Sbapt * They will also prevent passing a malformed object to the rwlock KPI by 159234949Sbapt * failing compilation as the rw_lock reserved member will not be found. 160234949Sbapt */ 161234949Sbapt#define rw_init(rw, n) \ 162234949Sbapt _rw_init_flags(&(rw)->rw_lock, n, 0) 163234949Sbapt#define rw_init_flags(rw, n, o) \ 164234949Sbapt _rw_init_flags(&(rw)->rw_lock, n, o) 165234949Sbapt#define rw_destroy(rw) \ 166234949Sbapt _rw_destroy(&(rw)->rw_lock) 167234949Sbapt#define rw_wowned(rw) \ 168234949Sbapt _rw_wowned(&(rw)->rw_lock) 169234949Sbapt#define _rw_wlock(rw, f, l) \ 170234949Sbapt _rw_wlock_cookie(&(rw)->rw_lock, f, l) 171234949Sbapt#define _rw_try_wlock(rw, f, l) \ 172234949Sbapt __rw_try_wlock(&(rw)->rw_lock, f, l) 173234949Sbapt#define _rw_wunlock(rw, f, l) \ 174234949Sbapt _rw_wunlock_cookie(&(rw)->rw_lock, f, l) 175234949Sbapt#define _rw_try_rlock(rw, f, l) \ 176234949Sbapt __rw_try_rlock(&(rw)->rw_lock, f, l) 177234949Sbapt#if LOCK_DEBUG > 0 178234949Sbapt#define _rw_rlock(rw, f, l) \ 179234949Sbapt __rw_rlock(&(rw)->rw_lock, f, l) 180234949Sbapt#define _rw_runlock(rw, f, l) \ 181234949Sbapt _rw_runlock_cookie(&(rw)->rw_lock, f, l) 182234949Sbapt#else 183234949Sbapt#define _rw_rlock(rw, f, l) \ 184234949Sbapt __rw_rlock_int((struct rwlock *)rw) 185234949Sbapt#define _rw_runlock(rw, f, l) \ 186234949Sbapt _rw_runlock_cookie_int((struct rwlock *)rw) 187234949Sbapt#endif 188234949Sbapt#if LOCK_DEBUG > 0 189234949Sbapt#define _rw_wlock_hard(rw, v, f, l) \ 190234949Sbapt __rw_wlock_hard(&(rw)->rw_lock, v, f, l) 191234949Sbapt#define _rw_wunlock_hard(rw, v, f, l) \ 192234949Sbapt __rw_wunlock_hard(&(rw)->rw_lock, v, f, l) 193234949Sbapt#define _rw_try_upgrade(rw, f, l) \ 194234949Sbapt __rw_try_upgrade(&(rw)->rw_lock, f, l) 195234949Sbapt#define _rw_downgrade(rw, f, l) \ 196234949Sbapt __rw_downgrade(&(rw)->rw_lock, f, l) 197234949Sbapt#else 198234949Sbapt#define _rw_wlock_hard(rw, v, f, l) \ 199234949Sbapt __rw_wlock_hard(&(rw)->rw_lock, v) 200234949Sbapt#define _rw_wunlock_hard(rw, v, f, l) \ 201234949Sbapt __rw_wunlock_hard(&(rw)->rw_lock, v) 202234949Sbapt#define _rw_try_upgrade(rw, f, l) \ 203234949Sbapt __rw_try_upgrade_int(rw) 204234949Sbapt#define _rw_downgrade(rw, f, l) \ 205234949Sbapt __rw_downgrade_int(rw) 206234949Sbapt#endif 207234949Sbapt#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 208234949Sbapt#define _rw_assert(rw, w, f, l) \ 209234949Sbapt __rw_assert(&(rw)->rw_lock, w, f, l) 210234949Sbapt#endif 211234949Sbapt 212234949Sbapt/* 213234949Sbapt * Public interface for lock operations. 214234949Sbapt */ 215234949Sbapt 216234949Sbapt#ifndef LOCK_DEBUG 217234949Sbapt#error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/rwlock.h> 218234949Sbapt#endif 219234949Sbapt#if LOCK_DEBUG > 0 || defined(RWLOCK_NOINLINE) 220234949Sbapt#define rw_wlock(rw) _rw_wlock((rw), LOCK_FILE, LOCK_LINE) 221234949Sbapt#define rw_wunlock(rw) _rw_wunlock((rw), LOCK_FILE, LOCK_LINE) 222234949Sbapt#else 223234949Sbapt#define rw_wlock(rw) \ 224234949Sbapt __rw_wlock((rw), curthread, LOCK_FILE, LOCK_LINE) 225234949Sbapt#define rw_wunlock(rw) \ 226234949Sbapt __rw_wunlock((rw), curthread, LOCK_FILE, LOCK_LINE) 227234949Sbapt#endif 228234949Sbapt#define rw_rlock(rw) _rw_rlock((rw), LOCK_FILE, LOCK_LINE) 229234949Sbapt#define rw_runlock(rw) _rw_runlock((rw), LOCK_FILE, LOCK_LINE) 230234949Sbapt#define rw_try_rlock(rw) _rw_try_rlock((rw), LOCK_FILE, LOCK_LINE) 231234949Sbapt#define rw_try_upgrade(rw) _rw_try_upgrade((rw), LOCK_FILE, LOCK_LINE) 232234949Sbapt#define rw_try_wlock(rw) _rw_try_wlock((rw), LOCK_FILE, LOCK_LINE) 233234949Sbapt#define rw_downgrade(rw) _rw_downgrade((rw), LOCK_FILE, LOCK_LINE) 234234949Sbapt#define rw_unlock(rw) __extension__ ({ \ 235234949Sbapt if (rw_wowned(rw)) \ 236234949Sbapt rw_wunlock(rw); \ 237234949Sbapt else \ 238234949Sbapt rw_runlock(rw); \ 239234949Sbapt (void)0; /* ensure void type for expression */ \ 240234949Sbapt}) 241234949Sbapt#define rw_sleep(chan, rw, pri, wmesg, timo) \ 242234949Sbapt _sleep((chan), &(rw)->lock_object, (pri), (wmesg), \ 243234949Sbapt tick_sbt * (timo), 0, C_HARDCLOCK) 244234949Sbapt 245234949Sbapt#define rw_initialized(rw) lock_initialized(&(rw)->lock_object) 246234949Sbapt 247234949Sbaptstruct rw_args { 248234949Sbapt void *ra_rw; 249234949Sbapt const char *ra_desc; 250234949Sbapt int ra_flags; 251234949Sbapt}; 252234949Sbapt 253234949Sbapt#define RW_SYSINIT_FLAGS(name, rw, desc, flags) \ 254234949Sbapt static struct rw_args name##_args = { \ 255234949Sbapt (rw), \ 256234949Sbapt (desc), \ 257234949Sbapt (flags), \ 258234949Sbapt }; \ 259234949Sbapt SYSINIT(name##_rw_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 260234949Sbapt rw_sysinit, &name##_args); \ 261234949Sbapt SYSUNINIT(name##_rw_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 262234949Sbapt _rw_destroy, __DEVOLATILE(void *, &(rw)->rw_lock)) 263234949Sbapt 264234949Sbapt#define RW_SYSINIT(name, rw, desc) RW_SYSINIT_FLAGS(name, rw, desc, 0) 265234949Sbapt 266234949Sbapt/* 267234949Sbapt * Options passed to rw_init_flags(). 268234949Sbapt */ 269234949Sbapt#define RW_DUPOK 0x01 270234949Sbapt#define RW_NOPROFILE 0x02 271234949Sbapt#define RW_NOWITNESS 0x04 272234949Sbapt#define RW_QUIET 0x08 273234949Sbapt#define RW_RECURSE 0x10 274234949Sbapt#define RW_NEW 0x20 275234949Sbapt 276234949Sbapt/* 277234949Sbapt * The INVARIANTS-enabled rw_assert() functionality. 278234949Sbapt * 279234949Sbapt * The constants need to be defined for INVARIANT_SUPPORT infrastructure 280234949Sbapt * support as _rw_assert() itself uses them and the latter implies that 281234949Sbapt * _rw_assert() must build. 282234949Sbapt */ 283234949Sbapt#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 284234949Sbapt#define RA_LOCKED LA_LOCKED 285234949Sbapt#define RA_RLOCKED LA_SLOCKED 286234949Sbapt#define RA_WLOCKED LA_XLOCKED 287234949Sbapt#define RA_UNLOCKED LA_UNLOCKED 288234949Sbapt#define RA_RECURSED LA_RECURSED 289234949Sbapt#define RA_NOTRECURSED LA_NOTRECURSED 290234949Sbapt#endif 291234949Sbapt 292234949Sbapt#ifdef INVARIANTS 293234949Sbapt#define rw_assert(rw, what) _rw_assert((rw), (what), LOCK_FILE, LOCK_LINE) 294234949Sbapt#else 295234949Sbapt#define rw_assert(rw, what) 296234949Sbapt#endif 297234949Sbapt 298234949Sbapt#endif /* _KERNEL */ 299234949Sbapt#endif /* !_SYS_RWLOCK_H_ */ 300234949Sbapt