sx.h revision 219819
1251881Speter/*- 2251881Speter * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3251881Speter * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4251881Speter * All rights reserved. 5251881Speter * 6251881Speter * Redistribution and use in source and binary forms, with or without 7251881Speter * modification, are permitted provided that the following conditions 8251881Speter * are met: 9251881Speter * 1. Redistributions of source code must retain the above copyright 10251881Speter * notice(s), this list of conditions and the following disclaimer as 11251881Speter * the first lines of this file unmodified other than the possible 12251881Speter * addition of one or more copyright notices. 13251881Speter * 2. Redistributions in binary form must reproduce the above copyright 14251881Speter * notice(s), this list of conditions and the following disclaimer in the 15251881Speter * documentation and/or other materials provided with the distribution. 16251881Speter * 17251881Speter * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18251881Speter * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19251881Speter * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20251881Speter * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21251881Speter * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22251881Speter * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23251881Speter * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24251881Speter * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25251881Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26251881Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27251881Speter * DAMAGE. 28251881Speter * 29251881Speter * $FreeBSD: head/sys/sys/sx.h 219819 2011-03-21 09:40:01Z jeff $ 30251881Speter */ 31251881Speter 32251881Speter#ifndef _SYS_SX_H_ 33251881Speter#define _SYS_SX_H_ 34251881Speter 35251881Speter#include <sys/_lock.h> 36362181Sdim#include <sys/_sx.h> 37362181Sdim 38362181Sdim#ifdef _KERNEL 39362181Sdim#include <sys/pcpu.h> 40362181Sdim#include <sys/lock_profile.h> 41362181Sdim#include <sys/lockstat.h> 42362181Sdim#include <machine/atomic.h> 43251881Speter#endif 44251881Speter 45251881Speter/* 46251881Speter * In general, the sx locks and rwlocks use very similar algorithms. 47251881Speter * The main difference in the implementations is how threads are 48251881Speter * blocked when a lock is unavailable. For this, sx locks use sleep 49251881Speter * queues which do not support priority propagation, and rwlocks use 50251881Speter * turnstiles which do. 51251881Speter * 52251881Speter * The sx_lock field consists of several fields. The low bit 53251881Speter * indicates if the lock is locked with a shared or exclusive lock. A 54251881Speter * value of 0 indicates an exclusive lock, and a value of 1 indicates 55251881Speter * a shared lock. Bit 1 is a boolean indicating if there are any 56251881Speter * threads waiting for a shared lock. Bit 2 is a boolean indicating 57251881Speter * if there are any threads waiting for an exclusive lock. Bit 3 is a 58251881Speter * boolean indicating if an exclusive lock is recursively held. The 59251881Speter * rest of the variable's definition is dependent on the value of the 60251881Speter * first bit. For an exclusive lock, it is a pointer to the thread 61251881Speter * holding the lock, similar to the mtx_lock field of mutexes. For 62251881Speter * shared locks, it is a count of read locks that are held. 63251881Speter * 64251881Speter * When the lock is not locked by any thread, it is encoded as a 65251881Speter * shared lock with zero waiters. 66251881Speter */ 67251881Speter 68251881Speter#define SX_LOCK_SHARED 0x01 69251881Speter#define SX_LOCK_SHARED_WAITERS 0x02 70251881Speter#define SX_LOCK_EXCLUSIVE_WAITERS 0x04 71251881Speter#define SX_LOCK_RECURSED 0x08 72251881Speter#define SX_LOCK_FLAGMASK \ 73251881Speter (SX_LOCK_SHARED | SX_LOCK_SHARED_WAITERS | \ 74251881Speter SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED) 75251881Speter 76251881Speter#define SX_OWNER(x) ((x) & ~SX_LOCK_FLAGMASK) 77251881Speter#define SX_SHARERS_SHIFT 4 78251881Speter#define SX_SHARERS(x) (SX_OWNER(x) >> SX_SHARERS_SHIFT) 79251881Speter#define SX_SHARERS_LOCK(x) \ 80251881Speter ((x) << SX_SHARERS_SHIFT | SX_LOCK_SHARED) 81251881Speter#define SX_ONE_SHARER (1 << SX_SHARERS_SHIFT) 82251881Speter 83251881Speter#define SX_LOCK_UNLOCKED SX_SHARERS_LOCK(0) 84251881Speter#define SX_LOCK_DESTROYED \ 85251881Speter (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS) 86251881Speter 87251881Speter#ifdef _KERNEL 88251881Speter 89251881Speter/* 90251881Speter * Function prototipes. Routines that start with an underscore are not part 91251881Speter * of the public interface and are wrappered with a macro. 92251881Speter */ 93251881Spetervoid sx_sysinit(void *arg); 94251881Speter#define sx_init(sx, desc) sx_init_flags((sx), (desc), 0) 95251881Spetervoid sx_init_flags(struct sx *sx, const char *description, int opts); 96251881Spetervoid sx_destroy(struct sx *sx); 97251881Speterint _sx_slock(struct sx *sx, int opts, const char *file, int line); 98251881Speterint _sx_xlock(struct sx *sx, int opts, const char *file, int line); 99251881Speterint _sx_try_slock(struct sx *sx, const char *file, int line); 100251881Speterint _sx_try_xlock(struct sx *sx, const char *file, int line); 101251881Spetervoid _sx_sunlock(struct sx *sx, const char *file, int line); 102251881Spetervoid _sx_xunlock(struct sx *sx, const char *file, int line); 103251881Speterint _sx_try_upgrade(struct sx *sx, const char *file, int line); 104251881Spetervoid _sx_downgrade(struct sx *sx, const char *file, int line); 105251881Speterint _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, 106251881Speter const char *file, int line); 107251881Speterint _sx_slock_hard(struct sx *sx, int opts, const char *file, int line); 108251881Spetervoid _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int 109251881Speter line); 110251881Spetervoid _sx_sunlock_hard(struct sx *sx, const char *file, int line); 111251881Speter#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 112251881Spetervoid _sx_assert(struct sx *sx, int what, const char *file, int line); 113251881Speter#endif 114251881Speter#ifdef DDB 115362181Sdimint sx_chain(struct thread *td, struct thread **ownerp); 116251881Speter#endif 117251881Speter 118251881Speterstruct sx_args { 119251881Speter struct sx *sa_sx; 120251881Speter const char *sa_desc; 121251881Speter int sa_flags; 122251881Speter}; 123251881Speter 124362181Sdim#define SX_SYSINIT_FLAGS(name, sxa, desc, flags) \ 125251881Speter static struct sx_args name##_args = { \ 126251881Speter (sxa), \ 127251881Speter (desc), \ 128251881Speter (flags) \ 129251881Speter }; \ 130251881Speter SYSINIT(name##_sx_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 131251881Speter sx_sysinit, &name##_args); \ 132251881Speter SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 133362181Sdim sx_destroy, (sxa)) 134251881Speter 135251881Speter#define SX_SYSINIT(name, sxa, desc) SX_SYSINIT_FLAGS(name, sxa, desc, 0) 136251881Speter 137251881Speter/* 138251881Speter * Full lock operations that are suitable to be inlined in non-debug kernels. 139251881Speter * If the lock can't be acquired or released trivially then the work is 140251881Speter * deferred to 'tougher' functions. 141251881Speter */ 142251881Speter 143251881Speter/* Acquire an exclusive lock. */ 144251881Speterstatic __inline int 145251881Speter__sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file, 146251881Speter int line) 147251881Speter{ 148251881Speter uintptr_t tid = (uintptr_t)td; 149251881Speter int error = 0; 150251881Speter 151251881Speter if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) 152251881Speter error = _sx_xlock_hard(sx, tid, opts, file, line); 153251881Speter else 154251881Speter LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, 155251881Speter sx, 0, 0, file, line); 156251881Speter 157251881Speter return (error); 158251881Speter} 159251881Speter 160251881Speter/* Release an exclusive lock. */ 161251881Speterstatic __inline void 162251881Speter__sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line) 163251881Speter{ 164251881Speter uintptr_t tid = (uintptr_t)td; 165251881Speter 166251881Speter if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) 167251881Speter _sx_xunlock_hard(sx, tid, file, line); 168251881Speter} 169251881Speter 170251881Speter/* Acquire a shared lock. */ 171251881Speterstatic __inline int 172251881Speter__sx_slock(struct sx *sx, int opts, const char *file, int line) 173251881Speter{ 174251881Speter uintptr_t x = sx->sx_lock; 175251881Speter int error = 0; 176251881Speter 177251881Speter if (!(x & SX_LOCK_SHARED) || 178251881Speter !atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) 179251881Speter error = _sx_slock_hard(sx, opts, file, line); 180251881Speter else 181251881Speter LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 0, 182251881Speter 0, file, line); 183251881Speter 184251881Speter return (error); 185251881Speter} 186251881Speter 187251881Speter/* 188251881Speter * Release a shared lock. We can just drop a single shared lock so 189251881Speter * long as we aren't trying to drop the last shared lock when other 190251881Speter * threads are waiting for an exclusive lock. This takes advantage of 191251881Speter * the fact that an unlocked lock is encoded as a shared lock with a 192251881Speter * count of 0. 193251881Speter */ 194251881Speterstatic __inline void 195251881Speter__sx_sunlock(struct sx *sx, const char *file, int line) 196251881Speter{ 197251881Speter uintptr_t x = sx->sx_lock; 198251881Speter 199251881Speter if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) || 200251881Speter !atomic_cmpset_rel_ptr(&sx->sx_lock, x, x - SX_ONE_SHARER)) 201251881Speter _sx_sunlock_hard(sx, file, line); 202251881Speter} 203251881Speter 204251881Speter/* 205251881Speter * Public interface for lock operations. 206251881Speter */ 207251881Speter#ifndef LOCK_DEBUG 208251881Speter#error "LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>" 209251881Speter#endif 210251881Speter#if (LOCK_DEBUG > 0) || defined(SX_NOINLINE) 211251881Speter#define sx_xlock(sx) (void)_sx_xlock((sx), 0, LOCK_FILE, LOCK_LINE) 212251881Speter#define sx_xlock_sig(sx) \ 213251881Speter _sx_xlock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 214251881Speter#define sx_xunlock(sx) _sx_xunlock((sx), LOCK_FILE, LOCK_LINE) 215251881Speter#define sx_slock(sx) (void)_sx_slock((sx), 0, LOCK_FILE, LOCK_LINE) 216251881Speter#define sx_slock_sig(sx) \ 217251881Speter _sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 218251881Speter#define sx_sunlock(sx) _sx_sunlock((sx), LOCK_FILE, LOCK_LINE) 219251881Speter#else 220251881Speter#define sx_xlock(sx) \ 221251881Speter (void)__sx_xlock((sx), curthread, 0, LOCK_FILE, LOCK_LINE) 222251881Speter#define sx_xlock_sig(sx) \ 223251881Speter __sx_xlock((sx), curthread, SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 224251881Speter#define sx_xunlock(sx) \ 225251881Speter __sx_xunlock((sx), curthread, LOCK_FILE, LOCK_LINE) 226251881Speter#define sx_slock(sx) (void)__sx_slock((sx), 0, LOCK_FILE, LOCK_LINE) 227251881Speter#define sx_slock_sig(sx) \ 228251881Speter __sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 229251881Speter#define sx_sunlock(sx) __sx_sunlock((sx), LOCK_FILE, LOCK_LINE) 230251881Speter#endif /* LOCK_DEBUG > 0 || SX_NOINLINE */ 231251881Speter#define sx_try_slock(sx) _sx_try_slock((sx), LOCK_FILE, LOCK_LINE) 232251881Speter#define sx_try_xlock(sx) _sx_try_xlock((sx), LOCK_FILE, LOCK_LINE) 233251881Speter#define sx_try_upgrade(sx) _sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE) 234251881Speter#define sx_downgrade(sx) _sx_downgrade((sx), LOCK_FILE, LOCK_LINE) 235251881Speter 236251881Speter/* 237251881Speter * Return a pointer to the owning thread if the lock is exclusively 238251881Speter * locked. 239251881Speter */ 240251881Speter#define sx_xholder(sx) \ 241251881Speter ((sx)->sx_lock & SX_LOCK_SHARED ? NULL : \ 242251881Speter (struct thread *)SX_OWNER((sx)->sx_lock)) 243251881Speter 244251881Speter#define sx_xlocked(sx) \ 245251881Speter (((sx)->sx_lock & ~(SX_LOCK_FLAGMASK & ~SX_LOCK_SHARED)) == \ 246251881Speter (uintptr_t)curthread) 247251881Speter 248251881Speter#define sx_unlock(sx) do { \ 249251881Speter if (sx_xlocked(sx)) \ 250251881Speter sx_xunlock(sx); \ 251251881Speter else \ 252251881Speter sx_sunlock(sx); \ 253251881Speter} while (0) 254251881Speter 255251881Speter#define sx_sleep(chan, sx, pri, wmesg, timo) \ 256251881Speter _sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo)) 257251881Speter 258251881Speter/* 259251881Speter * Options passed to sx_init_flags(). 260251881Speter */ 261251881Speter#define SX_DUPOK 0x01 262251881Speter#define SX_NOPROFILE 0x02 263251881Speter#define SX_NOWITNESS 0x04 264251881Speter#define SX_QUIET 0x08 265251881Speter#define SX_NOADAPTIVE 0x10 266251881Speter#define SX_RECURSE 0x20 267251881Speter 268251881Speter/* 269251881Speter * Options passed to sx_*lock_hard(). 270251881Speter */ 271251881Speter#define SX_INTERRUPTIBLE 0x40 272251881Speter 273251881Speter#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 274251881Speter#define SA_LOCKED LA_LOCKED 275251881Speter#define SA_SLOCKED LA_SLOCKED 276251881Speter#define SA_XLOCKED LA_XLOCKED 277251881Speter#define SA_UNLOCKED LA_UNLOCKED 278251881Speter#define SA_RECURSED LA_RECURSED 279251881Speter#define SA_NOTRECURSED LA_NOTRECURSED 280251881Speter 281251881Speter/* Backwards compatability. */ 282251881Speter#define SX_LOCKED LA_LOCKED 283251881Speter#define SX_SLOCKED LA_SLOCKED 284251881Speter#define SX_XLOCKED LA_XLOCKED 285251881Speter#define SX_UNLOCKED LA_UNLOCKED 286251881Speter#define SX_RECURSED LA_RECURSED 287251881Speter#define SX_NOTRECURSED LA_NOTRECURSED 288251881Speter#endif 289251881Speter 290251881Speter#ifdef INVARIANTS 291251881Speter#define sx_assert(sx, what) _sx_assert((sx), (what), LOCK_FILE, LOCK_LINE) 292251881Speter#else 293251881Speter#define sx_assert(sx, what) (void)0 294251881Speter#endif 295251881Speter 296251881Speter#endif /* _KERNEL */ 297251881Speter 298251881Speter#endif /* !_SYS_SX_H_ */ 299251881Speter