sx.h revision 193011
11541Srgrimes/*- 21541Srgrimes * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 31541Srgrimes * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 41541Srgrimes * All rights reserved. 551138Salfred * 6136831Srwatson * Redistribution and use in source and binary forms, with or without 71541Srgrimes * modification, are permitted provided that the following conditions 81541Srgrimes * are met: 9106149Sdwmalone * 1. Redistributions of source code must retain the above copyright 101541Srgrimes * notice(s), this list of conditions and the following disclaimer as 1164002Speter * the first lines of this file unmodified other than the possible 121541Srgrimes * addition of one or more copyright notices. 131541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 141541Srgrimes * notice(s), this list of conditions and the following disclaimer in the 151541Srgrimes * documentation and/or other materials provided with the distribution. 161541Srgrimes * 171541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 181541Srgrimes * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 191541Srgrimes * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 201541Srgrimes * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 211541Srgrimes * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 221541Srgrimes * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 231541Srgrimes * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 241541Srgrimes * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 251541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 261541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 271541Srgrimes * DAMAGE. 28122540Smckusick * 291541Srgrimes * $FreeBSD: head/sys/sys/sx.h 193011 2009-05-29 01:49:27Z attilio $ 301541Srgrimes */ 311541Srgrimes 321541Srgrimes#ifndef _SYS_SX_H_ 331541Srgrimes#define _SYS_SX_H_ 341541Srgrimes 351541Srgrimes#include <sys/_lock.h> 361541Srgrimes#include <sys/_sx.h> 371541Srgrimes 381541Srgrimes#ifdef _KERNEL 391541Srgrimes#include <sys/pcpu.h> 401541Srgrimes#include <sys/lock_profile.h> 411541Srgrimes#include <sys/lockstat.h> 421541Srgrimes#include <machine/atomic.h> 431541Srgrimes#endif 441541Srgrimes 451541Srgrimes/* 461541Srgrimes * In general, the sx locks and rwlocks use very similar algorithms. 471541Srgrimes * The main difference in the implementations is how threads are 481541Srgrimes * blocked when a lock is unavailable. For this, sx locks use sleep 491541Srgrimes * queues which do not support priority propagation, and rwlocks use 501541Srgrimes * turnstiles which do. 511541Srgrimes * 521541Srgrimes * The sx_lock field consists of several fields. The low bit 531541Srgrimes * indicates if the lock is locked with a shared or exclusive lock. A 541541Srgrimes * value of 0 indicates an exclusive lock, and a value of 1 indicates 551541Srgrimes * a shared lock. Bit 1 is a boolean indicating if there are any 5652150Smarcel * threads waiting for a shared lock. Bit 2 is a boolean indicating 571541Srgrimes * if there are any threads waiting for an exclusive lock. Bit 3 is a 5852150Smarcel * boolean indicating if an exclusive lock is recursively held. The 591541Srgrimes * rest of the variable's definition is dependent on the value of the 601541Srgrimes * first bit. For an exclusive lock, it is a pointer to the thread 611541Srgrimes * holding the lock, similar to the mtx_lock field of mutexes. For 6252150Smarcel * shared locks, it is a count of read locks that are held. 631541Srgrimes * 641541Srgrimes * When the lock is not locked by any thread, it is encoded as a 651541Srgrimes * shared lock with zero waiters. 661541Srgrimes * 671541Srgrimes * A note about memory barriers. Exclusive locks need to use the same 681541Srgrimes * memory barriers as mutexes: _acq when acquiring an exclusive lock 691541Srgrimes * and _rel when releasing an exclusive lock. On the other side, 701541Srgrimes * shared lock needs to use an _acq barrier when acquiring the lock 711541Srgrimes * but, since they don't update any locked data, no memory barrier is 721541Srgrimes * needed when releasing a shared lock. 731541Srgrimes */ 741541Srgrimes 751541Srgrimes#define SX_LOCK_SHARED 0x01 761541Srgrimes#define SX_LOCK_SHARED_WAITERS 0x02 771541Srgrimes#define SX_LOCK_EXCLUSIVE_WAITERS 0x04 781541Srgrimes#define SX_LOCK_RECURSED 0x08 791541Srgrimes#define SX_LOCK_FLAGMASK \ 801541Srgrimes (SX_LOCK_SHARED | SX_LOCK_SHARED_WAITERS | \ 811541Srgrimes SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED) 821541Srgrimes 831541Srgrimes#define SX_OWNER(x) ((x) & ~SX_LOCK_FLAGMASK) 841541Srgrimes#define SX_SHARERS_SHIFT 4 851541Srgrimes#define SX_SHARERS(x) (SX_OWNER(x) >> SX_SHARERS_SHIFT) 861541Srgrimes#define SX_SHARERS_LOCK(x) \ 871541Srgrimes ((x) << SX_SHARERS_SHIFT | SX_LOCK_SHARED) 881541Srgrimes#define SX_ONE_SHARER (1 << SX_SHARERS_SHIFT) 891541Srgrimes 901541Srgrimes#define SX_LOCK_UNLOCKED SX_SHARERS_LOCK(0) 911541Srgrimes#define SX_LOCK_DESTROYED \ 921541Srgrimes (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS) 931541Srgrimes 941541Srgrimes#ifdef _KERNEL 951541Srgrimes 961541Srgrimes/* 971541Srgrimes * Function prototipes. Routines that start with an underscore are not part 981541Srgrimes * of the public interface and are wrappered with a macro. 991541Srgrimes */ 1001541Srgrimesvoid sx_sysinit(void *arg); 1011541Srgrimes#define sx_init(sx, desc) sx_init_flags((sx), (desc), 0) 1021541Srgrimesvoid sx_init_flags(struct sx *sx, const char *description, int opts); 1031541Srgrimesvoid sx_destroy(struct sx *sx); 1041541Srgrimesint _sx_slock(struct sx *sx, int opts, const char *file, int line); 1051541Srgrimesint _sx_xlock(struct sx *sx, int opts, const char *file, int line); 1061541Srgrimesint _sx_try_slock(struct sx *sx, const char *file, int line); 1071541Srgrimesint _sx_try_xlock(struct sx *sx, const char *file, int line); 1081541Srgrimesvoid _sx_sunlock(struct sx *sx, const char *file, int line); 1091541Srgrimesvoid _sx_xunlock(struct sx *sx, const char *file, int line); 1101541Srgrimesint _sx_try_upgrade(struct sx *sx, const char *file, int line); 1111541Srgrimesvoid _sx_downgrade(struct sx *sx, const char *file, int line); 1121541Srgrimesint _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, 113105950Speter const char *file, int line); 1141541Srgrimesint _sx_slock_hard(struct sx *sx, int opts, const char *file, int line); 1151541Srgrimesvoid _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int 1161541Srgrimes line); 1171541Srgrimesvoid _sx_sunlock_hard(struct sx *sx, const char *file, int line); 1181541Srgrimes#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 1191541Srgrimesvoid _sx_assert(struct sx *sx, int what, const char *file, int line); 1201541Srgrimes#endif 12152150Smarcel#ifdef DDB 1221541Srgrimesint sx_chain(struct thread *td, struct thread **ownerp); 1231541Srgrimes#endif 1241541Srgrimes 1251541Srgrimesstruct sx_args { 1261541Srgrimes struct sx *sa_sx; 1271541Srgrimes const char *sa_desc; 1281541Srgrimes}; 12914220Speter 1301541Srgrimes#define SX_SYSINIT(name, sxa, desc) \ 1311541Srgrimes static struct sx_args name##_args = { \ 1321541Srgrimes (sxa), \ 1331541Srgrimes (desc) \ 1341541Srgrimes }; \ 1351541Srgrimes SYSINIT(name##_sx_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 1368019Sache sx_sysinit, &name##_args); \ 1378019Sache SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 1381541Srgrimes sx_destroy, (sxa)) 1391541Srgrimes 1401541Srgrimes/* 1411541Srgrimes * Full lock operations that are suitable to be inlined in non-debug kernels. 1421541Srgrimes * If the lock can't be acquired or released trivially then the work is 1431541Srgrimes * deferred to 'tougher' functions. 1441541Srgrimes */ 1451541Srgrimes 1461541Srgrimes/* Acquire an exclusive lock. */ 1471541Srgrimesstatic __inline int 1481541Srgrimes__sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file, 1491541Srgrimes int line) 1501541Srgrimes{ 1511541Srgrimes uintptr_t tid = (uintptr_t)td; 1521541Srgrimes int error = 0; 1531541Srgrimes 1541541Srgrimes if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) 1551541Srgrimes error = _sx_xlock_hard(sx, tid, opts, file, line); 1561541Srgrimes else 1571541Srgrimes LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, 1581541Srgrimes sx, 0, 0, file, line); 1591541Srgrimes 1601541Srgrimes return (error); 16114220Speter} 16214220Speter 16314220Speter/* Release an exclusive lock. */ 1641541Srgrimesstatic __inline void 1651541Srgrimes__sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line) 1661541Srgrimes{ 167122540Smckusick uintptr_t tid = (uintptr_t)td; 168122540Smckusick 1691541Srgrimes if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) 170127891Sdfr _sx_xunlock_hard(sx, tid, file, line); 1711541Srgrimes} 1721549Srgrimes 1731549Srgrimes/* Acquire a shared lock. */ 1741549Srgrimesstatic __inline int 1751549Srgrimes__sx_slock(struct sx *sx, int opts, const char *file, int line) 1762442Sdg{ 1771541Srgrimes uintptr_t x = sx->sx_lock; 1781541Srgrimes int error = 0; 1792729Sdfr 1802729Sdfr if (!(x & SX_LOCK_SHARED) || 1811541Srgrimes !atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) 1821541Srgrimes error = _sx_slock_hard(sx, opts, file, line); 18345065Salc else 18445065Salc LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 0, 1852858Swollman 0, file, line); 1862297Swollman 18714220Speter return (error); 18814220Speter} 18914220Speter 1901541Srgrimes/* 1911541Srgrimes * Release a shared lock. We can just drop a single shared lock so 1921541Srgrimes * long as we aren't trying to drop the last shared lock when other 1931541Srgrimes * threads are waiting for an exclusive lock. This takes advantage of 19432889Sphk * the fact that an unlocked lock is encoded as a shared lock with a 19532889Sphk * count of 0. 19632889Sphk */ 19732889Sphkstatic __inline void 1981541Srgrimes__sx_sunlock(struct sx *sx, const char *file, int line) 1991541Srgrimes{ 2001541Srgrimes uintptr_t x = sx->sx_lock; 2011541Srgrimes 2021541Srgrimes if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) || 2031541Srgrimes !atomic_cmpset_ptr(&sx->sx_lock, x, x - SX_ONE_SHARER)) 2041541Srgrimes _sx_sunlock_hard(sx, file, line); 2051541Srgrimes} 2061541Srgrimes 2071541Srgrimes/* 2081541Srgrimes * Public interface for lock operations. 2091541Srgrimes */ 2101541Srgrimes#ifndef LOCK_DEBUG 2111541Srgrimes#error "LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>" 2121541Srgrimes#endif 2131541Srgrimes#if (LOCK_DEBUG > 0) || defined(SX_NOINLINE) 2141541Srgrimes#define sx_xlock(sx) (void)_sx_xlock((sx), 0, LOCK_FILE, LOCK_LINE) 21535938Sdyson#define sx_xlock_sig(sx) \ 21635938Sdyson _sx_xlock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 21728400Speter#define sx_xunlock(sx) _sx_xunlock((sx), LOCK_FILE, LOCK_LINE) 21825582Speter#define sx_slock(sx) (void)_sx_slock((sx), 0, LOCK_FILE, LOCK_LINE) 21929349Speter#define sx_slock_sig(sx) \ 2202124Sdg _sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 2212124Sdg#define sx_sunlock(sx) _sx_sunlock((sx), LOCK_FILE, LOCK_LINE) 2222124Sdg#else 2232124Sdg#define sx_xlock(sx) \ 2242124Sdg (void)__sx_xlock((sx), curthread, 0, LOCK_FILE, LOCK_LINE) 2252124Sdg#define sx_xlock_sig(sx) \ 2262124Sdg __sx_xlock((sx), curthread, SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 2272124Sdg#define sx_xunlock(sx) \ 2282124Sdg __sx_xunlock((sx), curthread, LOCK_FILE, LOCK_LINE) 2292124Sdg#define sx_slock(sx) (void)__sx_slock((sx), 0, LOCK_FILE, LOCK_LINE) 23012865Speter#define sx_slock_sig(sx) \ 23112865Speter __sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE) 23212865Speter#define sx_sunlock(sx) __sx_sunlock((sx), LOCK_FILE, LOCK_LINE) 23359829Speter#endif /* LOCK_DEBUG > 0 || SX_NOINLINE */ 23412865Speter#define sx_try_slock(sx) _sx_try_slock((sx), LOCK_FILE, LOCK_LINE) 23512865Speter#define sx_try_xlock(sx) _sx_try_xlock((sx), LOCK_FILE, LOCK_LINE) 23612865Speter#define sx_try_upgrade(sx) _sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE) 23712865Speter#define sx_downgrade(sx) _sx_downgrade((sx), LOCK_FILE, LOCK_LINE) 23812865Speter 23912865Speter/* 24012865Speter * Return a pointer to the owning thread if the lock is exclusively 24112865Speter * locked. 24225582Speter */ 24325582Speter#define sx_xholder(sx) \ 24425582Speter ((sx)->sx_lock & SX_LOCK_SHARED ? NULL : \ 24525582Speter (struct thread *)SX_OWNER((sx)->sx_lock)) 24625582Speter 24725582Speter#define sx_xlocked(sx) \ 24825582Speter (((sx)->sx_lock & ~(SX_LOCK_FLAGMASK & ~SX_LOCK_SHARED)) == \ 24925582Speter (uintptr_t)curthread) 25025582Speter 25114220Speter#define sx_unlock(sx) do { \ 25214220Speter if (sx_xlocked(sx)) \ 25314220Speter sx_xunlock(sx); \ 25414220Speter else \ 25514220Speter sx_sunlock(sx); \ 25614220Speter} while (0) 25714220Speter 258137875Smarks#define sx_sleep(chan, sx, pri, wmesg, timo) \ 25914220Speter _sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo)) 26014220Speter 26114220Speter/* 26229349Speter * Options passed to sx_init_flags(). 26324452Speter */ 26424440Speter#define SX_DUPOK 0x01 26525537Sdfr#define SX_NOPROFILE 0x02 26625537Sdfr#define SX_NOWITNESS 0x04 26725537Sdfr#define SX_QUIET 0x08 26825537Sdfr#define SX_NOADAPTIVE 0x10 26925537Sdfr#define SX_RECURSE 0x20 27025537Sdfr 27125537Sdfr/* 27225537Sdfr * Options passed to sx_*lock_hard(). 27325537Sdfr */ 27425537Sdfr#define SX_INTERRUPTIBLE 0x40 27525537Sdfr 27625537Sdfr#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 27725537Sdfr#define SA_LOCKED LA_LOCKED 27825537Sdfr#define SA_SLOCKED LA_SLOCKED 27925537Sdfr#define SA_XLOCKED LA_XLOCKED 28025537Sdfr#define SA_UNLOCKED LA_UNLOCKED 28125537Sdfr#define SA_RECURSED LA_RECURSED 28235938Sdyson#define SA_NOTRECURSED LA_NOTRECURSED 28325537Sdfr 28435938Sdyson/* Backwards compatability. */ 28535938Sdyson#define SX_LOCKED LA_LOCKED 28635938Sdyson#define SX_SLOCKED LA_SLOCKED 28735938Sdyson#define SX_XLOCKED LA_XLOCKED 28835938Sdyson#define SX_UNLOCKED LA_UNLOCKED 28935938Sdyson#define SX_RECURSED LA_RECURSED 29035938Sdyson#define SX_NOTRECURSED LA_NOTRECURSED 29125537Sdfr#endif 29225537Sdfr 29325537Sdfr#ifdef INVARIANTS 29425537Sdfr#define sx_assert(sx, what) _sx_assert((sx), (what), LOCK_FILE, LOCK_LINE) 29525537Sdfr#else 29625537Sdfr#define sx_assert(sx, what) 29725537Sdfr#endif 29825537Sdfr 29925537Sdfr#endif /* _KERNEL */ 30025537Sdfr 30125537Sdfr#endif /* !_SYS_SX_H_ */ 30225537Sdfr