sx.h revision 169394
138363Swpaul/*- 238363Swpaul * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 338363Swpaul * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 438363Swpaul * All rights reserved. 538363Swpaul * 638363Swpaul * Redistribution and use in source and binary forms, with or without 738363Swpaul * modification, are permitted provided that the following conditions 838363Swpaul * are met: 938363Swpaul * 1. Redistributions of source code must retain the above copyright 1038363Swpaul * notice(s), this list of conditions and the following disclaimer as 1138363Swpaul * the first lines of this file unmodified other than the possible 1238363Swpaul * addition of one or more copyright notices. 1338363Swpaul * 2. Redistributions in binary form must reproduce the above copyright 1438363Swpaul * notice(s), this list of conditions and the following disclaimer in the 1538363Swpaul * documentation and/or other materials provided with the distribution. 1638363Swpaul * 1738363Swpaul * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1838363Swpaul * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1938363Swpaul * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 2038363Swpaul * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2138363Swpaul * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2238363Swpaul * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2338363Swpaul * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2438363Swpaul * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2538363Swpaul * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2638363Swpaul * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2738363Swpaul * DAMAGE. 2838363Swpaul * 2938363Swpaul * $FreeBSD: head/sys/sys/sx.h 169394 2007-05-08 21:51:37Z jhb $ 3038363Swpaul */ 3138363Swpaul 3246204Swpaul#ifndef _SYS_SX_H_ 3338363Swpaul#define _SYS_SX_H_ 3438363Swpaul 3538363Swpaul#include <sys/_lock.h> 3638363Swpaul#include <sys/_sx.h> 3738363Swpaul 3838363Swpaul#ifdef _KERNEL 3938363Swpaul#include <machine/atomic.h> 4038363Swpaul#endif 4138363Swpaul 4238363Swpaul/* 4338363Swpaul * In general, the sx locks and rwlocks use very similar algorithms. 4438363Swpaul * The main difference in the implementations is how threads are 4538363Swpaul * blocked when a lock is unavailable. For this, sx locks use sleep 4638363Swpaul * queues which do not support priority propagation, and rwlocks use 4738363Swpaul * turnstiles which do. 4838363Swpaul * 4938363Swpaul * The sx_lock field consists of several fields. The low bit 5038363Swpaul * indicates if the lock is locked with a shared or exclusive lock. A 5138363Swpaul * value of 0 indicates an exclusive lock, and a value of 1 indicates 5238363Swpaul * a shared lock. Bit 1 is a boolean indicating if there are any 5338363Swpaul * threads waiting for a shared lock. Bit 2 is a boolean indicating 5438363Swpaul * if there are any threads waiting for an exclusive lock. Bit 3 is a 5538363Swpaul * boolean indicating if an exclusive lock is recursively held. The 5638363Swpaul * rest of the variable's definition is dependent on the value of the 5738363Swpaul * first bit. For an exclusive lock, it is a pointer to the thread 5838363Swpaul * holding the lock, similar to the mtx_lock field of mutexes. For 5938363Swpaul * shared locks, it is a count of read locks that are held. 6038363Swpaul * 6138363Swpaul * When the lock is not locked by any thread, it is encoded as a 6238363Swpaul * shared lock with zero waiters. 6338363Swpaul * 6438363Swpaul * A note about memory barriers. Exclusive locks need to use the same 6538363Swpaul * memory barriers as mutexes: _acq when acquiring an exclusive lock 6638363Swpaul * and _rel when releasing an exclusive lock. On the other side, 6738363Swpaul * shared lock needs to use an _acq barrier when acquiring the lock 6838363Swpaul * but, since they don't update any locked data, no memory barrier is 6938363Swpaul * needed when releasing a shared lock. 7038363Swpaul */ 7138363Swpaul 7238363Swpaul#define SX_LOCK_SHARED 0x01 7338363Swpaul#define SX_LOCK_SHARED_WAITERS 0x02 7438363Swpaul#define SX_LOCK_EXCLUSIVE_WAITERS 0x04 7538363Swpaul#define SX_LOCK_RECURSED 0x08 7638363Swpaul#define SX_LOCK_FLAGMASK \ 7738363Swpaul (SX_LOCK_SHARED | SX_LOCK_SHARED_WAITERS | \ 7838363Swpaul SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED) 7938363Swpaul 8038363Swpaul#define SX_OWNER(x) ((x) & ~SX_LOCK_FLAGMASK) 8138363Swpaul#define SX_SHARERS_SHIFT 4 8238363Swpaul#define SX_SHARERS(x) (SX_OWNER(x) >> SX_SHARERS_SHIFT) 8338363Swpaul#define SX_SHARERS_LOCK(x) \ 8438363Swpaul ((x) << SX_SHARERS_SHIFT | SX_LOCK_SHARED) 8538363Swpaul#define SX_ONE_SHARER (1 << SX_SHARERS_SHIFT) 8638363Swpaul 8738363Swpaul#define SX_LOCK_UNLOCKED SX_SHARERS_LOCK(0) 8838363Swpaul#define SX_LOCK_DESTROYED \ 8938363Swpaul (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS) 9038363Swpaul 9138363Swpaul#ifdef _KERNEL 9238363Swpaul 9338363Swpaul/* 9438363Swpaul * Full lock operations that are suitable to be inlined in non-debug kernels. 9538363Swpaul * If the lock can't be acquired or released trivially then the work is 9638363Swpaul * deferred to 'tougher' functions. 9738363Swpaul */ 9838363Swpaul 9938363Swpaul/* Acquire an exclusive lock. */ 10038363Swpaul#define __sx_xlock(sx, tid, file, line) do { \ 10138363Swpaul uintptr_t _tid = (uintptr_t)(tid); \ 10238363Swpaul \ 10338363Swpaul if (!atomic_cmpset_acq_ptr(&(sx)->sx_lock, SX_LOCK_UNLOCKED, \ 10438363Swpaul _tid)) { \ 10538363Swpaul _sx_xlock_hard((sx), _tid, (file), (line)); \ 10638363Swpaul } else \ 10738363Swpaul lock_profile_obtain_lock_success(&(sx)->lock_object, 0, \ 10838363Swpaul 0, (file), (line)); \ 10938363Swpaul} while (0) 11038363Swpaul 11138363Swpaul/* Release an exclusive lock. */ 11238363Swpaul#define __sx_xunlock(sx, tid, file, line) do { \ 11338363Swpaul uintptr_t _tid = (uintptr_t)(tid); \ 11438363Swpaul \ 11538363Swpaul if (!atomic_cmpset_rel_ptr(&(sx)->sx_lock, _tid, \ 11638363Swpaul SX_LOCK_UNLOCKED)) \ 11738363Swpaul _sx_xunlock_hard((sx), _tid, (file), (line)); \ 11838363Swpaul} while (0) 11938363Swpaul 12038363Swpaul/* Acquire a shared lock. */ 12138363Swpaul#define __sx_slock(sx, file, line) do { \ 12238363Swpaul uintptr_t x = (sx)->sx_lock; \ 12338363Swpaul \ 12438363Swpaul if (!(x & SX_LOCK_SHARED) || \ 12538363Swpaul !atomic_cmpset_acq_ptr(&(sx)->sx_lock, x, \ 12638363Swpaul x + SX_ONE_SHARER)) { \ 12738363Swpaul _sx_slock_hard((sx), (file), (line)); \ 12838363Swpaul } else \ 12938363Swpaul lock_profile_obtain_lock_success(&(sx)->lock_object, 0, \ 13038363Swpaul 0, (file), (line)); \ 13138363Swpaul} while (0) 13238363Swpaul 13338363Swpaul/* 13438363Swpaul * Release a shared lock. We can just drop a single shared lock so 13538363Swpaul * long as we aren't trying to drop the last shared lock when other 13638363Swpaul * threads are waiting for an exclusive lock. This takes advantage of 13738363Swpaul * the fact that an unlocked lock is encoded as a shared lock with a 13838363Swpaul * count of 0. 13938363Swpaul */ 14038363Swpaul#define __sx_sunlock(sx, file, line) do { \ 14138363Swpaul uintptr_t x = (sx)->sx_lock; \ 14238363Swpaul \ 14338363Swpaul if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) || \ 14438363Swpaul !atomic_cmpset_ptr(&(sx)->sx_lock, x, x - SX_ONE_SHARER)) \ 14538363Swpaul _sx_sunlock_hard((sx), (file), (line)); \ 14638363Swpaul} while (0) 14738363Swpaul 14838363Swpaul/* 14938363Swpaul * Function prototipes. Routines that start with an underscore are not part 15038363Swpaul * of the public interface and are wrappered with a macro. 15138363Swpaul */ 15238363Swpaulvoid sx_sysinit(void *arg); 15338363Swpaul#define sx_init(sx, desc) sx_init_flags((sx), (desc), 0) 15438363Swpaulvoid sx_init_flags(struct sx *sx, const char *description, int opts); 15538363Swpaulvoid sx_destroy(struct sx *sx); 15638363Swpaulvoid _sx_slock(struct sx *sx, const char *file, int line); 15738363Swpaulvoid _sx_xlock(struct sx *sx, const char *file, int line); 15838363Swpaulint _sx_try_slock(struct sx *sx, const char *file, int line); 15938363Swpaulint _sx_try_xlock(struct sx *sx, const char *file, int line); 16038363Swpaulvoid _sx_sunlock(struct sx *sx, const char *file, int line); 16138363Swpaulvoid _sx_xunlock(struct sx *sx, const char *file, int line); 16238363Swpaulint _sx_try_upgrade(struct sx *sx, const char *file, int line); 16338363Swpaulvoid _sx_downgrade(struct sx *sx, const char *file, int line); 16438363Swpaulvoid _sx_xlock_hard(struct sx *sx, uintptr_t tid, const char *file, int 16538363Swpaul line); 16638363Swpaulvoid _sx_slock_hard(struct sx *sx, const char *file, int line); 16738363Swpaulvoid _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int 16838363Swpaul line); 16938363Swpaulvoid _sx_sunlock_hard(struct sx *sx, const char *file, int line); 17038363Swpaul#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 17138363Swpaulvoid _sx_assert(struct sx *sx, int what, const char *file, int line); 17238363Swpaul#endif 17338363Swpaul#ifdef DDB 17438526Swpaulint sx_chain(struct thread *td, struct thread **ownerp); 17538526Swpaul#endif 17638526Swpaul 17738526Swpaulstruct sx_args { 17838526Swpaul struct sx *sa_sx; 17938526Swpaul const char *sa_desc; 18038526Swpaul}; 18138363Swpaul 18238363Swpaul#define SX_SYSINIT(name, sxa, desc) \ 18338363Swpaul static struct sx_args name##_args = { \ 18438363Swpaul (sxa), \ 18538363Swpaul (desc) \ 18638363Swpaul }; \ 18738363Swpaul SYSINIT(name##_sx_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 18838363Swpaul sx_sysinit, &name##_args); \ 18938363Swpaul SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 19038363Swpaul sx_destroy, (sxa)) 19138363Swpaul 19238363Swpaul/* 19338363Swpaul * Public interface for lock operations. 19438363Swpaul */ 19538363Swpaul#ifndef LOCK_DEBUG 19638363Swpaul#error "LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>" 19738363Swpaul#endif 19838363Swpaul#if (LOCK_DEBUG > 0) || defined(SX_NOINLINE) 19938363Swpaul#define sx_xlock(sx) _sx_xlock((sx), LOCK_FILE, LOCK_LINE) 20038363Swpaul#define sx_xunlock(sx) _sx_xunlock((sx), LOCK_FILE, LOCK_LINE) 20138363Swpaul#define sx_slock(sx) _sx_slock((sx), LOCK_FILE, LOCK_LINE) 20238363Swpaul#define sx_sunlock(sx) _sx_sunlock((sx), LOCK_FILE, LOCK_LINE) 20338363Swpaul#else 20438363Swpaul#define sx_xlock(sx) \ 20538363Swpaul __sx_xlock((sx), curthread, LOCK_FILE, LOCK_LINE) 20638363Swpaul#define sx_xunlock(sx) \ 20738363Swpaul __sx_xunlock((sx), curthread, LOCK_FILE, LOCK_LINE) 20838363Swpaul#define sx_slock(sx) __sx_slock((sx), LOCK_FILE, LOCK_LINE) 20938363Swpaul#define sx_sunlock(sx) __sx_sunlock((sx), LOCK_FILE, LOCK_LINE) 21038363Swpaul#endif /* LOCK_DEBUG > 0 || SX_NOINLINE */ 21138363Swpaul#define sx_try_slock(sx) _sx_try_slock((sx), LOCK_FILE, LOCK_LINE) 21238363Swpaul#define sx_try_xlock(sx) _sx_try_xlock((sx), LOCK_FILE, LOCK_LINE) 21338363Swpaul#define sx_try_upgrade(sx) _sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE) 21438363Swpaul#define sx_downgrade(sx) _sx_downgrade((sx), LOCK_FILE, LOCK_LINE) 21538363Swpaul 21638363Swpaul#define sx_xlocked(sx) \ 21738363Swpaul (((sx)->sx_lock & ~(SX_LOCK_FLAGMASK & ~SX_LOCK_SHARED)) == \ 21838363Swpaul (uintptr_t)curthread) 21938363Swpaul 22038363Swpaul#define sx_unlock(sx) do { \ 22138363Swpaul if (sx_xlocked(sx)) \ 22238363Swpaul sx_xunlock(sx); \ 22338363Swpaul else \ 22438363Swpaul sx_sunlock(sx); \ 22538363Swpaul} while (0) 22638363Swpaul 22738363Swpaul#define sx_sleep(chan, sx, pri, wmesg, timo) \ 22838363Swpaul _sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo)) 22938363Swpaul 23038363Swpaul/* 23138363Swpaul * Options passed to sx_init_flags(). 23238363Swpaul */ 23338363Swpaul#define SX_DUPOK 0x01 23438363Swpaul#define SX_NOPROFILE 0x02 23538363Swpaul#define SX_NOWITNESS 0x04 23638363Swpaul#define SX_QUIET 0x08 23738363Swpaul#define SX_ADAPTIVESPIN 0x10 23838363Swpaul 23938363Swpaul/* 24038363Swpaul * XXX: These options should be renamed as SA_* 24138363Swpaul */ 24238363Swpaul#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 24338363Swpaul#define SX_LOCKED LA_LOCKED 24438363Swpaul#define SX_SLOCKED LA_SLOCKED 24538363Swpaul#define SX_XLOCKED LA_XLOCKED 24638363Swpaul#define SX_UNLOCKED LA_UNLOCKED 24738363Swpaul#define SX_RECURSED LA_RECURSED 24838363Swpaul#define SX_NOTRECURSED LA_NOTRECURSED 24938363Swpaul#endif 25038363Swpaul 25138363Swpaul#ifdef INVARIANTS 25238363Swpaul#define sx_assert(sx, what) _sx_assert((sx), (what), LOCK_FILE, LOCK_LINE) 25338363Swpaul#else 25438363Swpaul#define sx_assert(sx, what) 25538363Swpaul#endif 25638363Swpaul 25738363Swpaul#endif /* _KERNEL */ 25838363Swpaul 25938363Swpaul#endif /* !_SYS_SX_H_ */ 26038363Swpaul