174912Sjhb/*- 274912Sjhb * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 324269Speter * 424269Speter * Redistribution and use in source and binary forms, with or without 524269Speter * modification, are permitted provided that the following conditions 624269Speter * are met: 724269Speter * 1. Redistributions of source code must retain the above copyright 824269Speter * notice, this list of conditions and the following disclaimer. 924269Speter * 2. Redistributions in binary form must reproduce the above copyright 1024269Speter * notice, this list of conditions and the following disclaimer in the 1124269Speter * documentation and/or other materials provided with the distribution. 1274912Sjhb * 3. Berkeley Software Design Inc's name may not be used to endorse or 1374912Sjhb * promote products derived from this software without specific prior 1474912Sjhb * written permission. 1524269Speter * 1674912Sjhb * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1724269Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1824269Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1974912Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2024269Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2124269Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2224269Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2324269Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2424269Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2524269Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2624269Speter * SUCH DAMAGE. 2724269Speter * 28212408Sobrien * from BSDI Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp 2950477Speter * $FreeBSD: releng/11.0/sys/sys/lock.h 303953 2016-08-11 09:28:49Z mjg $ 3024269Speter */ 3124269Speter 3274912Sjhb#ifndef _SYS_LOCK_H_ 3374912Sjhb#define _SYS_LOCK_H_ 3424269Speter 3574912Sjhb#include <sys/queue.h> 3676166Smarkm#include <sys/_lock.h> 37258858Sattilio#include <sys/ktr_class.h> 3874912Sjhb 39178165Sattiliostruct lock_list_entry; 40102506Sbdestruct thread; 41102506Sbde 4224269Speter/* 4374912Sjhb * Lock classes. Each lock has a class which describes characteristics 4474912Sjhb * common to all types of locks of a given class. 4524269Speter * 4674912Sjhb * Spin locks in general must always protect against preemption, as it is 4774912Sjhb * an error to perform any type of context switch while holding a spin lock. 4874912Sjhb * Also, for an individual lock to be recursable, its class must allow 4974912Sjhb * recursion and the lock itself must explicitly allow recursion. 50167368Sjhb * 51167368Sjhb * The 'lc_ddb_show' function pointer is used to dump class-specific 52167368Sjhb * data for the 'show lock' DDB command. The 'lc_lock' and 53167368Sjhb * 'lc_unlock' function pointers are used in sleep(9) and cv_wait(9) 54167368Sjhb * to lock and unlock locks while blocking on a sleep queue. The 55167368Sjhb * return value of 'lc_unlock' will be passed to 'lc_lock' on resume 56167368Sjhb * to allow communication of state between the two routines. 5724269Speter */ 5874912Sjhb 5983045Sobrienstruct lock_class { 60255745Sdavide const char *lc_name; 61255745Sdavide u_int lc_flags; 62255745Sdavide void (*lc_assert)(const struct lock_object *lock, int what); 63255745Sdavide void (*lc_ddb_show)(const struct lock_object *lock); 64255745Sdavide void (*lc_lock)(struct lock_object *lock, uintptr_t how); 65255745Sdavide int (*lc_owner)(const struct lock_object *lock, 66255745Sdavide struct thread **owner); 67255745Sdavide uintptr_t (*lc_unlock)(struct lock_object *lock); 6874912Sjhb}; 6974912Sjhb 7074912Sjhb#define LC_SLEEPLOCK 0x00000001 /* Sleep lock. */ 7174912Sjhb#define LC_SPINLOCK 0x00000002 /* Spin lock. */ 7274912Sjhb#define LC_SLEEPABLE 0x00000004 /* Sleeping allowed with this lock. */ 7374912Sjhb#define LC_RECURSABLE 0x00000008 /* Locks of this type may recurse. */ 7482244Sjhb#define LC_UPGRADABLE 0x00000010 /* Upgrades and downgrades permitted. */ 7574912Sjhb 7674912Sjhb#define LO_CLASSFLAGS 0x0000ffff /* Class specific flags. */ 7774912Sjhb#define LO_INITIALIZED 0x00010000 /* Lock has been initialized. */ 7874912Sjhb#define LO_WITNESS 0x00020000 /* Should witness monitor this lock. */ 7974912Sjhb#define LO_QUIET 0x00040000 /* Don't log locking operations. */ 8074912Sjhb#define LO_RECURSABLE 0x00080000 /* Lock may recurse. */ 8174912Sjhb#define LO_SLEEPABLE 0x00100000 /* Lock may be held while sleeping. */ 8282244Sjhb#define LO_UPGRADABLE 0x00200000 /* Lock may be upgraded/downgraded. */ 8393273Sjeff#define LO_DUPOK 0x00400000 /* Don't check for duplicate acquires */ 84250411Smarcel#define LO_IS_VNODE 0x00800000 /* Tell WITNESS about a VNODE lock */ 85154077Sjhb#define LO_CLASSMASK 0x0f000000 /* Class index bitmask. */ 86164159Skmacy#define LO_NOPROFILE 0x10000000 /* Don't profile this lock */ 87275751Sdchagin#define LO_NEW 0x20000000 /* Don't check for double-init */ 8874912Sjhb 89154077Sjhb/* 90154077Sjhb * Lock classes are statically assigned an index into the gobal lock_classes 91154077Sjhb * array. Debugging code looks up the lock class for a given lock object 92154077Sjhb * by indexing the array. 93154077Sjhb */ 94154077Sjhb#define LO_CLASSSHIFT 24 95154077Sjhb#define LO_CLASSINDEX(lock) ((((lock)->lo_flags) & LO_CLASSMASK) >> LO_CLASSSHIFT) 96154077Sjhb#define LOCK_CLASS(lock) (lock_classes[LO_CLASSINDEX((lock))]) 97154484Sjhb#define LOCK_CLASS_MAX (LO_CLASSMASK >> LO_CLASSSHIFT) 98154077Sjhb 9924269Speter/* 10074912Sjhb * Option flags passed to lock operations that witness also needs to know 10174912Sjhb * about or that are generic across all locks. 10224269Speter */ 103125160Sjhb#define LOP_NEWORDER 0x00000001 /* Define a new lock order. */ 10474912Sjhb#define LOP_QUIET 0x00000002 /* Don't log locking operations. */ 10574912Sjhb#define LOP_TRYLOCK 0x00000004 /* Don't check lock order. */ 10676272Sjhb#define LOP_EXCLUSIVE 0x00000008 /* Exclusive lock. */ 107145421Sjeff#define LOP_DUPOK 0x00000010 /* Don't check for duplicate acquires */ 10824269Speter 10978871Sjhb/* Flags passed to witness_assert. */ 110176249Sattilio#define LA_MASKASSERT 0x000000ff /* Mask for witness defined asserts. */ 11178871Sjhb#define LA_UNLOCKED 0x00000000 /* Lock is unlocked. */ 11278871Sjhb#define LA_LOCKED 0x00000001 /* Lock is at least share locked. */ 11378871Sjhb#define LA_SLOCKED 0x00000002 /* Lock is exactly share locked. */ 11478871Sjhb#define LA_XLOCKED 0x00000004 /* Lock is exclusively locked. */ 11578871Sjhb#define LA_RECURSED 0x00000008 /* Lock is recursed. */ 11678871Sjhb#define LA_NOTRECURSED 0x00000010 /* Lock is not recursed. */ 11778871Sjhb 11874912Sjhb#ifdef _KERNEL 11924269Speter/* 12085186Sjhb * If any of WITNESS, INVARIANTS, or KTR_LOCK KTR tracing has been enabled, 12185186Sjhb * then turn on LOCK_DEBUG. When this option is on, extra debugging 12285186Sjhb * facilities such as tracking the file and line number of lock operations 12385186Sjhb * are enabled. Also, mutex locking operations are not inlined to avoid 12485186Sjhb * bloat from all the extra debugging code. We also have to turn on all the 12585186Sjhb * calling conventions for this debugging code in modules so that modules can 12685186Sjhb * work with both debug and non-debug kernels. 12785186Sjhb */ 128258858Sattilio#if defined(KLD_MODULE) || defined(WITNESS) || defined(INVARIANTS) || defined(INVARIANT_SUPPORT) || defined(LOCK_PROFILING) || (defined(KTR) && (KTR_COMPILE & KTR_LOCK)) 12985186Sjhb#define LOCK_DEBUG 1 13085186Sjhb#else 13185186Sjhb#define LOCK_DEBUG 0 13285186Sjhb#endif 13385186Sjhb 13485186Sjhb/* 13583593Sjhb * In the LOCK_DEBUG case, use the filename and line numbers for debugging 13683593Sjhb * operations. Otherwise, use default values to avoid the unneeded bloat. 13783593Sjhb */ 13885186Sjhb#if LOCK_DEBUG > 0 13983593Sjhb#define LOCK_FILE __FILE__ 14083593Sjhb#define LOCK_LINE __LINE__ 14183593Sjhb#else 14283593Sjhb#define LOCK_FILE NULL 14383593Sjhb#define LOCK_LINE 0 14483593Sjhb#endif 14583593Sjhb 14683593Sjhb/* 14774912Sjhb * Macros for KTR_LOCK tracing. 14824269Speter * 14974912Sjhb * opname - name of this operation (LOCK/UNLOCK/SLOCK, etc.) 15074912Sjhb * lo - struct lock_object * for this lock 15174912Sjhb * flags - flags passed to the lock operation 15274912Sjhb * recurse - this locks recursion level (or 0 if class is not recursable) 15374912Sjhb * result - result of a try lock operation 15474912Sjhb * file - file name 15574912Sjhb * line - line number 15624269Speter */ 15774912Sjhb#define LOCK_LOG_TEST(lo, flags) \ 15874912Sjhb (((flags) & LOP_QUIET) == 0 && ((lo)->lo_flags & LO_QUIET) == 0) 15924269Speter 16074912Sjhb#define LOCK_LOG_LOCK(opname, lo, flags, recurse, file, line) do { \ 16174912Sjhb if (LOCK_LOG_TEST((lo), (flags))) \ 162196204Skib CTR6(KTR_LOCK, opname " (%s) %s %p r = %d at %s:%d", \ 163154077Sjhb LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \ 164196204Skib (lo), (u_int)(recurse), (file), (line)); \ 16574912Sjhb} while (0) 16674912Sjhb 16774912Sjhb#define LOCK_LOG_TRY(opname, lo, flags, result, file, line) do { \ 16874912Sjhb if (LOCK_LOG_TEST((lo), (flags))) \ 169196204Skib CTR6(KTR_LOCK, "TRY_" opname " (%s) %s %p result=%d at %s:%d",\ 170154077Sjhb LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \ 171196204Skib (lo), (u_int)(result), (file), (line)); \ 17274912Sjhb} while (0) 17374912Sjhb 17474912Sjhb#define LOCK_LOG_INIT(lo, flags) do { \ 17574912Sjhb if (LOCK_LOG_TEST((lo), (flags))) \ 17688661Sjake CTR4(KTR_LOCK, "%s: %p (%s) %s", __func__, (lo), \ 177154077Sjhb LOCK_CLASS(lo)->lc_name, (lo)->lo_name); \ 17874912Sjhb} while (0) 17974912Sjhb 18074912Sjhb#define LOCK_LOG_DESTROY(lo, flags) LOCK_LOG_INIT(lo, flags) 18174912Sjhb 182261238Sjmg#define lock_initialized(lo) ((lo)->lo_flags & LO_INITIALIZED) 183154484Sjhb 18424269Speter/* 18574912Sjhb * Helpful macros for quickly coming up with assertions with informative 18674912Sjhb * panic messages. 18724269Speter */ 18874912Sjhb#define MPASS(ex) MPASS4(ex, #ex, __FILE__, __LINE__) 18974912Sjhb#define MPASS2(ex, what) MPASS4(ex, what, __FILE__, __LINE__) 19074912Sjhb#define MPASS3(ex, file, line) MPASS4(ex, #ex, file, line) 19174912Sjhb#define MPASS4(ex, what, file, line) \ 19274912Sjhb KASSERT((ex), ("Assertion %s failed at %s:%d", what, file, line)) 19324269Speter 19474912Sjhbextern struct lock_class lock_class_mtx_sleep; 19574912Sjhbextern struct lock_class lock_class_mtx_spin; 19674912Sjhbextern struct lock_class lock_class_sx; 197154941Sjhbextern struct lock_class lock_class_rw; 198173444Supsextern struct lock_class lock_class_rm; 199252209Sjhbextern struct lock_class lock_class_rm_sleepable; 200164246Skmacyextern struct lock_class lock_class_lockmgr; 20170140Sjake 202154077Sjhbextern struct lock_class *lock_classes[]; 203154077Sjhb 204303953Smjgstruct lock_delay_config { 205303953Smjg u_int initial; 206303953Smjg u_int step; 207303953Smjg u_int min; 208303953Smjg u_int max; 209303953Smjg}; 210303953Smjg 211303953Smjgstruct lock_delay_arg { 212303953Smjg struct lock_delay_config *config; 213303953Smjg u_int delay; 214303953Smjg u_int spin_cnt; 215303953Smjg}; 216303953Smjg 217303953Smjgstatic inline void 218303953Smjglock_delay_arg_init(struct lock_delay_arg *la, struct lock_delay_config *lc) { 219303953Smjg la->config = lc; 220303953Smjg la->delay = 0; 221303953Smjg la->spin_cnt = 0; 222303953Smjg} 223303953Smjg 224303953Smjg#define LOCK_DELAY_SYSINIT(func) \ 225303953Smjg SYSINIT(func##_ld, SI_SUB_LOCK, SI_ORDER_ANY, func, NULL) 226303953Smjg 227166455Sbmsvoid lock_init(struct lock_object *, struct lock_class *, 228207925Sattilio const char *, const char *, int); 229166455Sbmsvoid lock_destroy(struct lock_object *); 230303953Smjgvoid lock_delay(struct lock_delay_arg *); 231144637Sjhbvoid spinlock_enter(void); 232144637Sjhbvoid spinlock_exit(void); 233179025Sattiliovoid witness_init(struct lock_object *, const char *); 23474912Sjhbvoid witness_destroy(struct lock_object *); 235125160Sjhbint witness_defineorder(struct lock_object *, struct lock_object *); 236182914Sjhbvoid witness_checkorder(struct lock_object *, int, const char *, int, 237207925Sattilio struct lock_object *); 23874912Sjhbvoid witness_lock(struct lock_object *, int, const char *, int); 23982244Sjhbvoid witness_upgrade(struct lock_object *, int, const char *, int); 24082244Sjhbvoid witness_downgrade(struct lock_object *, int, const char *, int); 24174912Sjhbvoid witness_unlock(struct lock_object *, int, const char *, int); 24274912Sjhbvoid witness_save(struct lock_object *, const char **, int *); 24374912Sjhbvoid witness_restore(struct lock_object *, const char *, int); 244207929Sattilioint witness_list_locks(struct lock_list_entry **, 245207929Sattilio int (*)(const char *, ...)); 246111881Sjhbint witness_warn(int, struct lock_object *, const char *, ...); 247227588Spjdvoid witness_assert(const struct lock_object *, int, const char *, int); 248207929Sattiliovoid witness_display_spinlock(struct lock_object *, struct thread *, 249207929Sattilio int (*)(const char *, ...)); 250102448Siedowseint witness_line(struct lock_object *); 251187511Sthompsavoid witness_norelease(struct lock_object *); 252187511Sthompsavoid witness_releaseok(struct lock_object *); 253102448Siedowseconst char *witness_file(struct lock_object *); 254181695Sattiliovoid witness_thread_exit(struct thread *); 25524269Speter 25674912Sjhb#ifdef WITNESS 257111881Sjhb 258111881Sjhb/* Flags for witness_warn(). */ 259111881Sjhb#define WARN_GIANTOK 0x01 /* Giant is exempt from this check. */ 260111881Sjhb#define WARN_PANIC 0x02 /* Panic if check fails. */ 261111881Sjhb#define WARN_SLEEPOK 0x04 /* Sleepable locks are exempt from check. */ 262111881Sjhb 263179025Sattilio#define WITNESS_INIT(lock, type) \ 264179025Sattilio witness_init((lock), (type)) 26566615Sjasone 26674912Sjhb#define WITNESS_DESTROY(lock) \ 26774912Sjhb witness_destroy(lock) 26824269Speter 269182914Sjhb#define WITNESS_CHECKORDER(lock, flags, file, line, interlock) \ 270182914Sjhb witness_checkorder((lock), (flags), (file), (line), (interlock)) 271125160Sjhb 272125160Sjhb#define WITNESS_DEFINEORDER(lock1, lock2) \ 273125160Sjhb witness_defineorder((struct lock_object *)(lock1), \ 274125160Sjhb (struct lock_object *)(lock2)) 275125160Sjhb 27674912Sjhb#define WITNESS_LOCK(lock, flags, file, line) \ 27774912Sjhb witness_lock((lock), (flags), (file), (line)) 27874912Sjhb 27982244Sjhb#define WITNESS_UPGRADE(lock, flags, file, line) \ 28082244Sjhb witness_upgrade((lock), (flags), (file), (line)) 28182244Sjhb 28282244Sjhb#define WITNESS_DOWNGRADE(lock, flags, file, line) \ 28382244Sjhb witness_downgrade((lock), (flags), (file), (line)) 28482244Sjhb 28574912Sjhb#define WITNESS_UNLOCK(lock, flags, file, line) \ 28674912Sjhb witness_unlock((lock), (flags), (file), (line)) 28774912Sjhb 288154934Sjhb#define WITNESS_CHECK(flags, lock, fmt, ...) \ 289154934Sjhb witness_warn((flags), (lock), (fmt), ## __VA_ARGS__) 290154934Sjhb 291111881Sjhb#define WITNESS_WARN(flags, lock, fmt, ...) \ 292111881Sjhb witness_warn((flags), (lock), (fmt), ## __VA_ARGS__) 29374912Sjhb 29474912Sjhb#define WITNESS_SAVE_DECL(n) \ 29574912Sjhb const char * __CONCAT(n, __wf); \ 29674912Sjhb int __CONCAT(n, __wl) 29774912Sjhb 29874912Sjhb#define WITNESS_SAVE(lock, n) \ 29974912Sjhb witness_save((lock), &__CONCAT(n, __wf), &__CONCAT(n, __wl)) 30074912Sjhb 30174912Sjhb#define WITNESS_RESTORE(lock, n) \ 30274912Sjhb witness_restore((lock), __CONCAT(n, __wf), __CONCAT(n, __wl)) 30374912Sjhb 304187511Sthompsa#define WITNESS_NORELEASE(lock) \ 305187511Sthompsa witness_norelease(&(lock)->lock_object) 306187511Sthompsa 307187511Sthompsa#define WITNESS_RELEASEOK(lock) \ 308187511Sthompsa witness_releaseok(&(lock)->lock_object) 309187511Sthompsa 310102448Siedowse#define WITNESS_FILE(lock) \ 311102448Siedowse witness_file(lock) 312102448Siedowse 313102448Siedowse#define WITNESS_LINE(lock) \ 314102448Siedowse witness_line(lock) 315102448Siedowse 31674912Sjhb#else /* WITNESS */ 317194578Srdivacky#define WITNESS_INIT(lock, type) (void)0 318194578Srdivacky#define WITNESS_DESTROY(lock) (void)0 319125160Sjhb#define WITNESS_DEFINEORDER(lock1, lock2) 0 320194578Srdivacky#define WITNESS_CHECKORDER(lock, flags, file, line, interlock) (void)0 321194578Srdivacky#define WITNESS_LOCK(lock, flags, file, line) (void)0 322194578Srdivacky#define WITNESS_UPGRADE(lock, flags, file, line) (void)0 323194578Srdivacky#define WITNESS_DOWNGRADE(lock, flags, file, line) (void)0 324194578Srdivacky#define WITNESS_UNLOCK(lock, flags, file, line) (void)0 325154934Sjhb#define WITNESS_CHECK(flags, lock, fmt, ...) 0 326194578Srdivacky#define WITNESS_WARN(flags, lock, fmt, ...) (void)0 327194578Srdivacky#define WITNESS_SAVE_DECL(n) (void)0 328194578Srdivacky#define WITNESS_SAVE(lock, n) (void)0 329194578Srdivacky#define WITNESS_RESTORE(lock, n) (void)0 330194578Srdivacky#define WITNESS_NORELEASE(lock) (void)0 331194578Srdivacky#define WITNESS_RELEASEOK(lock) (void)0 332102448Siedowse#define WITNESS_FILE(lock) ("?") 333102448Siedowse#define WITNESS_LINE(lock) (0) 33474912Sjhb#endif /* WITNESS */ 33574912Sjhb 33674912Sjhb#endif /* _KERNEL */ 33774912Sjhb#endif /* _SYS_LOCK_H_ */ 338