165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone */ 3165557Sjasone 3265557Sjasone/* 3386411Sjhb * Machine independent bits of mutex implementation. 3472200Sbmilekic */ 3572200Sbmilekic 36116182Sobrien#include <sys/cdefs.h> 37116182Sobrien__FBSDID("$FreeBSD$"); 38116182Sobrien 3997081Sjhb#include "opt_adaptive_mutexes.h" 4068790Sjhb#include "opt_ddb.h" 41164159Skmacy#include "opt_global.h" 42236238Sfabient#include "opt_hwpmc_hooks.h" 43192853Ssson#include "opt_kdtrace.h" 44134649Sscottl#include "opt_sched.h" 4567676Sjhb 4665557Sjasone#include <sys/param.h> 4793609Sdes#include <sys/systm.h> 4867352Sjhb#include <sys/bus.h> 49148557Sps#include <sys/conf.h> 50131927Smarcel#include <sys/kdb.h> 5167352Sjhb#include <sys/kernel.h> 5293609Sdes#include <sys/ktr.h> 5376166Smarkm#include <sys/lock.h> 5467352Sjhb#include <sys/malloc.h> 5574912Sjhb#include <sys/mutex.h> 5665557Sjasone#include <sys/proc.h> 5778766Sjhb#include <sys/resourcevar.h> 58104964Sjeff#include <sys/sched.h> 5993609Sdes#include <sys/sbuf.h> 6067676Sjhb#include <sys/sysctl.h> 61122514Sjhb#include <sys/turnstile.h> 6267352Sjhb#include <sys/vmmeter.h> 63164159Skmacy#include <sys/lock_profile.h> 6465557Sjasone 6567352Sjhb#include <machine/atomic.h> 6667352Sjhb#include <machine/bus.h> 6765557Sjasone#include <machine/cpu.h> 6867352Sjhb 6968790Sjhb#include <ddb/ddb.h> 7068790Sjhb 71151450Sjhb#include <fs/devfs/devfs_int.h> 72151450Sjhb 7367352Sjhb#include <vm/vm.h> 7467352Sjhb#include <vm/vm_extern.h> 7567352Sjhb 76167801Sjhb#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 77167801Sjhb#define ADAPTIVE_MUTEXES 78167801Sjhb#endif 79167801Sjhb 80236238Sfabient#ifdef HWPMC_HOOKS 81236238Sfabient#include <sys/pmckern.h> 82236238SfabientPMC_SOFT_DEFINE( , , lock, failed); 83236238Sfabient#endif 84236238Sfabient 8565557Sjasone/* 8672200Sbmilekic * Internal utility macros. 8771352Sjasone */ 8872200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 8971352Sjasone 90169393Sjhb#define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED) 91169393Sjhb 92159208Sjhb#define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 9371352Sjasone 94173733Sattiliostatic void assert_mtx(struct lock_object *lock, int what); 95153395Sjhb#ifdef DDB 96153395Sjhbstatic void db_show_mtx(struct lock_object *lock); 97153395Sjhb#endif 98167368Sjhbstatic void lock_mtx(struct lock_object *lock, int how); 99167368Sjhbstatic void lock_spin(struct lock_object *lock, int how); 100192853Ssson#ifdef KDTRACE_HOOKS 101192853Sssonstatic int owner_mtx(struct lock_object *lock, struct thread **owner); 102192853Ssson#endif 103167368Sjhbstatic int unlock_mtx(struct lock_object *lock); 104167368Sjhbstatic int unlock_spin(struct lock_object *lock); 105153395Sjhb 10671352Sjasone/* 10774912Sjhb * Lock classes for sleep and spin mutexes. 10871352Sjasone */ 10974912Sjhbstruct lock_class lock_class_mtx_sleep = { 110167365Sjhb .lc_name = "sleep mutex", 111167365Sjhb .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 112173733Sattilio .lc_assert = assert_mtx, 113153395Sjhb#ifdef DDB 114167365Sjhb .lc_ddb_show = db_show_mtx, 115153395Sjhb#endif 116167368Sjhb .lc_lock = lock_mtx, 117167368Sjhb .lc_unlock = unlock_mtx, 118192853Ssson#ifdef KDTRACE_HOOKS 119192853Ssson .lc_owner = owner_mtx, 120192853Ssson#endif 12174912Sjhb}; 12274912Sjhbstruct lock_class lock_class_mtx_spin = { 123167365Sjhb .lc_name = "spin mutex", 124167365Sjhb .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 125173733Sattilio .lc_assert = assert_mtx, 126153395Sjhb#ifdef DDB 127167365Sjhb .lc_ddb_show = db_show_mtx, 128153395Sjhb#endif 129167368Sjhb .lc_lock = lock_spin, 130167368Sjhb .lc_unlock = unlock_spin, 131192853Ssson#ifdef KDTRACE_HOOKS 132192853Ssson .lc_owner = owner_mtx, 133192853Ssson#endif 13474912Sjhb}; 13571352Sjasone 13671352Sjasone/* 13793702Sjhb * System-wide mutexes 13893702Sjhb */ 139170295Sjeffstruct mtx blocked_lock; 14093702Sjhbstruct mtx Giant; 14193702Sjhb 142196970Sphkvoid 143173733Sattilioassert_mtx(struct lock_object *lock, int what) 144173733Sattilio{ 145173733Sattilio 146173733Sattilio mtx_assert((struct mtx *)lock, what); 147173733Sattilio} 148173733Sattilio 149196970Sphkvoid 150167368Sjhblock_mtx(struct lock_object *lock, int how) 151167368Sjhb{ 152167368Sjhb 153167368Sjhb mtx_lock((struct mtx *)lock); 154167368Sjhb} 155167368Sjhb 156196970Sphkvoid 157167368Sjhblock_spin(struct lock_object *lock, int how) 158167368Sjhb{ 159167368Sjhb 160167368Sjhb panic("spin locks can only use msleep_spin"); 161167368Sjhb} 162167368Sjhb 163196970Sphkint 164167368Sjhbunlock_mtx(struct lock_object *lock) 165167368Sjhb{ 166167368Sjhb struct mtx *m; 167167368Sjhb 168167368Sjhb m = (struct mtx *)lock; 169167368Sjhb mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 170167368Sjhb mtx_unlock(m); 171167368Sjhb return (0); 172167368Sjhb} 173167368Sjhb 174196970Sphkint 175167368Sjhbunlock_spin(struct lock_object *lock) 176167368Sjhb{ 177167368Sjhb 178167368Sjhb panic("spin locks can only use msleep_spin"); 179167368Sjhb} 180167368Sjhb 181192853Ssson#ifdef KDTRACE_HOOKS 182192853Sssonint 183192853Sssonowner_mtx(struct lock_object *lock, struct thread **owner) 184192853Ssson{ 185192853Ssson struct mtx *m = (struct mtx *)lock; 186192853Ssson 187192853Ssson *owner = mtx_owner(m); 188192853Ssson return (mtx_unowned(m) == 0); 189192853Ssson} 190192853Ssson#endif 191192853Ssson 19271352Sjasone/* 19374900Sjhb * Function versions of the inlined __mtx_* macros. These are used by 19474900Sjhb * modules and can also be called from assembly language if needed. 19574900Sjhb */ 19674900Sjhbvoid 19774900Sjhb_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 19874900Sjhb{ 19974900Sjhb 200235404Savg if (SCHEDULER_STOPPED()) 201235404Savg return; 202255862Sjhb KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 203255862Sjhb ("mtx_lock() by idle thread %p on sleep mutex %s @ %s:%d", 204255862Sjhb curthread, m->lock_object.lo_name, file, line)); 205160766Sjhb KASSERT(m->mtx_lock != MTX_DESTROYED, 206160766Sjhb ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 207167787Sjhb KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 208167787Sjhb ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 209102907Sjhb file, line)); 210167787Sjhb WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 211182914Sjhb file, line, NULL); 212164159Skmacy 213215054Sjhb __mtx_lock(m, curthread, opts, file, line); 214167787Sjhb LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 21583841Sjhb line); 216167787Sjhb WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 217160771Sjhb curthread->td_locks++; 21874900Sjhb} 21974900Sjhb 22074900Sjhbvoid 22174900Sjhb_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 22274900Sjhb{ 223235404Savg 224235404Savg if (SCHEDULER_STOPPED()) 225235404Savg return; 226160766Sjhb KASSERT(m->mtx_lock != MTX_DESTROYED, 227160766Sjhb ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 228167787Sjhb KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 229167787Sjhb ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 230102907Sjhb file, line)); 231160771Sjhb curthread->td_locks--; 232167787Sjhb WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 233167787Sjhb LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 234102907Sjhb line); 23583947Sjhb mtx_assert(m, MA_OWNED); 236167163Skmacy 237168329Skmacy if (m->mtx_recurse == 0) 238192853Ssson LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m); 239215054Sjhb __mtx_unlock(m, curthread, opts, file, line); 24074900Sjhb} 24174900Sjhb 24274900Sjhbvoid 24374900Sjhb_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 24474900Sjhb{ 245176260Sjhb 246235404Savg if (SCHEDULER_STOPPED()) 247235404Savg return; 248160766Sjhb KASSERT(m->mtx_lock != MTX_DESTROYED, 249160766Sjhb ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 250167787Sjhb KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 251102907Sjhb ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 252167787Sjhb m->lock_object.lo_name, file, line)); 253176260Sjhb if (mtx_owned(m)) 254176260Sjhb KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 255176260Sjhb ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", 256176260Sjhb m->lock_object.lo_name, file, line)); 257167787Sjhb WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 258182914Sjhb file, line, NULL); 259215054Sjhb __mtx_lock_spin(m, curthread, opts, file, line); 260167787Sjhb LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 26183841Sjhb line); 262167787Sjhb WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 26374900Sjhb} 26474900Sjhb 26574900Sjhbvoid 26674900Sjhb_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 26774900Sjhb{ 268167163Skmacy 269235404Savg if (SCHEDULER_STOPPED()) 270235404Savg return; 271160766Sjhb KASSERT(m->mtx_lock != MTX_DESTROYED, 272160766Sjhb ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 273167787Sjhb KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 274102907Sjhb ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 275167787Sjhb m->lock_object.lo_name, file, line)); 276167787Sjhb WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 277167787Sjhb LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 27883841Sjhb line); 279102907Sjhb mtx_assert(m, MA_OWNED); 280167163Skmacy 281215054Sjhb __mtx_unlock_spin(m); 28274900Sjhb} 28374900Sjhb 28474900Sjhb/* 28572200Sbmilekic * The important part of mtx_trylock{,_flags}() 286124161Sjhb * Tries to acquire lock `m.' If this function is called on a mutex that 287124161Sjhb * is already owned, it will recursively acquire the lock. 28871352Sjasone */ 28972200Sbmilekicint 29072200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 29171352Sjasone{ 292189846Sjeff#ifdef LOCK_PROFILING 293164159Skmacy uint64_t waittime = 0; 294189846Sjeff int contested = 0; 295189846Sjeff#endif 296189846Sjeff int rval; 297182909Sjhb 298235404Savg if (SCHEDULER_STOPPED()) 299235404Savg return (1); 300235404Savg 301255862Sjhb KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 302255862Sjhb ("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d", 303255862Sjhb curthread, m->lock_object.lo_name, file, line)); 304160766Sjhb KASSERT(m->mtx_lock != MTX_DESTROYED, 305160766Sjhb ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 306167787Sjhb KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 307167787Sjhb ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 308149737Sjhb file, line)); 30971352Sjasone 310167787Sjhb if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) { 311124161Sjhb m->mtx_recurse++; 312124161Sjhb atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 313124161Sjhb rval = 1; 314124161Sjhb } else 315215054Sjhb rval = _mtx_obtain_lock(m, (uintptr_t)curthread); 31672200Sbmilekic 317167787Sjhb LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line); 318160771Sjhb if (rval) { 319167787Sjhb WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 32076272Sjhb file, line); 321160771Sjhb curthread->td_locks++; 322167012Skmacy if (m->mtx_recurse == 0) 323192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, 324192853Ssson m, contested, waittime, file, line); 325164159Skmacy 326160771Sjhb } 32771352Sjasone 32874912Sjhb return (rval); 32971352Sjasone} 33071352Sjasone 33171352Sjasone/* 33272200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 33371352Sjasone * 33472200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 33572200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 33671352Sjasone */ 33772200Sbmilekicvoid 338148067Sjhb_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, 339133137Sjhb int line) 34071352Sjasone{ 341170295Sjeff struct turnstile *ts; 342189846Sjeff uintptr_t v; 343167801Sjhb#ifdef ADAPTIVE_MUTEXES 344157763Sjhb volatile struct thread *owner; 34597081Sjhb#endif 346102450Siedowse#ifdef KTR 347102450Siedowse int cont_logged = 0; 348102450Siedowse#endif 349189846Sjeff#ifdef LOCK_PROFILING 350168329Skmacy int contested = 0; 351168329Skmacy uint64_t waittime = 0; 352189846Sjeff#endif 353192853Ssson#ifdef KDTRACE_HOOKS 354192853Ssson uint64_t spin_cnt = 0; 355192853Ssson uint64_t sleep_cnt = 0; 356192853Ssson int64_t sleep_time = 0; 357192853Ssson#endif 358182909Sjhb 359235404Savg if (SCHEDULER_STOPPED()) 360235404Savg return; 361235404Savg 362111880Sjhb if (mtx_owned(m)) { 363167787Sjhb KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 364124161Sjhb ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 365167787Sjhb m->lock_object.lo_name, file, line)); 36672200Sbmilekic m->mtx_recurse++; 36772200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 368167787Sjhb if (LOCK_LOG_TEST(&m->lock_object, opts)) 36972344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 37072200Sbmilekic return; 37171352Sjasone } 37271352Sjasone 373236238Sfabient#ifdef HWPMC_HOOKS 374236238Sfabient PMC_SOFT_CALL( , , lock, failed); 375236238Sfabient#endif 376168329Skmacy lock_profile_obtain_lock_failed(&m->lock_object, 377168329Skmacy &contested, &waittime); 378167787Sjhb if (LOCK_LOG_TEST(&m->lock_object, opts)) 37972994Sjhb CTR4(KTR_LOCK, 38072994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 381167787Sjhb m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 38271352Sjasone 383215054Sjhb while (!_mtx_obtain_lock(m, tid)) { 384192853Ssson#ifdef KDTRACE_HOOKS 385192853Ssson spin_cnt++; 386192853Ssson#endif 387173960Sattilio#ifdef ADAPTIVE_MUTEXES 388173960Sattilio /* 389173960Sattilio * If the owner is running on another CPU, spin until the 390173960Sattilio * owner stops running or the state of the lock changes. 391173960Sattilio */ 392173960Sattilio v = m->mtx_lock; 393173960Sattilio if (v != MTX_UNOWNED) { 394173960Sattilio owner = (struct thread *)(v & ~MTX_FLAGMASK); 395173960Sattilio if (TD_IS_RUNNING(owner)) { 396173960Sattilio if (LOCK_LOG_TEST(&m->lock_object, 0)) 397173960Sattilio CTR3(KTR_LOCK, 398173960Sattilio "%s: spinning on %p held by %p", 399173960Sattilio __func__, m, owner); 400173960Sattilio while (mtx_owner(m) == owner && 401192853Ssson TD_IS_RUNNING(owner)) { 402173960Sattilio cpu_spinwait(); 403192853Ssson#ifdef KDTRACE_HOOKS 404192853Ssson spin_cnt++; 405192853Ssson#endif 406192853Ssson } 407173960Sattilio continue; 408173960Sattilio } 409173960Sattilio } 410173960Sattilio#endif 411173960Sattilio 412170295Sjeff ts = turnstile_trywait(&m->lock_object); 413111880Sjhb v = m->mtx_lock; 414111880Sjhb 41572200Sbmilekic /* 41672200Sbmilekic * Check if the lock has been released while spinning for 417122514Sjhb * the turnstile chain lock. 41872200Sbmilekic */ 419111880Sjhb if (v == MTX_UNOWNED) { 420170295Sjeff turnstile_cancel(ts); 42172200Sbmilekic continue; 42271352Sjasone } 42371352Sjasone 424173960Sattilio#ifdef ADAPTIVE_MUTEXES 42569376Sjhb /* 426193035Sjhb * The current lock owner might have started executing 427193035Sjhb * on another CPU (or the lock could have changed 428193035Sjhb * owners) while we were waiting on the turnstile 429193035Sjhb * chain lock. If so, drop the turnstile lock and try 430193035Sjhb * again. 43169376Sjhb */ 432173960Sattilio owner = (struct thread *)(v & ~MTX_FLAGMASK); 433173960Sattilio if (TD_IS_RUNNING(owner)) { 434170295Sjeff turnstile_cancel(ts); 43572200Sbmilekic continue; 43672200Sbmilekic } 437173960Sattilio#endif 43867352Sjhb 43972200Sbmilekic /* 440173960Sattilio * If the mutex isn't already contested and a failure occurs 441173960Sattilio * setting the contested bit, the mutex was either released 442173960Sattilio * or the state of the MTX_RECURSED bit changed. 44397081Sjhb */ 444173960Sattilio if ((v & MTX_CONTESTED) == 0 && 445173960Sattilio !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 446170295Sjeff turnstile_cancel(ts); 44797081Sjhb continue; 44897081Sjhb } 44997081Sjhb 45097081Sjhb /* 45193692Sjhb * We definitely must sleep for this lock. 45272200Sbmilekic */ 45372200Sbmilekic mtx_assert(m, MA_NOTOWNED); 45467352Sjhb 455102450Siedowse#ifdef KTR 456102450Siedowse if (!cont_logged) { 457102450Siedowse CTR6(KTR_CONTENTION, 458102450Siedowse "contention: %p at %s:%d wants %s, taken by %s:%d", 459167787Sjhb (void *)tid, file, line, m->lock_object.lo_name, 460167787Sjhb WITNESS_FILE(&m->lock_object), 461167787Sjhb WITNESS_LINE(&m->lock_object)); 462102450Siedowse cont_logged = 1; 463102450Siedowse } 464102450Siedowse#endif 46567352Sjhb 46672200Sbmilekic /* 467122514Sjhb * Block on the turnstile. 46872200Sbmilekic */ 469192853Ssson#ifdef KDTRACE_HOOKS 470192853Ssson sleep_time -= lockstat_nsecs(); 471192853Ssson#endif 472170295Sjeff turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 473192853Ssson#ifdef KDTRACE_HOOKS 474192853Ssson sleep_time += lockstat_nsecs(); 475192853Ssson sleep_cnt++; 476192853Ssson#endif 47772200Sbmilekic } 478102450Siedowse#ifdef KTR 479102450Siedowse if (cont_logged) { 480102450Siedowse CTR4(KTR_CONTENTION, 481102450Siedowse "contention end: %s acquired by %p at %s:%d", 482167787Sjhb m->lock_object.lo_name, (void *)tid, file, line); 483102450Siedowse } 484102450Siedowse#endif 485192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested, 486182909Sjhb waittime, file, line); 487192853Ssson#ifdef KDTRACE_HOOKS 488192853Ssson if (sleep_time) 489192853Ssson LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time); 490192853Ssson 491192853Ssson /* 492192853Ssson * Only record the loops spinning and not sleeping. 493192853Ssson */ 494192853Ssson if (spin_cnt > sleep_cnt) 495192853Ssson LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt)); 496192853Ssson#endif 49772200Sbmilekic} 49872200Sbmilekic 499170295Sjeffstatic void 500170295Sjeff_mtx_lock_spin_failed(struct mtx *m) 501170295Sjeff{ 502170295Sjeff struct thread *td; 503170295Sjeff 504170295Sjeff td = mtx_owner(m); 505170295Sjeff 506170295Sjeff /* If the mutex is unlocked, try again. */ 507170295Sjeff if (td == NULL) 508170295Sjeff return; 509170327Skib 510170295Sjeff printf( "spin lock %p (%s) held by %p (tid %d) too long\n", 511170295Sjeff m, m->lock_object.lo_name, td, td->td_tid); 512170295Sjeff#ifdef WITNESS 513207929Sattilio witness_display_spinlock(&m->lock_object, td, printf); 514170295Sjeff#endif 515170295Sjeff panic("spin lock held too long"); 516170295Sjeff} 517170295Sjeff 518170327Skib#ifdef SMP 51972200Sbmilekic/* 52072200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 52172200Sbmilekic * 52272200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 52372200Sbmilekic * is handled inline. 52472200Sbmilekic */ 52572200Sbmilekicvoid 526148067Sjhb_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, 527133137Sjhb int line) 52872200Sbmilekic{ 529189846Sjeff int i = 0; 530189846Sjeff#ifdef LOCK_PROFILING 531189846Sjeff int contested = 0; 532168329Skmacy uint64_t waittime = 0; 533189846Sjeff#endif 534182909Sjhb 535235404Savg if (SCHEDULER_STOPPED()) 536235404Savg return; 537235404Savg 538167787Sjhb if (LOCK_LOG_TEST(&m->lock_object, opts)) 53972344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 54072200Sbmilekic 541236238Sfabient#ifdef HWPMC_HOOKS 542236238Sfabient PMC_SOFT_CALL( , , lock, failed); 543236238Sfabient#endif 544168329Skmacy lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 545215054Sjhb while (!_mtx_obtain_lock(m, tid)) { 54672200Sbmilekic 54775568Sjhb /* Give interrupts a chance while we spin. */ 548144637Sjhb spinlock_exit(); 54972200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 55097086Sjhb if (i++ < 10000000) { 551133084Smux cpu_spinwait(); 55272200Sbmilekic continue; 55397086Sjhb } 554161336Sjhb if (i < 60000000 || kdb_active || panicstr != NULL) 55572200Sbmilekic DELAY(1); 556170295Sjeff else 557170295Sjeff _mtx_lock_spin_failed(m); 558133084Smux cpu_spinwait(); 55967352Sjhb } 560144637Sjhb spinlock_enter(); 56167352Sjhb } 56272200Sbmilekic 563167787Sjhb if (LOCK_LOG_TEST(&m->lock_object, opts)) 56472200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 56572200Sbmilekic 566192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, 567192853Ssson contested, waittime, (file), (line)); 568192853Ssson LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i); 56967352Sjhb} 570139733Sjhb#endif /* SMP */ 57167352Sjhb 572170295Sjeffvoid 573170295Sjeff_thread_lock_flags(struct thread *td, int opts, const char *file, int line) 574170295Sjeff{ 575170295Sjeff struct mtx *m; 576170295Sjeff uintptr_t tid; 577189846Sjeff int i; 578189846Sjeff#ifdef LOCK_PROFILING 579189846Sjeff int contested = 0; 580189846Sjeff uint64_t waittime = 0; 581189846Sjeff#endif 582192853Ssson#ifdef KDTRACE_HOOKS 583192853Ssson uint64_t spin_cnt = 0; 584192853Ssson#endif 585170295Sjeff 586189846Sjeff i = 0; 587170295Sjeff tid = (uintptr_t)curthread; 588235404Savg 589235404Savg if (SCHEDULER_STOPPED()) 590235404Savg return; 591235404Savg 592170295Sjeff for (;;) { 593170295Sjeffretry: 594170295Sjeff spinlock_enter(); 595170358Sjeff m = td->td_lock; 596176257Sjhb KASSERT(m->mtx_lock != MTX_DESTROYED, 597176257Sjhb ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 598176257Sjhb KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 599176257Sjhb ("thread_lock() of sleep mutex %s @ %s:%d", 600176257Sjhb m->lock_object.lo_name, file, line)); 601176260Sjhb if (mtx_owned(m)) 602176260Sjhb KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 603176260Sjhb ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 604176260Sjhb m->lock_object.lo_name, file, line)); 605170295Sjeff WITNESS_CHECKORDER(&m->lock_object, 606182914Sjhb opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 607215054Sjhb while (!_mtx_obtain_lock(m, tid)) { 608192853Ssson#ifdef KDTRACE_HOOKS 609192853Ssson spin_cnt++; 610192853Ssson#endif 611170295Sjeff if (m->mtx_lock == tid) { 612170295Sjeff m->mtx_recurse++; 613170295Sjeff break; 614170295Sjeff } 615236238Sfabient#ifdef HWPMC_HOOKS 616236238Sfabient PMC_SOFT_CALL( , , lock, failed); 617236238Sfabient#endif 618174629Sjeff lock_profile_obtain_lock_failed(&m->lock_object, 619174629Sjeff &contested, &waittime); 620170295Sjeff /* Give interrupts a chance while we spin. */ 621170295Sjeff spinlock_exit(); 622170295Sjeff while (m->mtx_lock != MTX_UNOWNED) { 623170295Sjeff if (i++ < 10000000) 624170295Sjeff cpu_spinwait(); 625170295Sjeff else if (i < 60000000 || 626170295Sjeff kdb_active || panicstr != NULL) 627170295Sjeff DELAY(1); 628170295Sjeff else 629170295Sjeff _mtx_lock_spin_failed(m); 630170295Sjeff cpu_spinwait(); 631170295Sjeff if (m != td->td_lock) 632170295Sjeff goto retry; 633170295Sjeff } 634170295Sjeff spinlock_enter(); 635170295Sjeff } 636170295Sjeff if (m == td->td_lock) 637170295Sjeff break; 638215054Sjhb __mtx_unlock_spin(m); /* does spinlock_exit() */ 639192853Ssson#ifdef KDTRACE_HOOKS 640192853Ssson spin_cnt++; 641192853Ssson#endif 642170295Sjeff } 643174629Sjeff if (m->mtx_recurse == 0) 644192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, 645192853Ssson m, contested, waittime, (file), (line)); 646176257Sjhb LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 647176257Sjhb line); 648170295Sjeff WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 649192853Ssson LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt); 650170295Sjeff} 651170295Sjeff 652170295Sjeffstruct mtx * 653170295Sjeffthread_lock_block(struct thread *td) 654170295Sjeff{ 655170295Sjeff struct mtx *lock; 656170295Sjeff 657170295Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 658170358Sjeff lock = td->td_lock; 659170295Sjeff td->td_lock = &blocked_lock; 660170295Sjeff mtx_unlock_spin(lock); 661170295Sjeff 662170295Sjeff return (lock); 663170295Sjeff} 664170295Sjeff 665170295Sjeffvoid 666170295Sjeffthread_lock_unblock(struct thread *td, struct mtx *new) 667170295Sjeff{ 668170295Sjeff mtx_assert(new, MA_OWNED); 669170295Sjeff MPASS(td->td_lock == &blocked_lock); 670170465Smjacob atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new); 671170295Sjeff} 672170295Sjeff 673170295Sjeffvoid 674170295Sjeffthread_lock_set(struct thread *td, struct mtx *new) 675170295Sjeff{ 676170295Sjeff struct mtx *lock; 677170295Sjeff 678170295Sjeff mtx_assert(new, MA_OWNED); 679170295Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 680170358Sjeff lock = td->td_lock; 681170295Sjeff td->td_lock = new; 682170295Sjeff mtx_unlock_spin(lock); 683170295Sjeff} 684170295Sjeff 68572200Sbmilekic/* 68672200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 68772200Sbmilekic * 68872200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 68972200Sbmilekic * need to wake up a blocked thread). 69072200Sbmilekic */ 69167352Sjhbvoid 69272200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 69367352Sjhb{ 694122514Sjhb struct turnstile *ts; 69567352Sjhb 696235404Savg if (SCHEDULER_STOPPED()) 697235404Savg return; 698235404Savg 69972200Sbmilekic if (mtx_recursed(m)) { 70072200Sbmilekic if (--(m->mtx_recurse) == 0) 70172200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 702167787Sjhb if (LOCK_LOG_TEST(&m->lock_object, opts)) 70372200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 70472200Sbmilekic return; 70572200Sbmilekic } 70672200Sbmilekic 707170295Sjeff /* 708170295Sjeff * We have to lock the chain before the turnstile so this turnstile 709170295Sjeff * can be removed from the hash list if it is empty. 710170295Sjeff */ 711170295Sjeff turnstile_chain_lock(&m->lock_object); 712167787Sjhb ts = turnstile_lookup(&m->lock_object); 713167787Sjhb if (LOCK_LOG_TEST(&m->lock_object, opts)) 71472200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 715122514Sjhb MPASS(ts != NULL); 716154937Sjhb turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 717215054Sjhb _mtx_release_lock_quick(m); 718182909Sjhb 719170295Sjeff /* 720170295Sjeff * This turnstile is now no longer associated with the mutex. We can 721170295Sjeff * unlock the chain lock so a new turnstile may take it's place. 722170295Sjeff */ 723154937Sjhb turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 724170295Sjeff turnstile_chain_unlock(&m->lock_object); 72567352Sjhb} 72667352Sjhb 72772200Sbmilekic/* 72872200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 729215054Sjhb * See the __mtx_unlock_spin() macro for the details. 73072200Sbmilekic */ 73172200Sbmilekic 73272200Sbmilekic/* 73372994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 73472200Sbmilekic */ 73572996Sjhb#ifdef INVARIANT_SUPPORT 73671352Sjasonevoid 73771360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 73871352Sjasone{ 73980748Sjhb 740148557Sps if (panicstr != NULL || dumping) 74180748Sjhb return; 74273033Sjake switch (what) { 74371352Sjasone case MA_OWNED: 74471352Sjasone case MA_OWNED | MA_RECURSED: 74571352Sjasone case MA_OWNED | MA_NOTRECURSED: 74673033Sjake if (!mtx_owned(m)) 74771352Sjasone panic("mutex %s not owned at %s:%d", 748167787Sjhb m->lock_object.lo_name, file, line); 74973033Sjake if (mtx_recursed(m)) { 75073033Sjake if ((what & MA_NOTRECURSED) != 0) 75171352Sjasone panic("mutex %s recursed at %s:%d", 752167787Sjhb m->lock_object.lo_name, file, line); 75373033Sjake } else if ((what & MA_RECURSED) != 0) { 75471352Sjasone panic("mutex %s unrecursed at %s:%d", 755167787Sjhb m->lock_object.lo_name, file, line); 75671352Sjasone } 75771352Sjasone break; 75871352Sjasone case MA_NOTOWNED: 75973033Sjake if (mtx_owned(m)) 76071352Sjasone panic("mutex %s owned at %s:%d", 761167787Sjhb m->lock_object.lo_name, file, line); 76271352Sjasone break; 76371352Sjasone default: 76471360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 76571352Sjasone } 76671352Sjasone} 76771352Sjasone#endif 76871352Sjasone 76972200Sbmilekic/* 77072200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 77174912Sjhb * 77274912Sjhb * Most of these checks have been moved off into the LO_INITIALIZED flag 77374912Sjhb * maintained by the witness code. 77472200Sbmilekic */ 77567352Sjhb#ifdef MUTEX_DEBUG 77667352Sjhb 77792723Salfredvoid mtx_validate(struct mtx *); 77867352Sjhb 77974912Sjhbvoid 78074912Sjhbmtx_validate(struct mtx *m) 78167352Sjhb{ 78267352Sjhb 78367352Sjhb/* 784105919Sphk * XXX: When kernacc() does not require Giant we can reenable this check 785105919Sphk */ 786105919Sphk#ifdef notyet 78782304Sbmilekic /* 78882304Sbmilekic * Can't call kernacc() from early init386(), especially when 78982304Sbmilekic * initializing Giant mutex, because some stuff in kernacc() 79082304Sbmilekic * requires Giant itself. 791105782Sdes */ 79282302Sbmilekic if (!cold) 79382302Sbmilekic if (!kernacc((caddr_t)m, sizeof(m), 79482302Sbmilekic VM_PROT_READ | VM_PROT_WRITE)) 79582302Sbmilekic panic("Can't read and write to mutex %p", m); 79667352Sjhb#endif 79767352Sjhb} 79867352Sjhb#endif 79967352Sjhb 80072200Sbmilekic/* 80193672Sarr * General init routine used by the MTX_SYSINIT() macro. 80293672Sarr */ 80393672Sarrvoid 80493672Sarrmtx_sysinit(void *arg) 80593672Sarr{ 80693672Sarr struct mtx_args *margs = arg; 80793672Sarr 80893813Sjhb mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 80993672Sarr} 81093672Sarr 81193672Sarr/* 81272200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 81393813Sjhb * `opts' with options contained in `opts' and name `name.' The optional 81493813Sjhb * lock type `type' is used as a general lock category name for use with 81593813Sjhb * witness. 816105782Sdes */ 81767352Sjhbvoid 81893813Sjhbmtx_init(struct mtx *m, const char *name, const char *type, int opts) 81967352Sjhb{ 820154484Sjhb struct lock_class *class; 821154484Sjhb int flags; 82272200Sbmilekic 82374912Sjhb MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 824164159Skmacy MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); 825196334Sattilio ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock, 826196334Sattilio ("%s: mtx_lock not aligned for %s: %p", __func__, name, 827196334Sattilio &m->mtx_lock)); 82872200Sbmilekic 82967352Sjhb#ifdef MUTEX_DEBUG 83072200Sbmilekic /* Diagnostic and error correction */ 83174912Sjhb mtx_validate(m); 83269429Sjhb#endif 83367352Sjhb 834154484Sjhb /* Determine lock class and lock flags. */ 83574912Sjhb if (opts & MTX_SPIN) 836154484Sjhb class = &lock_class_mtx_spin; 83774912Sjhb else 838154484Sjhb class = &lock_class_mtx_sleep; 839154484Sjhb flags = 0; 84074912Sjhb if (opts & MTX_QUIET) 841154484Sjhb flags |= LO_QUIET; 84274912Sjhb if (opts & MTX_RECURSE) 843154484Sjhb flags |= LO_RECURSABLE; 84474912Sjhb if ((opts & MTX_NOWITNESS) == 0) 845154484Sjhb flags |= LO_WITNESS; 84693273Sjeff if (opts & MTX_DUPOK) 847154484Sjhb flags |= LO_DUPOK; 848164159Skmacy if (opts & MTX_NOPROFILE) 849164159Skmacy flags |= LO_NOPROFILE; 85072200Sbmilekic 851154484Sjhb /* Initialize mutex. */ 85267352Sjhb m->mtx_lock = MTX_UNOWNED; 853154484Sjhb m->mtx_recurse = 0; 85472200Sbmilekic 855167787Sjhb lock_init(&m->lock_object, class, name, type, flags); 85667352Sjhb} 85767352Sjhb 85872200Sbmilekic/* 85974912Sjhb * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 86074912Sjhb * passed in as a flag here because if the corresponding mtx_init() was 86174912Sjhb * called with MTX_QUIET set, then it will already be set in the mutex's 86274912Sjhb * flags. 86372200Sbmilekic */ 86467352Sjhbvoid 86567352Sjhbmtx_destroy(struct mtx *m) 86667352Sjhb{ 86767352Sjhb 86874912Sjhb if (!mtx_owned(m)) 86974912Sjhb MPASS(mtx_unowned(m)); 87074912Sjhb else { 87171228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 87272200Sbmilekic 873154103Sscottl /* Perform the non-mtx related part of mtx_unlock_spin(). */ 874167787Sjhb if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) 875154103Sscottl spinlock_exit(); 876160771Sjhb else 877160771Sjhb curthread->td_locks--; 878154103Sscottl 879189789Sjeff lock_profile_release_lock(&m->lock_object); 88074912Sjhb /* Tell witness this isn't locked to make it happy. */ 881167787Sjhb WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__, 88288900Sjhb __LINE__); 88371320Sjasone } 88471320Sjasone 885160766Sjhb m->mtx_lock = MTX_DESTROYED; 886167787Sjhb lock_destroy(&m->lock_object); 88771320Sjasone} 88885564Sdillon 88985564Sdillon/* 89093702Sjhb * Intialize the mutex code and system mutexes. This is called from the MD 89193702Sjhb * startup code prior to mi_startup(). The per-CPU data space needs to be 89293702Sjhb * setup before this is called. 89393702Sjhb */ 89493702Sjhbvoid 89593702Sjhbmutex_init(void) 89693702Sjhb{ 89793702Sjhb 898122514Sjhb /* Setup turnstiles so that sleep mutexes work. */ 899122514Sjhb init_turnstiles(); 900122514Sjhb 90193702Sjhb /* 90293702Sjhb * Initialize mutexes. 90393702Sjhb */ 90493813Sjhb mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 905170295Sjeff mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN); 906170295Sjeff blocked_lock.mtx_lock = 0xdeadc0de; /* Always blocked. */ 90793813Sjhb mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 908170295Sjeff mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE); 909151450Sjhb mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 91093702Sjhb mtx_lock(&Giant); 91193702Sjhb} 912153395Sjhb 913153395Sjhb#ifdef DDB 914153395Sjhbvoid 915153395Sjhbdb_show_mtx(struct lock_object *lock) 916153395Sjhb{ 917153395Sjhb struct thread *td; 918153395Sjhb struct mtx *m; 919153395Sjhb 920153395Sjhb m = (struct mtx *)lock; 921153395Sjhb 922153395Sjhb db_printf(" flags: {"); 923154484Sjhb if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 924153395Sjhb db_printf("SPIN"); 925153395Sjhb else 926153395Sjhb db_printf("DEF"); 927167787Sjhb if (m->lock_object.lo_flags & LO_RECURSABLE) 928153395Sjhb db_printf(", RECURSE"); 929167787Sjhb if (m->lock_object.lo_flags & LO_DUPOK) 930153395Sjhb db_printf(", DUPOK"); 931153395Sjhb db_printf("}\n"); 932153395Sjhb db_printf(" state: {"); 933153395Sjhb if (mtx_unowned(m)) 934153395Sjhb db_printf("UNOWNED"); 935169393Sjhb else if (mtx_destroyed(m)) 936169393Sjhb db_printf("DESTROYED"); 937153395Sjhb else { 938153395Sjhb db_printf("OWNED"); 939153395Sjhb if (m->mtx_lock & MTX_CONTESTED) 940153395Sjhb db_printf(", CONTESTED"); 941153395Sjhb if (m->mtx_lock & MTX_RECURSED) 942153395Sjhb db_printf(", RECURSED"); 943153395Sjhb } 944153395Sjhb db_printf("}\n"); 945169393Sjhb if (!mtx_unowned(m) && !mtx_destroyed(m)) { 946153395Sjhb td = mtx_owner(m); 947153395Sjhb db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 948173600Sjulian td->td_tid, td->td_proc->p_pid, td->td_name); 949153395Sjhb if (mtx_recursed(m)) 950153395Sjhb db_printf(" recursed: %d\n", m->mtx_recurse); 951153395Sjhb } 952153395Sjhb} 953153395Sjhb#endif 954