subr_turnstile.c revision 116182
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone */ 3165557Sjasone 3265557Sjasone/* 3386411Sjhb * Machine independent bits of mutex implementation. 3472200Sbmilekic */ 3572200Sbmilekic 36116182Sobrien#include <sys/cdefs.h> 37116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/subr_turnstile.c 116182 2003-06-11 00:56:59Z obrien $"); 38116182Sobrien 3997081Sjhb#include "opt_adaptive_mutexes.h" 4068790Sjhb#include "opt_ddb.h" 4167676Sjhb 4265557Sjasone#include <sys/param.h> 4393609Sdes#include <sys/systm.h> 4467352Sjhb#include <sys/bus.h> 4567352Sjhb#include <sys/kernel.h> 4693609Sdes#include <sys/ktr.h> 4776166Smarkm#include <sys/lock.h> 4867352Sjhb#include <sys/malloc.h> 4974912Sjhb#include <sys/mutex.h> 5065557Sjasone#include <sys/proc.h> 5178766Sjhb#include <sys/resourcevar.h> 52104964Sjeff#include <sys/sched.h> 5393609Sdes#include <sys/sbuf.h> 5467676Sjhb#include <sys/sysctl.h> 5567352Sjhb#include <sys/vmmeter.h> 5665557Sjasone 5767352Sjhb#include <machine/atomic.h> 5867352Sjhb#include <machine/bus.h> 5967352Sjhb#include <machine/clock.h> 6065557Sjasone#include <machine/cpu.h> 6167352Sjhb 6268790Sjhb#include <ddb/ddb.h> 6368790Sjhb 6467352Sjhb#include <vm/vm.h> 6567352Sjhb#include <vm/vm_extern.h> 6667352Sjhb 6765557Sjasone/* 6872200Sbmilekic * Internal utility macros. 6971352Sjasone */ 7072200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 7171352Sjasone 7272200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 7383366Sjulian : (struct thread *)((m)->mtx_lock & MTX_FLAGMASK)) 7471352Sjasone 7571352Sjasone/* 7674912Sjhb * Lock classes for sleep and spin mutexes. 7771352Sjasone */ 7874912Sjhbstruct lock_class lock_class_mtx_sleep = { 7974912Sjhb "sleep mutex", 8074912Sjhb LC_SLEEPLOCK | LC_RECURSABLE 8174912Sjhb}; 8274912Sjhbstruct lock_class lock_class_mtx_spin = { 8374912Sjhb "spin mutex", 8474912Sjhb LC_SPINLOCK | LC_RECURSABLE 8574912Sjhb}; 8671352Sjasone 8771352Sjasone/* 8893702Sjhb * System-wide mutexes 8993702Sjhb */ 9093702Sjhbstruct mtx sched_lock; 9193702Sjhbstruct mtx Giant; 9293702Sjhb 9393702Sjhb/* 9472200Sbmilekic * Prototypes for non-exported routines. 9572200Sbmilekic */ 9683366Sjulianstatic void propagate_priority(struct thread *); 9767352Sjhb 9867352Sjhbstatic void 9983366Sjulianpropagate_priority(struct thread *td) 10067352Sjhb{ 10190538Sjulian int pri = td->td_priority; 10283366Sjulian struct mtx *m = td->td_blocked; 10367352Sjhb 10469376Sjhb mtx_assert(&sched_lock, MA_OWNED); 10567352Sjhb for (;;) { 10683366Sjulian struct thread *td1; 10767352Sjhb 10883366Sjulian td = mtx_owner(m); 10967352Sjhb 11083366Sjulian if (td == NULL) { 11167352Sjhb /* 11267352Sjhb * This really isn't quite right. Really 11383366Sjulian * ought to bump priority of thread that 11467352Sjhb * next acquires the mutex. 11567352Sjhb */ 11667352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 11767352Sjhb return; 11867352Sjhb } 11972200Sbmilekic 12099072Sjulian MPASS(td->td_proc != NULL); 12183366Sjulian MPASS(td->td_proc->p_magic == P_MAGIC); 122103216Sjulian KASSERT(!TD_IS_SLEEPING(td), ("sleeping thread owns a mutex")); 12390538Sjulian if (td->td_priority <= pri) /* lower is higher priority */ 12467352Sjhb return; 12569376Sjhb 12669376Sjhb 12769376Sjhb /* 12867352Sjhb * If lock holder is actually running, just bump priority. 12967352Sjhb */ 130103216Sjulian if (TD_IS_RUNNING(td)) { 13199072Sjulian td->td_priority = pri; 13267352Sjhb return; 13367352Sjhb } 13472376Sjake 13573912Sjhb#ifndef SMP 13667352Sjhb /* 13783366Sjulian * For UP, we check to see if td is curthread (this shouldn't 13873912Sjhb * ever happen however as it would mean we are in a deadlock.) 13973912Sjhb */ 14083366Sjulian KASSERT(td != curthread, ("Deadlock detected")); 14173912Sjhb#endif 14273912Sjhb 14373912Sjhb /* 14483366Sjulian * If on run queue move to new run queue, and quit. 14583366Sjulian * XXXKSE this gets a lot more complicated under threads 14683366Sjulian * but try anyhow. 14767352Sjhb */ 148103216Sjulian if (TD_ON_RUNQ(td)) { 14983366Sjulian MPASS(td->td_blocked == NULL); 150104964Sjeff sched_prio(td, pri); 15167352Sjhb return; 15267352Sjhb } 15399072Sjulian /* 15499072Sjulian * Adjust for any other cases. 15599072Sjulian */ 15699072Sjulian td->td_priority = pri; 15767352Sjhb 15867352Sjhb /* 15969376Sjhb * If we aren't blocked on a mutex, we should be. 16067352Sjhb */ 161104387Sjhb KASSERT(TD_ON_LOCK(td), ( 16269376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 16399072Sjulian td->td_proc->p_pid, td->td_proc->p_comm, td->td_state, 16474912Sjhb m->mtx_object.lo_name)); 16567352Sjhb 16667352Sjhb /* 16783366Sjulian * Pick up the mutex that td is blocked on. 16867352Sjhb */ 16983366Sjulian m = td->td_blocked; 17067352Sjhb MPASS(m != NULL); 17167352Sjhb 17267352Sjhb /* 17383366Sjulian * Check if the thread needs to be moved up on 17467352Sjhb * the blocked chain 17567352Sjhb */ 17683366Sjulian if (td == TAILQ_FIRST(&m->mtx_blocked)) { 17769376Sjhb continue; 17869376Sjhb } 17972200Sbmilekic 180104387Sjhb td1 = TAILQ_PREV(td, threadqueue, td_lockq); 18190538Sjulian if (td1->td_priority <= pri) { 18267352Sjhb continue; 18367352Sjhb } 18467352Sjhb 18567352Sjhb /* 18683366Sjulian * Remove thread from blocked chain and determine where 18783366Sjulian * it should be moved up to. Since we know that td1 has 18883366Sjulian * a lower priority than td, we know that at least one 18983366Sjulian * thread in the chain has a lower priority and that 19083366Sjulian * td1 will thus not be NULL after the loop. 19167352Sjhb */ 192104387Sjhb TAILQ_REMOVE(&m->mtx_blocked, td, td_lockq); 193104387Sjhb TAILQ_FOREACH(td1, &m->mtx_blocked, td_lockq) { 19483366Sjulian MPASS(td1->td_proc->p_magic == P_MAGIC); 19590538Sjulian if (td1->td_priority > pri) 19667352Sjhb break; 19767352Sjhb } 19872200Sbmilekic 19983366Sjulian MPASS(td1 != NULL); 200104387Sjhb TAILQ_INSERT_BEFORE(td1, td, td_lockq); 20167352Sjhb CTR4(KTR_LOCK, 20271560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 20383366Sjulian td, td1, m, m->mtx_object.lo_name); 20467352Sjhb } 20567352Sjhb} 20667352Sjhb 20793609Sdes#ifdef MUTEX_PROFILING 20893609SdesSYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging"); 20993609SdesSYSCTL_NODE(_debug_mutex, OID_AUTO, prof, CTLFLAG_RD, NULL, "mutex profiling"); 21093609Sdesstatic int mutex_prof_enable = 0; 21193609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, enable, CTLFLAG_RW, 21293609Sdes &mutex_prof_enable, 0, "Enable tracing of mutex holdtime"); 21393609Sdes 21493609Sdesstruct mutex_prof { 215105644Sdes const char *name; 216105644Sdes const char *file; 217105644Sdes int line; 218109654Sdes uintmax_t cnt_max; 219109654Sdes uintmax_t cnt_tot; 220109654Sdes uintmax_t cnt_cur; 22193705Sdes struct mutex_prof *next; 22293609Sdes}; 22393609Sdes 22471352Sjasone/* 22593609Sdes * mprof_buf is a static pool of profiling records to avoid possible 22693609Sdes * reentrance of the memory allocation functions. 22793609Sdes * 22893609Sdes * Note: NUM_MPROF_BUFFERS must be smaller than MPROF_HASH_SIZE. 22993609Sdes */ 230105644Sdes#define NUM_MPROF_BUFFERS 1000 23193609Sdesstatic struct mutex_prof mprof_buf[NUM_MPROF_BUFFERS]; 23293609Sdesstatic int first_free_mprof_buf; 233105644Sdes#define MPROF_HASH_SIZE 1009 23493609Sdesstatic struct mutex_prof *mprof_hash[MPROF_HASH_SIZE]; 235111508Smtm/* SWAG: sbuf size = avg stat. line size * number of locks */ 236111508Smtm#define MPROF_SBUF_SIZE 256 * 400 23793609Sdes 23893609Sdesstatic int mutex_prof_acquisitions; 23993609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, acquisitions, CTLFLAG_RD, 24093609Sdes &mutex_prof_acquisitions, 0, "Number of mutex acquistions recorded"); 24193609Sdesstatic int mutex_prof_records; 24293609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, records, CTLFLAG_RD, 24393609Sdes &mutex_prof_records, 0, "Number of profiling records"); 24493609Sdesstatic int mutex_prof_maxrecords = NUM_MPROF_BUFFERS; 24593609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, maxrecords, CTLFLAG_RD, 24693609Sdes &mutex_prof_maxrecords, 0, "Maximum number of profiling records"); 24793609Sdesstatic int mutex_prof_rejected; 24893609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, rejected, CTLFLAG_RD, 24993609Sdes &mutex_prof_rejected, 0, "Number of rejected profiling records"); 25093609Sdesstatic int mutex_prof_hashsize = MPROF_HASH_SIZE; 25193609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, hashsize, CTLFLAG_RD, 25293609Sdes &mutex_prof_hashsize, 0, "Hash size"); 25393609Sdesstatic int mutex_prof_collisions = 0; 25493609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, collisions, CTLFLAG_RD, 25593609Sdes &mutex_prof_collisions, 0, "Number of hash collisions"); 25693609Sdes 25793609Sdes/* 25893609Sdes * mprof_mtx protects the profiling buffers and the hash. 25993609Sdes */ 26093609Sdesstatic struct mtx mprof_mtx; 26193705SdesMTX_SYSINIT(mprof, &mprof_mtx, "mutex profiling lock", MTX_SPIN | MTX_QUIET); 26293609Sdes 26393667Sdesstatic u_int64_t 26493667Sdesnanoseconds(void) 26593667Sdes{ 26693667Sdes struct timespec tv; 26793667Sdes 26893667Sdes nanotime(&tv); 26993667Sdes return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec); 27093667Sdes} 27193667Sdes 27293609Sdesstatic int 27393609Sdesdump_mutex_prof_stats(SYSCTL_HANDLER_ARGS) 27493609Sdes{ 27593609Sdes struct sbuf *sb; 27693609Sdes int error, i; 277111508Smtm static int multiplier = 1; 27893609Sdes 27993609Sdes if (first_free_mprof_buf == 0) 280105644Sdes return (SYSCTL_OUT(req, "No locking recorded", 281105644Sdes sizeof("No locking recorded"))); 28293609Sdes 283111508Smtmretry_sbufops: 284111508Smtm sb = sbuf_new(NULL, NULL, MPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN); 285105644Sdes sbuf_printf(sb, "%6s %12s %11s %5s %s\n", 286105644Sdes "max", "total", "count", "avg", "name"); 287105644Sdes /* 288105644Sdes * XXX this spinlock seems to be by far the largest perpetrator 289105644Sdes * of spinlock latency (1.6 msec on an Athlon1600 was recorded 290105644Sdes * even before I pessimized it further by moving the average 291105644Sdes * computation here). 292105644Sdes */ 29393609Sdes mtx_lock_spin(&mprof_mtx); 294111508Smtm for (i = 0; i < first_free_mprof_buf; ++i) { 295105644Sdes sbuf_printf(sb, "%6ju %12ju %11ju %5ju %s:%d (%s)\n", 296109654Sdes mprof_buf[i].cnt_max / 1000, 297109654Sdes mprof_buf[i].cnt_tot / 1000, 298109654Sdes mprof_buf[i].cnt_cur, 299109654Sdes mprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 : 300109654Sdes mprof_buf[i].cnt_tot / (mprof_buf[i].cnt_cur * 1000), 30193609Sdes mprof_buf[i].file, mprof_buf[i].line, mprof_buf[i].name); 302111508Smtm if (sbuf_overflowed(sb)) { 303111508Smtm mtx_unlock_spin(&mprof_mtx); 304111508Smtm sbuf_delete(sb); 305111508Smtm multiplier++; 306111508Smtm goto retry_sbufops; 307111508Smtm } 308111508Smtm } 30993609Sdes mtx_unlock_spin(&mprof_mtx); 31093609Sdes sbuf_finish(sb); 31193609Sdes error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 31293609Sdes sbuf_delete(sb); 31393609Sdes return (error); 31493609Sdes} 315105644SdesSYSCTL_PROC(_debug_mutex_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD, 31693609Sdes NULL, 0, dump_mutex_prof_stats, "A", "Mutex profiling statistics"); 31793609Sdes#endif 31893609Sdes 31993609Sdes/* 32074900Sjhb * Function versions of the inlined __mtx_* macros. These are used by 32174900Sjhb * modules and can also be called from assembly language if needed. 32274900Sjhb */ 32374900Sjhbvoid 32474900Sjhb_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 32574900Sjhb{ 32674900Sjhb 32783841Sjhb MPASS(curthread != NULL); 328102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, 329102907Sjhb ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 330102907Sjhb file, line)); 33183841Sjhb _get_sleep_lock(m, curthread, opts, file, line); 33283841Sjhb LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 33383841Sjhb line); 33483841Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 33593609Sdes#ifdef MUTEX_PROFILING 33693609Sdes /* don't reset the timer when/if recursing */ 33799324Sdes if (m->mtx_acqtime == 0) { 33899324Sdes m->mtx_filename = file; 33999324Sdes m->mtx_lineno = line; 34099324Sdes m->mtx_acqtime = mutex_prof_enable ? nanoseconds() : 0; 34193609Sdes ++mutex_prof_acquisitions; 34293609Sdes } 34393609Sdes#endif 34474900Sjhb} 34574900Sjhb 34674900Sjhbvoid 34774900Sjhb_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 34874900Sjhb{ 34974900Sjhb 35083841Sjhb MPASS(curthread != NULL); 351102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, 352102907Sjhb ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 353102907Sjhb file, line)); 354105782Sdes WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 355102907Sjhb LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 356102907Sjhb line); 35783947Sjhb mtx_assert(m, MA_OWNED); 35893609Sdes#ifdef MUTEX_PROFILING 35999324Sdes if (m->mtx_acqtime != 0) { 36093609Sdes static const char *unknown = "(unknown)"; 36193609Sdes struct mutex_prof *mpp; 36293667Sdes u_int64_t acqtime, now; 36393609Sdes const char *p, *q; 36493705Sdes volatile u_int hash; 36593609Sdes 36693667Sdes now = nanoseconds(); 36799324Sdes acqtime = m->mtx_acqtime; 36899324Sdes m->mtx_acqtime = 0; 36993667Sdes if (now <= acqtime) 37093609Sdes goto out; 371111508Smtm for (p = m->mtx_filename; 372111508Smtm p != NULL && strncmp(p, "../", 3) == 0; p += 3) 37393609Sdes /* nothing */ ; 37493609Sdes if (p == NULL || *p == '\0') 37593609Sdes p = unknown; 37699324Sdes for (hash = m->mtx_lineno, q = p; *q != '\0'; ++q) 37793609Sdes hash = (hash * 2 + *q) % MPROF_HASH_SIZE; 37893609Sdes mtx_lock_spin(&mprof_mtx); 37993705Sdes for (mpp = mprof_hash[hash]; mpp != NULL; mpp = mpp->next) 38099324Sdes if (mpp->line == m->mtx_lineno && 38199324Sdes strcmp(mpp->file, p) == 0) 38293609Sdes break; 38393609Sdes if (mpp == NULL) { 38493609Sdes /* Just exit if we cannot get a trace buffer */ 38593609Sdes if (first_free_mprof_buf >= NUM_MPROF_BUFFERS) { 38693609Sdes ++mutex_prof_rejected; 38793609Sdes goto unlock; 38893609Sdes } 38993609Sdes mpp = &mprof_buf[first_free_mprof_buf++]; 39093609Sdes mpp->name = mtx_name(m); 39193609Sdes mpp->file = p; 39299324Sdes mpp->line = m->mtx_lineno; 39393705Sdes mpp->next = mprof_hash[hash]; 39493705Sdes if (mprof_hash[hash] != NULL) 39593705Sdes ++mutex_prof_collisions; 396105782Sdes mprof_hash[hash] = mpp; 39793609Sdes ++mutex_prof_records; 39893609Sdes } 39993609Sdes /* 40093609Sdes * Record if the mutex has been held longer now than ever 401105644Sdes * before. 40293609Sdes */ 403109654Sdes if (now - acqtime > mpp->cnt_max) 404109654Sdes mpp->cnt_max = now - acqtime; 405109654Sdes mpp->cnt_tot += now - acqtime; 406109654Sdes mpp->cnt_cur++; 40793609Sdesunlock: 40893609Sdes mtx_unlock_spin(&mprof_mtx); 40993609Sdes } 41093609Sdesout: 41193609Sdes#endif 41283841Sjhb _rel_sleep_lock(m, curthread, opts, file, line); 41374900Sjhb} 41474900Sjhb 41574900Sjhbvoid 41674900Sjhb_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 41774900Sjhb{ 41874900Sjhb 41983841Sjhb MPASS(curthread != NULL); 420102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, 421102907Sjhb ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 422102907Sjhb m->mtx_object.lo_name, file, line)); 423100754Sjhb#if defined(SMP) || LOCK_DEBUG > 0 || 1 42483841Sjhb _get_spin_lock(m, curthread, opts, file, line); 42597079Sjhb#else 42697079Sjhb critical_enter(); 42797079Sjhb#endif 42883841Sjhb LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 42983841Sjhb line); 43083841Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 43174900Sjhb} 43274900Sjhb 43374900Sjhbvoid 43474900Sjhb_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 43574900Sjhb{ 43674900Sjhb 43783841Sjhb MPASS(curthread != NULL); 438102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, 439102907Sjhb ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 440102907Sjhb m->mtx_object.lo_name, file, line)); 441105782Sdes WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 44283841Sjhb LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 44383841Sjhb line); 444102907Sjhb mtx_assert(m, MA_OWNED); 445100754Sjhb#if defined(SMP) || LOCK_DEBUG > 0 || 1 44683841Sjhb _rel_spin_lock(m); 44797079Sjhb#else 44897079Sjhb critical_exit(); 44997079Sjhb#endif 45074900Sjhb} 45174900Sjhb 45274900Sjhb/* 45372200Sbmilekic * The important part of mtx_trylock{,_flags}() 454111885Sjhb * Tries to acquire lock `m.' We do NOT handle recursion here. If this 455111885Sjhb * function is called on a recursed mutex, it will return failure and 456111885Sjhb * will not recursively acquire the lock. You are expected to know what 457111885Sjhb * you are doing. 45871352Sjasone */ 45972200Sbmilekicint 46072200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 46171352Sjasone{ 46272200Sbmilekic int rval; 46371352Sjasone 46483366Sjulian MPASS(curthread != NULL); 46571352Sjasone 46683366Sjulian rval = _obtain_lock(m, curthread); 46772200Sbmilekic 46874912Sjhb LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 469111879Sjhb if (rval) 47076272Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 47176272Sjhb file, line); 47271352Sjasone 47374912Sjhb return (rval); 47471352Sjasone} 47571352Sjasone 47671352Sjasone/* 47772200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 47871352Sjasone * 47972200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 48072200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 48171352Sjasone */ 48272200Sbmilekicvoid 48372200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 48471352Sjasone{ 48583366Sjulian struct thread *td = curthread; 486111880Sjhb struct thread *td1; 48797081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 48897081Sjhb struct thread *owner; 48997081Sjhb#endif 490111880Sjhb uintptr_t v; 491102450Siedowse#ifdef KTR 492102450Siedowse int cont_logged = 0; 493102450Siedowse#endif 49471352Sjasone 495111880Sjhb if (mtx_owned(m)) { 49672200Sbmilekic m->mtx_recurse++; 49772200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 49874912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 49972344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 50072200Sbmilekic return; 50171352Sjasone } 50271352Sjasone 50374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 50472994Sjhb CTR4(KTR_LOCK, 50572994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 50674912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 50771352Sjasone 50883366Sjulian while (!_obtain_lock(m, td)) { 50971352Sjasone 51072200Sbmilekic mtx_lock_spin(&sched_lock); 511111880Sjhb v = m->mtx_lock; 512111880Sjhb 51372200Sbmilekic /* 51472200Sbmilekic * Check if the lock has been released while spinning for 51572200Sbmilekic * the sched_lock. 51672200Sbmilekic */ 517111880Sjhb if (v == MTX_UNOWNED) { 51872200Sbmilekic mtx_unlock_spin(&sched_lock); 51997086Sjhb#ifdef __i386__ 52097139Sjhb ia32_pause(); 52197086Sjhb#endif 52272200Sbmilekic continue; 52371352Sjasone } 52471352Sjasone 52572200Sbmilekic /* 52672200Sbmilekic * The mutex was marked contested on release. This means that 52783366Sjulian * there are threads blocked on it. 52872200Sbmilekic */ 52972200Sbmilekic if (v == MTX_CONTESTED) { 53083366Sjulian td1 = TAILQ_FIRST(&m->mtx_blocked); 53183366Sjulian MPASS(td1 != NULL); 53283366Sjulian m->mtx_lock = (uintptr_t)td | MTX_CONTESTED; 53367352Sjhb 53490538Sjulian if (td1->td_priority < td->td_priority) 535105782Sdes td->td_priority = td1->td_priority; 53672200Sbmilekic mtx_unlock_spin(&sched_lock); 53767352Sjhb return; 53867352Sjhb } 53969376Sjhb 54069376Sjhb /* 54172200Sbmilekic * If the mutex isn't already contested and a failure occurs 54272200Sbmilekic * setting the contested bit, the mutex was either released 54372200Sbmilekic * or the state of the MTX_RECURSED bit changed. 54469376Sjhb */ 54572200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 54672200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 54772200Sbmilekic (void *)(v | MTX_CONTESTED))) { 54872200Sbmilekic mtx_unlock_spin(&sched_lock); 54997086Sjhb#ifdef __i386__ 55097139Sjhb ia32_pause(); 55197086Sjhb#endif 55272200Sbmilekic continue; 55372200Sbmilekic } 55467352Sjhb 55597081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 55672200Sbmilekic /* 55797081Sjhb * If the current owner of the lock is executing on another 55897081Sjhb * CPU, spin instead of blocking. 55997081Sjhb */ 56097081Sjhb owner = (struct thread *)(v & MTX_FLAGMASK); 561113632Sjhb if (m != &Giant && TD_IS_RUNNING(owner)) { 56297081Sjhb mtx_unlock_spin(&sched_lock); 563113632Sjhb while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) { 56497086Sjhb#ifdef __i386__ 56597837Sjhb ia32_pause(); 56697086Sjhb#endif 56797837Sjhb } 56897081Sjhb continue; 56997081Sjhb } 57097081Sjhb#endif /* SMP && ADAPTIVE_MUTEXES */ 57197081Sjhb 57297081Sjhb /* 57393692Sjhb * We definitely must sleep for this lock. 57472200Sbmilekic */ 57572200Sbmilekic mtx_assert(m, MA_NOTOWNED); 57667352Sjhb 57767352Sjhb#ifdef notyet 57872200Sbmilekic /* 57972200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 58072200Sbmilekic * must clean up before going to sleep. 58172200Sbmilekic */ 58283366Sjulian if (td->td_ithd != NULL) { 58383366Sjulian struct ithd *it = td->td_ithd; 58467352Sjhb 58572200Sbmilekic if (it->it_interrupted) { 58674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 58772200Sbmilekic CTR2(KTR_LOCK, 58872994Sjhb "_mtx_lock_sleep: %p interrupted %p", 58972200Sbmilekic it, it->it_interrupted); 59072200Sbmilekic intr_thd_fixup(it); 59167352Sjhb } 59272200Sbmilekic } 59367352Sjhb#endif 59467352Sjhb 59572200Sbmilekic /* 59672200Sbmilekic * Put us on the list of threads blocked on this mutex. 59772200Sbmilekic */ 59872200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 59990418Sjhb td1 = mtx_owner(m); 60083366Sjulian LIST_INSERT_HEAD(&td1->td_contested, m, mtx_contested); 601104387Sjhb TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_lockq); 60272200Sbmilekic } else { 603104387Sjhb TAILQ_FOREACH(td1, &m->mtx_blocked, td_lockq) 60490538Sjulian if (td1->td_priority > td->td_priority) 60572200Sbmilekic break; 60683366Sjulian if (td1) 607104387Sjhb TAILQ_INSERT_BEFORE(td1, td, td_lockq); 60872200Sbmilekic else 609104387Sjhb TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_lockq); 61072200Sbmilekic } 611102450Siedowse#ifdef KTR 612102450Siedowse if (!cont_logged) { 613102450Siedowse CTR6(KTR_CONTENTION, 614102450Siedowse "contention: %p at %s:%d wants %s, taken by %s:%d", 615102450Siedowse td, file, line, m->mtx_object.lo_name, 616102450Siedowse WITNESS_FILE(&m->mtx_object), 617102450Siedowse WITNESS_LINE(&m->mtx_object)); 618102450Siedowse cont_logged = 1; 619102450Siedowse } 620102450Siedowse#endif 62167352Sjhb 62272200Sbmilekic /* 62372200Sbmilekic * Save who we're blocked on. 62472200Sbmilekic */ 62583366Sjulian td->td_blocked = m; 626104387Sjhb td->td_lockname = m->mtx_object.lo_name; 627104387Sjhb TD_SET_LOCK(td); 62883366Sjulian propagate_priority(td); 62967352Sjhb 63074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 63172200Sbmilekic CTR3(KTR_LOCK, 63283366Sjulian "_mtx_lock_sleep: p %p blocked on [%p] %s", td, m, 63374912Sjhb m->mtx_object.lo_name); 63472200Sbmilekic 63583366Sjulian td->td_proc->p_stats->p_ru.ru_nvcsw++; 63672200Sbmilekic mi_switch(); 63772200Sbmilekic 63874912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 63972200Sbmilekic CTR3(KTR_LOCK, 64072200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 64183366Sjulian td, m, m->mtx_object.lo_name); 64272200Sbmilekic 64372200Sbmilekic mtx_unlock_spin(&sched_lock); 64472200Sbmilekic } 64572200Sbmilekic 646102450Siedowse#ifdef KTR 647102450Siedowse if (cont_logged) { 648102450Siedowse CTR4(KTR_CONTENTION, 649102450Siedowse "contention end: %s acquired by %p at %s:%d", 650102450Siedowse m->mtx_object.lo_name, td, file, line); 651102450Siedowse } 652102450Siedowse#endif 65372200Sbmilekic return; 65472200Sbmilekic} 65572200Sbmilekic 65672200Sbmilekic/* 65772200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 65872200Sbmilekic * 65972200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 66072200Sbmilekic * is handled inline. 66172200Sbmilekic */ 66272200Sbmilekicvoid 66388088Sjhb_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) 66472200Sbmilekic{ 66572200Sbmilekic int i = 0; 66672200Sbmilekic 66774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 66872344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 66972200Sbmilekic 67072200Sbmilekic for (;;) { 67183366Sjulian if (_obtain_lock(m, curthread)) 67272200Sbmilekic break; 67372200Sbmilekic 67475568Sjhb /* Give interrupts a chance while we spin. */ 67588088Sjhb critical_exit(); 67672200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 67797086Sjhb if (i++ < 10000000) { 67897086Sjhb#ifdef __i386__ 67997139Sjhb ia32_pause(); 68097086Sjhb#endif 68172200Sbmilekic continue; 68297086Sjhb } 68397084Sjhb if (i < 60000000) 68472200Sbmilekic DELAY(1); 68567352Sjhb#ifdef DDB 68672200Sbmilekic else if (!db_active) 68767352Sjhb#else 68872200Sbmilekic else 68967352Sjhb#endif 69097082Sjhb panic("spin lock %s held by %p for > 5 seconds", 69197082Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock); 69297086Sjhb#ifdef __i386__ 69397139Sjhb ia32_pause(); 69497086Sjhb#endif 69567352Sjhb } 69688088Sjhb critical_enter(); 69767352Sjhb } 69872200Sbmilekic 69974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 70072200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 70172200Sbmilekic 70272200Sbmilekic return; 70367352Sjhb} 70467352Sjhb 70572200Sbmilekic/* 70672200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 70772200Sbmilekic * 70872200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 70972200Sbmilekic * need to wake up a blocked thread). 71072200Sbmilekic */ 71167352Sjhbvoid 71272200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 71367352Sjhb{ 71483366Sjulian struct thread *td, *td1; 71567352Sjhb struct mtx *m1; 71667352Sjhb int pri; 71767352Sjhb 71883366Sjulian td = curthread; 71972200Sbmilekic 72072200Sbmilekic if (mtx_recursed(m)) { 72172200Sbmilekic if (--(m->mtx_recurse) == 0) 72272200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 72374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 72472200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 72572200Sbmilekic return; 72672200Sbmilekic } 72772200Sbmilekic 72872200Sbmilekic mtx_lock_spin(&sched_lock); 72974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 73072200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 73172200Sbmilekic 73283366Sjulian td1 = TAILQ_FIRST(&m->mtx_blocked); 73397081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 73497081Sjhb if (td1 == NULL) { 73597081Sjhb _release_lock_quick(m); 73697081Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 73797081Sjhb CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 73897081Sjhb mtx_unlock_spin(&sched_lock); 73997081Sjhb return; 74097081Sjhb } 74197081Sjhb#endif 74283366Sjulian MPASS(td->td_proc->p_magic == P_MAGIC); 74383366Sjulian MPASS(td1->td_proc->p_magic == P_MAGIC); 74472200Sbmilekic 745104387Sjhb TAILQ_REMOVE(&m->mtx_blocked, td1, td_lockq); 74672200Sbmilekic 74772200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 74872200Sbmilekic LIST_REMOVE(m, mtx_contested); 74972200Sbmilekic _release_lock_quick(m); 75074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 75172200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 75272200Sbmilekic } else 75372200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 75472200Sbmilekic 75572376Sjake pri = PRI_MAX; 75683366Sjulian LIST_FOREACH(m1, &td->td_contested, mtx_contested) { 75790538Sjulian int cp = TAILQ_FIRST(&m1->mtx_blocked)->td_priority; 75872200Sbmilekic if (cp < pri) 75972200Sbmilekic pri = cp; 76072200Sbmilekic } 76172200Sbmilekic 76290538Sjulian if (pri > td->td_base_pri) 76390538Sjulian pri = td->td_base_pri; 76490538Sjulian td->td_priority = pri; 76572200Sbmilekic 76674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 76772200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 76883366Sjulian m, td1); 76972200Sbmilekic 77083366Sjulian td1->td_blocked = NULL; 771104387Sjhb TD_CLR_LOCK(td1); 772104160Sjulian if (!TD_CAN_RUN(td1)) { 773104160Sjulian mtx_unlock_spin(&sched_lock); 774104160Sjulian return; 775104160Sjulian } 776104161Sjulian setrunqueue(td1); 77772200Sbmilekic 77890538Sjulian if (td->td_critnest == 1 && td1->td_priority < pri) { 77967352Sjhb#ifdef notyet 78083366Sjulian if (td->td_ithd != NULL) { 78183366Sjulian struct ithd *it = td->td_ithd; 78267352Sjhb 78372200Sbmilekic if (it->it_interrupted) { 78474912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 78572200Sbmilekic CTR2(KTR_LOCK, 78672994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 78772200Sbmilekic it, it->it_interrupted); 78872200Sbmilekic intr_thd_fixup(it); 78967352Sjhb } 79072200Sbmilekic } 79167352Sjhb#endif 79274912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 79372200Sbmilekic CTR2(KTR_LOCK, 79472200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 79572200Sbmilekic (void *)m->mtx_lock); 79672200Sbmilekic 79783366Sjulian td->td_proc->p_stats->p_ru.ru_nivcsw++; 79872200Sbmilekic mi_switch(); 79974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 80072200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 80172200Sbmilekic m, (void *)m->mtx_lock); 80267352Sjhb } 80372200Sbmilekic 80472200Sbmilekic mtx_unlock_spin(&sched_lock); 80572200Sbmilekic 80672200Sbmilekic return; 80767352Sjhb} 80867352Sjhb 80972200Sbmilekic/* 81072200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 811105782Sdes * See the _rel_spin_lock() macro for the details. 81272200Sbmilekic */ 81372200Sbmilekic 81472200Sbmilekic/* 81572994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 81672200Sbmilekic */ 81772996Sjhb#ifdef INVARIANT_SUPPORT 81871352Sjasonevoid 81971360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 82071352Sjasone{ 82180748Sjhb 82280748Sjhb if (panicstr != NULL) 82380748Sjhb return; 82473033Sjake switch (what) { 82571352Sjasone case MA_OWNED: 82671352Sjasone case MA_OWNED | MA_RECURSED: 82771352Sjasone case MA_OWNED | MA_NOTRECURSED: 82873033Sjake if (!mtx_owned(m)) 82971352Sjasone panic("mutex %s not owned at %s:%d", 83074912Sjhb m->mtx_object.lo_name, file, line); 83173033Sjake if (mtx_recursed(m)) { 83273033Sjake if ((what & MA_NOTRECURSED) != 0) 83371352Sjasone panic("mutex %s recursed at %s:%d", 83474912Sjhb m->mtx_object.lo_name, file, line); 83573033Sjake } else if ((what & MA_RECURSED) != 0) { 83671352Sjasone panic("mutex %s unrecursed at %s:%d", 83774912Sjhb m->mtx_object.lo_name, file, line); 83871352Sjasone } 83971352Sjasone break; 84071352Sjasone case MA_NOTOWNED: 84173033Sjake if (mtx_owned(m)) 84271352Sjasone panic("mutex %s owned at %s:%d", 84374912Sjhb m->mtx_object.lo_name, file, line); 84471352Sjasone break; 84571352Sjasone default: 84671360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 84771352Sjasone } 84871352Sjasone} 84971352Sjasone#endif 85071352Sjasone 85172200Sbmilekic/* 85272200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 85374912Sjhb * 85474912Sjhb * Most of these checks have been moved off into the LO_INITIALIZED flag 85574912Sjhb * maintained by the witness code. 85672200Sbmilekic */ 85767352Sjhb#ifdef MUTEX_DEBUG 85867352Sjhb 85992723Salfredvoid mtx_validate(struct mtx *); 86067352Sjhb 86174912Sjhbvoid 86274912Sjhbmtx_validate(struct mtx *m) 86367352Sjhb{ 86467352Sjhb 86567352Sjhb/* 866105919Sphk * XXX: When kernacc() does not require Giant we can reenable this check 867105919Sphk */ 868105919Sphk#ifdef notyet 869105919Sphk/* 87067352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 87167352Sjhb * we can re-enable the kernacc() checks. 87267352Sjhb */ 87367352Sjhb#ifndef __alpha__ 87482304Sbmilekic /* 87582304Sbmilekic * Can't call kernacc() from early init386(), especially when 87682304Sbmilekic * initializing Giant mutex, because some stuff in kernacc() 87782304Sbmilekic * requires Giant itself. 878105782Sdes */ 87982302Sbmilekic if (!cold) 88082302Sbmilekic if (!kernacc((caddr_t)m, sizeof(m), 88182302Sbmilekic VM_PROT_READ | VM_PROT_WRITE)) 88282302Sbmilekic panic("Can't read and write to mutex %p", m); 88367352Sjhb#endif 884105919Sphk#endif 88567352Sjhb} 88667352Sjhb#endif 88767352Sjhb 88872200Sbmilekic/* 88993672Sarr * General init routine used by the MTX_SYSINIT() macro. 89093672Sarr */ 89193672Sarrvoid 89293672Sarrmtx_sysinit(void *arg) 89393672Sarr{ 89493672Sarr struct mtx_args *margs = arg; 89593672Sarr 89693813Sjhb mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 89793672Sarr} 89893672Sarr 89993672Sarr/* 90072200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 90193813Sjhb * `opts' with options contained in `opts' and name `name.' The optional 90293813Sjhb * lock type `type' is used as a general lock category name for use with 90393813Sjhb * witness. 904105782Sdes */ 90567352Sjhbvoid 90693813Sjhbmtx_init(struct mtx *m, const char *name, const char *type, int opts) 90767352Sjhb{ 90874912Sjhb struct lock_object *lock; 90972200Sbmilekic 91074912Sjhb MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 911112108Sjhb MTX_NOWITNESS | MTX_DUPOK)) == 0); 91272200Sbmilekic 91367352Sjhb#ifdef MUTEX_DEBUG 91472200Sbmilekic /* Diagnostic and error correction */ 91574912Sjhb mtx_validate(m); 91669429Sjhb#endif 91767352Sjhb 91885205Sjhb lock = &m->mtx_object; 91985205Sjhb KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, 920115568Sphk ("mutex \"%s\" %p already initialized", name, m)); 92174912Sjhb bzero(m, sizeof(*m)); 92274912Sjhb if (opts & MTX_SPIN) 92374912Sjhb lock->lo_class = &lock_class_mtx_spin; 92474912Sjhb else 92574912Sjhb lock->lo_class = &lock_class_mtx_sleep; 92693813Sjhb lock->lo_name = name; 92793813Sjhb lock->lo_type = type != NULL ? type : name; 92874912Sjhb if (opts & MTX_QUIET) 92974912Sjhb lock->lo_flags = LO_QUIET; 93074912Sjhb if (opts & MTX_RECURSE) 93174912Sjhb lock->lo_flags |= LO_RECURSABLE; 93274912Sjhb if ((opts & MTX_NOWITNESS) == 0) 93374912Sjhb lock->lo_flags |= LO_WITNESS; 93493273Sjeff if (opts & MTX_DUPOK) 93593273Sjeff lock->lo_flags |= LO_DUPOK; 93672200Sbmilekic 93767352Sjhb m->mtx_lock = MTX_UNOWNED; 93874912Sjhb TAILQ_INIT(&m->mtx_blocked); 93972200Sbmilekic 94074912Sjhb LOCK_LOG_INIT(lock, opts); 94172200Sbmilekic 94274912Sjhb WITNESS_INIT(lock); 94367352Sjhb} 94467352Sjhb 94572200Sbmilekic/* 94674912Sjhb * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 94774912Sjhb * passed in as a flag here because if the corresponding mtx_init() was 94874912Sjhb * called with MTX_QUIET set, then it will already be set in the mutex's 94974912Sjhb * flags. 95072200Sbmilekic */ 95167352Sjhbvoid 95267352Sjhbmtx_destroy(struct mtx *m) 95367352Sjhb{ 95467352Sjhb 95574912Sjhb LOCK_LOG_DESTROY(&m->mtx_object, 0); 95672200Sbmilekic 95774912Sjhb if (!mtx_owned(m)) 95874912Sjhb MPASS(mtx_unowned(m)); 95974912Sjhb else { 96071228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 96172200Sbmilekic 96274912Sjhb /* Tell witness this isn't locked to make it happy. */ 96388900Sjhb WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, 96488900Sjhb __LINE__); 96571320Sjasone } 96671320Sjasone 96774912Sjhb WITNESS_DESTROY(&m->mtx_object); 96871320Sjasone} 96985564Sdillon 97085564Sdillon/* 97193702Sjhb * Intialize the mutex code and system mutexes. This is called from the MD 97293702Sjhb * startup code prior to mi_startup(). The per-CPU data space needs to be 97393702Sjhb * setup before this is called. 97493702Sjhb */ 97593702Sjhbvoid 97693702Sjhbmutex_init(void) 97793702Sjhb{ 97893702Sjhb 97993702Sjhb /* Setup thread0 so that mutexes work. */ 98093702Sjhb LIST_INIT(&thread0.td_contested); 98193702Sjhb 98293702Sjhb /* 98393702Sjhb * Initialize mutexes. 98493702Sjhb */ 98593813Sjhb mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 98693813Sjhb mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 98793813Sjhb mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 98893702Sjhb mtx_lock(&Giant); 98993702Sjhb} 990