subr_turnstile.c revision 104160
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone * $FreeBSD: head/sys/kern/subr_turnstile.c 104160 2002-09-29 23:23:25Z julian $ 3165557Sjasone */ 3265557Sjasone 3365557Sjasone/* 3486411Sjhb * Machine independent bits of mutex implementation. 3572200Sbmilekic */ 3672200Sbmilekic 3797081Sjhb#include "opt_adaptive_mutexes.h" 3868790Sjhb#include "opt_ddb.h" 3967676Sjhb 4065557Sjasone#include <sys/param.h> 4193609Sdes#include <sys/systm.h> 4267352Sjhb#include <sys/bus.h> 4367352Sjhb#include <sys/kernel.h> 4493609Sdes#include <sys/ktr.h> 4576166Smarkm#include <sys/lock.h> 4667352Sjhb#include <sys/malloc.h> 4774912Sjhb#include <sys/mutex.h> 4865557Sjasone#include <sys/proc.h> 4978766Sjhb#include <sys/resourcevar.h> 5093609Sdes#include <sys/sbuf.h> 5197156Sdes#include <sys/stdint.h> 5267676Sjhb#include <sys/sysctl.h> 5367352Sjhb#include <sys/vmmeter.h> 5465557Sjasone 5567352Sjhb#include <machine/atomic.h> 5667352Sjhb#include <machine/bus.h> 5767352Sjhb#include <machine/clock.h> 5865557Sjasone#include <machine/cpu.h> 5967352Sjhb 6068790Sjhb#include <ddb/ddb.h> 6168790Sjhb 6267352Sjhb#include <vm/vm.h> 6367352Sjhb#include <vm/vm_extern.h> 6467352Sjhb 6565557Sjasone/* 6672200Sbmilekic * Internal utility macros. 6771352Sjasone */ 6872200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 6971352Sjasone 7072200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 7183366Sjulian : (struct thread *)((m)->mtx_lock & MTX_FLAGMASK)) 7271352Sjasone 7397839Sjhb/* XXXKSE This test will change. */ 7497839Sjhb#define thread_running(td) \ 7597836Sjhb ((td)->td_kse != NULL && (td)->td_kse->ke_oncpu != NOCPU) 7697836Sjhb 7771352Sjasone/* 7874912Sjhb * Lock classes for sleep and spin mutexes. 7971352Sjasone */ 8074912Sjhbstruct lock_class lock_class_mtx_sleep = { 8174912Sjhb "sleep mutex", 8274912Sjhb LC_SLEEPLOCK | LC_RECURSABLE 8374912Sjhb}; 8474912Sjhbstruct lock_class lock_class_mtx_spin = { 8574912Sjhb "spin mutex", 8674912Sjhb LC_SPINLOCK | LC_RECURSABLE 8774912Sjhb}; 8871352Sjasone 8971352Sjasone/* 9093702Sjhb * System-wide mutexes 9193702Sjhb */ 9293702Sjhbstruct mtx sched_lock; 9393702Sjhbstruct mtx Giant; 9493702Sjhb 9593702Sjhb/* 9672200Sbmilekic * Prototypes for non-exported routines. 9772200Sbmilekic */ 9883366Sjulianstatic void propagate_priority(struct thread *); 9967352Sjhb 10067352Sjhbstatic void 10183366Sjulianpropagate_priority(struct thread *td) 10267352Sjhb{ 10390538Sjulian int pri = td->td_priority; 10483366Sjulian struct mtx *m = td->td_blocked; 10567352Sjhb 10669376Sjhb mtx_assert(&sched_lock, MA_OWNED); 10767352Sjhb for (;;) { 10883366Sjulian struct thread *td1; 10967352Sjhb 11083366Sjulian td = mtx_owner(m); 11167352Sjhb 11283366Sjulian if (td == NULL) { 11367352Sjhb /* 11467352Sjhb * This really isn't quite right. Really 11583366Sjulian * ought to bump priority of thread that 11667352Sjhb * next acquires the mutex. 11767352Sjhb */ 11867352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 11967352Sjhb return; 12067352Sjhb } 12172200Sbmilekic 12299072Sjulian MPASS(td->td_proc != NULL); 12383366Sjulian MPASS(td->td_proc->p_magic == P_MAGIC); 124103216Sjulian KASSERT(!TD_IS_SLEEPING(td), ("sleeping thread owns a mutex")); 12590538Sjulian if (td->td_priority <= pri) /* lower is higher priority */ 12667352Sjhb return; 12769376Sjhb 12869376Sjhb 12969376Sjhb /* 13067352Sjhb * If lock holder is actually running, just bump priority. 13167352Sjhb */ 132103216Sjulian if (TD_IS_RUNNING(td)) { 13399072Sjulian td->td_priority = pri; 13467352Sjhb return; 13567352Sjhb } 13672376Sjake 13773912Sjhb#ifndef SMP 13867352Sjhb /* 13983366Sjulian * For UP, we check to see if td is curthread (this shouldn't 14073912Sjhb * ever happen however as it would mean we are in a deadlock.) 14173912Sjhb */ 14283366Sjulian KASSERT(td != curthread, ("Deadlock detected")); 14373912Sjhb#endif 14473912Sjhb 14573912Sjhb /* 14683366Sjulian * If on run queue move to new run queue, and quit. 14783366Sjulian * XXXKSE this gets a lot more complicated under threads 14883366Sjulian * but try anyhow. 14999072Sjulian * We should have a special call to do this more efficiently. 15067352Sjhb */ 151103216Sjulian if (TD_ON_RUNQ(td)) { 15283366Sjulian MPASS(td->td_blocked == NULL); 15383366Sjulian remrunqueue(td); 15499072Sjulian td->td_priority = pri; 15583366Sjulian setrunqueue(td); 15667352Sjhb return; 15767352Sjhb } 15899072Sjulian /* 15999072Sjulian * Adjust for any other cases. 16099072Sjulian */ 16199072Sjulian td->td_priority = pri; 16267352Sjhb 16367352Sjhb /* 16469376Sjhb * If we aren't blocked on a mutex, we should be. 16567352Sjhb */ 166103216Sjulian KASSERT(TD_ON_MUTEX(td), ( 16769376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 16899072Sjulian td->td_proc->p_pid, td->td_proc->p_comm, td->td_state, 16974912Sjhb m->mtx_object.lo_name)); 17067352Sjhb 17167352Sjhb /* 17283366Sjulian * Pick up the mutex that td is blocked on. 17367352Sjhb */ 17483366Sjulian m = td->td_blocked; 17567352Sjhb MPASS(m != NULL); 17667352Sjhb 17767352Sjhb /* 17883366Sjulian * Check if the thread needs to be moved up on 17967352Sjhb * the blocked chain 18067352Sjhb */ 18183366Sjulian if (td == TAILQ_FIRST(&m->mtx_blocked)) { 18269376Sjhb continue; 18369376Sjhb } 18472200Sbmilekic 18583366Sjulian td1 = TAILQ_PREV(td, threadqueue, td_blkq); 18690538Sjulian if (td1->td_priority <= pri) { 18767352Sjhb continue; 18867352Sjhb } 18967352Sjhb 19067352Sjhb /* 19183366Sjulian * Remove thread from blocked chain and determine where 19283366Sjulian * it should be moved up to. Since we know that td1 has 19383366Sjulian * a lower priority than td, we know that at least one 19483366Sjulian * thread in the chain has a lower priority and that 19583366Sjulian * td1 will thus not be NULL after the loop. 19667352Sjhb */ 19783366Sjulian TAILQ_REMOVE(&m->mtx_blocked, td, td_blkq); 19883366Sjulian TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) { 19983366Sjulian MPASS(td1->td_proc->p_magic == P_MAGIC); 20090538Sjulian if (td1->td_priority > pri) 20167352Sjhb break; 20267352Sjhb } 20372200Sbmilekic 20483366Sjulian MPASS(td1 != NULL); 20583366Sjulian TAILQ_INSERT_BEFORE(td1, td, td_blkq); 20667352Sjhb CTR4(KTR_LOCK, 20771560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 20883366Sjulian td, td1, m, m->mtx_object.lo_name); 20967352Sjhb } 21067352Sjhb} 21167352Sjhb 21293609Sdes#ifdef MUTEX_PROFILING 21393609SdesSYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging"); 21493609SdesSYSCTL_NODE(_debug_mutex, OID_AUTO, prof, CTLFLAG_RD, NULL, "mutex profiling"); 21593609Sdesstatic int mutex_prof_enable = 0; 21693609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, enable, CTLFLAG_RW, 21793609Sdes &mutex_prof_enable, 0, "Enable tracing of mutex holdtime"); 21893609Sdes 21993609Sdesstruct mutex_prof { 22093609Sdes const char *name; 22193609Sdes const char *file; 22293609Sdes int line; 22393609Sdes#define MPROF_MAX 0 22493609Sdes#define MPROF_TOT 1 22593609Sdes#define MPROF_CNT 2 22693609Sdes#define MPROF_AVG 3 22797156Sdes uintmax_t counter[4]; 22893705Sdes struct mutex_prof *next; 22993609Sdes}; 23093609Sdes 23171352Sjasone/* 23293609Sdes * mprof_buf is a static pool of profiling records to avoid possible 23393609Sdes * reentrance of the memory allocation functions. 23493609Sdes * 23593609Sdes * Note: NUM_MPROF_BUFFERS must be smaller than MPROF_HASH_SIZE. 23693609Sdes */ 23793705Sdes#define NUM_MPROF_BUFFERS 1000 23893609Sdesstatic struct mutex_prof mprof_buf[NUM_MPROF_BUFFERS]; 23993609Sdesstatic int first_free_mprof_buf; 24093705Sdes#define MPROF_HASH_SIZE 1009 24193609Sdesstatic struct mutex_prof *mprof_hash[MPROF_HASH_SIZE]; 24293609Sdes 24393609Sdesstatic int mutex_prof_acquisitions; 24493609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, acquisitions, CTLFLAG_RD, 24593609Sdes &mutex_prof_acquisitions, 0, "Number of mutex acquistions recorded"); 24693609Sdesstatic int mutex_prof_records; 24793609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, records, CTLFLAG_RD, 24893609Sdes &mutex_prof_records, 0, "Number of profiling records"); 24993609Sdesstatic int mutex_prof_maxrecords = NUM_MPROF_BUFFERS; 25093609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, maxrecords, CTLFLAG_RD, 25193609Sdes &mutex_prof_maxrecords, 0, "Maximum number of profiling records"); 25293609Sdesstatic int mutex_prof_rejected; 25393609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, rejected, CTLFLAG_RD, 25493609Sdes &mutex_prof_rejected, 0, "Number of rejected profiling records"); 25593609Sdesstatic int mutex_prof_hashsize = MPROF_HASH_SIZE; 25693609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, hashsize, CTLFLAG_RD, 25793609Sdes &mutex_prof_hashsize, 0, "Hash size"); 25893609Sdesstatic int mutex_prof_collisions = 0; 25993609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, collisions, CTLFLAG_RD, 26093609Sdes &mutex_prof_collisions, 0, "Number of hash collisions"); 26193609Sdes 26293609Sdes/* 26393609Sdes * mprof_mtx protects the profiling buffers and the hash. 26493609Sdes */ 26593609Sdesstatic struct mtx mprof_mtx; 26693705SdesMTX_SYSINIT(mprof, &mprof_mtx, "mutex profiling lock", MTX_SPIN | MTX_QUIET); 26793609Sdes 26893667Sdesstatic u_int64_t 26993667Sdesnanoseconds(void) 27093667Sdes{ 27193667Sdes struct timespec tv; 27293667Sdes 27393667Sdes nanotime(&tv); 27493667Sdes return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec); 27593667Sdes} 27693667Sdes 27793609Sdesstatic int 27893609Sdesdump_mutex_prof_stats(SYSCTL_HANDLER_ARGS) 27993609Sdes{ 28093609Sdes struct sbuf *sb; 28193609Sdes int error, i; 28293609Sdes 28393609Sdes if (first_free_mprof_buf == 0) 28493609Sdes return SYSCTL_OUT(req, "No locking recorded", 28593609Sdes sizeof("No locking recorded")); 28693609Sdes 28793609Sdes sb = sbuf_new(NULL, NULL, 1024, SBUF_AUTOEXTEND); 28893609Sdes sbuf_printf(sb, "%12s %12s %12s %12s %s\n", 28993609Sdes "max", "total", "count", "average", "name"); 29093609Sdes mtx_lock_spin(&mprof_mtx); 29193609Sdes for (i = 0; i < first_free_mprof_buf; ++i) 29297156Sdes sbuf_printf(sb, "%12ju %12ju %12ju %12ju %s:%d (%s)\n", 29393667Sdes mprof_buf[i].counter[MPROF_MAX] / 1000, 29493667Sdes mprof_buf[i].counter[MPROF_TOT] / 1000, 29593667Sdes mprof_buf[i].counter[MPROF_CNT], 29693667Sdes mprof_buf[i].counter[MPROF_AVG] / 1000, 29793609Sdes mprof_buf[i].file, mprof_buf[i].line, mprof_buf[i].name); 29893609Sdes mtx_unlock_spin(&mprof_mtx); 29993609Sdes sbuf_finish(sb); 30093609Sdes error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 30193609Sdes sbuf_delete(sb); 30293609Sdes return (error); 30393609Sdes} 30493609SdesSYSCTL_PROC(_debug_mutex_prof, OID_AUTO, stats, CTLTYPE_STRING|CTLFLAG_RD, 30593609Sdes NULL, 0, dump_mutex_prof_stats, "A", "Mutex profiling statistics"); 30693609Sdes#endif 30793609Sdes 30893609Sdes/* 30974900Sjhb * Function versions of the inlined __mtx_* macros. These are used by 31074900Sjhb * modules and can also be called from assembly language if needed. 31174900Sjhb */ 31274900Sjhbvoid 31374900Sjhb_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 31474900Sjhb{ 31574900Sjhb 31683841Sjhb MPASS(curthread != NULL); 317102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, 318102907Sjhb ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 319102907Sjhb file, line)); 32083841Sjhb _get_sleep_lock(m, curthread, opts, file, line); 32183841Sjhb LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 32283841Sjhb line); 32383841Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 32493609Sdes#ifdef MUTEX_PROFILING 32593609Sdes /* don't reset the timer when/if recursing */ 32699324Sdes if (m->mtx_acqtime == 0) { 32799324Sdes m->mtx_filename = file; 32899324Sdes m->mtx_lineno = line; 32999324Sdes m->mtx_acqtime = mutex_prof_enable ? nanoseconds() : 0; 33093609Sdes ++mutex_prof_acquisitions; 33193609Sdes } 33293609Sdes#endif 33374900Sjhb} 33474900Sjhb 33574900Sjhbvoid 33674900Sjhb_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 33774900Sjhb{ 33874900Sjhb 33983841Sjhb MPASS(curthread != NULL); 340102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, 341102907Sjhb ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 342102907Sjhb file, line)); 343102907Sjhb WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 344102907Sjhb LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 345102907Sjhb line); 34683947Sjhb mtx_assert(m, MA_OWNED); 34793609Sdes#ifdef MUTEX_PROFILING 34899324Sdes if (m->mtx_acqtime != 0) { 34993609Sdes static const char *unknown = "(unknown)"; 35093609Sdes struct mutex_prof *mpp; 35193667Sdes u_int64_t acqtime, now; 35293609Sdes const char *p, *q; 35393705Sdes volatile u_int hash; 35493609Sdes 35593667Sdes now = nanoseconds(); 35699324Sdes acqtime = m->mtx_acqtime; 35799324Sdes m->mtx_acqtime = 0; 35893667Sdes if (now <= acqtime) 35993609Sdes goto out; 36099324Sdes for (p = m->mtx_filename; strncmp(p, "../", 3) == 0; p += 3) 36193609Sdes /* nothing */ ; 36293609Sdes if (p == NULL || *p == '\0') 36393609Sdes p = unknown; 36499324Sdes for (hash = m->mtx_lineno, q = p; *q != '\0'; ++q) 36593609Sdes hash = (hash * 2 + *q) % MPROF_HASH_SIZE; 36693609Sdes mtx_lock_spin(&mprof_mtx); 36793705Sdes for (mpp = mprof_hash[hash]; mpp != NULL; mpp = mpp->next) 36899324Sdes if (mpp->line == m->mtx_lineno && 36999324Sdes strcmp(mpp->file, p) == 0) 37093609Sdes break; 37193609Sdes if (mpp == NULL) { 37293609Sdes /* Just exit if we cannot get a trace buffer */ 37393609Sdes if (first_free_mprof_buf >= NUM_MPROF_BUFFERS) { 37493609Sdes ++mutex_prof_rejected; 37593609Sdes goto unlock; 37693609Sdes } 37793609Sdes mpp = &mprof_buf[first_free_mprof_buf++]; 37893609Sdes mpp->name = mtx_name(m); 37993609Sdes mpp->file = p; 38099324Sdes mpp->line = m->mtx_lineno; 38193705Sdes mpp->next = mprof_hash[hash]; 38293705Sdes if (mprof_hash[hash] != NULL) 38393705Sdes ++mutex_prof_collisions; 38493705Sdes mprof_hash[hash] = mpp; 38593609Sdes ++mutex_prof_records; 38693609Sdes } 38793609Sdes /* 38893609Sdes * Record if the mutex has been held longer now than ever 38993609Sdes * before 39093609Sdes */ 39193667Sdes if ((now - acqtime) > mpp->counter[MPROF_MAX]) 39293667Sdes mpp->counter[MPROF_MAX] = now - acqtime; 39393667Sdes mpp->counter[MPROF_TOT] += now - acqtime; 39493667Sdes mpp->counter[MPROF_CNT] += 1; 39593667Sdes mpp->counter[MPROF_AVG] = 39693667Sdes mpp->counter[MPROF_TOT] / mpp->counter[MPROF_CNT]; 39793609Sdesunlock: 39893609Sdes mtx_unlock_spin(&mprof_mtx); 39993609Sdes } 40093609Sdesout: 40193609Sdes#endif 40283841Sjhb _rel_sleep_lock(m, curthread, opts, file, line); 40374900Sjhb} 40474900Sjhb 40574900Sjhbvoid 40674900Sjhb_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 40774900Sjhb{ 40874900Sjhb 40983841Sjhb MPASS(curthread != NULL); 410102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, 411102907Sjhb ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 412102907Sjhb m->mtx_object.lo_name, file, line)); 413100754Sjhb#if defined(SMP) || LOCK_DEBUG > 0 || 1 41483841Sjhb _get_spin_lock(m, curthread, opts, file, line); 41597079Sjhb#else 41697079Sjhb critical_enter(); 41797079Sjhb#endif 41883841Sjhb LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 41983841Sjhb line); 42083841Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 42174900Sjhb} 42274900Sjhb 42374900Sjhbvoid 42474900Sjhb_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 42574900Sjhb{ 42674900Sjhb 42783841Sjhb MPASS(curthread != NULL); 428102907Sjhb KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, 429102907Sjhb ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 430102907Sjhb m->mtx_object.lo_name, file, line)); 43183841Sjhb WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 43283841Sjhb LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 43383841Sjhb line); 434102907Sjhb mtx_assert(m, MA_OWNED); 435100754Sjhb#if defined(SMP) || LOCK_DEBUG > 0 || 1 43683841Sjhb _rel_spin_lock(m); 43797079Sjhb#else 43897079Sjhb critical_exit(); 43997079Sjhb#endif 44074900Sjhb} 44174900Sjhb 44274900Sjhb/* 44372200Sbmilekic * The important part of mtx_trylock{,_flags}() 44472200Sbmilekic * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 44572200Sbmilekic * if we're called, it's because we know we don't already own this lock. 44671352Sjasone */ 44772200Sbmilekicint 44872200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 44971352Sjasone{ 45072200Sbmilekic int rval; 45171352Sjasone 45283366Sjulian MPASS(curthread != NULL); 45371352Sjasone 45483366Sjulian rval = _obtain_lock(m, curthread); 45572200Sbmilekic 45674912Sjhb LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 45774912Sjhb if (rval) { 45871352Sjasone /* 45972200Sbmilekic * We do not handle recursion in _mtx_trylock; see the 46072200Sbmilekic * note at the top of the routine. 46171352Sjasone */ 46272344Sbmilekic KASSERT(!mtx_recursed(m), 46372344Sbmilekic ("mtx_trylock() called on a recursed mutex")); 46476272Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 46576272Sjhb file, line); 46671352Sjasone } 46771352Sjasone 46874912Sjhb return (rval); 46971352Sjasone} 47071352Sjasone 47171352Sjasone/* 47272200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 47371352Sjasone * 47472200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 47572200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 47671352Sjasone */ 47772200Sbmilekicvoid 47872200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 47971352Sjasone{ 48083366Sjulian struct thread *td = curthread; 48197081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 48297081Sjhb struct thread *owner; 48397081Sjhb#endif 484102450Siedowse#ifdef KTR 485102450Siedowse int cont_logged = 0; 486102450Siedowse#endif 48771352Sjasone 48883366Sjulian if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)td) { 48972200Sbmilekic m->mtx_recurse++; 49072200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 49174912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 49272344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 49372200Sbmilekic return; 49471352Sjasone } 49571352Sjasone 49674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 49772994Sjhb CTR4(KTR_LOCK, 49872994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 49974912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 50071352Sjasone 50183366Sjulian while (!_obtain_lock(m, td)) { 50272200Sbmilekic uintptr_t v; 50383366Sjulian struct thread *td1; 50471352Sjasone 50572200Sbmilekic mtx_lock_spin(&sched_lock); 50672200Sbmilekic /* 50772200Sbmilekic * Check if the lock has been released while spinning for 50872200Sbmilekic * the sched_lock. 50972200Sbmilekic */ 51072200Sbmilekic if ((v = m->mtx_lock) == MTX_UNOWNED) { 51172200Sbmilekic mtx_unlock_spin(&sched_lock); 51297086Sjhb#ifdef __i386__ 51397139Sjhb ia32_pause(); 51497086Sjhb#endif 51572200Sbmilekic continue; 51671352Sjasone } 51771352Sjasone 51872200Sbmilekic /* 51972200Sbmilekic * The mutex was marked contested on release. This means that 52083366Sjulian * there are threads blocked on it. 52172200Sbmilekic */ 52272200Sbmilekic if (v == MTX_CONTESTED) { 52383366Sjulian td1 = TAILQ_FIRST(&m->mtx_blocked); 52483366Sjulian MPASS(td1 != NULL); 52583366Sjulian m->mtx_lock = (uintptr_t)td | MTX_CONTESTED; 52667352Sjhb 52790538Sjulian if (td1->td_priority < td->td_priority) 52890538Sjulian td->td_priority = td1->td_priority; 52972200Sbmilekic mtx_unlock_spin(&sched_lock); 53067352Sjhb return; 53167352Sjhb } 53269376Sjhb 53369376Sjhb /* 53472200Sbmilekic * If the mutex isn't already contested and a failure occurs 53572200Sbmilekic * setting the contested bit, the mutex was either released 53672200Sbmilekic * or the state of the MTX_RECURSED bit changed. 53769376Sjhb */ 53872200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 53972200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 54072200Sbmilekic (void *)(v | MTX_CONTESTED))) { 54172200Sbmilekic mtx_unlock_spin(&sched_lock); 54297086Sjhb#ifdef __i386__ 54397139Sjhb ia32_pause(); 54497086Sjhb#endif 54572200Sbmilekic continue; 54672200Sbmilekic } 54767352Sjhb 54897081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 54972200Sbmilekic /* 55097081Sjhb * If the current owner of the lock is executing on another 55197081Sjhb * CPU, spin instead of blocking. 55297081Sjhb */ 55397081Sjhb owner = (struct thread *)(v & MTX_FLAGMASK); 55497839Sjhb if (m != &Giant && thread_running(owner)) { 55597081Sjhb mtx_unlock_spin(&sched_lock); 55697839Sjhb while (mtx_owner(m) == owner && thread_running(owner)) { 55797086Sjhb#ifdef __i386__ 55897837Sjhb ia32_pause(); 55997086Sjhb#endif 56097837Sjhb } 56197081Sjhb continue; 56297081Sjhb } 56397081Sjhb#endif /* SMP && ADAPTIVE_MUTEXES */ 56497081Sjhb 56597081Sjhb /* 56693692Sjhb * We definitely must sleep for this lock. 56772200Sbmilekic */ 56872200Sbmilekic mtx_assert(m, MA_NOTOWNED); 56967352Sjhb 57067352Sjhb#ifdef notyet 57172200Sbmilekic /* 57272200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 57372200Sbmilekic * must clean up before going to sleep. 57472200Sbmilekic */ 57583366Sjulian if (td->td_ithd != NULL) { 57683366Sjulian struct ithd *it = td->td_ithd; 57767352Sjhb 57872200Sbmilekic if (it->it_interrupted) { 57974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 58072200Sbmilekic CTR2(KTR_LOCK, 58172994Sjhb "_mtx_lock_sleep: %p interrupted %p", 58272200Sbmilekic it, it->it_interrupted); 58372200Sbmilekic intr_thd_fixup(it); 58467352Sjhb } 58572200Sbmilekic } 58667352Sjhb#endif 58767352Sjhb 58872200Sbmilekic /* 58972200Sbmilekic * Put us on the list of threads blocked on this mutex. 59072200Sbmilekic */ 59172200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 59290418Sjhb td1 = mtx_owner(m); 59383366Sjulian LIST_INSERT_HEAD(&td1->td_contested, m, mtx_contested); 59483366Sjulian TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); 59572200Sbmilekic } else { 59683366Sjulian TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) 59790538Sjulian if (td1->td_priority > td->td_priority) 59872200Sbmilekic break; 59983366Sjulian if (td1) 60083366Sjulian TAILQ_INSERT_BEFORE(td1, td, td_blkq); 60172200Sbmilekic else 60283366Sjulian TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); 60372200Sbmilekic } 604102450Siedowse#ifdef KTR 605102450Siedowse if (!cont_logged) { 606102450Siedowse CTR6(KTR_CONTENTION, 607102450Siedowse "contention: %p at %s:%d wants %s, taken by %s:%d", 608102450Siedowse td, file, line, m->mtx_object.lo_name, 609102450Siedowse WITNESS_FILE(&m->mtx_object), 610102450Siedowse WITNESS_LINE(&m->mtx_object)); 611102450Siedowse cont_logged = 1; 612102450Siedowse } 613102450Siedowse#endif 61467352Sjhb 61572200Sbmilekic /* 61672200Sbmilekic * Save who we're blocked on. 61772200Sbmilekic */ 61883366Sjulian td->td_blocked = m; 61983366Sjulian td->td_mtxname = m->mtx_object.lo_name; 620103216Sjulian TD_SET_MUTEX(td); 62183366Sjulian propagate_priority(td); 62267352Sjhb 62374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 62472200Sbmilekic CTR3(KTR_LOCK, 62583366Sjulian "_mtx_lock_sleep: p %p blocked on [%p] %s", td, m, 62674912Sjhb m->mtx_object.lo_name); 62772200Sbmilekic 62883366Sjulian td->td_proc->p_stats->p_ru.ru_nvcsw++; 62972200Sbmilekic mi_switch(); 63072200Sbmilekic 63174912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 63272200Sbmilekic CTR3(KTR_LOCK, 63372200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 63483366Sjulian td, m, m->mtx_object.lo_name); 63572200Sbmilekic 63672200Sbmilekic mtx_unlock_spin(&sched_lock); 63772200Sbmilekic } 63872200Sbmilekic 639102450Siedowse#ifdef KTR 640102450Siedowse if (cont_logged) { 641102450Siedowse CTR4(KTR_CONTENTION, 642102450Siedowse "contention end: %s acquired by %p at %s:%d", 643102450Siedowse m->mtx_object.lo_name, td, file, line); 644102450Siedowse } 645102450Siedowse#endif 64672200Sbmilekic return; 64772200Sbmilekic} 64872200Sbmilekic 64972200Sbmilekic/* 65072200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 65172200Sbmilekic * 65272200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 65372200Sbmilekic * is handled inline. 65472200Sbmilekic */ 65572200Sbmilekicvoid 65688088Sjhb_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) 65772200Sbmilekic{ 65872200Sbmilekic int i = 0; 65972200Sbmilekic 66074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 66172344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 66272200Sbmilekic 66372200Sbmilekic for (;;) { 66483366Sjulian if (_obtain_lock(m, curthread)) 66572200Sbmilekic break; 66672200Sbmilekic 66775568Sjhb /* Give interrupts a chance while we spin. */ 66888088Sjhb critical_exit(); 66972200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 67097086Sjhb if (i++ < 10000000) { 67197086Sjhb#ifdef __i386__ 67297139Sjhb ia32_pause(); 67397086Sjhb#endif 67472200Sbmilekic continue; 67597086Sjhb } 67697084Sjhb if (i < 60000000) 67772200Sbmilekic DELAY(1); 67867352Sjhb#ifdef DDB 67972200Sbmilekic else if (!db_active) 68067352Sjhb#else 68172200Sbmilekic else 68267352Sjhb#endif 68397082Sjhb panic("spin lock %s held by %p for > 5 seconds", 68497082Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock); 68597086Sjhb#ifdef __i386__ 68697139Sjhb ia32_pause(); 68797086Sjhb#endif 68867352Sjhb } 68988088Sjhb critical_enter(); 69067352Sjhb } 69172200Sbmilekic 69274912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 69372200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 69472200Sbmilekic 69572200Sbmilekic return; 69667352Sjhb} 69767352Sjhb 69872200Sbmilekic/* 69972200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 70072200Sbmilekic * 70172200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 70272200Sbmilekic * need to wake up a blocked thread). 70372200Sbmilekic */ 70467352Sjhbvoid 70572200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 70667352Sjhb{ 70783366Sjulian struct thread *td, *td1; 70867352Sjhb struct mtx *m1; 70967352Sjhb int pri; 71067352Sjhb 71183366Sjulian td = curthread; 71272200Sbmilekic 71372200Sbmilekic if (mtx_recursed(m)) { 71472200Sbmilekic if (--(m->mtx_recurse) == 0) 71572200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 71674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 71772200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 71872200Sbmilekic return; 71972200Sbmilekic } 72072200Sbmilekic 72172200Sbmilekic mtx_lock_spin(&sched_lock); 72274912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 72372200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 72472200Sbmilekic 72583366Sjulian td1 = TAILQ_FIRST(&m->mtx_blocked); 72697081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 72797081Sjhb if (td1 == NULL) { 72897081Sjhb _release_lock_quick(m); 72997081Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 73097081Sjhb CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 73197081Sjhb mtx_unlock_spin(&sched_lock); 73297081Sjhb return; 73397081Sjhb } 73497081Sjhb#endif 73583366Sjulian MPASS(td->td_proc->p_magic == P_MAGIC); 73683366Sjulian MPASS(td1->td_proc->p_magic == P_MAGIC); 73772200Sbmilekic 73883366Sjulian TAILQ_REMOVE(&m->mtx_blocked, td1, td_blkq); 73972200Sbmilekic 74072200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 74172200Sbmilekic LIST_REMOVE(m, mtx_contested); 74272200Sbmilekic _release_lock_quick(m); 74374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 74472200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 74572200Sbmilekic } else 74672200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 74772200Sbmilekic 74872376Sjake pri = PRI_MAX; 74983366Sjulian LIST_FOREACH(m1, &td->td_contested, mtx_contested) { 75090538Sjulian int cp = TAILQ_FIRST(&m1->mtx_blocked)->td_priority; 75172200Sbmilekic if (cp < pri) 75272200Sbmilekic pri = cp; 75372200Sbmilekic } 75472200Sbmilekic 75590538Sjulian if (pri > td->td_base_pri) 75690538Sjulian pri = td->td_base_pri; 75790538Sjulian td->td_priority = pri; 75872200Sbmilekic 75974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 76072200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 76183366Sjulian m, td1); 76272200Sbmilekic 76383366Sjulian td1->td_blocked = NULL; 764103216Sjulian TD_CLR_MUTEX(td1); 765104160Sjulian if (!TD_CAN_RUN(td1)) { 766104160Sjulian mtx_unlock_spin(&sched_lock); 767104160Sjulian return; 768104160Sjulian } 76972200Sbmilekic 77090538Sjulian if (td->td_critnest == 1 && td1->td_priority < pri) { 77167352Sjhb#ifdef notyet 77283366Sjulian if (td->td_ithd != NULL) { 77383366Sjulian struct ithd *it = td->td_ithd; 77467352Sjhb 77572200Sbmilekic if (it->it_interrupted) { 77674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 77772200Sbmilekic CTR2(KTR_LOCK, 77872994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 77972200Sbmilekic it, it->it_interrupted); 78072200Sbmilekic intr_thd_fixup(it); 78167352Sjhb } 78272200Sbmilekic } 78367352Sjhb#endif 78474912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 78572200Sbmilekic CTR2(KTR_LOCK, 78672200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 78772200Sbmilekic (void *)m->mtx_lock); 78872200Sbmilekic 78983366Sjulian td->td_proc->p_stats->p_ru.ru_nivcsw++; 79072200Sbmilekic mi_switch(); 79174912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 79272200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 79372200Sbmilekic m, (void *)m->mtx_lock); 79467352Sjhb } 79572200Sbmilekic 79672200Sbmilekic mtx_unlock_spin(&sched_lock); 79772200Sbmilekic 79872200Sbmilekic return; 79967352Sjhb} 80067352Sjhb 80172200Sbmilekic/* 80272200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 80372200Sbmilekic * See the _rel_spin_lock() macro for the details. 80472200Sbmilekic */ 80572200Sbmilekic 80672200Sbmilekic/* 80772994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 80872200Sbmilekic */ 80972996Sjhb#ifdef INVARIANT_SUPPORT 81071352Sjasonevoid 81171360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 81271352Sjasone{ 81380748Sjhb 81480748Sjhb if (panicstr != NULL) 81580748Sjhb return; 81673033Sjake switch (what) { 81771352Sjasone case MA_OWNED: 81871352Sjasone case MA_OWNED | MA_RECURSED: 81971352Sjasone case MA_OWNED | MA_NOTRECURSED: 82073033Sjake if (!mtx_owned(m)) 82171352Sjasone panic("mutex %s not owned at %s:%d", 82274912Sjhb m->mtx_object.lo_name, file, line); 82373033Sjake if (mtx_recursed(m)) { 82473033Sjake if ((what & MA_NOTRECURSED) != 0) 82571352Sjasone panic("mutex %s recursed at %s:%d", 82674912Sjhb m->mtx_object.lo_name, file, line); 82773033Sjake } else if ((what & MA_RECURSED) != 0) { 82871352Sjasone panic("mutex %s unrecursed at %s:%d", 82974912Sjhb m->mtx_object.lo_name, file, line); 83071352Sjasone } 83171352Sjasone break; 83271352Sjasone case MA_NOTOWNED: 83373033Sjake if (mtx_owned(m)) 83471352Sjasone panic("mutex %s owned at %s:%d", 83574912Sjhb m->mtx_object.lo_name, file, line); 83671352Sjasone break; 83771352Sjasone default: 83871360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 83971352Sjasone } 84071352Sjasone} 84171352Sjasone#endif 84271352Sjasone 84372200Sbmilekic/* 84472200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 84574912Sjhb * 84674912Sjhb * Most of these checks have been moved off into the LO_INITIALIZED flag 84774912Sjhb * maintained by the witness code. 84872200Sbmilekic */ 84967352Sjhb#ifdef MUTEX_DEBUG 85067352Sjhb 85192723Salfredvoid mtx_validate(struct mtx *); 85267352Sjhb 85374912Sjhbvoid 85474912Sjhbmtx_validate(struct mtx *m) 85567352Sjhb{ 85667352Sjhb 85767352Sjhb/* 85867352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 85967352Sjhb * we can re-enable the kernacc() checks. 86067352Sjhb */ 86167352Sjhb#ifndef __alpha__ 86282304Sbmilekic /* 86382304Sbmilekic * Can't call kernacc() from early init386(), especially when 86482304Sbmilekic * initializing Giant mutex, because some stuff in kernacc() 86582304Sbmilekic * requires Giant itself. 86682304Sbmilekic */ 86782302Sbmilekic if (!cold) 86882302Sbmilekic if (!kernacc((caddr_t)m, sizeof(m), 86982302Sbmilekic VM_PROT_READ | VM_PROT_WRITE)) 87082302Sbmilekic panic("Can't read and write to mutex %p", m); 87167352Sjhb#endif 87267352Sjhb} 87367352Sjhb#endif 87467352Sjhb 87572200Sbmilekic/* 87693672Sarr * General init routine used by the MTX_SYSINIT() macro. 87793672Sarr */ 87893672Sarrvoid 87993672Sarrmtx_sysinit(void *arg) 88093672Sarr{ 88193672Sarr struct mtx_args *margs = arg; 88293672Sarr 88393813Sjhb mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 88493672Sarr} 88593672Sarr 88693672Sarr/* 88772200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 88893813Sjhb * `opts' with options contained in `opts' and name `name.' The optional 88993813Sjhb * lock type `type' is used as a general lock category name for use with 89093813Sjhb * witness. 89172200Sbmilekic */ 89267352Sjhbvoid 89393813Sjhbmtx_init(struct mtx *m, const char *name, const char *type, int opts) 89467352Sjhb{ 89574912Sjhb struct lock_object *lock; 89672200Sbmilekic 89774912Sjhb MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 89893273Sjeff MTX_SLEEPABLE | MTX_NOWITNESS | MTX_DUPOK)) == 0); 89972200Sbmilekic 90067352Sjhb#ifdef MUTEX_DEBUG 90172200Sbmilekic /* Diagnostic and error correction */ 90274912Sjhb mtx_validate(m); 90369429Sjhb#endif 90467352Sjhb 90585205Sjhb lock = &m->mtx_object; 90685205Sjhb KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, 90793813Sjhb ("mutex %s %p already initialized", name, m)); 90874912Sjhb bzero(m, sizeof(*m)); 90974912Sjhb if (opts & MTX_SPIN) 91074912Sjhb lock->lo_class = &lock_class_mtx_spin; 91174912Sjhb else 91274912Sjhb lock->lo_class = &lock_class_mtx_sleep; 91393813Sjhb lock->lo_name = name; 91493813Sjhb lock->lo_type = type != NULL ? type : name; 91574912Sjhb if (opts & MTX_QUIET) 91674912Sjhb lock->lo_flags = LO_QUIET; 91774912Sjhb if (opts & MTX_RECURSE) 91874912Sjhb lock->lo_flags |= LO_RECURSABLE; 91974912Sjhb if (opts & MTX_SLEEPABLE) 92074912Sjhb lock->lo_flags |= LO_SLEEPABLE; 92174912Sjhb if ((opts & MTX_NOWITNESS) == 0) 92274912Sjhb lock->lo_flags |= LO_WITNESS; 92393273Sjeff if (opts & MTX_DUPOK) 92493273Sjeff lock->lo_flags |= LO_DUPOK; 92572200Sbmilekic 92667352Sjhb m->mtx_lock = MTX_UNOWNED; 92774912Sjhb TAILQ_INIT(&m->mtx_blocked); 92872200Sbmilekic 92974912Sjhb LOCK_LOG_INIT(lock, opts); 93072200Sbmilekic 93174912Sjhb WITNESS_INIT(lock); 93267352Sjhb} 93367352Sjhb 93472200Sbmilekic/* 93574912Sjhb * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 93674912Sjhb * passed in as a flag here because if the corresponding mtx_init() was 93774912Sjhb * called with MTX_QUIET set, then it will already be set in the mutex's 93874912Sjhb * flags. 93972200Sbmilekic */ 94067352Sjhbvoid 94167352Sjhbmtx_destroy(struct mtx *m) 94267352Sjhb{ 94367352Sjhb 94474912Sjhb LOCK_LOG_DESTROY(&m->mtx_object, 0); 94572200Sbmilekic 94674912Sjhb if (!mtx_owned(m)) 94774912Sjhb MPASS(mtx_unowned(m)); 94874912Sjhb else { 94971228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 95072200Sbmilekic 95174912Sjhb /* Tell witness this isn't locked to make it happy. */ 95288900Sjhb WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, 95388900Sjhb __LINE__); 95471320Sjasone } 95571320Sjasone 95674912Sjhb WITNESS_DESTROY(&m->mtx_object); 95771320Sjasone} 95885564Sdillon 95985564Sdillon/* 96093702Sjhb * Intialize the mutex code and system mutexes. This is called from the MD 96193702Sjhb * startup code prior to mi_startup(). The per-CPU data space needs to be 96293702Sjhb * setup before this is called. 96393702Sjhb */ 96493702Sjhbvoid 96593702Sjhbmutex_init(void) 96693702Sjhb{ 96793702Sjhb 96893702Sjhb /* Setup thread0 so that mutexes work. */ 96993702Sjhb LIST_INIT(&thread0.td_contested); 97093702Sjhb 97193702Sjhb /* 97293702Sjhb * Initialize mutexes. 97393702Sjhb */ 97493813Sjhb mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 97593813Sjhb mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 97693813Sjhb mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 97793702Sjhb mtx_lock(&Giant); 97893702Sjhb} 97993702Sjhb 98093702Sjhb/* 98185564Sdillon * Encapsulated Giant mutex routines. These routines provide encapsulation 98285564Sdillon * control for the Giant mutex, allowing sysctls to be used to turn on and 98385564Sdillon * off Giant around certain subsystems. The default value for the sysctls 98485564Sdillon * are set to what developers believe is stable and working in regards to 98585564Sdillon * the Giant pushdown. Developers should not turn off Giant via these 98685564Sdillon * sysctls unless they know what they are doing. 98785564Sdillon * 98885564Sdillon * Callers of mtx_lock_giant() are expected to pass the return value to an 98985564Sdillon * accompanying mtx_unlock_giant() later on. If multiple subsystems are 99085564Sdillon * effected by a Giant wrap, all related sysctl variables must be zero for 99185564Sdillon * the subsystem call to operate without Giant (as determined by the caller). 99285564Sdillon */ 99385564Sdillon 99485564SdillonSYSCTL_NODE(_kern, OID_AUTO, giant, CTLFLAG_RD, NULL, "Giant mutex manipulation"); 99585564Sdillon 99685564Sdillonstatic int kern_giant_all = 0; 99785564SdillonSYSCTL_INT(_kern_giant, OID_AUTO, all, CTLFLAG_RW, &kern_giant_all, 0, ""); 99885564Sdillon 99985564Sdillonint kern_giant_proc = 1; /* Giant around PROC locks */ 100085564Sdillonint kern_giant_file = 1; /* Giant around struct file & filedesc */ 100190864Sdillonint kern_giant_ucred = 1; /* Giant around ucred */ 100285564SdillonSYSCTL_INT(_kern_giant, OID_AUTO, proc, CTLFLAG_RW, &kern_giant_proc, 0, ""); 100385564SdillonSYSCTL_INT(_kern_giant, OID_AUTO, file, CTLFLAG_RW, &kern_giant_file, 0, ""); 100490864SdillonSYSCTL_INT(_kern_giant, OID_AUTO, ucred, CTLFLAG_RW, &kern_giant_ucred, 0, ""); 100585564Sdillon 100685564Sdillonint 100785564Sdillonmtx_lock_giant(int sysctlvar) 100885564Sdillon{ 100985564Sdillon if (sysctlvar || kern_giant_all) { 101085564Sdillon mtx_lock(&Giant); 101185564Sdillon return(1); 101285564Sdillon } 101385564Sdillon return(0); 101485564Sdillon} 101585564Sdillon 101685564Sdillonvoid 101785564Sdillonmtx_unlock_giant(int s) 101885564Sdillon{ 101985564Sdillon if (s) 102085564Sdillon mtx_unlock(&Giant); 102185564Sdillon} 102285564Sdillon 1023