subr_turnstile.c revision 100754
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone * $FreeBSD: head/sys/kern/subr_turnstile.c 100754 2002-07-27 16:54:23Z jhb $ 3165557Sjasone */ 3265557Sjasone 3365557Sjasone/* 3486411Sjhb * Machine independent bits of mutex implementation. 3572200Sbmilekic */ 3672200Sbmilekic 3797081Sjhb#include "opt_adaptive_mutexes.h" 3868790Sjhb#include "opt_ddb.h" 3967676Sjhb 4065557Sjasone#include <sys/param.h> 4193609Sdes#include <sys/systm.h> 4267352Sjhb#include <sys/bus.h> 4367352Sjhb#include <sys/kernel.h> 4493609Sdes#include <sys/ktr.h> 4576166Smarkm#include <sys/lock.h> 4667352Sjhb#include <sys/malloc.h> 4774912Sjhb#include <sys/mutex.h> 4865557Sjasone#include <sys/proc.h> 4978766Sjhb#include <sys/resourcevar.h> 5093609Sdes#include <sys/sbuf.h> 5197156Sdes#include <sys/stdint.h> 5267676Sjhb#include <sys/sysctl.h> 5367352Sjhb#include <sys/vmmeter.h> 5465557Sjasone 5567352Sjhb#include <machine/atomic.h> 5667352Sjhb#include <machine/bus.h> 5767352Sjhb#include <machine/clock.h> 5865557Sjasone#include <machine/cpu.h> 5967352Sjhb 6068790Sjhb#include <ddb/ddb.h> 6168790Sjhb 6267352Sjhb#include <vm/vm.h> 6367352Sjhb#include <vm/vm_extern.h> 6467352Sjhb 6565557Sjasone/* 6672200Sbmilekic * Internal utility macros. 6771352Sjasone */ 6872200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 6971352Sjasone 7072200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 7183366Sjulian : (struct thread *)((m)->mtx_lock & MTX_FLAGMASK)) 7271352Sjasone 7397839Sjhb/* XXXKSE This test will change. */ 7497839Sjhb#define thread_running(td) \ 7597836Sjhb ((td)->td_kse != NULL && (td)->td_kse->ke_oncpu != NOCPU) 7697836Sjhb 7771352Sjasone/* 7874912Sjhb * Lock classes for sleep and spin mutexes. 7971352Sjasone */ 8074912Sjhbstruct lock_class lock_class_mtx_sleep = { 8174912Sjhb "sleep mutex", 8274912Sjhb LC_SLEEPLOCK | LC_RECURSABLE 8374912Sjhb}; 8474912Sjhbstruct lock_class lock_class_mtx_spin = { 8574912Sjhb "spin mutex", 8674912Sjhb LC_SPINLOCK | LC_RECURSABLE 8774912Sjhb}; 8871352Sjasone 8971352Sjasone/* 9093702Sjhb * System-wide mutexes 9193702Sjhb */ 9293702Sjhbstruct mtx sched_lock; 9393702Sjhbstruct mtx Giant; 9493702Sjhb 9593702Sjhb/* 9672200Sbmilekic * Prototypes for non-exported routines. 9772200Sbmilekic */ 9883366Sjulianstatic void propagate_priority(struct thread *); 9967352Sjhb 10067352Sjhbstatic void 10183366Sjulianpropagate_priority(struct thread *td) 10267352Sjhb{ 10390538Sjulian int pri = td->td_priority; 10483366Sjulian struct mtx *m = td->td_blocked; 10567352Sjhb 10669376Sjhb mtx_assert(&sched_lock, MA_OWNED); 10767352Sjhb for (;;) { 10883366Sjulian struct thread *td1; 10967352Sjhb 11083366Sjulian td = mtx_owner(m); 11167352Sjhb 11283366Sjulian if (td == NULL) { 11367352Sjhb /* 11467352Sjhb * This really isn't quite right. Really 11583366Sjulian * ought to bump priority of thread that 11667352Sjhb * next acquires the mutex. 11767352Sjhb */ 11867352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 11967352Sjhb return; 12067352Sjhb } 12172200Sbmilekic 12299072Sjulian KASSERT(td->td_state != TDS_SURPLUS, ("Mutex owner SURPLUS")); 12399072Sjulian MPASS(td->td_proc != NULL); 12483366Sjulian MPASS(td->td_proc->p_magic == P_MAGIC); 12599072Sjulian KASSERT(td->td_state != TDS_SLP, 12699072Sjulian ("sleeping thread owns a mutex")); 12790538Sjulian if (td->td_priority <= pri) /* lower is higher priority */ 12867352Sjhb return; 12969376Sjhb 13069376Sjhb 13169376Sjhb /* 13267352Sjhb * If lock holder is actually running, just bump priority. 13367352Sjhb */ 13499072Sjulian if (td->td_state == TDS_RUNNING) { 13599072Sjulian td->td_priority = pri; 13667352Sjhb return; 13767352Sjhb } 13872376Sjake 13973912Sjhb#ifndef SMP 14067352Sjhb /* 14183366Sjulian * For UP, we check to see if td is curthread (this shouldn't 14273912Sjhb * ever happen however as it would mean we are in a deadlock.) 14373912Sjhb */ 14483366Sjulian KASSERT(td != curthread, ("Deadlock detected")); 14573912Sjhb#endif 14673912Sjhb 14773912Sjhb /* 14883366Sjulian * If on run queue move to new run queue, and quit. 14983366Sjulian * XXXKSE this gets a lot more complicated under threads 15083366Sjulian * but try anyhow. 15199072Sjulian * We should have a special call to do this more efficiently. 15267352Sjhb */ 15399072Sjulian if (td->td_state == TDS_RUNQ) { 15483366Sjulian MPASS(td->td_blocked == NULL); 15583366Sjulian remrunqueue(td); 15699072Sjulian td->td_priority = pri; 15783366Sjulian setrunqueue(td); 15867352Sjhb return; 15967352Sjhb } 16099072Sjulian /* 16199072Sjulian * Adjust for any other cases. 16299072Sjulian */ 16399072Sjulian td->td_priority = pri; 16467352Sjhb 16567352Sjhb /* 16669376Sjhb * If we aren't blocked on a mutex, we should be. 16767352Sjhb */ 16899072Sjulian KASSERT(td->td_state == TDS_MTX, ( 16969376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 17099072Sjulian td->td_proc->p_pid, td->td_proc->p_comm, td->td_state, 17174912Sjhb m->mtx_object.lo_name)); 17267352Sjhb 17367352Sjhb /* 17483366Sjulian * Pick up the mutex that td is blocked on. 17567352Sjhb */ 17683366Sjulian m = td->td_blocked; 17767352Sjhb MPASS(m != NULL); 17867352Sjhb 17967352Sjhb /* 18083366Sjulian * Check if the thread needs to be moved up on 18167352Sjhb * the blocked chain 18267352Sjhb */ 18383366Sjulian if (td == TAILQ_FIRST(&m->mtx_blocked)) { 18469376Sjhb continue; 18569376Sjhb } 18672200Sbmilekic 18783366Sjulian td1 = TAILQ_PREV(td, threadqueue, td_blkq); 18890538Sjulian if (td1->td_priority <= pri) { 18967352Sjhb continue; 19067352Sjhb } 19167352Sjhb 19267352Sjhb /* 19383366Sjulian * Remove thread from blocked chain and determine where 19483366Sjulian * it should be moved up to. Since we know that td1 has 19583366Sjulian * a lower priority than td, we know that at least one 19683366Sjulian * thread in the chain has a lower priority and that 19783366Sjulian * td1 will thus not be NULL after the loop. 19867352Sjhb */ 19983366Sjulian TAILQ_REMOVE(&m->mtx_blocked, td, td_blkq); 20083366Sjulian TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) { 20183366Sjulian MPASS(td1->td_proc->p_magic == P_MAGIC); 20290538Sjulian if (td1->td_priority > pri) 20367352Sjhb break; 20467352Sjhb } 20572200Sbmilekic 20683366Sjulian MPASS(td1 != NULL); 20783366Sjulian TAILQ_INSERT_BEFORE(td1, td, td_blkq); 20867352Sjhb CTR4(KTR_LOCK, 20971560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 21083366Sjulian td, td1, m, m->mtx_object.lo_name); 21167352Sjhb } 21267352Sjhb} 21367352Sjhb 21493609Sdes#ifdef MUTEX_PROFILING 21593609SdesSYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging"); 21693609SdesSYSCTL_NODE(_debug_mutex, OID_AUTO, prof, CTLFLAG_RD, NULL, "mutex profiling"); 21793609Sdesstatic int mutex_prof_enable = 0; 21893609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, enable, CTLFLAG_RW, 21993609Sdes &mutex_prof_enable, 0, "Enable tracing of mutex holdtime"); 22093609Sdes 22193609Sdesstruct mutex_prof { 22293609Sdes const char *name; 22393609Sdes const char *file; 22493609Sdes int line; 22593609Sdes#define MPROF_MAX 0 22693609Sdes#define MPROF_TOT 1 22793609Sdes#define MPROF_CNT 2 22893609Sdes#define MPROF_AVG 3 22997156Sdes uintmax_t counter[4]; 23093705Sdes struct mutex_prof *next; 23193609Sdes}; 23293609Sdes 23371352Sjasone/* 23493609Sdes * mprof_buf is a static pool of profiling records to avoid possible 23593609Sdes * reentrance of the memory allocation functions. 23693609Sdes * 23793609Sdes * Note: NUM_MPROF_BUFFERS must be smaller than MPROF_HASH_SIZE. 23893609Sdes */ 23993705Sdes#define NUM_MPROF_BUFFERS 1000 24093609Sdesstatic struct mutex_prof mprof_buf[NUM_MPROF_BUFFERS]; 24193609Sdesstatic int first_free_mprof_buf; 24293705Sdes#define MPROF_HASH_SIZE 1009 24393609Sdesstatic struct mutex_prof *mprof_hash[MPROF_HASH_SIZE]; 24493609Sdes 24593609Sdesstatic int mutex_prof_acquisitions; 24693609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, acquisitions, CTLFLAG_RD, 24793609Sdes &mutex_prof_acquisitions, 0, "Number of mutex acquistions recorded"); 24893609Sdesstatic int mutex_prof_records; 24993609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, records, CTLFLAG_RD, 25093609Sdes &mutex_prof_records, 0, "Number of profiling records"); 25193609Sdesstatic int mutex_prof_maxrecords = NUM_MPROF_BUFFERS; 25293609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, maxrecords, CTLFLAG_RD, 25393609Sdes &mutex_prof_maxrecords, 0, "Maximum number of profiling records"); 25493609Sdesstatic int mutex_prof_rejected; 25593609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, rejected, CTLFLAG_RD, 25693609Sdes &mutex_prof_rejected, 0, "Number of rejected profiling records"); 25793609Sdesstatic int mutex_prof_hashsize = MPROF_HASH_SIZE; 25893609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, hashsize, CTLFLAG_RD, 25993609Sdes &mutex_prof_hashsize, 0, "Hash size"); 26093609Sdesstatic int mutex_prof_collisions = 0; 26193609SdesSYSCTL_INT(_debug_mutex_prof, OID_AUTO, collisions, CTLFLAG_RD, 26293609Sdes &mutex_prof_collisions, 0, "Number of hash collisions"); 26393609Sdes 26493609Sdes/* 26593609Sdes * mprof_mtx protects the profiling buffers and the hash. 26693609Sdes */ 26793609Sdesstatic struct mtx mprof_mtx; 26893705SdesMTX_SYSINIT(mprof, &mprof_mtx, "mutex profiling lock", MTX_SPIN | MTX_QUIET); 26993609Sdes 27093667Sdesstatic u_int64_t 27193667Sdesnanoseconds(void) 27293667Sdes{ 27393667Sdes struct timespec tv; 27493667Sdes 27593667Sdes nanotime(&tv); 27693667Sdes return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec); 27793667Sdes} 27893667Sdes 27993609Sdesstatic int 28093609Sdesdump_mutex_prof_stats(SYSCTL_HANDLER_ARGS) 28193609Sdes{ 28293609Sdes struct sbuf *sb; 28393609Sdes int error, i; 28493609Sdes 28593609Sdes if (first_free_mprof_buf == 0) 28693609Sdes return SYSCTL_OUT(req, "No locking recorded", 28793609Sdes sizeof("No locking recorded")); 28893609Sdes 28993609Sdes sb = sbuf_new(NULL, NULL, 1024, SBUF_AUTOEXTEND); 29093609Sdes sbuf_printf(sb, "%12s %12s %12s %12s %s\n", 29193609Sdes "max", "total", "count", "average", "name"); 29293609Sdes mtx_lock_spin(&mprof_mtx); 29393609Sdes for (i = 0; i < first_free_mprof_buf; ++i) 29497156Sdes sbuf_printf(sb, "%12ju %12ju %12ju %12ju %s:%d (%s)\n", 29593667Sdes mprof_buf[i].counter[MPROF_MAX] / 1000, 29693667Sdes mprof_buf[i].counter[MPROF_TOT] / 1000, 29793667Sdes mprof_buf[i].counter[MPROF_CNT], 29893667Sdes mprof_buf[i].counter[MPROF_AVG] / 1000, 29993609Sdes mprof_buf[i].file, mprof_buf[i].line, mprof_buf[i].name); 30093609Sdes mtx_unlock_spin(&mprof_mtx); 30193609Sdes sbuf_finish(sb); 30293609Sdes error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 30393609Sdes sbuf_delete(sb); 30493609Sdes return (error); 30593609Sdes} 30693609SdesSYSCTL_PROC(_debug_mutex_prof, OID_AUTO, stats, CTLTYPE_STRING|CTLFLAG_RD, 30793609Sdes NULL, 0, dump_mutex_prof_stats, "A", "Mutex profiling statistics"); 30893609Sdes#endif 30993609Sdes 31093609Sdes/* 31174900Sjhb * Function versions of the inlined __mtx_* macros. These are used by 31274900Sjhb * modules and can also be called from assembly language if needed. 31374900Sjhb */ 31474900Sjhbvoid 31574900Sjhb_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 31674900Sjhb{ 31774900Sjhb 31883841Sjhb MPASS(curthread != NULL); 31983841Sjhb _get_sleep_lock(m, curthread, opts, file, line); 32083841Sjhb LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 32183841Sjhb line); 32283841Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 32393609Sdes#ifdef MUTEX_PROFILING 32493609Sdes /* don't reset the timer when/if recursing */ 32599324Sdes if (m->mtx_acqtime == 0) { 32699324Sdes m->mtx_filename = file; 32799324Sdes m->mtx_lineno = line; 32899324Sdes m->mtx_acqtime = mutex_prof_enable ? nanoseconds() : 0; 32993609Sdes ++mutex_prof_acquisitions; 33093609Sdes } 33193609Sdes#endif 33274900Sjhb} 33374900Sjhb 33474900Sjhbvoid 33574900Sjhb_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 33674900Sjhb{ 33774900Sjhb 33883841Sjhb MPASS(curthread != NULL); 33983947Sjhb mtx_assert(m, MA_OWNED); 34093609Sdes#ifdef MUTEX_PROFILING 34199324Sdes if (m->mtx_acqtime != 0) { 34293609Sdes static const char *unknown = "(unknown)"; 34393609Sdes struct mutex_prof *mpp; 34493667Sdes u_int64_t acqtime, now; 34593609Sdes const char *p, *q; 34693705Sdes volatile u_int hash; 34793609Sdes 34893667Sdes now = nanoseconds(); 34999324Sdes acqtime = m->mtx_acqtime; 35099324Sdes m->mtx_acqtime = 0; 35193667Sdes if (now <= acqtime) 35293609Sdes goto out; 35399324Sdes for (p = m->mtx_filename; strncmp(p, "../", 3) == 0; p += 3) 35493609Sdes /* nothing */ ; 35593609Sdes if (p == NULL || *p == '\0') 35693609Sdes p = unknown; 35799324Sdes for (hash = m->mtx_lineno, q = p; *q != '\0'; ++q) 35893609Sdes hash = (hash * 2 + *q) % MPROF_HASH_SIZE; 35993609Sdes mtx_lock_spin(&mprof_mtx); 36093705Sdes for (mpp = mprof_hash[hash]; mpp != NULL; mpp = mpp->next) 36199324Sdes if (mpp->line == m->mtx_lineno && 36299324Sdes strcmp(mpp->file, p) == 0) 36393609Sdes break; 36493609Sdes if (mpp == NULL) { 36593609Sdes /* Just exit if we cannot get a trace buffer */ 36693609Sdes if (first_free_mprof_buf >= NUM_MPROF_BUFFERS) { 36793609Sdes ++mutex_prof_rejected; 36893609Sdes goto unlock; 36993609Sdes } 37093609Sdes mpp = &mprof_buf[first_free_mprof_buf++]; 37193609Sdes mpp->name = mtx_name(m); 37293609Sdes mpp->file = p; 37399324Sdes mpp->line = m->mtx_lineno; 37493705Sdes mpp->next = mprof_hash[hash]; 37593705Sdes if (mprof_hash[hash] != NULL) 37693705Sdes ++mutex_prof_collisions; 37793705Sdes mprof_hash[hash] = mpp; 37893609Sdes ++mutex_prof_records; 37993609Sdes } 38093609Sdes /* 38193609Sdes * Record if the mutex has been held longer now than ever 38293609Sdes * before 38393609Sdes */ 38493667Sdes if ((now - acqtime) > mpp->counter[MPROF_MAX]) 38593667Sdes mpp->counter[MPROF_MAX] = now - acqtime; 38693667Sdes mpp->counter[MPROF_TOT] += now - acqtime; 38793667Sdes mpp->counter[MPROF_CNT] += 1; 38893667Sdes mpp->counter[MPROF_AVG] = 38993667Sdes mpp->counter[MPROF_TOT] / mpp->counter[MPROF_CNT]; 39093609Sdesunlock: 39193609Sdes mtx_unlock_spin(&mprof_mtx); 39293609Sdes } 39393609Sdesout: 39493609Sdes#endif 39583841Sjhb WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 39683841Sjhb LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 39783841Sjhb line); 39883841Sjhb _rel_sleep_lock(m, curthread, opts, file, line); 39974900Sjhb} 40074900Sjhb 40174900Sjhbvoid 40274900Sjhb_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 40374900Sjhb{ 40474900Sjhb 40583841Sjhb MPASS(curthread != NULL); 406100754Sjhb#if defined(SMP) || LOCK_DEBUG > 0 || 1 40783841Sjhb _get_spin_lock(m, curthread, opts, file, line); 40897079Sjhb#else 40997079Sjhb critical_enter(); 41097079Sjhb#endif 41183841Sjhb LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 41283841Sjhb line); 41383841Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 41474900Sjhb} 41574900Sjhb 41674900Sjhbvoid 41774900Sjhb_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 41874900Sjhb{ 41974900Sjhb 42083841Sjhb MPASS(curthread != NULL); 42183947Sjhb mtx_assert(m, MA_OWNED); 42283841Sjhb WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 42383841Sjhb LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 42483841Sjhb line); 425100754Sjhb#if defined(SMP) || LOCK_DEBUG > 0 || 1 42683841Sjhb _rel_spin_lock(m); 42797079Sjhb#else 42897079Sjhb critical_exit(); 42997079Sjhb#endif 43074900Sjhb} 43174900Sjhb 43274900Sjhb/* 43372200Sbmilekic * The important part of mtx_trylock{,_flags}() 43472200Sbmilekic * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 43572200Sbmilekic * if we're called, it's because we know we don't already own this lock. 43671352Sjasone */ 43772200Sbmilekicint 43872200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 43971352Sjasone{ 44072200Sbmilekic int rval; 44171352Sjasone 44283366Sjulian MPASS(curthread != NULL); 44371352Sjasone 44483366Sjulian rval = _obtain_lock(m, curthread); 44572200Sbmilekic 44674912Sjhb LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 44774912Sjhb if (rval) { 44871352Sjasone /* 44972200Sbmilekic * We do not handle recursion in _mtx_trylock; see the 45072200Sbmilekic * note at the top of the routine. 45171352Sjasone */ 45272344Sbmilekic KASSERT(!mtx_recursed(m), 45372344Sbmilekic ("mtx_trylock() called on a recursed mutex")); 45476272Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 45576272Sjhb file, line); 45671352Sjasone } 45771352Sjasone 45874912Sjhb return (rval); 45971352Sjasone} 46071352Sjasone 46171352Sjasone/* 46272200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 46371352Sjasone * 46472200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 46572200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 46671352Sjasone */ 46772200Sbmilekicvoid 46872200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 46971352Sjasone{ 47083366Sjulian struct thread *td = curthread; 47197081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 47297081Sjhb struct thread *owner; 47397081Sjhb#endif 47471352Sjasone 47583366Sjulian if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)td) { 47672200Sbmilekic m->mtx_recurse++; 47772200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 47874912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 47972344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 48072200Sbmilekic return; 48171352Sjasone } 48271352Sjasone 48374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 48472994Sjhb CTR4(KTR_LOCK, 48572994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 48674912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 48771352Sjasone 48883366Sjulian while (!_obtain_lock(m, td)) { 48972200Sbmilekic uintptr_t v; 49083366Sjulian struct thread *td1; 49171352Sjasone 49272200Sbmilekic mtx_lock_spin(&sched_lock); 49372200Sbmilekic /* 49472200Sbmilekic * Check if the lock has been released while spinning for 49572200Sbmilekic * the sched_lock. 49672200Sbmilekic */ 49772200Sbmilekic if ((v = m->mtx_lock) == MTX_UNOWNED) { 49872200Sbmilekic mtx_unlock_spin(&sched_lock); 49997086Sjhb#ifdef __i386__ 50097139Sjhb ia32_pause(); 50197086Sjhb#endif 50272200Sbmilekic continue; 50371352Sjasone } 50471352Sjasone 50572200Sbmilekic /* 50672200Sbmilekic * The mutex was marked contested on release. This means that 50783366Sjulian * there are threads blocked on it. 50872200Sbmilekic */ 50972200Sbmilekic if (v == MTX_CONTESTED) { 51083366Sjulian td1 = TAILQ_FIRST(&m->mtx_blocked); 51183366Sjulian MPASS(td1 != NULL); 51283366Sjulian m->mtx_lock = (uintptr_t)td | MTX_CONTESTED; 51367352Sjhb 51490538Sjulian if (td1->td_priority < td->td_priority) 51590538Sjulian td->td_priority = td1->td_priority; 51672200Sbmilekic mtx_unlock_spin(&sched_lock); 51767352Sjhb return; 51867352Sjhb } 51969376Sjhb 52069376Sjhb /* 52172200Sbmilekic * If the mutex isn't already contested and a failure occurs 52272200Sbmilekic * setting the contested bit, the mutex was either released 52372200Sbmilekic * or the state of the MTX_RECURSED bit changed. 52469376Sjhb */ 52572200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 52672200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 52772200Sbmilekic (void *)(v | MTX_CONTESTED))) { 52872200Sbmilekic mtx_unlock_spin(&sched_lock); 52997086Sjhb#ifdef __i386__ 53097139Sjhb ia32_pause(); 53197086Sjhb#endif 53272200Sbmilekic continue; 53372200Sbmilekic } 53467352Sjhb 53597081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 53672200Sbmilekic /* 53797081Sjhb * If the current owner of the lock is executing on another 53897081Sjhb * CPU, spin instead of blocking. 53997081Sjhb */ 54097081Sjhb owner = (struct thread *)(v & MTX_FLAGMASK); 54197839Sjhb if (m != &Giant && thread_running(owner)) { 54297081Sjhb mtx_unlock_spin(&sched_lock); 54397839Sjhb while (mtx_owner(m) == owner && thread_running(owner)) { 54497086Sjhb#ifdef __i386__ 54597837Sjhb ia32_pause(); 54697086Sjhb#endif 54797837Sjhb } 54897081Sjhb continue; 54997081Sjhb } 55097081Sjhb#endif /* SMP && ADAPTIVE_MUTEXES */ 55197081Sjhb 55297081Sjhb /* 55393692Sjhb * We definitely must sleep for this lock. 55472200Sbmilekic */ 55572200Sbmilekic mtx_assert(m, MA_NOTOWNED); 55667352Sjhb 55767352Sjhb#ifdef notyet 55872200Sbmilekic /* 55972200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 56072200Sbmilekic * must clean up before going to sleep. 56172200Sbmilekic */ 56283366Sjulian if (td->td_ithd != NULL) { 56383366Sjulian struct ithd *it = td->td_ithd; 56467352Sjhb 56572200Sbmilekic if (it->it_interrupted) { 56674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 56772200Sbmilekic CTR2(KTR_LOCK, 56872994Sjhb "_mtx_lock_sleep: %p interrupted %p", 56972200Sbmilekic it, it->it_interrupted); 57072200Sbmilekic intr_thd_fixup(it); 57167352Sjhb } 57272200Sbmilekic } 57367352Sjhb#endif 57467352Sjhb 57572200Sbmilekic /* 57672200Sbmilekic * Put us on the list of threads blocked on this mutex. 57772200Sbmilekic */ 57872200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 57990418Sjhb td1 = mtx_owner(m); 58083366Sjulian LIST_INSERT_HEAD(&td1->td_contested, m, mtx_contested); 58183366Sjulian TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); 58272200Sbmilekic } else { 58383366Sjulian TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) 58490538Sjulian if (td1->td_priority > td->td_priority) 58572200Sbmilekic break; 58683366Sjulian if (td1) 58783366Sjulian TAILQ_INSERT_BEFORE(td1, td, td_blkq); 58872200Sbmilekic else 58983366Sjulian TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); 59072200Sbmilekic } 59167352Sjhb 59272200Sbmilekic /* 59372200Sbmilekic * Save who we're blocked on. 59472200Sbmilekic */ 59583366Sjulian td->td_blocked = m; 59683366Sjulian td->td_mtxname = m->mtx_object.lo_name; 59799072Sjulian td->td_state = TDS_MTX; 59883366Sjulian propagate_priority(td); 59967352Sjhb 60074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 60172200Sbmilekic CTR3(KTR_LOCK, 60283366Sjulian "_mtx_lock_sleep: p %p blocked on [%p] %s", td, m, 60374912Sjhb m->mtx_object.lo_name); 60472200Sbmilekic 60583366Sjulian td->td_proc->p_stats->p_ru.ru_nvcsw++; 60672200Sbmilekic mi_switch(); 60772200Sbmilekic 60874912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 60972200Sbmilekic CTR3(KTR_LOCK, 61072200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 61183366Sjulian td, m, m->mtx_object.lo_name); 61272200Sbmilekic 61372200Sbmilekic mtx_unlock_spin(&sched_lock); 61472200Sbmilekic } 61572200Sbmilekic 61672200Sbmilekic return; 61772200Sbmilekic} 61872200Sbmilekic 61972200Sbmilekic/* 62072200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 62172200Sbmilekic * 62272200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 62372200Sbmilekic * is handled inline. 62472200Sbmilekic */ 62572200Sbmilekicvoid 62688088Sjhb_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) 62772200Sbmilekic{ 62872200Sbmilekic int i = 0; 62972200Sbmilekic 63074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 63172344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 63272200Sbmilekic 63372200Sbmilekic for (;;) { 63483366Sjulian if (_obtain_lock(m, curthread)) 63572200Sbmilekic break; 63672200Sbmilekic 63775568Sjhb /* Give interrupts a chance while we spin. */ 63888088Sjhb critical_exit(); 63972200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 64097086Sjhb if (i++ < 10000000) { 64197086Sjhb#ifdef __i386__ 64297139Sjhb ia32_pause(); 64397086Sjhb#endif 64472200Sbmilekic continue; 64597086Sjhb } 64697084Sjhb if (i < 60000000) 64772200Sbmilekic DELAY(1); 64867352Sjhb#ifdef DDB 64972200Sbmilekic else if (!db_active) 65067352Sjhb#else 65172200Sbmilekic else 65267352Sjhb#endif 65397082Sjhb panic("spin lock %s held by %p for > 5 seconds", 65497082Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock); 65597086Sjhb#ifdef __i386__ 65697139Sjhb ia32_pause(); 65797086Sjhb#endif 65867352Sjhb } 65988088Sjhb critical_enter(); 66067352Sjhb } 66172200Sbmilekic 66274912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 66372200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 66472200Sbmilekic 66572200Sbmilekic return; 66667352Sjhb} 66767352Sjhb 66872200Sbmilekic/* 66972200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 67072200Sbmilekic * 67172200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 67272200Sbmilekic * need to wake up a blocked thread). 67372200Sbmilekic */ 67467352Sjhbvoid 67572200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 67667352Sjhb{ 67783366Sjulian struct thread *td, *td1; 67867352Sjhb struct mtx *m1; 67967352Sjhb int pri; 68067352Sjhb 68183366Sjulian td = curthread; 68272200Sbmilekic 68372200Sbmilekic if (mtx_recursed(m)) { 68472200Sbmilekic if (--(m->mtx_recurse) == 0) 68572200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 68674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 68772200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 68872200Sbmilekic return; 68972200Sbmilekic } 69072200Sbmilekic 69172200Sbmilekic mtx_lock_spin(&sched_lock); 69274912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 69372200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 69472200Sbmilekic 69583366Sjulian td1 = TAILQ_FIRST(&m->mtx_blocked); 69697081Sjhb#if defined(SMP) && defined(ADAPTIVE_MUTEXES) 69797081Sjhb if (td1 == NULL) { 69897081Sjhb _release_lock_quick(m); 69997081Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 70097081Sjhb CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 70197081Sjhb mtx_unlock_spin(&sched_lock); 70297081Sjhb return; 70397081Sjhb } 70497081Sjhb#endif 70583366Sjulian MPASS(td->td_proc->p_magic == P_MAGIC); 70683366Sjulian MPASS(td1->td_proc->p_magic == P_MAGIC); 70772200Sbmilekic 70883366Sjulian TAILQ_REMOVE(&m->mtx_blocked, td1, td_blkq); 70972200Sbmilekic 71072200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 71172200Sbmilekic LIST_REMOVE(m, mtx_contested); 71272200Sbmilekic _release_lock_quick(m); 71374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 71472200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 71572200Sbmilekic } else 71672200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 71772200Sbmilekic 71872376Sjake pri = PRI_MAX; 71983366Sjulian LIST_FOREACH(m1, &td->td_contested, mtx_contested) { 72090538Sjulian int cp = TAILQ_FIRST(&m1->mtx_blocked)->td_priority; 72172200Sbmilekic if (cp < pri) 72272200Sbmilekic pri = cp; 72372200Sbmilekic } 72472200Sbmilekic 72590538Sjulian if (pri > td->td_base_pri) 72690538Sjulian pri = td->td_base_pri; 72790538Sjulian td->td_priority = pri; 72872200Sbmilekic 72974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 73072200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 73183366Sjulian m, td1); 73272200Sbmilekic 73383366Sjulian td1->td_blocked = NULL; 73483366Sjulian setrunqueue(td1); 73572200Sbmilekic 73690538Sjulian if (td->td_critnest == 1 && td1->td_priority < pri) { 73767352Sjhb#ifdef notyet 73883366Sjulian if (td->td_ithd != NULL) { 73983366Sjulian struct ithd *it = td->td_ithd; 74067352Sjhb 74172200Sbmilekic if (it->it_interrupted) { 74274912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 74372200Sbmilekic CTR2(KTR_LOCK, 74472994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 74572200Sbmilekic it, it->it_interrupted); 74672200Sbmilekic intr_thd_fixup(it); 74767352Sjhb } 74872200Sbmilekic } 74967352Sjhb#endif 75074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 75172200Sbmilekic CTR2(KTR_LOCK, 75272200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 75372200Sbmilekic (void *)m->mtx_lock); 75472200Sbmilekic 75583366Sjulian td->td_proc->p_stats->p_ru.ru_nivcsw++; 75672200Sbmilekic mi_switch(); 75774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 75872200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 75972200Sbmilekic m, (void *)m->mtx_lock); 76067352Sjhb } 76172200Sbmilekic 76272200Sbmilekic mtx_unlock_spin(&sched_lock); 76372200Sbmilekic 76472200Sbmilekic return; 76567352Sjhb} 76667352Sjhb 76772200Sbmilekic/* 76872200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 76972200Sbmilekic * See the _rel_spin_lock() macro for the details. 77072200Sbmilekic */ 77172200Sbmilekic 77272200Sbmilekic/* 77372994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 77472200Sbmilekic */ 77572996Sjhb#ifdef INVARIANT_SUPPORT 77671352Sjasonevoid 77771360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 77871352Sjasone{ 77980748Sjhb 78080748Sjhb if (panicstr != NULL) 78180748Sjhb return; 78273033Sjake switch (what) { 78371352Sjasone case MA_OWNED: 78471352Sjasone case MA_OWNED | MA_RECURSED: 78571352Sjasone case MA_OWNED | MA_NOTRECURSED: 78673033Sjake if (!mtx_owned(m)) 78771352Sjasone panic("mutex %s not owned at %s:%d", 78874912Sjhb m->mtx_object.lo_name, file, line); 78973033Sjake if (mtx_recursed(m)) { 79073033Sjake if ((what & MA_NOTRECURSED) != 0) 79171352Sjasone panic("mutex %s recursed at %s:%d", 79274912Sjhb m->mtx_object.lo_name, file, line); 79373033Sjake } else if ((what & MA_RECURSED) != 0) { 79471352Sjasone panic("mutex %s unrecursed at %s:%d", 79574912Sjhb m->mtx_object.lo_name, file, line); 79671352Sjasone } 79771352Sjasone break; 79871352Sjasone case MA_NOTOWNED: 79973033Sjake if (mtx_owned(m)) 80071352Sjasone panic("mutex %s owned at %s:%d", 80174912Sjhb m->mtx_object.lo_name, file, line); 80271352Sjasone break; 80371352Sjasone default: 80471360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 80571352Sjasone } 80671352Sjasone} 80771352Sjasone#endif 80871352Sjasone 80972200Sbmilekic/* 81072200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 81174912Sjhb * 81274912Sjhb * Most of these checks have been moved off into the LO_INITIALIZED flag 81374912Sjhb * maintained by the witness code. 81472200Sbmilekic */ 81567352Sjhb#ifdef MUTEX_DEBUG 81667352Sjhb 81792723Salfredvoid mtx_validate(struct mtx *); 81867352Sjhb 81974912Sjhbvoid 82074912Sjhbmtx_validate(struct mtx *m) 82167352Sjhb{ 82267352Sjhb 82367352Sjhb/* 82467352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 82567352Sjhb * we can re-enable the kernacc() checks. 82667352Sjhb */ 82767352Sjhb#ifndef __alpha__ 82882304Sbmilekic /* 82982304Sbmilekic * Can't call kernacc() from early init386(), especially when 83082304Sbmilekic * initializing Giant mutex, because some stuff in kernacc() 83182304Sbmilekic * requires Giant itself. 83282304Sbmilekic */ 83382302Sbmilekic if (!cold) 83482302Sbmilekic if (!kernacc((caddr_t)m, sizeof(m), 83582302Sbmilekic VM_PROT_READ | VM_PROT_WRITE)) 83682302Sbmilekic panic("Can't read and write to mutex %p", m); 83767352Sjhb#endif 83867352Sjhb} 83967352Sjhb#endif 84067352Sjhb 84172200Sbmilekic/* 84293672Sarr * General init routine used by the MTX_SYSINIT() macro. 84393672Sarr */ 84493672Sarrvoid 84593672Sarrmtx_sysinit(void *arg) 84693672Sarr{ 84793672Sarr struct mtx_args *margs = arg; 84893672Sarr 84993813Sjhb mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 85093672Sarr} 85193672Sarr 85293672Sarr/* 85372200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 85493813Sjhb * `opts' with options contained in `opts' and name `name.' The optional 85593813Sjhb * lock type `type' is used as a general lock category name for use with 85693813Sjhb * witness. 85772200Sbmilekic */ 85867352Sjhbvoid 85993813Sjhbmtx_init(struct mtx *m, const char *name, const char *type, int opts) 86067352Sjhb{ 86174912Sjhb struct lock_object *lock; 86272200Sbmilekic 86374912Sjhb MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 86493273Sjeff MTX_SLEEPABLE | MTX_NOWITNESS | MTX_DUPOK)) == 0); 86572200Sbmilekic 86667352Sjhb#ifdef MUTEX_DEBUG 86772200Sbmilekic /* Diagnostic and error correction */ 86874912Sjhb mtx_validate(m); 86969429Sjhb#endif 87067352Sjhb 87185205Sjhb lock = &m->mtx_object; 87285205Sjhb KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, 87393813Sjhb ("mutex %s %p already initialized", name, m)); 87474912Sjhb bzero(m, sizeof(*m)); 87574912Sjhb if (opts & MTX_SPIN) 87674912Sjhb lock->lo_class = &lock_class_mtx_spin; 87774912Sjhb else 87874912Sjhb lock->lo_class = &lock_class_mtx_sleep; 87993813Sjhb lock->lo_name = name; 88093813Sjhb lock->lo_type = type != NULL ? type : name; 88174912Sjhb if (opts & MTX_QUIET) 88274912Sjhb lock->lo_flags = LO_QUIET; 88374912Sjhb if (opts & MTX_RECURSE) 88474912Sjhb lock->lo_flags |= LO_RECURSABLE; 88574912Sjhb if (opts & MTX_SLEEPABLE) 88674912Sjhb lock->lo_flags |= LO_SLEEPABLE; 88774912Sjhb if ((opts & MTX_NOWITNESS) == 0) 88874912Sjhb lock->lo_flags |= LO_WITNESS; 88993273Sjeff if (opts & MTX_DUPOK) 89093273Sjeff lock->lo_flags |= LO_DUPOK; 89172200Sbmilekic 89267352Sjhb m->mtx_lock = MTX_UNOWNED; 89374912Sjhb TAILQ_INIT(&m->mtx_blocked); 89472200Sbmilekic 89574912Sjhb LOCK_LOG_INIT(lock, opts); 89672200Sbmilekic 89774912Sjhb WITNESS_INIT(lock); 89867352Sjhb} 89967352Sjhb 90072200Sbmilekic/* 90174912Sjhb * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 90274912Sjhb * passed in as a flag here because if the corresponding mtx_init() was 90374912Sjhb * called with MTX_QUIET set, then it will already be set in the mutex's 90474912Sjhb * flags. 90572200Sbmilekic */ 90667352Sjhbvoid 90767352Sjhbmtx_destroy(struct mtx *m) 90867352Sjhb{ 90967352Sjhb 91074912Sjhb LOCK_LOG_DESTROY(&m->mtx_object, 0); 91172200Sbmilekic 91274912Sjhb if (!mtx_owned(m)) 91374912Sjhb MPASS(mtx_unowned(m)); 91474912Sjhb else { 91571228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 91672200Sbmilekic 91774912Sjhb /* Tell witness this isn't locked to make it happy. */ 91888900Sjhb WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, 91988900Sjhb __LINE__); 92071320Sjasone } 92171320Sjasone 92274912Sjhb WITNESS_DESTROY(&m->mtx_object); 92371320Sjasone} 92485564Sdillon 92585564Sdillon/* 92693702Sjhb * Intialize the mutex code and system mutexes. This is called from the MD 92793702Sjhb * startup code prior to mi_startup(). The per-CPU data space needs to be 92893702Sjhb * setup before this is called. 92993702Sjhb */ 93093702Sjhbvoid 93193702Sjhbmutex_init(void) 93293702Sjhb{ 93393702Sjhb 93493702Sjhb /* Setup thread0 so that mutexes work. */ 93593702Sjhb LIST_INIT(&thread0.td_contested); 93693702Sjhb 93793702Sjhb /* 93893702Sjhb * Initialize mutexes. 93993702Sjhb */ 94093813Sjhb mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 94193813Sjhb mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 94293813Sjhb mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 94393702Sjhb mtx_lock(&Giant); 94493702Sjhb} 94593702Sjhb 94693702Sjhb/* 94785564Sdillon * Encapsulated Giant mutex routines. These routines provide encapsulation 94885564Sdillon * control for the Giant mutex, allowing sysctls to be used to turn on and 94985564Sdillon * off Giant around certain subsystems. The default value for the sysctls 95085564Sdillon * are set to what developers believe is stable and working in regards to 95185564Sdillon * the Giant pushdown. Developers should not turn off Giant via these 95285564Sdillon * sysctls unless they know what they are doing. 95385564Sdillon * 95485564Sdillon * Callers of mtx_lock_giant() are expected to pass the return value to an 95585564Sdillon * accompanying mtx_unlock_giant() later on. If multiple subsystems are 95685564Sdillon * effected by a Giant wrap, all related sysctl variables must be zero for 95785564Sdillon * the subsystem call to operate without Giant (as determined by the caller). 95885564Sdillon */ 95985564Sdillon 96085564SdillonSYSCTL_NODE(_kern, OID_AUTO, giant, CTLFLAG_RD, NULL, "Giant mutex manipulation"); 96185564Sdillon 96285564Sdillonstatic int kern_giant_all = 0; 96385564SdillonSYSCTL_INT(_kern_giant, OID_AUTO, all, CTLFLAG_RW, &kern_giant_all, 0, ""); 96485564Sdillon 96585564Sdillonint kern_giant_proc = 1; /* Giant around PROC locks */ 96685564Sdillonint kern_giant_file = 1; /* Giant around struct file & filedesc */ 96790864Sdillonint kern_giant_ucred = 1; /* Giant around ucred */ 96885564SdillonSYSCTL_INT(_kern_giant, OID_AUTO, proc, CTLFLAG_RW, &kern_giant_proc, 0, ""); 96985564SdillonSYSCTL_INT(_kern_giant, OID_AUTO, file, CTLFLAG_RW, &kern_giant_file, 0, ""); 97090864SdillonSYSCTL_INT(_kern_giant, OID_AUTO, ucred, CTLFLAG_RW, &kern_giant_ucred, 0, ""); 97185564Sdillon 97285564Sdillonint 97385564Sdillonmtx_lock_giant(int sysctlvar) 97485564Sdillon{ 97585564Sdillon if (sysctlvar || kern_giant_all) { 97685564Sdillon mtx_lock(&Giant); 97785564Sdillon return(1); 97885564Sdillon } 97985564Sdillon return(0); 98085564Sdillon} 98185564Sdillon 98285564Sdillonvoid 98385564Sdillonmtx_unlock_giant(int s) 98485564Sdillon{ 98585564Sdillon if (s) 98685564Sdillon mtx_unlock(&Giant); 98785564Sdillon} 98885564Sdillon 989