subr_turnstile.c revision 80748
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone * $FreeBSD: head/sys/kern/subr_turnstile.c 80748 2001-07-31 17:45:50Z jhb $ 3165557Sjasone */ 3265557Sjasone 3365557Sjasone/* 3472200Sbmilekic * Machine independent bits of mutex implementation and implementation of 3572200Sbmilekic * `witness' structure & related debugging routines. 3672200Sbmilekic */ 3772200Sbmilekic 3872200Sbmilekic/* 3965557Sjasone * Main Entry: witness 4065557Sjasone * Pronunciation: 'wit-n&s 4165557Sjasone * Function: noun 4265557Sjasone * Etymology: Middle English witnesse, from Old English witnes knowledge, 4365557Sjasone * testimony, witness, from 2wit 4465557Sjasone * Date: before 12th century 4565557Sjasone * 1 : attestation of a fact or event : TESTIMONY 4665557Sjasone * 2 : one that gives evidence; specifically : one who testifies in 4765557Sjasone * a cause or before a judicial tribunal 4865557Sjasone * 3 : one asked to be present at a transaction so as to be able to 4965557Sjasone * testify to its having taken place 5065557Sjasone * 4 : one who has personal knowledge of something 5165557Sjasone * 5 a : something serving as evidence or proof : SIGN 5265557Sjasone * b : public affirmation by word or example of usually 5365557Sjasone * religious faith or conviction <the heroic witness to divine 5465557Sjasone * life -- Pilot> 5565557Sjasone * 6 capitalized : a member of the Jehovah's Witnesses 5665557Sjasone */ 5765557Sjasone 5868790Sjhb#include "opt_ddb.h" 5967676Sjhb 6065557Sjasone#include <sys/param.h> 6167352Sjhb#include <sys/bus.h> 6267352Sjhb#include <sys/kernel.h> 6376166Smarkm#include <sys/lock.h> 6467352Sjhb#include <sys/malloc.h> 6574912Sjhb#include <sys/mutex.h> 6665557Sjasone#include <sys/proc.h> 6778766Sjhb#include <sys/resourcevar.h> 6867676Sjhb#include <sys/sysctl.h> 6965557Sjasone#include <sys/systm.h> 7067352Sjhb#include <sys/vmmeter.h> 7165557Sjasone#include <sys/ktr.h> 7265557Sjasone 7367352Sjhb#include <machine/atomic.h> 7467352Sjhb#include <machine/bus.h> 7567352Sjhb#include <machine/clock.h> 7665557Sjasone#include <machine/cpu.h> 7767352Sjhb 7868790Sjhb#include <ddb/ddb.h> 7968790Sjhb 8067352Sjhb#include <vm/vm.h> 8167352Sjhb#include <vm/vm_extern.h> 8267352Sjhb 8365557Sjasone/* 8472200Sbmilekic * Internal utility macros. 8571352Sjasone */ 8672200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 8771352Sjasone 8872200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 8972200Sbmilekic : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) 9071352Sjasone 9172376Sjake#define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) 9271352Sjasone 9371352Sjasone/* 9474912Sjhb * Lock classes for sleep and spin mutexes. 9571352Sjasone */ 9674912Sjhbstruct lock_class lock_class_mtx_sleep = { 9774912Sjhb "sleep mutex", 9874912Sjhb LC_SLEEPLOCK | LC_RECURSABLE 9974912Sjhb}; 10074912Sjhbstruct lock_class lock_class_mtx_spin = { 10174912Sjhb "spin mutex", 10274912Sjhb LC_SPINLOCK | LC_RECURSABLE 10374912Sjhb}; 10471352Sjasone 10571352Sjasone/* 10672200Sbmilekic * Prototypes for non-exported routines. 10772200Sbmilekic */ 10871352Sjasonestatic void propagate_priority(struct proc *); 10967352Sjhb 11067352Sjhbstatic void 11167352Sjhbpropagate_priority(struct proc *p) 11267352Sjhb{ 11372376Sjake int pri = p->p_pri.pri_level; 11467352Sjhb struct mtx *m = p->p_blocked; 11567352Sjhb 11669376Sjhb mtx_assert(&sched_lock, MA_OWNED); 11767352Sjhb for (;;) { 11867352Sjhb struct proc *p1; 11967352Sjhb 12067352Sjhb p = mtx_owner(m); 12167352Sjhb 12267352Sjhb if (p == NULL) { 12367352Sjhb /* 12467352Sjhb * This really isn't quite right. Really 12567352Sjhb * ought to bump priority of process that 12667352Sjhb * next acquires the mutex. 12767352Sjhb */ 12867352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 12967352Sjhb return; 13067352Sjhb } 13172200Sbmilekic 13267352Sjhb MPASS(p->p_magic == P_MAGIC); 13369376Sjhb KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 13472376Sjake if (p->p_pri.pri_level <= pri) 13567352Sjhb return; 13669376Sjhb 13767352Sjhb /* 13869376Sjhb * Bump this process' priority. 13969376Sjhb */ 14069376Sjhb SET_PRIO(p, pri); 14169376Sjhb 14269376Sjhb /* 14367352Sjhb * If lock holder is actually running, just bump priority. 14467352Sjhb */ 14572836Sjhb if (p->p_oncpu != NOCPU) { 14675468Smarkm MPASS(p->p_stat == SRUN || p->p_stat == SZOMB || p->p_stat == SSTOP); 14767352Sjhb return; 14867352Sjhb } 14972376Sjake 15073912Sjhb#ifndef SMP 15167352Sjhb /* 15273912Sjhb * For UP, we check to see if p is curproc (this shouldn't 15373912Sjhb * ever happen however as it would mean we are in a deadlock.) 15473912Sjhb */ 15573912Sjhb KASSERT(p != curproc, ("Deadlock detected")); 15673912Sjhb#endif 15773912Sjhb 15873912Sjhb /* 15967352Sjhb * If on run queue move to new run queue, and 16067352Sjhb * quit. 16167352Sjhb */ 16267352Sjhb if (p->p_stat == SRUN) { 16367352Sjhb MPASS(p->p_blocked == NULL); 16467352Sjhb remrunqueue(p); 16567352Sjhb setrunqueue(p); 16667352Sjhb return; 16767352Sjhb } 16867352Sjhb 16967352Sjhb /* 17069376Sjhb * If we aren't blocked on a mutex, we should be. 17167352Sjhb */ 17269376Sjhb KASSERT(p->p_stat == SMTX, ( 17369376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 17469376Sjhb p->p_pid, p->p_comm, p->p_stat, 17574912Sjhb m->mtx_object.lo_name)); 17667352Sjhb 17767352Sjhb /* 17867352Sjhb * Pick up the mutex that p is blocked on. 17967352Sjhb */ 18067352Sjhb m = p->p_blocked; 18167352Sjhb MPASS(m != NULL); 18267352Sjhb 18367352Sjhb /* 18467352Sjhb * Check if the proc needs to be moved up on 18567352Sjhb * the blocked chain 18667352Sjhb */ 18769376Sjhb if (p == TAILQ_FIRST(&m->mtx_blocked)) { 18869376Sjhb continue; 18969376Sjhb } 19072200Sbmilekic 19172376Sjake p1 = TAILQ_PREV(p, procqueue, p_procq); 19272376Sjake if (p1->p_pri.pri_level <= pri) { 19367352Sjhb continue; 19467352Sjhb } 19567352Sjhb 19667352Sjhb /* 19769376Sjhb * Remove proc from blocked chain and determine where 19869376Sjhb * it should be moved up to. Since we know that p1 has 19969376Sjhb * a lower priority than p, we know that at least one 20069376Sjhb * process in the chain has a lower priority and that 20169376Sjhb * p1 will thus not be NULL after the loop. 20267352Sjhb */ 20367352Sjhb TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 20467352Sjhb TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 20567352Sjhb MPASS(p1->p_magic == P_MAGIC); 20672376Sjake if (p1->p_pri.pri_level > pri) 20767352Sjhb break; 20867352Sjhb } 20972200Sbmilekic 21069376Sjhb MPASS(p1 != NULL); 21169376Sjhb TAILQ_INSERT_BEFORE(p1, p, p_procq); 21267352Sjhb CTR4(KTR_LOCK, 21371560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 21474912Sjhb p, p1, m, m->mtx_object.lo_name); 21567352Sjhb } 21667352Sjhb} 21767352Sjhb 21871352Sjasone/* 21974900Sjhb * Function versions of the inlined __mtx_* macros. These are used by 22074900Sjhb * modules and can also be called from assembly language if needed. 22174900Sjhb */ 22274900Sjhbvoid 22374900Sjhb_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 22474900Sjhb{ 22574900Sjhb 22674900Sjhb __mtx_lock_flags(m, opts, file, line); 22774900Sjhb} 22874900Sjhb 22974900Sjhbvoid 23074900Sjhb_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 23174900Sjhb{ 23274900Sjhb 23374900Sjhb __mtx_unlock_flags(m, opts, file, line); 23474900Sjhb} 23574900Sjhb 23674900Sjhbvoid 23774900Sjhb_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 23874900Sjhb{ 23974900Sjhb 24074900Sjhb __mtx_lock_spin_flags(m, opts, file, line); 24174900Sjhb} 24274900Sjhb 24374900Sjhbvoid 24474900Sjhb_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 24574900Sjhb{ 24674900Sjhb 24774900Sjhb __mtx_unlock_spin_flags(m, opts, file, line); 24874900Sjhb} 24974900Sjhb 25074900Sjhb/* 25172200Sbmilekic * The important part of mtx_trylock{,_flags}() 25272200Sbmilekic * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 25372200Sbmilekic * if we're called, it's because we know we don't already own this lock. 25471352Sjasone */ 25572200Sbmilekicint 25672200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 25771352Sjasone{ 25872200Sbmilekic int rval; 25971352Sjasone 26072393Sbmilekic MPASS(curproc != NULL); 26171352Sjasone 26272200Sbmilekic /* 26372200Sbmilekic * _mtx_trylock does not accept MTX_NOSWITCH option. 26472200Sbmilekic */ 26572344Sbmilekic KASSERT((opts & MTX_NOSWITCH) == 0, 26672344Sbmilekic ("mtx_trylock() called with invalid option flag(s) %d", opts)); 26772200Sbmilekic 26872393Sbmilekic rval = _obtain_lock(m, curproc); 26972200Sbmilekic 27074912Sjhb LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 27174912Sjhb if (rval) { 27271352Sjasone /* 27372200Sbmilekic * We do not handle recursion in _mtx_trylock; see the 27472200Sbmilekic * note at the top of the routine. 27571352Sjasone */ 27672344Sbmilekic KASSERT(!mtx_recursed(m), 27772344Sbmilekic ("mtx_trylock() called on a recursed mutex")); 27876272Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 27976272Sjhb file, line); 28071352Sjasone } 28171352Sjasone 28274912Sjhb return (rval); 28371352Sjasone} 28471352Sjasone 28571352Sjasone/* 28672200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 28771352Sjasone * 28872200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 28972200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 29071352Sjasone */ 29172200Sbmilekicvoid 29272200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 29371352Sjasone{ 29472393Sbmilekic struct proc *p = curproc; 29571352Sjasone 29672200Sbmilekic if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 29772200Sbmilekic m->mtx_recurse++; 29872200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 29974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 30072344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 30172200Sbmilekic return; 30271352Sjasone } 30371352Sjasone 30474912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 30572994Sjhb CTR4(KTR_LOCK, 30672994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 30774912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 30871352Sjasone 30972200Sbmilekic while (!_obtain_lock(m, p)) { 31072200Sbmilekic uintptr_t v; 31172200Sbmilekic struct proc *p1; 31271352Sjasone 31372200Sbmilekic mtx_lock_spin(&sched_lock); 31472200Sbmilekic /* 31572200Sbmilekic * Check if the lock has been released while spinning for 31672200Sbmilekic * the sched_lock. 31772200Sbmilekic */ 31872200Sbmilekic if ((v = m->mtx_lock) == MTX_UNOWNED) { 31972200Sbmilekic mtx_unlock_spin(&sched_lock); 32072200Sbmilekic continue; 32171352Sjasone } 32271352Sjasone 32372200Sbmilekic /* 32472200Sbmilekic * The mutex was marked contested on release. This means that 32572200Sbmilekic * there are processes blocked on it. 32672200Sbmilekic */ 32772200Sbmilekic if (v == MTX_CONTESTED) { 32872200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 32972344Sbmilekic MPASS(p1 != NULL); 33072200Sbmilekic m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; 33167352Sjhb 33272376Sjake if (p1->p_pri.pri_level < p->p_pri.pri_level) 33372376Sjake SET_PRIO(p, p1->p_pri.pri_level); 33472200Sbmilekic mtx_unlock_spin(&sched_lock); 33567352Sjhb return; 33667352Sjhb } 33769376Sjhb 33869376Sjhb /* 33972200Sbmilekic * If the mutex isn't already contested and a failure occurs 34072200Sbmilekic * setting the contested bit, the mutex was either released 34172200Sbmilekic * or the state of the MTX_RECURSED bit changed. 34269376Sjhb */ 34372200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 34472200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 34572200Sbmilekic (void *)(v | MTX_CONTESTED))) { 34672200Sbmilekic mtx_unlock_spin(&sched_lock); 34772200Sbmilekic continue; 34872200Sbmilekic } 34967352Sjhb 35072200Sbmilekic /* 35172200Sbmilekic * We deffinately must sleep for this lock. 35272200Sbmilekic */ 35372200Sbmilekic mtx_assert(m, MA_NOTOWNED); 35467352Sjhb 35567352Sjhb#ifdef notyet 35672200Sbmilekic /* 35772200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 35872200Sbmilekic * must clean up before going to sleep. 35972200Sbmilekic */ 36072994Sjhb if (p->p_ithd != NULL) { 36172994Sjhb struct ithd *it = p->p_ithd; 36267352Sjhb 36372200Sbmilekic if (it->it_interrupted) { 36474912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 36572200Sbmilekic CTR2(KTR_LOCK, 36672994Sjhb "_mtx_lock_sleep: %p interrupted %p", 36772200Sbmilekic it, it->it_interrupted); 36872200Sbmilekic intr_thd_fixup(it); 36967352Sjhb } 37072200Sbmilekic } 37167352Sjhb#endif 37267352Sjhb 37372200Sbmilekic /* 37472200Sbmilekic * Put us on the list of threads blocked on this mutex. 37572200Sbmilekic */ 37672200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 37772200Sbmilekic p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); 37872200Sbmilekic LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); 37972200Sbmilekic TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 38072200Sbmilekic } else { 38172200Sbmilekic TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 38272376Sjake if (p1->p_pri.pri_level > p->p_pri.pri_level) 38372200Sbmilekic break; 38472200Sbmilekic if (p1) 38572200Sbmilekic TAILQ_INSERT_BEFORE(p1, p, p_procq); 38672200Sbmilekic else 38767352Sjhb TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 38872200Sbmilekic } 38967352Sjhb 39072200Sbmilekic /* 39172200Sbmilekic * Save who we're blocked on. 39272200Sbmilekic */ 39372200Sbmilekic p->p_blocked = m; 39474912Sjhb p->p_mtxname = m->mtx_object.lo_name; 39572200Sbmilekic p->p_stat = SMTX; 39672200Sbmilekic propagate_priority(p); 39767352Sjhb 39874912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 39972200Sbmilekic CTR3(KTR_LOCK, 40072200Sbmilekic "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, 40174912Sjhb m->mtx_object.lo_name); 40272200Sbmilekic 40378766Sjhb p->p_stats->p_ru.ru_nvcsw++; 40472200Sbmilekic mi_switch(); 40572200Sbmilekic 40674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 40772200Sbmilekic CTR3(KTR_LOCK, 40872200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 40974912Sjhb p, m, m->mtx_object.lo_name); 41072200Sbmilekic 41172200Sbmilekic mtx_unlock_spin(&sched_lock); 41272200Sbmilekic } 41372200Sbmilekic 41472200Sbmilekic return; 41572200Sbmilekic} 41672200Sbmilekic 41772200Sbmilekic/* 41872200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 41972200Sbmilekic * 42072200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 42172200Sbmilekic * is handled inline. 42272200Sbmilekic */ 42372200Sbmilekicvoid 42474900Sjhb_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, 42572200Sbmilekic int line) 42672200Sbmilekic{ 42772200Sbmilekic int i = 0; 42872200Sbmilekic 42974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 43072344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 43172200Sbmilekic 43272200Sbmilekic for (;;) { 43372393Sbmilekic if (_obtain_lock(m, curproc)) 43472200Sbmilekic break; 43572200Sbmilekic 43675568Sjhb /* Give interrupts a chance while we spin. */ 43775568Sjhb critical_exit(mtx_crit); 43872200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 43972200Sbmilekic if (i++ < 1000000) 44072200Sbmilekic continue; 44172200Sbmilekic if (i++ < 6000000) 44272200Sbmilekic DELAY(1); 44367352Sjhb#ifdef DDB 44472200Sbmilekic else if (!db_active) 44567352Sjhb#else 44672200Sbmilekic else 44767352Sjhb#endif 44872200Sbmilekic panic("spin lock %s held by %p for > 5 seconds", 44974912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock); 45067352Sjhb } 45175568Sjhb mtx_crit = critical_enter(); 45267352Sjhb } 45372200Sbmilekic 45474900Sjhb m->mtx_savecrit = mtx_crit; 45574912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 45672200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 45772200Sbmilekic 45872200Sbmilekic return; 45967352Sjhb} 46067352Sjhb 46172200Sbmilekic/* 46272200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 46372200Sbmilekic * 46472200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 46572200Sbmilekic * need to wake up a blocked thread). 46672200Sbmilekic */ 46767352Sjhbvoid 46872200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 46967352Sjhb{ 47067352Sjhb struct proc *p, *p1; 47167352Sjhb struct mtx *m1; 47267352Sjhb int pri; 47367352Sjhb 47472393Sbmilekic p = curproc; 47572200Sbmilekic 47672200Sbmilekic if (mtx_recursed(m)) { 47772200Sbmilekic if (--(m->mtx_recurse) == 0) 47872200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 47974912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 48072200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 48172200Sbmilekic return; 48272200Sbmilekic } 48372200Sbmilekic 48472200Sbmilekic mtx_lock_spin(&sched_lock); 48574912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 48672200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 48772200Sbmilekic 48872200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 48972200Sbmilekic MPASS(p->p_magic == P_MAGIC); 49072200Sbmilekic MPASS(p1->p_magic == P_MAGIC); 49172200Sbmilekic 49272200Sbmilekic TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 49372200Sbmilekic 49472200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 49572200Sbmilekic LIST_REMOVE(m, mtx_contested); 49672200Sbmilekic _release_lock_quick(m); 49774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 49872200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 49972200Sbmilekic } else 50072200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 50172200Sbmilekic 50272376Sjake pri = PRI_MAX; 50372200Sbmilekic LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 50472376Sjake int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level; 50572200Sbmilekic if (cp < pri) 50672200Sbmilekic pri = cp; 50772200Sbmilekic } 50872200Sbmilekic 50972376Sjake if (pri > p->p_pri.pri_native) 51072376Sjake pri = p->p_pri.pri_native; 51172200Sbmilekic SET_PRIO(p, pri); 51272200Sbmilekic 51374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 51472200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 51572200Sbmilekic m, p1); 51672200Sbmilekic 51772200Sbmilekic p1->p_blocked = NULL; 51872200Sbmilekic p1->p_stat = SRUN; 51972200Sbmilekic setrunqueue(p1); 52072200Sbmilekic 52172376Sjake if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) { 52267352Sjhb#ifdef notyet 52372994Sjhb if (p->p_ithd != NULL) { 52472994Sjhb struct ithd *it = p->p_ithd; 52567352Sjhb 52672200Sbmilekic if (it->it_interrupted) { 52774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 52872200Sbmilekic CTR2(KTR_LOCK, 52972994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 53072200Sbmilekic it, it->it_interrupted); 53172200Sbmilekic intr_thd_fixup(it); 53267352Sjhb } 53372200Sbmilekic } 53467352Sjhb#endif 53572200Sbmilekic setrunqueue(p); 53674912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 53772200Sbmilekic CTR2(KTR_LOCK, 53872200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 53972200Sbmilekic (void *)m->mtx_lock); 54072200Sbmilekic 54178766Sjhb p->p_stats->p_ru.ru_nivcsw++; 54272200Sbmilekic mi_switch(); 54374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 54472200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 54572200Sbmilekic m, (void *)m->mtx_lock); 54667352Sjhb } 54772200Sbmilekic 54872200Sbmilekic mtx_unlock_spin(&sched_lock); 54972200Sbmilekic 55072200Sbmilekic return; 55167352Sjhb} 55267352Sjhb 55372200Sbmilekic/* 55472200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 55572200Sbmilekic * See the _rel_spin_lock() macro for the details. 55672200Sbmilekic */ 55772200Sbmilekic 55872200Sbmilekic/* 55972994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 56072200Sbmilekic */ 56172996Sjhb#ifdef INVARIANT_SUPPORT 56271352Sjasonevoid 56371360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 56471352Sjasone{ 56580748Sjhb 56680748Sjhb if (panicstr != NULL) 56780748Sjhb return; 56873033Sjake switch (what) { 56971352Sjasone case MA_OWNED: 57071352Sjasone case MA_OWNED | MA_RECURSED: 57171352Sjasone case MA_OWNED | MA_NOTRECURSED: 57273033Sjake if (!mtx_owned(m)) 57371352Sjasone panic("mutex %s not owned at %s:%d", 57474912Sjhb m->mtx_object.lo_name, file, line); 57573033Sjake if (mtx_recursed(m)) { 57673033Sjake if ((what & MA_NOTRECURSED) != 0) 57771352Sjasone panic("mutex %s recursed at %s:%d", 57874912Sjhb m->mtx_object.lo_name, file, line); 57973033Sjake } else if ((what & MA_RECURSED) != 0) { 58071352Sjasone panic("mutex %s unrecursed at %s:%d", 58174912Sjhb m->mtx_object.lo_name, file, line); 58271352Sjasone } 58371352Sjasone break; 58471352Sjasone case MA_NOTOWNED: 58573033Sjake if (mtx_owned(m)) 58671352Sjasone panic("mutex %s owned at %s:%d", 58774912Sjhb m->mtx_object.lo_name, file, line); 58871352Sjasone break; 58971352Sjasone default: 59071360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 59171352Sjasone } 59271352Sjasone} 59371352Sjasone#endif 59471352Sjasone 59572200Sbmilekic/* 59672200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 59774912Sjhb * 59874912Sjhb * Most of these checks have been moved off into the LO_INITIALIZED flag 59974912Sjhb * maintained by the witness code. 60072200Sbmilekic */ 60167352Sjhb#ifdef MUTEX_DEBUG 60267352Sjhb 60374912Sjhbvoid mtx_validate __P((struct mtx *)); 60467352Sjhb 60574912Sjhbvoid 60674912Sjhbmtx_validate(struct mtx *m) 60767352Sjhb{ 60867352Sjhb 60967352Sjhb/* 61067352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 61167352Sjhb * we can re-enable the kernacc() checks. 61267352Sjhb */ 61367352Sjhb#ifndef __alpha__ 61474912Sjhb if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE)) 61574912Sjhb panic("Can't read and write to mutex %p", m); 61667352Sjhb#endif 61767352Sjhb} 61867352Sjhb#endif 61967352Sjhb 62072200Sbmilekic/* 62172200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 62272200Sbmilekic * `opts' with options contained in `opts' and description `description.' 62372200Sbmilekic */ 62467352Sjhbvoid 62572200Sbmilekicmtx_init(struct mtx *m, const char *description, int opts) 62667352Sjhb{ 62774912Sjhb struct lock_object *lock; 62872200Sbmilekic 62974912Sjhb MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 63074912Sjhb MTX_SLEEPABLE | MTX_NOWITNESS)) == 0); 63172200Sbmilekic 63267352Sjhb#ifdef MUTEX_DEBUG 63372200Sbmilekic /* Diagnostic and error correction */ 63474912Sjhb mtx_validate(m); 63569429Sjhb#endif 63667352Sjhb 63774912Sjhb bzero(m, sizeof(*m)); 63874912Sjhb lock = &m->mtx_object; 63974912Sjhb if (opts & MTX_SPIN) 64074912Sjhb lock->lo_class = &lock_class_mtx_spin; 64174912Sjhb else 64274912Sjhb lock->lo_class = &lock_class_mtx_sleep; 64374912Sjhb lock->lo_name = description; 64474912Sjhb if (opts & MTX_QUIET) 64574912Sjhb lock->lo_flags = LO_QUIET; 64674912Sjhb if (opts & MTX_RECURSE) 64774912Sjhb lock->lo_flags |= LO_RECURSABLE; 64874912Sjhb if (opts & MTX_SLEEPABLE) 64974912Sjhb lock->lo_flags |= LO_SLEEPABLE; 65074912Sjhb if ((opts & MTX_NOWITNESS) == 0) 65174912Sjhb lock->lo_flags |= LO_WITNESS; 65272200Sbmilekic 65367352Sjhb m->mtx_lock = MTX_UNOWNED; 65474912Sjhb TAILQ_INIT(&m->mtx_blocked); 65572200Sbmilekic 65674912Sjhb LOCK_LOG_INIT(lock, opts); 65772200Sbmilekic 65874912Sjhb WITNESS_INIT(lock); 65967352Sjhb} 66067352Sjhb 66172200Sbmilekic/* 66274912Sjhb * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 66374912Sjhb * passed in as a flag here because if the corresponding mtx_init() was 66474912Sjhb * called with MTX_QUIET set, then it will already be set in the mutex's 66574912Sjhb * flags. 66672200Sbmilekic */ 66767352Sjhbvoid 66867352Sjhbmtx_destroy(struct mtx *m) 66967352Sjhb{ 67067352Sjhb 67174912Sjhb LOCK_LOG_DESTROY(&m->mtx_object, 0); 67272200Sbmilekic 67374912Sjhb if (!mtx_owned(m)) 67474912Sjhb MPASS(mtx_unowned(m)); 67574912Sjhb else { 67671228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 67772200Sbmilekic 67874912Sjhb /* Tell witness this isn't locked to make it happy. */ 67976272Sjhb WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE | LOP_NOSWITCH, 68076272Sjhb __FILE__, __LINE__); 68171320Sjasone } 68271320Sjasone 68374912Sjhb WITNESS_DESTROY(&m->mtx_object); 68471320Sjasone} 685