subr_turnstile.c revision 76166
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone * $FreeBSD: head/sys/kern/subr_turnstile.c 76166 2001-05-01 08:13:21Z markm $ 3165557Sjasone */ 3265557Sjasone 3365557Sjasone/* 3472200Sbmilekic * Machine independent bits of mutex implementation and implementation of 3572200Sbmilekic * `witness' structure & related debugging routines. 3672200Sbmilekic */ 3772200Sbmilekic 3872200Sbmilekic/* 3965557Sjasone * Main Entry: witness 4065557Sjasone * Pronunciation: 'wit-n&s 4165557Sjasone * Function: noun 4265557Sjasone * Etymology: Middle English witnesse, from Old English witnes knowledge, 4365557Sjasone * testimony, witness, from 2wit 4465557Sjasone * Date: before 12th century 4565557Sjasone * 1 : attestation of a fact or event : TESTIMONY 4665557Sjasone * 2 : one that gives evidence; specifically : one who testifies in 4765557Sjasone * a cause or before a judicial tribunal 4865557Sjasone * 3 : one asked to be present at a transaction so as to be able to 4965557Sjasone * testify to its having taken place 5065557Sjasone * 4 : one who has personal knowledge of something 5165557Sjasone * 5 a : something serving as evidence or proof : SIGN 5265557Sjasone * b : public affirmation by word or example of usually 5365557Sjasone * religious faith or conviction <the heroic witness to divine 5465557Sjasone * life -- Pilot> 5565557Sjasone * 6 capitalized : a member of the Jehovah's Witnesses 5665557Sjasone */ 5765557Sjasone 5868790Sjhb#include "opt_ddb.h" 5967676Sjhb 6065557Sjasone#include <sys/param.h> 6167352Sjhb#include <sys/bus.h> 6267352Sjhb#include <sys/kernel.h> 6376166Smarkm#include <sys/lock.h> 6467352Sjhb#include <sys/malloc.h> 6574912Sjhb#include <sys/mutex.h> 6665557Sjasone#include <sys/proc.h> 6767676Sjhb#include <sys/sysctl.h> 6865557Sjasone#include <sys/systm.h> 6967352Sjhb#include <sys/vmmeter.h> 7065557Sjasone#include <sys/ktr.h> 7165557Sjasone 7267352Sjhb#include <machine/atomic.h> 7367352Sjhb#include <machine/bus.h> 7467352Sjhb#include <machine/clock.h> 7565557Sjasone#include <machine/cpu.h> 7667352Sjhb 7768790Sjhb#include <ddb/ddb.h> 7868790Sjhb 7967352Sjhb#include <vm/vm.h> 8067352Sjhb#include <vm/vm_extern.h> 8167352Sjhb 8265557Sjasone/* 8372200Sbmilekic * Internal utility macros. 8471352Sjasone */ 8572200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 8671352Sjasone 8772200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 8872200Sbmilekic : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) 8971352Sjasone 9072376Sjake#define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) 9171352Sjasone 9271352Sjasone/* 9374912Sjhb * Lock classes for sleep and spin mutexes. 9471352Sjasone */ 9574912Sjhbstruct lock_class lock_class_mtx_sleep = { 9674912Sjhb "sleep mutex", 9774912Sjhb LC_SLEEPLOCK | LC_RECURSABLE 9874912Sjhb}; 9974912Sjhbstruct lock_class lock_class_mtx_spin = { 10074912Sjhb "spin mutex", 10174912Sjhb LC_SPINLOCK | LC_RECURSABLE 10274912Sjhb}; 10371352Sjasone 10471352Sjasone/* 10572200Sbmilekic * Prototypes for non-exported routines. 10672200Sbmilekic */ 10771352Sjasonestatic void propagate_priority(struct proc *); 10867352Sjhb 10967352Sjhbstatic void 11067352Sjhbpropagate_priority(struct proc *p) 11167352Sjhb{ 11272376Sjake int pri = p->p_pri.pri_level; 11367352Sjhb struct mtx *m = p->p_blocked; 11467352Sjhb 11569376Sjhb mtx_assert(&sched_lock, MA_OWNED); 11667352Sjhb for (;;) { 11767352Sjhb struct proc *p1; 11867352Sjhb 11967352Sjhb p = mtx_owner(m); 12067352Sjhb 12167352Sjhb if (p == NULL) { 12267352Sjhb /* 12367352Sjhb * This really isn't quite right. Really 12467352Sjhb * ought to bump priority of process that 12567352Sjhb * next acquires the mutex. 12667352Sjhb */ 12767352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 12867352Sjhb return; 12967352Sjhb } 13072200Sbmilekic 13167352Sjhb MPASS(p->p_magic == P_MAGIC); 13269376Sjhb KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 13372376Sjake if (p->p_pri.pri_level <= pri) 13467352Sjhb return; 13569376Sjhb 13667352Sjhb /* 13769376Sjhb * Bump this process' priority. 13869376Sjhb */ 13969376Sjhb SET_PRIO(p, pri); 14069376Sjhb 14169376Sjhb /* 14267352Sjhb * If lock holder is actually running, just bump priority. 14367352Sjhb */ 14472836Sjhb if (p->p_oncpu != NOCPU) { 14575468Smarkm MPASS(p->p_stat == SRUN || p->p_stat == SZOMB || p->p_stat == SSTOP); 14667352Sjhb return; 14767352Sjhb } 14872376Sjake 14973912Sjhb#ifndef SMP 15067352Sjhb /* 15173912Sjhb * For UP, we check to see if p is curproc (this shouldn't 15273912Sjhb * ever happen however as it would mean we are in a deadlock.) 15373912Sjhb */ 15473912Sjhb KASSERT(p != curproc, ("Deadlock detected")); 15573912Sjhb#endif 15673912Sjhb 15773912Sjhb /* 15867352Sjhb * If on run queue move to new run queue, and 15967352Sjhb * quit. 16067352Sjhb */ 16167352Sjhb if (p->p_stat == SRUN) { 16267352Sjhb MPASS(p->p_blocked == NULL); 16367352Sjhb remrunqueue(p); 16467352Sjhb setrunqueue(p); 16567352Sjhb return; 16667352Sjhb } 16767352Sjhb 16867352Sjhb /* 16969376Sjhb * If we aren't blocked on a mutex, we should be. 17067352Sjhb */ 17169376Sjhb KASSERT(p->p_stat == SMTX, ( 17269376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 17369376Sjhb p->p_pid, p->p_comm, p->p_stat, 17474912Sjhb m->mtx_object.lo_name)); 17567352Sjhb 17667352Sjhb /* 17767352Sjhb * Pick up the mutex that p is blocked on. 17867352Sjhb */ 17967352Sjhb m = p->p_blocked; 18067352Sjhb MPASS(m != NULL); 18167352Sjhb 18267352Sjhb /* 18367352Sjhb * Check if the proc needs to be moved up on 18467352Sjhb * the blocked chain 18567352Sjhb */ 18669376Sjhb if (p == TAILQ_FIRST(&m->mtx_blocked)) { 18769376Sjhb continue; 18869376Sjhb } 18972200Sbmilekic 19072376Sjake p1 = TAILQ_PREV(p, procqueue, p_procq); 19172376Sjake if (p1->p_pri.pri_level <= pri) { 19267352Sjhb continue; 19367352Sjhb } 19467352Sjhb 19567352Sjhb /* 19669376Sjhb * Remove proc from blocked chain and determine where 19769376Sjhb * it should be moved up to. Since we know that p1 has 19869376Sjhb * a lower priority than p, we know that at least one 19969376Sjhb * process in the chain has a lower priority and that 20069376Sjhb * p1 will thus not be NULL after the loop. 20167352Sjhb */ 20267352Sjhb TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 20367352Sjhb TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 20467352Sjhb MPASS(p1->p_magic == P_MAGIC); 20572376Sjake if (p1->p_pri.pri_level > pri) 20667352Sjhb break; 20767352Sjhb } 20872200Sbmilekic 20969376Sjhb MPASS(p1 != NULL); 21069376Sjhb TAILQ_INSERT_BEFORE(p1, p, p_procq); 21167352Sjhb CTR4(KTR_LOCK, 21271560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 21374912Sjhb p, p1, m, m->mtx_object.lo_name); 21467352Sjhb } 21567352Sjhb} 21667352Sjhb 21771352Sjasone/* 21874900Sjhb * Function versions of the inlined __mtx_* macros. These are used by 21974900Sjhb * modules and can also be called from assembly language if needed. 22074900Sjhb */ 22174900Sjhbvoid 22274900Sjhb_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 22374900Sjhb{ 22474900Sjhb 22574900Sjhb __mtx_lock_flags(m, opts, file, line); 22674900Sjhb} 22774900Sjhb 22874900Sjhbvoid 22974900Sjhb_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 23074900Sjhb{ 23174900Sjhb 23274900Sjhb __mtx_unlock_flags(m, opts, file, line); 23374900Sjhb} 23474900Sjhb 23574900Sjhbvoid 23674900Sjhb_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 23774900Sjhb{ 23874900Sjhb 23974900Sjhb __mtx_lock_spin_flags(m, opts, file, line); 24074900Sjhb} 24174900Sjhb 24274900Sjhbvoid 24374900Sjhb_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 24474900Sjhb{ 24574900Sjhb 24674900Sjhb __mtx_unlock_spin_flags(m, opts, file, line); 24774900Sjhb} 24874900Sjhb 24974900Sjhb/* 25072200Sbmilekic * The important part of mtx_trylock{,_flags}() 25172200Sbmilekic * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 25272200Sbmilekic * if we're called, it's because we know we don't already own this lock. 25371352Sjasone */ 25472200Sbmilekicint 25572200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 25671352Sjasone{ 25772200Sbmilekic int rval; 25871352Sjasone 25972393Sbmilekic MPASS(curproc != NULL); 26071352Sjasone 26172200Sbmilekic /* 26272200Sbmilekic * _mtx_trylock does not accept MTX_NOSWITCH option. 26372200Sbmilekic */ 26472344Sbmilekic KASSERT((opts & MTX_NOSWITCH) == 0, 26572344Sbmilekic ("mtx_trylock() called with invalid option flag(s) %d", opts)); 26672200Sbmilekic 26772393Sbmilekic rval = _obtain_lock(m, curproc); 26872200Sbmilekic 26974912Sjhb LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 27074912Sjhb if (rval) { 27171352Sjasone /* 27272200Sbmilekic * We do not handle recursion in _mtx_trylock; see the 27372200Sbmilekic * note at the top of the routine. 27471352Sjasone */ 27572344Sbmilekic KASSERT(!mtx_recursed(m), 27672344Sbmilekic ("mtx_trylock() called on a recursed mutex")); 27774912Sjhb mtx_update_flags(m, 1); 27874912Sjhb WITNESS_LOCK(&m->mtx_object, opts | LOP_TRYLOCK, file, line); 27971352Sjasone } 28071352Sjasone 28174912Sjhb return (rval); 28271352Sjasone} 28371352Sjasone 28471352Sjasone/* 28572200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 28671352Sjasone * 28772200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 28872200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 28971352Sjasone */ 29072200Sbmilekicvoid 29172200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 29271352Sjasone{ 29372393Sbmilekic struct proc *p = curproc; 29471352Sjasone 29572200Sbmilekic if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 29672200Sbmilekic m->mtx_recurse++; 29772200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 29874912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 29972344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 30072200Sbmilekic return; 30171352Sjasone } 30271352Sjasone 30374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 30472994Sjhb CTR4(KTR_LOCK, 30572994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 30674912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 30771352Sjasone 30872200Sbmilekic while (!_obtain_lock(m, p)) { 30972200Sbmilekic uintptr_t v; 31072200Sbmilekic struct proc *p1; 31171352Sjasone 31272200Sbmilekic mtx_lock_spin(&sched_lock); 31372200Sbmilekic /* 31472200Sbmilekic * Check if the lock has been released while spinning for 31572200Sbmilekic * the sched_lock. 31672200Sbmilekic */ 31772200Sbmilekic if ((v = m->mtx_lock) == MTX_UNOWNED) { 31872200Sbmilekic mtx_unlock_spin(&sched_lock); 31972200Sbmilekic continue; 32071352Sjasone } 32171352Sjasone 32272200Sbmilekic /* 32372200Sbmilekic * The mutex was marked contested on release. This means that 32472200Sbmilekic * there are processes blocked on it. 32572200Sbmilekic */ 32672200Sbmilekic if (v == MTX_CONTESTED) { 32772200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 32872344Sbmilekic MPASS(p1 != NULL); 32972200Sbmilekic m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; 33067352Sjhb 33172376Sjake if (p1->p_pri.pri_level < p->p_pri.pri_level) 33272376Sjake SET_PRIO(p, p1->p_pri.pri_level); 33372200Sbmilekic mtx_unlock_spin(&sched_lock); 33467352Sjhb return; 33567352Sjhb } 33669376Sjhb 33769376Sjhb /* 33872200Sbmilekic * If the mutex isn't already contested and a failure occurs 33972200Sbmilekic * setting the contested bit, the mutex was either released 34072200Sbmilekic * or the state of the MTX_RECURSED bit changed. 34169376Sjhb */ 34272200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 34372200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 34472200Sbmilekic (void *)(v | MTX_CONTESTED))) { 34572200Sbmilekic mtx_unlock_spin(&sched_lock); 34672200Sbmilekic continue; 34772200Sbmilekic } 34867352Sjhb 34972200Sbmilekic /* 35072200Sbmilekic * We deffinately must sleep for this lock. 35172200Sbmilekic */ 35272200Sbmilekic mtx_assert(m, MA_NOTOWNED); 35367352Sjhb 35467352Sjhb#ifdef notyet 35572200Sbmilekic /* 35672200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 35772200Sbmilekic * must clean up before going to sleep. 35872200Sbmilekic */ 35972994Sjhb if (p->p_ithd != NULL) { 36072994Sjhb struct ithd *it = p->p_ithd; 36167352Sjhb 36272200Sbmilekic if (it->it_interrupted) { 36374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 36472200Sbmilekic CTR2(KTR_LOCK, 36572994Sjhb "_mtx_lock_sleep: %p interrupted %p", 36672200Sbmilekic it, it->it_interrupted); 36772200Sbmilekic intr_thd_fixup(it); 36867352Sjhb } 36972200Sbmilekic } 37067352Sjhb#endif 37167352Sjhb 37272200Sbmilekic /* 37372200Sbmilekic * Put us on the list of threads blocked on this mutex. 37472200Sbmilekic */ 37572200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 37672200Sbmilekic p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); 37772200Sbmilekic LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); 37872200Sbmilekic TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 37972200Sbmilekic } else { 38072200Sbmilekic TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 38172376Sjake if (p1->p_pri.pri_level > p->p_pri.pri_level) 38272200Sbmilekic break; 38372200Sbmilekic if (p1) 38472200Sbmilekic TAILQ_INSERT_BEFORE(p1, p, p_procq); 38572200Sbmilekic else 38667352Sjhb TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 38772200Sbmilekic } 38867352Sjhb 38972200Sbmilekic /* 39072200Sbmilekic * Save who we're blocked on. 39172200Sbmilekic */ 39272200Sbmilekic p->p_blocked = m; 39374912Sjhb p->p_mtxname = m->mtx_object.lo_name; 39472200Sbmilekic p->p_stat = SMTX; 39572200Sbmilekic propagate_priority(p); 39667352Sjhb 39774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 39872200Sbmilekic CTR3(KTR_LOCK, 39972200Sbmilekic "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, 40074912Sjhb m->mtx_object.lo_name); 40172200Sbmilekic 40272200Sbmilekic mi_switch(); 40372200Sbmilekic 40474912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 40572200Sbmilekic CTR3(KTR_LOCK, 40672200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 40774912Sjhb p, m, m->mtx_object.lo_name); 40872200Sbmilekic 40972200Sbmilekic mtx_unlock_spin(&sched_lock); 41072200Sbmilekic } 41172200Sbmilekic 41272200Sbmilekic return; 41372200Sbmilekic} 41472200Sbmilekic 41572200Sbmilekic/* 41672200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 41772200Sbmilekic * 41872200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 41972200Sbmilekic * is handled inline. 42072200Sbmilekic */ 42172200Sbmilekicvoid 42274900Sjhb_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, 42372200Sbmilekic int line) 42472200Sbmilekic{ 42572200Sbmilekic int i = 0; 42672200Sbmilekic 42774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 42872344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 42972200Sbmilekic 43072200Sbmilekic for (;;) { 43172393Sbmilekic if (_obtain_lock(m, curproc)) 43272200Sbmilekic break; 43372200Sbmilekic 43475568Sjhb /* Give interrupts a chance while we spin. */ 43575568Sjhb critical_exit(mtx_crit); 43672200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 43772200Sbmilekic if (i++ < 1000000) 43872200Sbmilekic continue; 43972200Sbmilekic if (i++ < 6000000) 44072200Sbmilekic DELAY(1); 44167352Sjhb#ifdef DDB 44272200Sbmilekic else if (!db_active) 44367352Sjhb#else 44472200Sbmilekic else 44567352Sjhb#endif 44672200Sbmilekic panic("spin lock %s held by %p for > 5 seconds", 44774912Sjhb m->mtx_object.lo_name, (void *)m->mtx_lock); 44867352Sjhb } 44975568Sjhb mtx_crit = critical_enter(); 45067352Sjhb } 45172200Sbmilekic 45274900Sjhb m->mtx_savecrit = mtx_crit; 45374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 45472200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 45572200Sbmilekic 45672200Sbmilekic return; 45767352Sjhb} 45867352Sjhb 45972200Sbmilekic/* 46072200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 46172200Sbmilekic * 46272200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 46372200Sbmilekic * need to wake up a blocked thread). 46472200Sbmilekic */ 46567352Sjhbvoid 46672200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 46767352Sjhb{ 46867352Sjhb struct proc *p, *p1; 46967352Sjhb struct mtx *m1; 47067352Sjhb int pri; 47167352Sjhb 47272393Sbmilekic p = curproc; 47372200Sbmilekic 47472200Sbmilekic if (mtx_recursed(m)) { 47572200Sbmilekic if (--(m->mtx_recurse) == 0) 47672200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 47774912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 47872200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 47972200Sbmilekic return; 48072200Sbmilekic } 48172200Sbmilekic 48272200Sbmilekic mtx_lock_spin(&sched_lock); 48374912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 48472200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 48572200Sbmilekic 48672200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 48772200Sbmilekic MPASS(p->p_magic == P_MAGIC); 48872200Sbmilekic MPASS(p1->p_magic == P_MAGIC); 48972200Sbmilekic 49072200Sbmilekic TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 49172200Sbmilekic 49272200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 49372200Sbmilekic LIST_REMOVE(m, mtx_contested); 49472200Sbmilekic _release_lock_quick(m); 49574912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 49672200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 49772200Sbmilekic } else 49872200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 49972200Sbmilekic 50072376Sjake pri = PRI_MAX; 50172200Sbmilekic LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 50272376Sjake int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level; 50372200Sbmilekic if (cp < pri) 50472200Sbmilekic pri = cp; 50572200Sbmilekic } 50672200Sbmilekic 50772376Sjake if (pri > p->p_pri.pri_native) 50872376Sjake pri = p->p_pri.pri_native; 50972200Sbmilekic SET_PRIO(p, pri); 51072200Sbmilekic 51174912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 51272200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 51372200Sbmilekic m, p1); 51472200Sbmilekic 51572200Sbmilekic p1->p_blocked = NULL; 51672200Sbmilekic p1->p_stat = SRUN; 51772200Sbmilekic setrunqueue(p1); 51872200Sbmilekic 51972376Sjake if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) { 52067352Sjhb#ifdef notyet 52172994Sjhb if (p->p_ithd != NULL) { 52272994Sjhb struct ithd *it = p->p_ithd; 52367352Sjhb 52472200Sbmilekic if (it->it_interrupted) { 52574912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 52672200Sbmilekic CTR2(KTR_LOCK, 52772994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 52872200Sbmilekic it, it->it_interrupted); 52972200Sbmilekic intr_thd_fixup(it); 53067352Sjhb } 53172200Sbmilekic } 53267352Sjhb#endif 53372200Sbmilekic setrunqueue(p); 53474912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 53572200Sbmilekic CTR2(KTR_LOCK, 53672200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 53772200Sbmilekic (void *)m->mtx_lock); 53872200Sbmilekic 53972200Sbmilekic mi_switch(); 54074912Sjhb if (LOCK_LOG_TEST(&m->mtx_object, opts)) 54172200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 54272200Sbmilekic m, (void *)m->mtx_lock); 54367352Sjhb } 54472200Sbmilekic 54572200Sbmilekic mtx_unlock_spin(&sched_lock); 54672200Sbmilekic 54772200Sbmilekic return; 54867352Sjhb} 54967352Sjhb 55072200Sbmilekic/* 55172200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 55272200Sbmilekic * See the _rel_spin_lock() macro for the details. 55372200Sbmilekic */ 55472200Sbmilekic 55574912Sjhb#ifdef WITNESS 55672200Sbmilekic/* 55774912Sjhb * Update the lock object flags before calling witness. Note that when we 55874912Sjhb * lock a mutex, this is called after getting the lock, but when unlocking 55974912Sjhb * a mutex, this function is called before releasing the lock. 56074912Sjhb */ 56174912Sjhbvoid 56274912Sjhb_mtx_update_flags(struct mtx *m, int locking) 56374912Sjhb{ 56474912Sjhb 56574912Sjhb mtx_assert(m, MA_OWNED); 56674912Sjhb if (locking) { 56774912Sjhb m->mtx_object.lo_flags |= LO_LOCKED; 56874912Sjhb if (mtx_recursed(m)) 56974912Sjhb m->mtx_object.lo_flags |= LO_RECURSED; 57074912Sjhb else 57174912Sjhb /* XXX: we shouldn't need this in theory. */ 57274912Sjhb m->mtx_object.lo_flags &= ~LO_RECURSED; 57374912Sjhb } else { 57474912Sjhb switch (m->mtx_recurse) { 57574912Sjhb case 0: 57674912Sjhb /* XXX: we shouldn't need the LO_RECURSED in theory. */ 57774912Sjhb m->mtx_object.lo_flags &= ~(LO_LOCKED | LO_RECURSED); 57874912Sjhb break; 57974912Sjhb case 1: 58074912Sjhb m->mtx_object.lo_flags &= ~(LO_RECURSED); 58174912Sjhb break; 58274912Sjhb default: 58374912Sjhb break; 58474912Sjhb } 58574912Sjhb } 58674912Sjhb} 58774912Sjhb#endif 58874912Sjhb 58974912Sjhb/* 59072994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 59172200Sbmilekic */ 59272996Sjhb#ifdef INVARIANT_SUPPORT 59371352Sjasonevoid 59471360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 59571352Sjasone{ 59673033Sjake switch (what) { 59771352Sjasone case MA_OWNED: 59871352Sjasone case MA_OWNED | MA_RECURSED: 59971352Sjasone case MA_OWNED | MA_NOTRECURSED: 60073033Sjake if (!mtx_owned(m)) 60171352Sjasone panic("mutex %s not owned at %s:%d", 60274912Sjhb m->mtx_object.lo_name, file, line); 60373033Sjake if (mtx_recursed(m)) { 60473033Sjake if ((what & MA_NOTRECURSED) != 0) 60571352Sjasone panic("mutex %s recursed at %s:%d", 60674912Sjhb m->mtx_object.lo_name, file, line); 60773033Sjake } else if ((what & MA_RECURSED) != 0) { 60871352Sjasone panic("mutex %s unrecursed at %s:%d", 60974912Sjhb m->mtx_object.lo_name, file, line); 61071352Sjasone } 61171352Sjasone break; 61271352Sjasone case MA_NOTOWNED: 61373033Sjake if (mtx_owned(m)) 61471352Sjasone panic("mutex %s owned at %s:%d", 61574912Sjhb m->mtx_object.lo_name, file, line); 61671352Sjasone break; 61771352Sjasone default: 61871360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 61971352Sjasone } 62071352Sjasone} 62171352Sjasone#endif 62271352Sjasone 62372200Sbmilekic/* 62472200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 62574912Sjhb * 62674912Sjhb * Most of these checks have been moved off into the LO_INITIALIZED flag 62774912Sjhb * maintained by the witness code. 62872200Sbmilekic */ 62967352Sjhb#ifdef MUTEX_DEBUG 63067352Sjhb 63174912Sjhbvoid mtx_validate __P((struct mtx *)); 63267352Sjhb 63374912Sjhbvoid 63474912Sjhbmtx_validate(struct mtx *m) 63567352Sjhb{ 63667352Sjhb 63767352Sjhb/* 63867352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 63967352Sjhb * we can re-enable the kernacc() checks. 64067352Sjhb */ 64167352Sjhb#ifndef __alpha__ 64274912Sjhb if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE)) 64374912Sjhb panic("Can't read and write to mutex %p", m); 64467352Sjhb#endif 64567352Sjhb} 64667352Sjhb#endif 64767352Sjhb 64872200Sbmilekic/* 64972200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 65072200Sbmilekic * `opts' with options contained in `opts' and description `description.' 65172200Sbmilekic */ 65267352Sjhbvoid 65372200Sbmilekicmtx_init(struct mtx *m, const char *description, int opts) 65467352Sjhb{ 65574912Sjhb struct lock_object *lock; 65672200Sbmilekic 65774912Sjhb MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 65874912Sjhb MTX_SLEEPABLE | MTX_NOWITNESS)) == 0); 65972200Sbmilekic 66067352Sjhb#ifdef MUTEX_DEBUG 66172200Sbmilekic /* Diagnostic and error correction */ 66274912Sjhb mtx_validate(m); 66369429Sjhb#endif 66467352Sjhb 66574912Sjhb bzero(m, sizeof(*m)); 66674912Sjhb lock = &m->mtx_object; 66774912Sjhb if (opts & MTX_SPIN) 66874912Sjhb lock->lo_class = &lock_class_mtx_spin; 66974912Sjhb else 67074912Sjhb lock->lo_class = &lock_class_mtx_sleep; 67174912Sjhb lock->lo_name = description; 67274912Sjhb if (opts & MTX_QUIET) 67374912Sjhb lock->lo_flags = LO_QUIET; 67474912Sjhb if (opts & MTX_RECURSE) 67574912Sjhb lock->lo_flags |= LO_RECURSABLE; 67674912Sjhb if (opts & MTX_SLEEPABLE) 67774912Sjhb lock->lo_flags |= LO_SLEEPABLE; 67874912Sjhb if ((opts & MTX_NOWITNESS) == 0) 67974912Sjhb lock->lo_flags |= LO_WITNESS; 68072200Sbmilekic 68167352Sjhb m->mtx_lock = MTX_UNOWNED; 68274912Sjhb TAILQ_INIT(&m->mtx_blocked); 68372200Sbmilekic 68474912Sjhb LOCK_LOG_INIT(lock, opts); 68572200Sbmilekic 68674912Sjhb WITNESS_INIT(lock); 68767352Sjhb} 68867352Sjhb 68972200Sbmilekic/* 69074912Sjhb * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 69174912Sjhb * passed in as a flag here because if the corresponding mtx_init() was 69274912Sjhb * called with MTX_QUIET set, then it will already be set in the mutex's 69374912Sjhb * flags. 69472200Sbmilekic */ 69567352Sjhbvoid 69667352Sjhbmtx_destroy(struct mtx *m) 69767352Sjhb{ 69867352Sjhb 69974912Sjhb LOCK_LOG_DESTROY(&m->mtx_object, 0); 70072200Sbmilekic 70174912Sjhb if (!mtx_owned(m)) 70274912Sjhb MPASS(mtx_unowned(m)); 70374912Sjhb else { 70471228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 70572200Sbmilekic 70674912Sjhb /* Tell witness this isn't locked to make it happy. */ 70774912Sjhb m->mtx_object.lo_flags &= ~LO_LOCKED; 70874912Sjhb WITNESS_UNLOCK(&m->mtx_object, MTX_NOSWITCH, __FILE__, 70974912Sjhb __LINE__); 71071320Sjasone } 71171320Sjasone 71274912Sjhb WITNESS_DESTROY(&m->mtx_object); 71371320Sjasone} 714