subr_witness.c revision 74016
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone * $FreeBSD: head/sys/kern/subr_witness.c 74016 2001-03-09 07:24:17Z jhb $ 3165557Sjasone */ 3265557Sjasone 3365557Sjasone/* 3472200Sbmilekic * Machine independent bits of mutex implementation and implementation of 3572200Sbmilekic * `witness' structure & related debugging routines. 3672200Sbmilekic */ 3772200Sbmilekic 3872200Sbmilekic/* 3965557Sjasone * Main Entry: witness 4065557Sjasone * Pronunciation: 'wit-n&s 4165557Sjasone * Function: noun 4265557Sjasone * Etymology: Middle English witnesse, from Old English witnes knowledge, 4365557Sjasone * testimony, witness, from 2wit 4465557Sjasone * Date: before 12th century 4565557Sjasone * 1 : attestation of a fact or event : TESTIMONY 4665557Sjasone * 2 : one that gives evidence; specifically : one who testifies in 4765557Sjasone * a cause or before a judicial tribunal 4865557Sjasone * 3 : one asked to be present at a transaction so as to be able to 4965557Sjasone * testify to its having taken place 5065557Sjasone * 4 : one who has personal knowledge of something 5165557Sjasone * 5 a : something serving as evidence or proof : SIGN 5265557Sjasone * b : public affirmation by word or example of usually 5365557Sjasone * religious faith or conviction <the heroic witness to divine 5465557Sjasone * life -- Pilot> 5565557Sjasone * 6 capitalized : a member of the Jehovah's Witnesses 5665557Sjasone */ 5765557Sjasone 5868790Sjhb#include "opt_ddb.h" 5967676Sjhb#include "opt_witness.h" 6067676Sjhb 6165557Sjasone#include <sys/param.h> 6267352Sjhb#include <sys/bus.h> 6367352Sjhb#include <sys/kernel.h> 6467352Sjhb#include <sys/malloc.h> 6565557Sjasone#include <sys/proc.h> 6667676Sjhb#include <sys/sysctl.h> 6765557Sjasone#include <sys/systm.h> 6867352Sjhb#include <sys/vmmeter.h> 6965557Sjasone#include <sys/ktr.h> 7065557Sjasone 7167352Sjhb#include <machine/atomic.h> 7267352Sjhb#include <machine/bus.h> 7367352Sjhb#include <machine/clock.h> 7465557Sjasone#include <machine/cpu.h> 7567352Sjhb 7668790Sjhb#include <ddb/ddb.h> 7768790Sjhb 7867352Sjhb#include <vm/vm.h> 7967352Sjhb#include <vm/vm_extern.h> 8067352Sjhb 8167352Sjhb#include <sys/mutex.h> 8265557Sjasone 8365557Sjasone/* 8472200Sbmilekic * The WITNESS-enabled mutex debug structure. 8567352Sjhb */ 8671352Sjasone#ifdef WITNESS 8771352Sjasonestruct mtx_debug { 8871352Sjasone struct witness *mtxd_witness; 8971352Sjasone LIST_ENTRY(mtx) mtxd_held; 9071352Sjasone const char *mtxd_file; 9171352Sjasone int mtxd_line; 9271352Sjasone}; 9371352Sjasone 9471560Sjhb#define mtx_held mtx_debug->mtxd_held 9571560Sjhb#define mtx_file mtx_debug->mtxd_file 9671560Sjhb#define mtx_line mtx_debug->mtxd_line 9771560Sjhb#define mtx_witness mtx_debug->mtxd_witness 9871352Sjasone#endif /* WITNESS */ 9971352Sjasone 10071352Sjasone/* 10172200Sbmilekic * Internal utility macros. 10271352Sjasone */ 10372200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 10471352Sjasone 10572200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 10672200Sbmilekic : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) 10771352Sjasone 10872376Sjake#define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) 10971352Sjasone 11071352Sjasone/* 11172200Sbmilekic * Early WITNESS-enabled declarations. 11271352Sjasone */ 11372200Sbmilekic#ifdef WITNESS 11471352Sjasone 11571352Sjasone/* 11672200Sbmilekic * Internal WITNESS routines which must be prototyped early. 11772200Sbmilekic * 11872200Sbmilekic * XXX: When/if witness code is cleaned up, it would be wise to place all 11972200Sbmilekic * witness prototyping early in this file. 12072200Sbmilekic */ 12172200Sbmilekicstatic void witness_init(struct mtx *, int flag); 12272200Sbmilekicstatic void witness_destroy(struct mtx *); 12372200Sbmilekicstatic void witness_display(void(*)(const char *fmt, ...)); 12471352Sjasone 12572200SbmilekicMALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure"); 12671352Sjasone 12767352Sjhb/* All mutexes in system (used for debug/panic) */ 12871560Sjhbstatic struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 }; 12972200Sbmilekic 13071320Sjasone/* 13172200Sbmilekic * This global is set to 0 once it becomes safe to use the witness code. 13271320Sjasone */ 13371320Sjasonestatic int witness_cold = 1; 13472200Sbmilekic 13569429Sjhb#else /* WITNESS */ 13671352Sjasone 13772200Sbmilekic/* XXX XXX XXX 13872200Sbmilekic * flag++ is sleazoid way of shuting up warning 13971352Sjasone */ 14071352Sjasone#define witness_init(m, flag) flag++ 14171352Sjasone#define witness_destroy(m) 14271352Sjasone#define witness_try_enter(m, t, f, l) 14369429Sjhb#endif /* WITNESS */ 14467352Sjhb 14572200Sbmilekic/* 14672200Sbmilekic * All mutex locks in system are kept on the all_mtx list. 14772200Sbmilekic */ 14871560Sjhbstatic struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", 14971560Sjhb TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), 15071560Sjhb { NULL, NULL }, &all_mtx, &all_mtx, 15171560Sjhb#ifdef WITNESS 15271560Sjhb &all_mtx_debug 15371560Sjhb#else 15471560Sjhb NULL 15571560Sjhb#endif 15671560Sjhb }; 15771560Sjhb 15872200Sbmilekic/* 15972200Sbmilekic * Global variables for book keeping. 16072200Sbmilekic */ 16167352Sjhbstatic int mtx_cur_cnt; 16267352Sjhbstatic int mtx_max_cnt; 16367352Sjhb 16472200Sbmilekic/* 16572344Sbmilekic * Couple of strings for KTR_LOCK tracing in order to avoid duplicates. 16672344Sbmilekic */ 16772344Sbmilekicchar STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d"; 16872344Sbmilekicchar STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d"; 16972344Sbmilekicchar STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d"; 17072344Sbmilekicchar STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d"; 17172344Sbmilekic 17272344Sbmilekic/* 17372200Sbmilekic * Prototypes for non-exported routines. 17472200Sbmilekic * 17572200Sbmilekic * NOTE: Prototypes for witness routines are placed at the bottom of the file. 17672200Sbmilekic */ 17771352Sjasonestatic void propagate_priority(struct proc *); 17867352Sjhb 17967352Sjhbstatic void 18067352Sjhbpropagate_priority(struct proc *p) 18167352Sjhb{ 18272376Sjake int pri = p->p_pri.pri_level; 18367352Sjhb struct mtx *m = p->p_blocked; 18467352Sjhb 18569376Sjhb mtx_assert(&sched_lock, MA_OWNED); 18667352Sjhb for (;;) { 18767352Sjhb struct proc *p1; 18867352Sjhb 18967352Sjhb p = mtx_owner(m); 19067352Sjhb 19167352Sjhb if (p == NULL) { 19267352Sjhb /* 19367352Sjhb * This really isn't quite right. Really 19467352Sjhb * ought to bump priority of process that 19567352Sjhb * next acquires the mutex. 19667352Sjhb */ 19767352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 19867352Sjhb return; 19967352Sjhb } 20072200Sbmilekic 20167352Sjhb MPASS(p->p_magic == P_MAGIC); 20269376Sjhb KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 20372376Sjake if (p->p_pri.pri_level <= pri) 20467352Sjhb return; 20569376Sjhb 20667352Sjhb /* 20769376Sjhb * Bump this process' priority. 20869376Sjhb */ 20969376Sjhb SET_PRIO(p, pri); 21069376Sjhb 21169376Sjhb /* 21267352Sjhb * If lock holder is actually running, just bump priority. 21367352Sjhb */ 21472836Sjhb if (p->p_oncpu != NOCPU) { 21567352Sjhb MPASS(p->p_stat == SRUN || p->p_stat == SZOMB); 21667352Sjhb return; 21767352Sjhb } 21872376Sjake 21973912Sjhb#ifndef SMP 22067352Sjhb /* 22173912Sjhb * For UP, we check to see if p is curproc (this shouldn't 22273912Sjhb * ever happen however as it would mean we are in a deadlock.) 22373912Sjhb */ 22473912Sjhb KASSERT(p != curproc, ("Deadlock detected")); 22573912Sjhb#endif 22673912Sjhb 22773912Sjhb /* 22867352Sjhb * If on run queue move to new run queue, and 22967352Sjhb * quit. 23067352Sjhb */ 23167352Sjhb if (p->p_stat == SRUN) { 23267352Sjhb MPASS(p->p_blocked == NULL); 23367352Sjhb remrunqueue(p); 23467352Sjhb setrunqueue(p); 23567352Sjhb return; 23667352Sjhb } 23767352Sjhb 23867352Sjhb /* 23969376Sjhb * If we aren't blocked on a mutex, we should be. 24067352Sjhb */ 24169376Sjhb KASSERT(p->p_stat == SMTX, ( 24269376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 24369376Sjhb p->p_pid, p->p_comm, p->p_stat, 24469376Sjhb m->mtx_description)); 24567352Sjhb 24667352Sjhb /* 24767352Sjhb * Pick up the mutex that p is blocked on. 24867352Sjhb */ 24967352Sjhb m = p->p_blocked; 25067352Sjhb MPASS(m != NULL); 25167352Sjhb 25267352Sjhb /* 25367352Sjhb * Check if the proc needs to be moved up on 25467352Sjhb * the blocked chain 25567352Sjhb */ 25669376Sjhb if (p == TAILQ_FIRST(&m->mtx_blocked)) { 25769376Sjhb continue; 25869376Sjhb } 25972200Sbmilekic 26072376Sjake p1 = TAILQ_PREV(p, procqueue, p_procq); 26172376Sjake if (p1->p_pri.pri_level <= pri) { 26267352Sjhb continue; 26367352Sjhb } 26467352Sjhb 26567352Sjhb /* 26669376Sjhb * Remove proc from blocked chain and determine where 26769376Sjhb * it should be moved up to. Since we know that p1 has 26869376Sjhb * a lower priority than p, we know that at least one 26969376Sjhb * process in the chain has a lower priority and that 27069376Sjhb * p1 will thus not be NULL after the loop. 27167352Sjhb */ 27267352Sjhb TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 27367352Sjhb TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 27467352Sjhb MPASS(p1->p_magic == P_MAGIC); 27572376Sjake if (p1->p_pri.pri_level > pri) 27667352Sjhb break; 27767352Sjhb } 27872200Sbmilekic 27969376Sjhb MPASS(p1 != NULL); 28069376Sjhb TAILQ_INSERT_BEFORE(p1, p, p_procq); 28167352Sjhb CTR4(KTR_LOCK, 28271560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 28367352Sjhb p, p1, m, m->mtx_description); 28467352Sjhb } 28567352Sjhb} 28667352Sjhb 28771352Sjasone/* 28872200Sbmilekic * The important part of mtx_trylock{,_flags}() 28972200Sbmilekic * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 29072200Sbmilekic * if we're called, it's because we know we don't already own this lock. 29171352Sjasone */ 29272200Sbmilekicint 29372200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 29471352Sjasone{ 29572200Sbmilekic int rval; 29671352Sjasone 29772393Sbmilekic MPASS(curproc != NULL); 29871352Sjasone 29972200Sbmilekic /* 30072200Sbmilekic * _mtx_trylock does not accept MTX_NOSWITCH option. 30172200Sbmilekic */ 30272344Sbmilekic KASSERT((opts & MTX_NOSWITCH) == 0, 30372344Sbmilekic ("mtx_trylock() called with invalid option flag(s) %d", opts)); 30472200Sbmilekic 30572393Sbmilekic rval = _obtain_lock(m, curproc); 30672200Sbmilekic 30772200Sbmilekic#ifdef WITNESS 30872200Sbmilekic if (rval && m->mtx_witness != NULL) { 30971352Sjasone /* 31072200Sbmilekic * We do not handle recursion in _mtx_trylock; see the 31172200Sbmilekic * note at the top of the routine. 31271352Sjasone */ 31372344Sbmilekic KASSERT(!mtx_recursed(m), 31472344Sbmilekic ("mtx_trylock() called on a recursed mutex")); 31572200Sbmilekic witness_try_enter(m, (opts | m->mtx_flags), file, line); 31671352Sjasone } 31772200Sbmilekic#endif /* WITNESS */ 31871352Sjasone 31972200Sbmilekic if ((opts & MTX_QUIET) == 0) 32072994Sjhb CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d", 32172200Sbmilekic m->mtx_description, m, rval, file, line); 32272200Sbmilekic 32372200Sbmilekic return rval; 32471352Sjasone} 32571352Sjasone 32671352Sjasone/* 32772200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 32871352Sjasone * 32972200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 33072200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 33171352Sjasone */ 33272200Sbmilekicvoid 33372200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 33471352Sjasone{ 33572393Sbmilekic struct proc *p = curproc; 33671352Sjasone 33772200Sbmilekic if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 33872200Sbmilekic m->mtx_recurse++; 33972200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 34072200Sbmilekic if ((opts & MTX_QUIET) == 0) 34172344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 34272200Sbmilekic return; 34371352Sjasone } 34471352Sjasone 34572200Sbmilekic if ((opts & MTX_QUIET) == 0) 34672994Sjhb CTR4(KTR_LOCK, 34772994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 34872994Sjhb m->mtx_description, (void *)m->mtx_lock, file, line); 34971352Sjasone 35072200Sbmilekic while (!_obtain_lock(m, p)) { 35172200Sbmilekic uintptr_t v; 35272200Sbmilekic struct proc *p1; 35371352Sjasone 35472200Sbmilekic mtx_lock_spin(&sched_lock); 35572200Sbmilekic /* 35672200Sbmilekic * Check if the lock has been released while spinning for 35772200Sbmilekic * the sched_lock. 35872200Sbmilekic */ 35972200Sbmilekic if ((v = m->mtx_lock) == MTX_UNOWNED) { 36072200Sbmilekic mtx_unlock_spin(&sched_lock); 36172200Sbmilekic continue; 36271352Sjasone } 36371352Sjasone 36472200Sbmilekic /* 36572200Sbmilekic * The mutex was marked contested on release. This means that 36672200Sbmilekic * there are processes blocked on it. 36772200Sbmilekic */ 36872200Sbmilekic if (v == MTX_CONTESTED) { 36972200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 37072344Sbmilekic MPASS(p1 != NULL); 37172200Sbmilekic m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; 37267352Sjhb 37372376Sjake if (p1->p_pri.pri_level < p->p_pri.pri_level) 37472376Sjake SET_PRIO(p, p1->p_pri.pri_level); 37572200Sbmilekic mtx_unlock_spin(&sched_lock); 37667352Sjhb return; 37767352Sjhb } 37869376Sjhb 37969376Sjhb /* 38072200Sbmilekic * If the mutex isn't already contested and a failure occurs 38172200Sbmilekic * setting the contested bit, the mutex was either released 38272200Sbmilekic * or the state of the MTX_RECURSED bit changed. 38369376Sjhb */ 38472200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 38572200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 38672200Sbmilekic (void *)(v | MTX_CONTESTED))) { 38772200Sbmilekic mtx_unlock_spin(&sched_lock); 38872200Sbmilekic continue; 38972200Sbmilekic } 39067352Sjhb 39172200Sbmilekic /* 39272200Sbmilekic * We deffinately must sleep for this lock. 39372200Sbmilekic */ 39472200Sbmilekic mtx_assert(m, MA_NOTOWNED); 39567352Sjhb 39667352Sjhb#ifdef notyet 39772200Sbmilekic /* 39872200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 39972200Sbmilekic * must clean up before going to sleep. 40072200Sbmilekic */ 40172994Sjhb if (p->p_ithd != NULL) { 40272994Sjhb struct ithd *it = p->p_ithd; 40367352Sjhb 40472200Sbmilekic if (it->it_interrupted) { 40572200Sbmilekic if ((opts & MTX_QUIET) == 0) 40672200Sbmilekic CTR2(KTR_LOCK, 40772994Sjhb "_mtx_lock_sleep: %p interrupted %p", 40872200Sbmilekic it, it->it_interrupted); 40972200Sbmilekic intr_thd_fixup(it); 41067352Sjhb } 41172200Sbmilekic } 41267352Sjhb#endif 41367352Sjhb 41472200Sbmilekic /* 41572200Sbmilekic * Put us on the list of threads blocked on this mutex. 41672200Sbmilekic */ 41772200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 41872200Sbmilekic p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); 41972200Sbmilekic LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); 42072200Sbmilekic TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 42172200Sbmilekic } else { 42272200Sbmilekic TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 42372376Sjake if (p1->p_pri.pri_level > p->p_pri.pri_level) 42472200Sbmilekic break; 42572200Sbmilekic if (p1) 42672200Sbmilekic TAILQ_INSERT_BEFORE(p1, p, p_procq); 42772200Sbmilekic else 42867352Sjhb TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 42972200Sbmilekic } 43067352Sjhb 43172200Sbmilekic /* 43272200Sbmilekic * Save who we're blocked on. 43372200Sbmilekic */ 43472200Sbmilekic p->p_blocked = m; 43572200Sbmilekic p->p_mtxname = m->mtx_description; 43672200Sbmilekic p->p_stat = SMTX; 43772200Sbmilekic propagate_priority(p); 43867352Sjhb 43972200Sbmilekic if ((opts & MTX_QUIET) == 0) 44072200Sbmilekic CTR3(KTR_LOCK, 44172200Sbmilekic "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, 44272200Sbmilekic m->mtx_description); 44372200Sbmilekic 44472200Sbmilekic mi_switch(); 44572200Sbmilekic 44672200Sbmilekic if ((opts & MTX_QUIET) == 0) 44772200Sbmilekic CTR3(KTR_LOCK, 44872200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 44972200Sbmilekic p, m, m->mtx_description); 45072200Sbmilekic 45172200Sbmilekic mtx_unlock_spin(&sched_lock); 45272200Sbmilekic } 45372200Sbmilekic 45472200Sbmilekic return; 45572200Sbmilekic} 45672200Sbmilekic 45772200Sbmilekic/* 45872200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 45972200Sbmilekic * 46072200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 46172200Sbmilekic * is handled inline. 46272200Sbmilekic */ 46372200Sbmilekicvoid 46472200Sbmilekic_mtx_lock_spin(struct mtx *m, int opts, u_int mtx_intr, const char *file, 46572200Sbmilekic int line) 46672200Sbmilekic{ 46772200Sbmilekic int i = 0; 46872200Sbmilekic 46972200Sbmilekic if ((opts & MTX_QUIET) == 0) 47072344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 47172200Sbmilekic 47272200Sbmilekic for (;;) { 47372393Sbmilekic if (_obtain_lock(m, curproc)) 47472200Sbmilekic break; 47572200Sbmilekic 47672200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 47772200Sbmilekic if (i++ < 1000000) 47872200Sbmilekic continue; 47972200Sbmilekic if (i++ < 6000000) 48072200Sbmilekic DELAY(1); 48167352Sjhb#ifdef DDB 48272200Sbmilekic else if (!db_active) 48367352Sjhb#else 48472200Sbmilekic else 48567352Sjhb#endif 48672200Sbmilekic panic("spin lock %s held by %p for > 5 seconds", 48772200Sbmilekic m->mtx_description, (void *)m->mtx_lock); 48867352Sjhb } 48967352Sjhb } 49072200Sbmilekic 49172200Sbmilekic m->mtx_saveintr = mtx_intr; 49272200Sbmilekic if ((opts & MTX_QUIET) == 0) 49372200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 49472200Sbmilekic 49572200Sbmilekic return; 49667352Sjhb} 49767352Sjhb 49872200Sbmilekic/* 49972200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 50072200Sbmilekic * 50172200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 50272200Sbmilekic * need to wake up a blocked thread). 50372200Sbmilekic */ 50467352Sjhbvoid 50572200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 50667352Sjhb{ 50767352Sjhb struct proc *p, *p1; 50867352Sjhb struct mtx *m1; 50967352Sjhb int pri; 51067352Sjhb 51172393Sbmilekic p = curproc; 51272200Sbmilekic 51372200Sbmilekic if (mtx_recursed(m)) { 51472200Sbmilekic if (--(m->mtx_recurse) == 0) 51572200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 51672200Sbmilekic if ((opts & MTX_QUIET) == 0) 51772200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 51872200Sbmilekic return; 51972200Sbmilekic } 52072200Sbmilekic 52172200Sbmilekic mtx_lock_spin(&sched_lock); 52272200Sbmilekic if ((opts & MTX_QUIET) == 0) 52372200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 52472200Sbmilekic 52572200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 52672200Sbmilekic MPASS(p->p_magic == P_MAGIC); 52772200Sbmilekic MPASS(p1->p_magic == P_MAGIC); 52872200Sbmilekic 52972200Sbmilekic TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 53072200Sbmilekic 53172200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 53272200Sbmilekic LIST_REMOVE(m, mtx_contested); 53372200Sbmilekic _release_lock_quick(m); 53472200Sbmilekic if ((opts & MTX_QUIET) == 0) 53572200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 53672200Sbmilekic } else 53772200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 53872200Sbmilekic 53972376Sjake pri = PRI_MAX; 54072200Sbmilekic LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 54172376Sjake int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level; 54272200Sbmilekic if (cp < pri) 54372200Sbmilekic pri = cp; 54472200Sbmilekic } 54572200Sbmilekic 54672376Sjake if (pri > p->p_pri.pri_native) 54772376Sjake pri = p->p_pri.pri_native; 54872200Sbmilekic SET_PRIO(p, pri); 54972200Sbmilekic 55072200Sbmilekic if ((opts & MTX_QUIET) == 0) 55172200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 55272200Sbmilekic m, p1); 55372200Sbmilekic 55472200Sbmilekic p1->p_blocked = NULL; 55572200Sbmilekic p1->p_stat = SRUN; 55672200Sbmilekic setrunqueue(p1); 55772200Sbmilekic 55872376Sjake if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) { 55967352Sjhb#ifdef notyet 56072994Sjhb if (p->p_ithd != NULL) { 56172994Sjhb struct ithd *it = p->p_ithd; 56267352Sjhb 56372200Sbmilekic if (it->it_interrupted) { 56472200Sbmilekic if ((opts & MTX_QUIET) == 0) 56572200Sbmilekic CTR2(KTR_LOCK, 56672994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 56772200Sbmilekic it, it->it_interrupted); 56872200Sbmilekic intr_thd_fixup(it); 56967352Sjhb } 57072200Sbmilekic } 57167352Sjhb#endif 57272200Sbmilekic setrunqueue(p); 57372200Sbmilekic if ((opts & MTX_QUIET) == 0) 57472200Sbmilekic CTR2(KTR_LOCK, 57572200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 57672200Sbmilekic (void *)m->mtx_lock); 57772200Sbmilekic 57872200Sbmilekic mi_switch(); 57972200Sbmilekic if ((opts & MTX_QUIET) == 0) 58072200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 58172200Sbmilekic m, (void *)m->mtx_lock); 58267352Sjhb } 58372200Sbmilekic 58472200Sbmilekic mtx_unlock_spin(&sched_lock); 58572200Sbmilekic 58672200Sbmilekic return; 58767352Sjhb} 58867352Sjhb 58972200Sbmilekic/* 59072200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 59172200Sbmilekic * See the _rel_spin_lock() macro for the details. 59272200Sbmilekic */ 59372200Sbmilekic 59472200Sbmilekic/* 59572994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 59672200Sbmilekic */ 59772996Sjhb#ifdef INVARIANT_SUPPORT 59871352Sjasonevoid 59971360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 60071352Sjasone{ 60173033Sjake switch (what) { 60271352Sjasone case MA_OWNED: 60371352Sjasone case MA_OWNED | MA_RECURSED: 60471352Sjasone case MA_OWNED | MA_NOTRECURSED: 60573033Sjake if (!mtx_owned(m)) 60671352Sjasone panic("mutex %s not owned at %s:%d", 60773033Sjake m->mtx_description, file, line); 60873033Sjake if (mtx_recursed(m)) { 60973033Sjake if ((what & MA_NOTRECURSED) != 0) 61071352Sjasone panic("mutex %s recursed at %s:%d", 61173033Sjake m->mtx_description, file, line); 61273033Sjake } else if ((what & MA_RECURSED) != 0) { 61371352Sjasone panic("mutex %s unrecursed at %s:%d", 61473033Sjake m->mtx_description, file, line); 61571352Sjasone } 61671352Sjasone break; 61771352Sjasone case MA_NOTOWNED: 61873033Sjake if (mtx_owned(m)) 61971352Sjasone panic("mutex %s owned at %s:%d", 62073033Sjake m->mtx_description, file, line); 62171352Sjasone break; 62271352Sjasone default: 62371360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 62471352Sjasone } 62571352Sjasone} 62671352Sjasone#endif 62771352Sjasone 62872200Sbmilekic/* 62972200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 63072200Sbmilekic */ 63167352Sjhb#define MV_DESTROY 0 /* validate before destory */ 63267352Sjhb#define MV_INIT 1 /* validate before init */ 63367352Sjhb 63467352Sjhb#ifdef MUTEX_DEBUG 63567352Sjhb 63667352Sjhbint mtx_validate __P((struct mtx *, int)); 63767352Sjhb 63867352Sjhbint 63967352Sjhbmtx_validate(struct mtx *m, int when) 64067352Sjhb{ 64167352Sjhb struct mtx *mp; 64267352Sjhb int i; 64367352Sjhb int retval = 0; 64467352Sjhb 64571320Sjasone#ifdef WITNESS 64671320Sjasone if (witness_cold) 64771320Sjasone return 0; 64871320Sjasone#endif 64967352Sjhb if (m == &all_mtx || cold) 65067352Sjhb return 0; 65167352Sjhb 65272200Sbmilekic mtx_lock(&all_mtx); 65367352Sjhb/* 65467352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 65567352Sjhb * we can re-enable the kernacc() checks. 65667352Sjhb */ 65767352Sjhb#ifndef __alpha__ 65867352Sjhb MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), 65967352Sjhb VM_PROT_READ) == 1); 66067352Sjhb#endif 66167352Sjhb MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); 66267352Sjhb for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 66367352Sjhb#ifndef __alpha__ 66467352Sjhb if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), 66567352Sjhb VM_PROT_READ) != 1) { 66667352Sjhb panic("mtx_validate: mp=%p mp->mtx_next=%p", 66767352Sjhb mp, mp->mtx_next); 66867352Sjhb } 66967352Sjhb#endif 67067352Sjhb i++; 67167352Sjhb if (i > mtx_cur_cnt) { 67267352Sjhb panic("mtx_validate: too many in chain, known=%d\n", 67367352Sjhb mtx_cur_cnt); 67467352Sjhb } 67567352Sjhb } 67667352Sjhb MPASS(i == mtx_cur_cnt); 67767352Sjhb switch (when) { 67867352Sjhb case MV_DESTROY: 67967352Sjhb for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 68067352Sjhb if (mp == m) 68167352Sjhb break; 68267352Sjhb MPASS(mp == m); 68367352Sjhb break; 68467352Sjhb case MV_INIT: 68567352Sjhb for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 68667352Sjhb if (mp == m) { 68767352Sjhb /* 68867352Sjhb * Not good. This mutex already exists. 68967352Sjhb */ 69067352Sjhb printf("re-initing existing mutex %s\n", 69167352Sjhb m->mtx_description); 69267352Sjhb MPASS(m->mtx_lock == MTX_UNOWNED); 69367352Sjhb retval = 1; 69467352Sjhb } 69567352Sjhb } 69672200Sbmilekic mtx_unlock(&all_mtx); 69767352Sjhb return (retval); 69867352Sjhb} 69967352Sjhb#endif 70067352Sjhb 70172200Sbmilekic/* 70272200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 70372200Sbmilekic * `opts' with options contained in `opts' and description `description.' 70472200Sbmilekic * Place on "all_mtx" queue. 70572200Sbmilekic */ 70667352Sjhbvoid 70772200Sbmilekicmtx_init(struct mtx *m, const char *description, int opts) 70867352Sjhb{ 70972200Sbmilekic 71072200Sbmilekic if ((opts & MTX_QUIET) == 0) 71172200Sbmilekic CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description); 71272200Sbmilekic 71367352Sjhb#ifdef MUTEX_DEBUG 71472200Sbmilekic /* Diagnostic and error correction */ 71572200Sbmilekic if (mtx_validate(m, MV_INIT)) 71667352Sjhb return; 71769429Sjhb#endif 71867352Sjhb 71967352Sjhb bzero((void *)m, sizeof *m); 72067352Sjhb TAILQ_INIT(&m->mtx_blocked); 72172200Sbmilekic 72269429Sjhb#ifdef WITNESS 72371320Sjasone if (!witness_cold) { 72471560Sjhb m->mtx_debug = malloc(sizeof(struct mtx_debug), 72572200Sbmilekic M_WITNESS, M_NOWAIT | M_ZERO); 72671560Sjhb MPASS(m->mtx_debug != NULL); 72771320Sjasone } 72871560Sjhb#endif 72971320Sjasone 73072200Sbmilekic m->mtx_description = description; 73172200Sbmilekic m->mtx_flags = opts; 73267352Sjhb m->mtx_lock = MTX_UNOWNED; 73372200Sbmilekic 73467352Sjhb /* Put on all mutex queue */ 73572200Sbmilekic mtx_lock(&all_mtx); 73667352Sjhb m->mtx_next = &all_mtx; 73767352Sjhb m->mtx_prev = all_mtx.mtx_prev; 73867352Sjhb m->mtx_prev->mtx_next = m; 73967352Sjhb all_mtx.mtx_prev = m; 74067352Sjhb if (++mtx_cur_cnt > mtx_max_cnt) 74167352Sjhb mtx_max_cnt = mtx_cur_cnt; 74272200Sbmilekic mtx_unlock(&all_mtx); 74372200Sbmilekic 74471320Sjasone#ifdef WITNESS 74571320Sjasone if (!witness_cold) 74672200Sbmilekic witness_init(m, opts); 74771320Sjasone#endif 74867352Sjhb} 74967352Sjhb 75072200Sbmilekic/* 75172200Sbmilekic * Remove lock `m' from all_mtx queue. 75272200Sbmilekic */ 75367352Sjhbvoid 75467352Sjhbmtx_destroy(struct mtx *m) 75567352Sjhb{ 75667352Sjhb 75771320Sjasone#ifdef WITNESS 75871320Sjasone KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", 75971320Sjasone __FUNCTION__)); 76071320Sjasone#endif 76172200Sbmilekic 76271560Sjhb CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description); 76372200Sbmilekic 76467352Sjhb#ifdef MUTEX_DEBUG 76567352Sjhb if (m->mtx_next == NULL) 76667352Sjhb panic("mtx_destroy: %p (%s) already destroyed", 76767352Sjhb m, m->mtx_description); 76867352Sjhb 76967352Sjhb if (!mtx_owned(m)) { 77067352Sjhb MPASS(m->mtx_lock == MTX_UNOWNED); 77167352Sjhb } else { 77271228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 77367352Sjhb } 77472200Sbmilekic 77572200Sbmilekic /* diagnostic */ 77672200Sbmilekic mtx_validate(m, MV_DESTROY); 77767352Sjhb#endif 77867352Sjhb 77967352Sjhb#ifdef WITNESS 78067352Sjhb if (m->mtx_witness) 78167352Sjhb witness_destroy(m); 78267352Sjhb#endif /* WITNESS */ 78367352Sjhb 78467352Sjhb /* Remove from the all mutex queue */ 78572200Sbmilekic mtx_lock(&all_mtx); 78667352Sjhb m->mtx_next->mtx_prev = m->mtx_prev; 78767352Sjhb m->mtx_prev->mtx_next = m->mtx_next; 78872200Sbmilekic 78967352Sjhb#ifdef MUTEX_DEBUG 79067352Sjhb m->mtx_next = m->mtx_prev = NULL; 79169429Sjhb#endif 79272200Sbmilekic 79369429Sjhb#ifdef WITNESS 79472200Sbmilekic free(m->mtx_debug, M_WITNESS); 79571560Sjhb m->mtx_debug = NULL; 79667352Sjhb#endif 79772200Sbmilekic 79867352Sjhb mtx_cur_cnt--; 79972200Sbmilekic mtx_unlock(&all_mtx); 80067352Sjhb} 80167352Sjhb 80272200Sbmilekic 80371560Sjhb/* 80472200Sbmilekic * The WITNESS-enabled diagnostic code. 80571560Sjhb */ 80671560Sjhb#ifdef WITNESS 80771320Sjasonestatic void 80871320Sjasonewitness_fixup(void *dummy __unused) 80971320Sjasone{ 81071320Sjasone struct mtx *mp; 81171320Sjasone 81271560Sjhb /* 81371560Sjhb * We have to release Giant before initializing its witness 81471560Sjhb * structure so that WITNESS doesn't get confused. 81571560Sjhb */ 81672200Sbmilekic mtx_unlock(&Giant); 81771560Sjhb mtx_assert(&Giant, MA_NOTOWNED); 81871560Sjhb 81972200Sbmilekic mtx_lock(&all_mtx); 82072200Sbmilekic 82171320Sjasone /* Iterate through all mutexes and finish up mutex initialization. */ 82271320Sjasone for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 82371320Sjasone 82471560Sjhb mp->mtx_debug = malloc(sizeof(struct mtx_debug), 82572200Sbmilekic M_WITNESS, M_NOWAIT | M_ZERO); 82671560Sjhb MPASS(mp->mtx_debug != NULL); 82771320Sjasone 82871320Sjasone witness_init(mp, mp->mtx_flags); 82971320Sjasone } 83072200Sbmilekic mtx_unlock(&all_mtx); 83171320Sjasone 83271320Sjasone /* Mark the witness code as being ready for use. */ 83371320Sjasone atomic_store_rel_int(&witness_cold, 0); 83471560Sjhb 83572200Sbmilekic mtx_lock(&Giant); 83671320Sjasone} 83771320SjasoneSYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) 83871320Sjasone 83965557Sjasone#define WITNESS_COUNT 200 84065557Sjasone#define WITNESS_NCHILDREN 2 84165557Sjasone 84267401Sjhbint witness_watch = 1; 84365557Sjasone 84465856Sjhbstruct witness { 84565557Sjasone struct witness *w_next; 84667404Sjhb const char *w_description; 84765624Sjasone const char *w_file; 84865557Sjasone int w_line; 84965557Sjasone struct witness *w_morechildren; 85065557Sjasone u_char w_childcnt; 85165557Sjasone u_char w_Giant_squawked:1; 85265557Sjasone u_char w_other_squawked:1; 85365557Sjasone u_char w_same_squawked:1; 85471228Sbmilekic u_char w_spin:1; /* MTX_SPIN type mutex. */ 85565557Sjasone u_int w_level; 85665557Sjasone struct witness *w_children[WITNESS_NCHILDREN]; 85765856Sjhb}; 85865557Sjasone 85965856Sjhbstruct witness_blessed { 86065557Sjasone char *b_lock1; 86165557Sjasone char *b_lock2; 86265856Sjhb}; 86365557Sjasone 86467676Sjhb#ifdef DDB 86565557Sjasone/* 86667676Sjhb * When DDB is enabled and witness_ddb is set to 1, it will cause the system to 86765557Sjasone * drop into kdebug() when: 86865557Sjasone * - a lock heirarchy violation occurs 86965557Sjasone * - locks are held when going to sleep. 87065557Sjasone */ 87171560Sjhbint witness_ddb; 87267676Sjhb#ifdef WITNESS_DDB 87371560SjhbTUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb); 87467676Sjhb#else 87571560SjhbTUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb); 87665557Sjasone#endif 87767676SjhbSYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, ""); 87867676Sjhb#endif /* DDB */ 87965557Sjasone 88071560Sjhbint witness_skipspin; 88167676Sjhb#ifdef WITNESS_SKIPSPIN 88271560SjhbTUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin); 88367676Sjhb#else 88471560SjhbTUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); 88565557Sjasone#endif 88667676SjhbSYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, 88767676Sjhb ""); 88865557Sjasone 88972200Sbmilekic/* 89072200Sbmilekic * Witness-enabled globals 89172200Sbmilekic */ 89271320Sjasonestatic struct mtx w_mtx; 89365856Sjhbstatic struct witness *w_free; 89465856Sjhbstatic struct witness *w_all; 89565856Sjhbstatic int w_inited; 89665856Sjhbstatic int witness_dead; /* fatal error, probably no memory */ 89765557Sjasone 89865856Sjhbstatic struct witness w_data[WITNESS_COUNT]; 89965557Sjasone 90072200Sbmilekic/* 90172200Sbmilekic * Internal witness routine prototypes 90272200Sbmilekic */ 90372200Sbmilekicstatic struct witness *enroll(const char *description, int flag); 90472200Sbmilekicstatic int itismychild(struct witness *parent, struct witness *child); 90572200Sbmilekicstatic void removechild(struct witness *parent, struct witness *child); 90672200Sbmilekicstatic int isitmychild(struct witness *parent, struct witness *child); 90772200Sbmilekicstatic int isitmydescendant(struct witness *parent, struct witness *child); 90872200Sbmilekicstatic int dup_ok(struct witness *); 90972200Sbmilekicstatic int blessed(struct witness *, struct witness *); 91072200Sbmilekicstatic void 91172200Sbmilekic witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *); 91272200Sbmilekicstatic void witness_leveldescendents(struct witness *parent, int level); 91372200Sbmilekicstatic void witness_levelall(void); 91472200Sbmilekicstatic struct witness * witness_get(void); 91572200Sbmilekicstatic void witness_free(struct witness *m); 91665557Sjasone 91765557Sjasonestatic char *ignore_list[] = { 91865557Sjasone "witness lock", 91965557Sjasone NULL 92065557Sjasone}; 92165557Sjasone 92265557Sjasonestatic char *spin_order_list[] = { 92372224Sjhb#if defined(__i386__) && defined (SMP) 92472224Sjhb "com", 92572224Sjhb#endif 92669362Sjhb "sio", 92772224Sjhb#ifdef __i386__ 92872224Sjhb "cy", 92972224Sjhb#endif 93073238Sjulian "ng_node", 93173238Sjulian "ng_worklist", 93272836Sjhb "ithread table lock", 93372836Sjhb "ithread list lock", 93465557Sjasone "sched lock", 93568808Sjhb#ifdef __i386__ 93667676Sjhb "clk", 93768808Sjhb#endif 93868889Sjake "callout", 93965557Sjasone /* 94065557Sjasone * leaf locks 94165557Sjasone */ 94272224Sjhb#ifdef SMP 94371576Sjasone#ifdef __i386__ 94471576Sjasone "ap boot", 94571576Sjasone "imen", 94671576Sjasone#endif 94771576Sjasone "smp rendezvous", 94872224Sjhb#endif 94965557Sjasone NULL 95065557Sjasone}; 95165557Sjasone 95265557Sjasonestatic char *order_list[] = { 95372256Sjhb "Giant", "proctree", "allproc", "process lock", "uidinfo hash", 95472256Sjhb "uidinfo struct", NULL, 95565557Sjasone NULL 95665557Sjasone}; 95765557Sjasone 95865557Sjasonestatic char *dup_list[] = { 95973912Sjhb "process lock", 96065557Sjasone NULL 96165557Sjasone}; 96265557Sjasone 96365557Sjasonestatic char *sleep_list[] = { 96468862Sjake "Giant", 96565557Sjasone NULL 96665557Sjasone}; 96765557Sjasone 96865557Sjasone/* 96965557Sjasone * Pairs of locks which have been blessed 97065557Sjasone * Don't complain about order problems with blessed locks 97165557Sjasone */ 97265856Sjhbstatic struct witness_blessed blessed_list[] = { 97365557Sjasone}; 97472200Sbmilekicstatic int blessed_count = 97572200Sbmilekic sizeof(blessed_list) / sizeof(struct witness_blessed); 97665557Sjasone 97771352Sjasonestatic void 97865856Sjhbwitness_init(struct mtx *m, int flag) 97965557Sjasone{ 98065557Sjasone m->mtx_witness = enroll(m->mtx_description, flag); 98165557Sjasone} 98265557Sjasone 98371352Sjasonestatic void 98465856Sjhbwitness_destroy(struct mtx *m) 98565557Sjasone{ 98665856Sjhb struct mtx *m1; 98765557Sjasone struct proc *p; 98872393Sbmilekic p = curproc; 98972224Sjhb LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) { 99065557Sjasone if (m1 == m) { 99165557Sjasone LIST_REMOVE(m, mtx_held); 99265557Sjasone break; 99365557Sjasone } 99465557Sjasone } 99565557Sjasone return; 99665557Sjasone 99765557Sjasone} 99865557Sjasone 99971352Sjasonestatic void 100071352Sjasonewitness_display(void(*prnt)(const char *fmt, ...)) 100171352Sjasone{ 100271352Sjasone struct witness *w, *w1; 100372224Sjhb int level, found; 100471352Sjasone 100571352Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 100671352Sjasone witness_levelall(); 100771352Sjasone 100872224Sjhb /* 100972224Sjhb * First, handle sleep mutexes which have been acquired at least 101072224Sjhb * once. 101172224Sjhb */ 101272224Sjhb prnt("Sleep mutexes:\n"); 101371352Sjasone for (w = w_all; w; w = w->w_next) { 101472224Sjhb if (w->w_file == NULL || w->w_spin) 101571352Sjasone continue; 101671352Sjasone for (w1 = w_all; w1; w1 = w1->w_next) { 101771352Sjasone if (isitmychild(w1, w)) 101871352Sjasone break; 101971352Sjasone } 102071352Sjasone if (w1 != NULL) 102171352Sjasone continue; 102271352Sjasone /* 102371352Sjasone * This lock has no anscestors, display its descendants. 102471352Sjasone */ 102571352Sjasone witness_displaydescendants(prnt, w); 102671352Sjasone } 102772224Sjhb 102872224Sjhb /* 102972224Sjhb * Now do spin mutexes which have been acquired at least once. 103072224Sjhb */ 103172224Sjhb prnt("\nSpin mutexes:\n"); 103272224Sjhb level = 0; 103372224Sjhb while (level < sizeof(spin_order_list) / sizeof(char *)) { 103472224Sjhb found = 0; 103572224Sjhb for (w = w_all; w; w = w->w_next) { 103672224Sjhb if (w->w_file == NULL || !w->w_spin) 103772224Sjhb continue; 103872224Sjhb if (w->w_level == 1 << level) { 103972224Sjhb witness_displaydescendants(prnt, w); 104072224Sjhb level++; 104172224Sjhb found = 1; 104272224Sjhb } 104372224Sjhb } 104472224Sjhb if (found == 0) 104572224Sjhb level++; 104672224Sjhb } 104772224Sjhb 104872224Sjhb /* 104972224Sjhb * Finally, any mutexes which have not been acquired yet. 105072224Sjhb */ 105172224Sjhb prnt("\nMutexes which were never acquired:\n"); 105271352Sjasone for (w = w_all; w; w = w->w_next) { 105371352Sjasone if (w->w_file != NULL) 105471352Sjasone continue; 105571352Sjasone prnt("%s\n", w->w_description); 105671352Sjasone } 105771352Sjasone} 105871352Sjasone 105965557Sjasonevoid 106065856Sjhbwitness_enter(struct mtx *m, int flags, const char *file, int line) 106165557Sjasone{ 106265856Sjhb struct witness *w, *w1; 106365856Sjhb struct mtx *m1; 106465557Sjasone struct proc *p; 106565557Sjasone int i; 106667676Sjhb#ifdef DDB 106767676Sjhb int go_into_ddb = 0; 106867676Sjhb#endif /* DDB */ 106965557Sjasone 107071352Sjasone if (witness_cold || m->mtx_witness == NULL || panicstr) 107171320Sjasone return; 107265557Sjasone w = m->mtx_witness; 107372393Sbmilekic p = curproc; 107465557Sjasone 107565557Sjasone if (flags & MTX_SPIN) { 107671560Sjhb if ((m->mtx_flags & MTX_SPIN) == 0) 107765651Sjasone panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" 107865651Sjasone " %s:%d", m->mtx_description, file, line); 107971228Sbmilekic if (mtx_recursed(m)) { 108071560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 108171228Sbmilekic panic("mutex_enter: recursion on non-recursive" 108271228Sbmilekic " mutex %s @ %s:%d", m->mtx_description, 108371228Sbmilekic file, line); 108465557Sjasone return; 108571228Sbmilekic } 108672200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 108770861Sjake i = PCPU_GET(witness_spin_check); 108865557Sjasone if (i != 0 && w->w_level < i) { 108972200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 109065651Sjasone panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" 109165651Sjasone " %s:%d already holding %s:%x", 109265557Sjasone m->mtx_description, w->w_level, file, line, 109365557Sjasone spin_order_list[ffs(i)-1], i); 109465557Sjasone } 109565557Sjasone PCPU_SET(witness_spin_check, i | w->w_level); 109672200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 109774016Sjhb p->p_spinlocks++; 109874016Sjhb MPASS(p->p_spinlocks > 0); 109969361Sjhb w->w_file = file; 110069361Sjhb w->w_line = line; 110169361Sjhb m->mtx_line = line; 110269361Sjhb m->mtx_file = file; 110365557Sjasone return; 110465557Sjasone } 110571560Sjhb if ((m->mtx_flags & MTX_SPIN) != 0) 110665557Sjasone panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 110765557Sjasone m->mtx_description, file, line); 110865557Sjasone 110971228Sbmilekic if (mtx_recursed(m)) { 111071560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 111171228Sbmilekic panic("mutex_enter: recursion on non-recursive" 111271228Sbmilekic " mutex %s @ %s:%d", m->mtx_description, 111371228Sbmilekic file, line); 111465557Sjasone return; 111571228Sbmilekic } 111665557Sjasone if (witness_dead) 111765557Sjasone goto out; 111869998Sjhb if (cold) 111965557Sjasone goto out; 112065557Sjasone 112174016Sjhb if (p->p_spinlocks != 0) 112272200Sbmilekic panic("blockable mtx_lock() of %s when not legal @ %s:%d", 112365557Sjasone m->mtx_description, file, line); 112465557Sjasone /* 112565557Sjasone * Is this the first mutex acquired 112665557Sjasone */ 112765557Sjasone if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) 112865557Sjasone goto out; 112965557Sjasone 113065557Sjasone if ((w1 = m1->mtx_witness) == w) { 113165557Sjasone if (w->w_same_squawked || dup_ok(w)) 113265557Sjasone goto out; 113365557Sjasone w->w_same_squawked = 1; 113465557Sjasone printf("acquring duplicate lock of same type: \"%s\"\n", 113565557Sjasone m->mtx_description); 113665557Sjasone printf(" 1st @ %s:%d\n", w->w_file, w->w_line); 113765557Sjasone printf(" 2nd @ %s:%d\n", file, line); 113867676Sjhb#ifdef DDB 113967676Sjhb go_into_ddb = 1; 114067676Sjhb#endif /* DDB */ 114165557Sjasone goto out; 114265557Sjasone } 114365557Sjasone MPASS(!mtx_owned(&w_mtx)); 114472200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 114565557Sjasone /* 114665557Sjasone * If we have a known higher number just say ok 114765557Sjasone */ 114865557Sjasone if (witness_watch > 1 && w->w_level > w1->w_level) { 114972200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 115065557Sjasone goto out; 115165557Sjasone } 115265557Sjasone if (isitmydescendant(m1->mtx_witness, w)) { 115372200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 115465557Sjasone goto out; 115565557Sjasone } 115665557Sjasone for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { 115765557Sjasone 115867352Sjhb MPASS(i < 200); 115965557Sjasone w1 = m1->mtx_witness; 116065557Sjasone if (isitmydescendant(w, w1)) { 116172200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 116265557Sjasone if (blessed(w, w1)) 116365557Sjasone goto out; 116465557Sjasone if (m1 == &Giant) { 116565557Sjasone if (w1->w_Giant_squawked) 116665557Sjasone goto out; 116765557Sjasone else 116865557Sjasone w1->w_Giant_squawked = 1; 116965557Sjasone } else { 117065557Sjasone if (w1->w_other_squawked) 117165557Sjasone goto out; 117265557Sjasone else 117365557Sjasone w1->w_other_squawked = 1; 117465557Sjasone } 117565557Sjasone printf("lock order reversal\n"); 117665557Sjasone printf(" 1st %s last acquired @ %s:%d\n", 117765557Sjasone w->w_description, w->w_file, w->w_line); 117865557Sjasone printf(" 2nd %p %s @ %s:%d\n", 117965557Sjasone m1, w1->w_description, w1->w_file, w1->w_line); 118065557Sjasone printf(" 3rd %p %s @ %s:%d\n", 118165557Sjasone m, w->w_description, file, line); 118267676Sjhb#ifdef DDB 118367676Sjhb go_into_ddb = 1; 118467676Sjhb#endif /* DDB */ 118565557Sjasone goto out; 118665557Sjasone } 118765557Sjasone } 118865557Sjasone m1 = LIST_FIRST(&p->p_heldmtx); 118965557Sjasone if (!itismychild(m1->mtx_witness, w)) 119072200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 119165557Sjasone 119265557Sjasoneout: 119367676Sjhb#ifdef DDB 119467676Sjhb if (witness_ddb && go_into_ddb) 119567676Sjhb Debugger("witness_enter"); 119667676Sjhb#endif /* DDB */ 119765557Sjasone w->w_file = file; 119865557Sjasone w->w_line = line; 119965557Sjasone m->mtx_line = line; 120065557Sjasone m->mtx_file = file; 120165557Sjasone 120265557Sjasone /* 120368582Sjhb * If this pays off it likely means that a mutex being witnessed 120465557Sjasone * is acquired in hardclock. Put it in the ignore list. It is 120565557Sjasone * likely not the mutex this assert fails on. 120665557Sjasone */ 120767352Sjhb MPASS(m->mtx_held.le_prev == NULL); 120865557Sjasone LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 120965557Sjasone} 121065557Sjasone 121165557Sjasonevoid 121265856Sjhbwitness_try_enter(struct mtx *m, int flags, const char *file, int line) 121365557Sjasone{ 121465557Sjasone struct proc *p; 121565856Sjhb struct witness *w = m->mtx_witness; 121665557Sjasone 121771320Sjasone if (witness_cold) 121871320Sjasone return; 121969998Sjhb if (panicstr) 122069998Sjhb return; 122165557Sjasone if (flags & MTX_SPIN) { 122271560Sjhb if ((m->mtx_flags & MTX_SPIN) == 0) 122365557Sjasone panic("mutex_try_enter: " 122465557Sjasone "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", 122565557Sjasone m->mtx_description, file, line); 122671228Sbmilekic if (mtx_recursed(m)) { 122771560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 122871228Sbmilekic panic("mutex_try_enter: recursion on" 122971228Sbmilekic " non-recursive mutex %s @ %s:%d", 123071228Sbmilekic m->mtx_description, file, line); 123165557Sjasone return; 123271228Sbmilekic } 123372200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 123470861Sjake PCPU_SET(witness_spin_check, 123570861Sjake PCPU_GET(witness_spin_check) | w->w_level); 123672200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 123769361Sjhb w->w_file = file; 123869361Sjhb w->w_line = line; 123969361Sjhb m->mtx_line = line; 124069361Sjhb m->mtx_file = file; 124165557Sjasone return; 124265557Sjasone } 124365557Sjasone 124471560Sjhb if ((m->mtx_flags & MTX_SPIN) != 0) 124565557Sjasone panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 124665557Sjasone m->mtx_description, file, line); 124765557Sjasone 124871228Sbmilekic if (mtx_recursed(m)) { 124971560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 125071228Sbmilekic panic("mutex_try_enter: recursion on non-recursive" 125171228Sbmilekic " mutex %s @ %s:%d", m->mtx_description, file, 125271228Sbmilekic line); 125365557Sjasone return; 125471228Sbmilekic } 125565557Sjasone w->w_file = file; 125665557Sjasone w->w_line = line; 125765557Sjasone m->mtx_line = line; 125865557Sjasone m->mtx_file = file; 125972393Sbmilekic p = curproc; 126067352Sjhb MPASS(m->mtx_held.le_prev == NULL); 126165557Sjasone LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 126265557Sjasone} 126365557Sjasone 126465557Sjasonevoid 126571352Sjasonewitness_exit(struct mtx *m, int flags, const char *file, int line) 126665557Sjasone{ 126771352Sjasone struct witness *w; 126874016Sjhb struct proc *p; 126965557Sjasone 127071352Sjasone if (witness_cold || m->mtx_witness == NULL || panicstr) 127171352Sjasone return; 127271352Sjasone w = m->mtx_witness; 127374016Sjhb p = curproc; 127465557Sjasone 127571352Sjasone if (flags & MTX_SPIN) { 127671560Sjhb if ((m->mtx_flags & MTX_SPIN) == 0) 127771352Sjasone panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" 127871352Sjasone " %s:%d", m->mtx_description, file, line); 127971352Sjasone if (mtx_recursed(m)) { 128071560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 128171352Sjasone panic("mutex_exit: recursion on non-recursive" 128271352Sjasone " mutex %s @ %s:%d", m->mtx_description, 128371352Sjasone file, line); 128471352Sjasone return; 128565557Sjasone } 128672200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 128771352Sjasone PCPU_SET(witness_spin_check, 128871352Sjasone PCPU_GET(witness_spin_check) & ~w->w_level); 128972200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 129074016Sjhb MPASS(p->p_spinlocks > 0); 129174016Sjhb p->p_spinlocks--; 129271352Sjasone return; 129365557Sjasone } 129471560Sjhb if ((m->mtx_flags & MTX_SPIN) != 0) 129571352Sjasone panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 129671352Sjasone m->mtx_description, file, line); 129771352Sjasone 129871352Sjasone if (mtx_recursed(m)) { 129971560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 130071352Sjasone panic("mutex_exit: recursion on non-recursive" 130171352Sjasone " mutex %s @ %s:%d", m->mtx_description, 130271352Sjasone file, line); 130371352Sjasone return; 130465557Sjasone } 130571352Sjasone 130674016Sjhb if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) 130772200Sbmilekic panic("switchable mtx_unlock() of %s when not legal @ %s:%d", 130871352Sjasone m->mtx_description, file, line); 130971352Sjasone LIST_REMOVE(m, mtx_held); 131071352Sjasone m->mtx_held.le_prev = NULL; 131165557Sjasone} 131265557Sjasone 131365557Sjasoneint 131465856Sjhbwitness_sleep(int check_only, struct mtx *mtx, const char *file, int line) 131565557Sjasone{ 131665856Sjhb struct mtx *m; 131765557Sjasone struct proc *p; 131865557Sjasone char **sleep; 131965557Sjasone int n = 0; 132065557Sjasone 132171320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 132272393Sbmilekic p = curproc; 132372224Sjhb LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { 132465557Sjasone if (m == mtx) 132565557Sjasone continue; 132665557Sjasone for (sleep = sleep_list; *sleep!= NULL; sleep++) 132765557Sjasone if (strcmp(m->mtx_description, *sleep) == 0) 132865557Sjasone goto next; 132972224Sjhb if (n == 0) 133072224Sjhb printf("Whee!\n"); 133165557Sjasone printf("%s:%d: %s with \"%s\" locked from %s:%d\n", 133265557Sjasone file, line, check_only ? "could sleep" : "sleeping", 133365557Sjasone m->mtx_description, 133465557Sjasone m->mtx_witness->w_file, m->mtx_witness->w_line); 133565557Sjasone n++; 133665557Sjasone next: 133765557Sjasone } 133867676Sjhb#ifdef DDB 133967676Sjhb if (witness_ddb && n) 134067676Sjhb Debugger("witness_sleep"); 134167676Sjhb#endif /* DDB */ 134265557Sjasone return (n); 134365557Sjasone} 134465557Sjasone 134565856Sjhbstatic struct witness * 134667404Sjhbenroll(const char *description, int flag) 134765557Sjasone{ 134865557Sjasone int i; 134965856Sjhb struct witness *w, *w1; 135065557Sjasone char **ignore; 135165557Sjasone char **order; 135265557Sjasone 135365557Sjasone if (!witness_watch) 135465557Sjasone return (NULL); 135565557Sjasone for (ignore = ignore_list; *ignore != NULL; ignore++) 135665557Sjasone if (strcmp(description, *ignore) == 0) 135765557Sjasone return (NULL); 135865557Sjasone 135965557Sjasone if (w_inited == 0) { 136071320Sjasone mtx_init(&w_mtx, "witness lock", MTX_SPIN); 136165557Sjasone for (i = 0; i < WITNESS_COUNT; i++) { 136265557Sjasone w = &w_data[i]; 136365557Sjasone witness_free(w); 136465557Sjasone } 136565557Sjasone w_inited = 1; 136665557Sjasone for (order = order_list; *order != NULL; order++) { 136765557Sjasone w = enroll(*order, MTX_DEF); 136865557Sjasone w->w_file = "order list"; 136965557Sjasone for (order++; *order != NULL; order++) { 137065557Sjasone w1 = enroll(*order, MTX_DEF); 137165557Sjasone w1->w_file = "order list"; 137265557Sjasone itismychild(w, w1); 137365557Sjasone w = w1; 137465557Sjasone } 137565557Sjasone } 137665557Sjasone } 137765557Sjasone if ((flag & MTX_SPIN) && witness_skipspin) 137865557Sjasone return (NULL); 137972200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 138065557Sjasone for (w = w_all; w; w = w->w_next) { 138165557Sjasone if (strcmp(description, w->w_description) == 0) { 138272200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 138365557Sjasone return (w); 138465557Sjasone } 138565557Sjasone } 138665557Sjasone if ((w = witness_get()) == NULL) 138765557Sjasone return (NULL); 138865557Sjasone w->w_next = w_all; 138965557Sjasone w_all = w; 139065557Sjasone w->w_description = description; 139172200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 139265557Sjasone if (flag & MTX_SPIN) { 139365557Sjasone w->w_spin = 1; 139465557Sjasone 139565557Sjasone i = 1; 139665557Sjasone for (order = spin_order_list; *order != NULL; order++) { 139765557Sjasone if (strcmp(description, *order) == 0) 139865557Sjasone break; 139965557Sjasone i <<= 1; 140065557Sjasone } 140165557Sjasone if (*order == NULL) 140265557Sjasone panic("spin lock %s not in order list", description); 140365557Sjasone w->w_level = i; 140471560Sjhb } 140571228Sbmilekic 140665557Sjasone return (w); 140765557Sjasone} 140865557Sjasone 140965557Sjasonestatic int 141065856Sjhbitismychild(struct witness *parent, struct witness *child) 141165557Sjasone{ 141265557Sjasone static int recursed; 141365557Sjasone 141465557Sjasone /* 141565557Sjasone * Insert "child" after "parent" 141665557Sjasone */ 141765557Sjasone while (parent->w_morechildren) 141865557Sjasone parent = parent->w_morechildren; 141965557Sjasone 142065557Sjasone if (parent->w_childcnt == WITNESS_NCHILDREN) { 142165557Sjasone if ((parent->w_morechildren = witness_get()) == NULL) 142265557Sjasone return (1); 142365557Sjasone parent = parent->w_morechildren; 142465557Sjasone } 142567352Sjhb MPASS(child != NULL); 142665557Sjasone parent->w_children[parent->w_childcnt++] = child; 142765557Sjasone /* 142865557Sjasone * now prune whole tree 142965557Sjasone */ 143065557Sjasone if (recursed) 143165557Sjasone return (0); 143265557Sjasone recursed = 1; 143365557Sjasone for (child = w_all; child != NULL; child = child->w_next) { 143465557Sjasone for (parent = w_all; parent != NULL; 143565557Sjasone parent = parent->w_next) { 143665557Sjasone if (!isitmychild(parent, child)) 143765557Sjasone continue; 143865557Sjasone removechild(parent, child); 143965557Sjasone if (isitmydescendant(parent, child)) 144065557Sjasone continue; 144165557Sjasone itismychild(parent, child); 144265557Sjasone } 144365557Sjasone } 144465557Sjasone recursed = 0; 144565557Sjasone witness_levelall(); 144665557Sjasone return (0); 144765557Sjasone} 144865557Sjasone 144965557Sjasonestatic void 145065856Sjhbremovechild(struct witness *parent, struct witness *child) 145165557Sjasone{ 145265856Sjhb struct witness *w, *w1; 145365557Sjasone int i; 145465557Sjasone 145565557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) 145665557Sjasone for (i = 0; i < w->w_childcnt; i++) 145765557Sjasone if (w->w_children[i] == child) 145865557Sjasone goto found; 145965557Sjasone return; 146065557Sjasonefound: 146165557Sjasone for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) 146265557Sjasone continue; 146365557Sjasone w->w_children[i] = w1->w_children[--w1->w_childcnt]; 146467352Sjhb MPASS(w->w_children[i] != NULL); 146565557Sjasone 146665557Sjasone if (w1->w_childcnt != 0) 146765557Sjasone return; 146865557Sjasone 146965557Sjasone if (w1 == parent) 147065557Sjasone return; 147165557Sjasone for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) 147265557Sjasone continue; 147365557Sjasone w->w_morechildren = 0; 147465557Sjasone witness_free(w1); 147565557Sjasone} 147665557Sjasone 147765557Sjasonestatic int 147865856Sjhbisitmychild(struct witness *parent, struct witness *child) 147965557Sjasone{ 148065856Sjhb struct witness *w; 148165557Sjasone int i; 148265557Sjasone 148365557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) { 148465557Sjasone for (i = 0; i < w->w_childcnt; i++) { 148565557Sjasone if (w->w_children[i] == child) 148665557Sjasone return (1); 148765557Sjasone } 148865557Sjasone } 148965557Sjasone return (0); 149065557Sjasone} 149165557Sjasone 149265557Sjasonestatic int 149365856Sjhbisitmydescendant(struct witness *parent, struct witness *child) 149465557Sjasone{ 149565856Sjhb struct witness *w; 149665557Sjasone int i; 149765557Sjasone int j; 149865557Sjasone 149965557Sjasone for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { 150067352Sjhb MPASS(j < 1000); 150165557Sjasone for (i = 0; i < w->w_childcnt; i++) { 150265557Sjasone if (w->w_children[i] == child) 150365557Sjasone return (1); 150465557Sjasone } 150565557Sjasone for (i = 0; i < w->w_childcnt; i++) { 150665557Sjasone if (isitmydescendant(w->w_children[i], child)) 150765557Sjasone return (1); 150865557Sjasone } 150965557Sjasone } 151065557Sjasone return (0); 151165557Sjasone} 151265557Sjasone 151365557Sjasonevoid 151465557Sjasonewitness_levelall (void) 151565557Sjasone{ 151665856Sjhb struct witness *w, *w1; 151765557Sjasone 151865557Sjasone for (w = w_all; w; w = w->w_next) 151971228Sbmilekic if (!(w->w_spin)) 152065557Sjasone w->w_level = 0; 152165557Sjasone for (w = w_all; w; w = w->w_next) { 152265557Sjasone if (w->w_spin) 152365557Sjasone continue; 152465557Sjasone for (w1 = w_all; w1; w1 = w1->w_next) { 152565557Sjasone if (isitmychild(w1, w)) 152665557Sjasone break; 152765557Sjasone } 152865557Sjasone if (w1 != NULL) 152965557Sjasone continue; 153065557Sjasone witness_leveldescendents(w, 0); 153165557Sjasone } 153265557Sjasone} 153365557Sjasone 153465557Sjasonestatic void 153565856Sjhbwitness_leveldescendents(struct witness *parent, int level) 153665557Sjasone{ 153765557Sjasone int i; 153865856Sjhb struct witness *w; 153965557Sjasone 154065557Sjasone if (parent->w_level < level) 154165557Sjasone parent->w_level = level; 154265557Sjasone level++; 154365557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) 154465557Sjasone for (i = 0; i < w->w_childcnt; i++) 154565557Sjasone witness_leveldescendents(w->w_children[i], level); 154665557Sjasone} 154765557Sjasone 154865557Sjasonestatic void 154965856Sjhbwitness_displaydescendants(void(*prnt)(const char *fmt, ...), 155065856Sjhb struct witness *parent) 155165557Sjasone{ 155265856Sjhb struct witness *w; 155365557Sjasone int i; 155472224Sjhb int level; 155565557Sjasone 155672224Sjhb level = parent->w_spin ? ffs(parent->w_level) : parent->w_level; 155772224Sjhb 155865557Sjasone prnt("%d", level); 155965557Sjasone if (level < 10) 156065557Sjasone prnt(" "); 156165557Sjasone for (i = 0; i < level; i++) 156265557Sjasone prnt(" "); 156365557Sjasone prnt("%s", parent->w_description); 156472224Sjhb if (parent->w_file != NULL) 156572224Sjhb prnt(" -- last acquired @ %s:%d\n", parent->w_file, 156672224Sjhb parent->w_line); 156765557Sjasone 156865557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) 156965557Sjasone for (i = 0; i < w->w_childcnt; i++) 157065557Sjasone witness_displaydescendants(prnt, w->w_children[i]); 157165557Sjasone } 157265557Sjasone 157365557Sjasonestatic int 157465856Sjhbdup_ok(struct witness *w) 157565557Sjasone{ 157665557Sjasone char **dup; 157765557Sjasone 157865557Sjasone for (dup = dup_list; *dup!= NULL; dup++) 157965557Sjasone if (strcmp(w->w_description, *dup) == 0) 158065557Sjasone return (1); 158165557Sjasone return (0); 158265557Sjasone} 158365557Sjasone 158465557Sjasonestatic int 158565856Sjhbblessed(struct witness *w1, struct witness *w2) 158665557Sjasone{ 158765557Sjasone int i; 158865856Sjhb struct witness_blessed *b; 158965557Sjasone 159065557Sjasone for (i = 0; i < blessed_count; i++) { 159165557Sjasone b = &blessed_list[i]; 159265557Sjasone if (strcmp(w1->w_description, b->b_lock1) == 0) { 159365557Sjasone if (strcmp(w2->w_description, b->b_lock2) == 0) 159465557Sjasone return (1); 159565557Sjasone continue; 159665557Sjasone } 159765557Sjasone if (strcmp(w1->w_description, b->b_lock2) == 0) 159865557Sjasone if (strcmp(w2->w_description, b->b_lock1) == 0) 159965557Sjasone return (1); 160065557Sjasone } 160165557Sjasone return (0); 160265557Sjasone} 160365557Sjasone 160465856Sjhbstatic struct witness * 160565557Sjasonewitness_get() 160665557Sjasone{ 160765856Sjhb struct witness *w; 160865557Sjasone 160965557Sjasone if ((w = w_free) == NULL) { 161065557Sjasone witness_dead = 1; 161172200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 161265557Sjasone printf("witness exhausted\n"); 161365557Sjasone return (NULL); 161465557Sjasone } 161565557Sjasone w_free = w->w_next; 161665856Sjhb bzero(w, sizeof(*w)); 161765557Sjasone return (w); 161865557Sjasone} 161965557Sjasone 162065557Sjasonestatic void 162165856Sjhbwitness_free(struct witness *w) 162265557Sjasone{ 162365557Sjasone w->w_next = w_free; 162465557Sjasone w_free = w; 162565557Sjasone} 162665557Sjasone 162769881Sjakeint 162865557Sjasonewitness_list(struct proc *p) 162965557Sjasone{ 163065856Sjhb struct mtx *m; 163169881Sjake int nheld; 163265557Sjasone 163371320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 163469881Sjake nheld = 0; 163572224Sjhb LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { 163665557Sjasone printf("\t\"%s\" (%p) locked at %s:%d\n", 163765557Sjasone m->mtx_description, m, 163865557Sjasone m->mtx_witness->w_file, m->mtx_witness->w_line); 163969881Sjake nheld++; 164065557Sjasone } 164169881Sjake 164269881Sjake return (nheld); 164365557Sjasone} 164465557Sjasone 164571709Sjhb#ifdef DDB 164671709Sjhb 164772224SjhbDB_SHOW_COMMAND(mutexes, db_witness_list) 164871709Sjhb{ 164971709Sjhb 165072393Sbmilekic witness_list(curproc); 165171709Sjhb} 165271709Sjhb 165372224SjhbDB_SHOW_COMMAND(witness, db_witness_display) 165472224Sjhb{ 165572224Sjhb 165672224Sjhb witness_display(db_printf); 165772224Sjhb} 165871709Sjhb#endif 165971709Sjhb 166065557Sjasonevoid 166165856Sjhbwitness_save(struct mtx *m, const char **filep, int *linep) 166265557Sjasone{ 166371320Sjasone 166471320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 166571352Sjasone if (m->mtx_witness == NULL) 166671352Sjasone return; 166771352Sjasone 166865557Sjasone *filep = m->mtx_witness->w_file; 166965557Sjasone *linep = m->mtx_witness->w_line; 167065557Sjasone} 167165557Sjasone 167265557Sjasonevoid 167365856Sjhbwitness_restore(struct mtx *m, const char *file, int line) 167465557Sjasone{ 167571320Sjasone 167671320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 167771352Sjasone if (m->mtx_witness == NULL) 167871352Sjasone return; 167971352Sjasone 168065557Sjasone m->mtx_witness->w_file = file; 168165557Sjasone m->mtx_witness->w_line = line; 168265557Sjasone} 168365557Sjasone 168469429Sjhb#endif /* WITNESS */ 1685