subr_witness.c revision 73912
165557Sjasone/*- 265557Sjasone * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 365557Sjasone * 465557Sjasone * Redistribution and use in source and binary forms, with or without 565557Sjasone * modification, are permitted provided that the following conditions 665557Sjasone * are met: 765557Sjasone * 1. Redistributions of source code must retain the above copyright 865557Sjasone * notice, this list of conditions and the following disclaimer. 965557Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1065557Sjasone * notice, this list of conditions and the following disclaimer in the 1165557Sjasone * documentation and/or other materials provided with the distribution. 1265557Sjasone * 3. Berkeley Software Design Inc's name may not be used to endorse or 1365557Sjasone * promote products derived from this software without specific prior 1465557Sjasone * written permission. 1565557Sjasone * 1665557Sjasone * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1765557Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1865557Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1965557Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2065557Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2165557Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2265557Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2365557Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2465557Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2565557Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2665557Sjasone * SUCH DAMAGE. 2765557Sjasone * 2865557Sjasone * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 2967352Sjhb * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 3065557Sjasone * $FreeBSD: head/sys/kern/subr_witness.c 73912 2001-03-07 02:45:15Z jhb $ 3165557Sjasone */ 3265557Sjasone 3365557Sjasone/* 3472200Sbmilekic * Machine independent bits of mutex implementation and implementation of 3572200Sbmilekic * `witness' structure & related debugging routines. 3672200Sbmilekic */ 3772200Sbmilekic 3872200Sbmilekic/* 3965557Sjasone * Main Entry: witness 4065557Sjasone * Pronunciation: 'wit-n&s 4165557Sjasone * Function: noun 4265557Sjasone * Etymology: Middle English witnesse, from Old English witnes knowledge, 4365557Sjasone * testimony, witness, from 2wit 4465557Sjasone * Date: before 12th century 4565557Sjasone * 1 : attestation of a fact or event : TESTIMONY 4665557Sjasone * 2 : one that gives evidence; specifically : one who testifies in 4765557Sjasone * a cause or before a judicial tribunal 4865557Sjasone * 3 : one asked to be present at a transaction so as to be able to 4965557Sjasone * testify to its having taken place 5065557Sjasone * 4 : one who has personal knowledge of something 5165557Sjasone * 5 a : something serving as evidence or proof : SIGN 5265557Sjasone * b : public affirmation by word or example of usually 5365557Sjasone * religious faith or conviction <the heroic witness to divine 5465557Sjasone * life -- Pilot> 5565557Sjasone * 6 capitalized : a member of the Jehovah's Witnesses 5665557Sjasone */ 5765557Sjasone 5868790Sjhb#include "opt_ddb.h" 5967676Sjhb#include "opt_witness.h" 6067676Sjhb 6165557Sjasone#include <sys/param.h> 6267352Sjhb#include <sys/bus.h> 6367352Sjhb#include <sys/kernel.h> 6467352Sjhb#include <sys/malloc.h> 6565557Sjasone#include <sys/proc.h> 6667676Sjhb#include <sys/sysctl.h> 6765557Sjasone#include <sys/systm.h> 6867352Sjhb#include <sys/vmmeter.h> 6965557Sjasone#include <sys/ktr.h> 7065557Sjasone 7167352Sjhb#include <machine/atomic.h> 7267352Sjhb#include <machine/bus.h> 7367352Sjhb#include <machine/clock.h> 7465557Sjasone#include <machine/cpu.h> 7567352Sjhb 7668790Sjhb#include <ddb/ddb.h> 7768790Sjhb 7867352Sjhb#include <vm/vm.h> 7967352Sjhb#include <vm/vm_extern.h> 8067352Sjhb 8167352Sjhb#include <sys/mutex.h> 8265557Sjasone 8365557Sjasone/* 8472200Sbmilekic * The WITNESS-enabled mutex debug structure. 8567352Sjhb */ 8671352Sjasone#ifdef WITNESS 8771352Sjasonestruct mtx_debug { 8871352Sjasone struct witness *mtxd_witness; 8971352Sjasone LIST_ENTRY(mtx) mtxd_held; 9071352Sjasone const char *mtxd_file; 9171352Sjasone int mtxd_line; 9271352Sjasone}; 9371352Sjasone 9471560Sjhb#define mtx_held mtx_debug->mtxd_held 9571560Sjhb#define mtx_file mtx_debug->mtxd_file 9671560Sjhb#define mtx_line mtx_debug->mtxd_line 9771560Sjhb#define mtx_witness mtx_debug->mtxd_witness 9871352Sjasone#endif /* WITNESS */ 9971352Sjasone 10071352Sjasone/* 10172200Sbmilekic * Internal utility macros. 10271352Sjasone */ 10372200Sbmilekic#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 10471352Sjasone 10572200Sbmilekic#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 10672200Sbmilekic : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) 10771352Sjasone 10872376Sjake#define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) 10971352Sjasone 11071352Sjasone/* 11172200Sbmilekic * Early WITNESS-enabled declarations. 11271352Sjasone */ 11372200Sbmilekic#ifdef WITNESS 11471352Sjasone 11571352Sjasone/* 11672200Sbmilekic * Internal WITNESS routines which must be prototyped early. 11772200Sbmilekic * 11872200Sbmilekic * XXX: When/if witness code is cleaned up, it would be wise to place all 11972200Sbmilekic * witness prototyping early in this file. 12072200Sbmilekic */ 12172200Sbmilekicstatic void witness_init(struct mtx *, int flag); 12272200Sbmilekicstatic void witness_destroy(struct mtx *); 12372200Sbmilekicstatic void witness_display(void(*)(const char *fmt, ...)); 12471352Sjasone 12572200SbmilekicMALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure"); 12671352Sjasone 12767352Sjhb/* All mutexes in system (used for debug/panic) */ 12871560Sjhbstatic struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 }; 12972200Sbmilekic 13071320Sjasone/* 13172200Sbmilekic * This global is set to 0 once it becomes safe to use the witness code. 13271320Sjasone */ 13371320Sjasonestatic int witness_cold = 1; 13472200Sbmilekic 13569429Sjhb#else /* WITNESS */ 13671352Sjasone 13772200Sbmilekic/* XXX XXX XXX 13872200Sbmilekic * flag++ is sleazoid way of shuting up warning 13971352Sjasone */ 14071352Sjasone#define witness_init(m, flag) flag++ 14171352Sjasone#define witness_destroy(m) 14271352Sjasone#define witness_try_enter(m, t, f, l) 14369429Sjhb#endif /* WITNESS */ 14467352Sjhb 14572200Sbmilekic/* 14672200Sbmilekic * All mutex locks in system are kept on the all_mtx list. 14772200Sbmilekic */ 14871560Sjhbstatic struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", 14971560Sjhb TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), 15071560Sjhb { NULL, NULL }, &all_mtx, &all_mtx, 15171560Sjhb#ifdef WITNESS 15271560Sjhb &all_mtx_debug 15371560Sjhb#else 15471560Sjhb NULL 15571560Sjhb#endif 15671560Sjhb }; 15771560Sjhb 15872200Sbmilekic/* 15972200Sbmilekic * Global variables for book keeping. 16072200Sbmilekic */ 16167352Sjhbstatic int mtx_cur_cnt; 16267352Sjhbstatic int mtx_max_cnt; 16367352Sjhb 16472200Sbmilekic/* 16572344Sbmilekic * Couple of strings for KTR_LOCK tracing in order to avoid duplicates. 16672344Sbmilekic */ 16772344Sbmilekicchar STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d"; 16872344Sbmilekicchar STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d"; 16972344Sbmilekicchar STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d"; 17072344Sbmilekicchar STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d"; 17172344Sbmilekic 17272344Sbmilekic/* 17372200Sbmilekic * Prototypes for non-exported routines. 17472200Sbmilekic * 17572200Sbmilekic * NOTE: Prototypes for witness routines are placed at the bottom of the file. 17672200Sbmilekic */ 17771352Sjasonestatic void propagate_priority(struct proc *); 17867352Sjhb 17967352Sjhbstatic void 18067352Sjhbpropagate_priority(struct proc *p) 18167352Sjhb{ 18272376Sjake int pri = p->p_pri.pri_level; 18367352Sjhb struct mtx *m = p->p_blocked; 18467352Sjhb 18569376Sjhb mtx_assert(&sched_lock, MA_OWNED); 18667352Sjhb for (;;) { 18767352Sjhb struct proc *p1; 18867352Sjhb 18967352Sjhb p = mtx_owner(m); 19067352Sjhb 19167352Sjhb if (p == NULL) { 19267352Sjhb /* 19367352Sjhb * This really isn't quite right. Really 19467352Sjhb * ought to bump priority of process that 19567352Sjhb * next acquires the mutex. 19667352Sjhb */ 19767352Sjhb MPASS(m->mtx_lock == MTX_CONTESTED); 19867352Sjhb return; 19967352Sjhb } 20072200Sbmilekic 20167352Sjhb MPASS(p->p_magic == P_MAGIC); 20269376Sjhb KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 20372376Sjake if (p->p_pri.pri_level <= pri) 20467352Sjhb return; 20569376Sjhb 20667352Sjhb /* 20769376Sjhb * Bump this process' priority. 20869376Sjhb */ 20969376Sjhb SET_PRIO(p, pri); 21069376Sjhb 21169376Sjhb /* 21267352Sjhb * If lock holder is actually running, just bump priority. 21367352Sjhb */ 21472836Sjhb if (p->p_oncpu != NOCPU) { 21567352Sjhb MPASS(p->p_stat == SRUN || p->p_stat == SZOMB); 21667352Sjhb return; 21767352Sjhb } 21872376Sjake 21973912Sjhb#ifndef SMP 22067352Sjhb /* 22173912Sjhb * For UP, we check to see if p is curproc (this shouldn't 22273912Sjhb * ever happen however as it would mean we are in a deadlock.) 22373912Sjhb */ 22473912Sjhb KASSERT(p != curproc, ("Deadlock detected")); 22573912Sjhb#endif 22673912Sjhb 22773912Sjhb /* 22867352Sjhb * If on run queue move to new run queue, and 22967352Sjhb * quit. 23067352Sjhb */ 23167352Sjhb if (p->p_stat == SRUN) { 23267352Sjhb MPASS(p->p_blocked == NULL); 23367352Sjhb remrunqueue(p); 23467352Sjhb setrunqueue(p); 23567352Sjhb return; 23667352Sjhb } 23767352Sjhb 23867352Sjhb /* 23969376Sjhb * If we aren't blocked on a mutex, we should be. 24067352Sjhb */ 24169376Sjhb KASSERT(p->p_stat == SMTX, ( 24269376Sjhb "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 24369376Sjhb p->p_pid, p->p_comm, p->p_stat, 24469376Sjhb m->mtx_description)); 24567352Sjhb 24667352Sjhb /* 24767352Sjhb * Pick up the mutex that p is blocked on. 24867352Sjhb */ 24967352Sjhb m = p->p_blocked; 25067352Sjhb MPASS(m != NULL); 25167352Sjhb 25267352Sjhb /* 25367352Sjhb * Check if the proc needs to be moved up on 25467352Sjhb * the blocked chain 25567352Sjhb */ 25669376Sjhb if (p == TAILQ_FIRST(&m->mtx_blocked)) { 25769376Sjhb continue; 25869376Sjhb } 25972200Sbmilekic 26072376Sjake p1 = TAILQ_PREV(p, procqueue, p_procq); 26172376Sjake if (p1->p_pri.pri_level <= pri) { 26267352Sjhb continue; 26367352Sjhb } 26467352Sjhb 26567352Sjhb /* 26669376Sjhb * Remove proc from blocked chain and determine where 26769376Sjhb * it should be moved up to. Since we know that p1 has 26869376Sjhb * a lower priority than p, we know that at least one 26969376Sjhb * process in the chain has a lower priority and that 27069376Sjhb * p1 will thus not be NULL after the loop. 27167352Sjhb */ 27267352Sjhb TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 27367352Sjhb TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 27467352Sjhb MPASS(p1->p_magic == P_MAGIC); 27572376Sjake if (p1->p_pri.pri_level > pri) 27667352Sjhb break; 27767352Sjhb } 27872200Sbmilekic 27969376Sjhb MPASS(p1 != NULL); 28069376Sjhb TAILQ_INSERT_BEFORE(p1, p, p_procq); 28167352Sjhb CTR4(KTR_LOCK, 28271560Sjhb "propagate_priority: p %p moved before %p on [%p] %s", 28367352Sjhb p, p1, m, m->mtx_description); 28467352Sjhb } 28567352Sjhb} 28667352Sjhb 28771352Sjasone/* 28872200Sbmilekic * The important part of mtx_trylock{,_flags}() 28972200Sbmilekic * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 29072200Sbmilekic * if we're called, it's because we know we don't already own this lock. 29171352Sjasone */ 29272200Sbmilekicint 29372200Sbmilekic_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 29471352Sjasone{ 29572200Sbmilekic int rval; 29671352Sjasone 29772393Sbmilekic MPASS(curproc != NULL); 29871352Sjasone 29972200Sbmilekic /* 30072200Sbmilekic * _mtx_trylock does not accept MTX_NOSWITCH option. 30172200Sbmilekic */ 30272344Sbmilekic KASSERT((opts & MTX_NOSWITCH) == 0, 30372344Sbmilekic ("mtx_trylock() called with invalid option flag(s) %d", opts)); 30472200Sbmilekic 30572393Sbmilekic rval = _obtain_lock(m, curproc); 30672200Sbmilekic 30772200Sbmilekic#ifdef WITNESS 30872200Sbmilekic if (rval && m->mtx_witness != NULL) { 30971352Sjasone /* 31072200Sbmilekic * We do not handle recursion in _mtx_trylock; see the 31172200Sbmilekic * note at the top of the routine. 31271352Sjasone */ 31372344Sbmilekic KASSERT(!mtx_recursed(m), 31472344Sbmilekic ("mtx_trylock() called on a recursed mutex")); 31572200Sbmilekic witness_try_enter(m, (opts | m->mtx_flags), file, line); 31671352Sjasone } 31772200Sbmilekic#endif /* WITNESS */ 31871352Sjasone 31972200Sbmilekic if ((opts & MTX_QUIET) == 0) 32072994Sjhb CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d", 32172200Sbmilekic m->mtx_description, m, rval, file, line); 32272200Sbmilekic 32372200Sbmilekic return rval; 32471352Sjasone} 32571352Sjasone 32671352Sjasone/* 32772200Sbmilekic * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 32871352Sjasone * 32972200Sbmilekic * We call this if the lock is either contested (i.e. we need to go to 33072200Sbmilekic * sleep waiting for it), or if we need to recurse on it. 33171352Sjasone */ 33272200Sbmilekicvoid 33372200Sbmilekic_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 33471352Sjasone{ 33572393Sbmilekic struct proc *p = curproc; 33671352Sjasone 33772200Sbmilekic if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 33872200Sbmilekic m->mtx_recurse++; 33972200Sbmilekic atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 34072200Sbmilekic if ((opts & MTX_QUIET) == 0) 34172344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 34272200Sbmilekic return; 34371352Sjasone } 34471352Sjasone 34572200Sbmilekic if ((opts & MTX_QUIET) == 0) 34672994Sjhb CTR4(KTR_LOCK, 34772994Sjhb "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 34872994Sjhb m->mtx_description, (void *)m->mtx_lock, file, line); 34971352Sjasone 35072200Sbmilekic while (!_obtain_lock(m, p)) { 35172200Sbmilekic uintptr_t v; 35272200Sbmilekic struct proc *p1; 35371352Sjasone 35472200Sbmilekic mtx_lock_spin(&sched_lock); 35572200Sbmilekic /* 35672200Sbmilekic * Check if the lock has been released while spinning for 35772200Sbmilekic * the sched_lock. 35872200Sbmilekic */ 35972200Sbmilekic if ((v = m->mtx_lock) == MTX_UNOWNED) { 36072200Sbmilekic mtx_unlock_spin(&sched_lock); 36172200Sbmilekic continue; 36271352Sjasone } 36371352Sjasone 36472200Sbmilekic /* 36572200Sbmilekic * The mutex was marked contested on release. This means that 36672200Sbmilekic * there are processes blocked on it. 36772200Sbmilekic */ 36872200Sbmilekic if (v == MTX_CONTESTED) { 36972200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 37072344Sbmilekic MPASS(p1 != NULL); 37172200Sbmilekic m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; 37267352Sjhb 37372376Sjake if (p1->p_pri.pri_level < p->p_pri.pri_level) 37472376Sjake SET_PRIO(p, p1->p_pri.pri_level); 37572200Sbmilekic mtx_unlock_spin(&sched_lock); 37667352Sjhb return; 37767352Sjhb } 37869376Sjhb 37969376Sjhb /* 38072200Sbmilekic * If the mutex isn't already contested and a failure occurs 38172200Sbmilekic * setting the contested bit, the mutex was either released 38272200Sbmilekic * or the state of the MTX_RECURSED bit changed. 38369376Sjhb */ 38472200Sbmilekic if ((v & MTX_CONTESTED) == 0 && 38572200Sbmilekic !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 38672200Sbmilekic (void *)(v | MTX_CONTESTED))) { 38772200Sbmilekic mtx_unlock_spin(&sched_lock); 38872200Sbmilekic continue; 38972200Sbmilekic } 39067352Sjhb 39172200Sbmilekic /* 39272200Sbmilekic * We deffinately must sleep for this lock. 39372200Sbmilekic */ 39472200Sbmilekic mtx_assert(m, MA_NOTOWNED); 39567352Sjhb 39667352Sjhb#ifdef notyet 39772200Sbmilekic /* 39872200Sbmilekic * If we're borrowing an interrupted thread's VM context, we 39972200Sbmilekic * must clean up before going to sleep. 40072200Sbmilekic */ 40172994Sjhb if (p->p_ithd != NULL) { 40272994Sjhb struct ithd *it = p->p_ithd; 40367352Sjhb 40472200Sbmilekic if (it->it_interrupted) { 40572200Sbmilekic if ((opts & MTX_QUIET) == 0) 40672200Sbmilekic CTR2(KTR_LOCK, 40772994Sjhb "_mtx_lock_sleep: %p interrupted %p", 40872200Sbmilekic it, it->it_interrupted); 40972200Sbmilekic intr_thd_fixup(it); 41067352Sjhb } 41172200Sbmilekic } 41267352Sjhb#endif 41367352Sjhb 41472200Sbmilekic /* 41572200Sbmilekic * Put us on the list of threads blocked on this mutex. 41672200Sbmilekic */ 41772200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 41872200Sbmilekic p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); 41972200Sbmilekic LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); 42072200Sbmilekic TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 42172200Sbmilekic } else { 42272200Sbmilekic TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 42372376Sjake if (p1->p_pri.pri_level > p->p_pri.pri_level) 42472200Sbmilekic break; 42572200Sbmilekic if (p1) 42672200Sbmilekic TAILQ_INSERT_BEFORE(p1, p, p_procq); 42772200Sbmilekic else 42867352Sjhb TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 42972200Sbmilekic } 43067352Sjhb 43172200Sbmilekic /* 43272200Sbmilekic * Save who we're blocked on. 43372200Sbmilekic */ 43472200Sbmilekic p->p_blocked = m; 43572200Sbmilekic p->p_mtxname = m->mtx_description; 43672200Sbmilekic p->p_stat = SMTX; 43772200Sbmilekic propagate_priority(p); 43867352Sjhb 43972200Sbmilekic if ((opts & MTX_QUIET) == 0) 44072200Sbmilekic CTR3(KTR_LOCK, 44172200Sbmilekic "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, 44272200Sbmilekic m->mtx_description); 44372200Sbmilekic 44472200Sbmilekic mi_switch(); 44572200Sbmilekic 44672200Sbmilekic if ((opts & MTX_QUIET) == 0) 44772200Sbmilekic CTR3(KTR_LOCK, 44872200Sbmilekic "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 44972200Sbmilekic p, m, m->mtx_description); 45072200Sbmilekic 45172200Sbmilekic mtx_unlock_spin(&sched_lock); 45272200Sbmilekic } 45372200Sbmilekic 45472200Sbmilekic return; 45572200Sbmilekic} 45672200Sbmilekic 45772200Sbmilekic/* 45872200Sbmilekic * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 45972200Sbmilekic * 46072200Sbmilekic * This is only called if we need to actually spin for the lock. Recursion 46172200Sbmilekic * is handled inline. 46272200Sbmilekic */ 46372200Sbmilekicvoid 46472200Sbmilekic_mtx_lock_spin(struct mtx *m, int opts, u_int mtx_intr, const char *file, 46572200Sbmilekic int line) 46672200Sbmilekic{ 46772200Sbmilekic int i = 0; 46872200Sbmilekic 46972200Sbmilekic if ((opts & MTX_QUIET) == 0) 47072344Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 47172200Sbmilekic 47272200Sbmilekic for (;;) { 47372393Sbmilekic if (_obtain_lock(m, curproc)) 47472200Sbmilekic break; 47572200Sbmilekic 47672200Sbmilekic while (m->mtx_lock != MTX_UNOWNED) { 47772200Sbmilekic if (i++ < 1000000) 47872200Sbmilekic continue; 47972200Sbmilekic if (i++ < 6000000) 48072200Sbmilekic DELAY(1); 48167352Sjhb#ifdef DDB 48272200Sbmilekic else if (!db_active) 48367352Sjhb#else 48472200Sbmilekic else 48567352Sjhb#endif 48672200Sbmilekic panic("spin lock %s held by %p for > 5 seconds", 48772200Sbmilekic m->mtx_description, (void *)m->mtx_lock); 48867352Sjhb } 48967352Sjhb } 49072200Sbmilekic 49172200Sbmilekic m->mtx_saveintr = mtx_intr; 49272200Sbmilekic if ((opts & MTX_QUIET) == 0) 49372200Sbmilekic CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 49472200Sbmilekic 49572200Sbmilekic return; 49667352Sjhb} 49767352Sjhb 49872200Sbmilekic/* 49972200Sbmilekic * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 50072200Sbmilekic * 50172200Sbmilekic * We are only called here if the lock is recursed or contested (i.e. we 50272200Sbmilekic * need to wake up a blocked thread). 50372200Sbmilekic */ 50467352Sjhbvoid 50572200Sbmilekic_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 50667352Sjhb{ 50767352Sjhb struct proc *p, *p1; 50867352Sjhb struct mtx *m1; 50967352Sjhb int pri; 51067352Sjhb 51172393Sbmilekic p = curproc; 51272200Sbmilekic 51372200Sbmilekic if (mtx_recursed(m)) { 51472200Sbmilekic if (--(m->mtx_recurse) == 0) 51572200Sbmilekic atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 51672200Sbmilekic if ((opts & MTX_QUIET) == 0) 51772200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 51872200Sbmilekic return; 51972200Sbmilekic } 52072200Sbmilekic 52172200Sbmilekic mtx_lock_spin(&sched_lock); 52272200Sbmilekic if ((opts & MTX_QUIET) == 0) 52372200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 52472200Sbmilekic 52572200Sbmilekic p1 = TAILQ_FIRST(&m->mtx_blocked); 52672200Sbmilekic MPASS(p->p_magic == P_MAGIC); 52772200Sbmilekic MPASS(p1->p_magic == P_MAGIC); 52872200Sbmilekic 52972200Sbmilekic TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 53072200Sbmilekic 53172200Sbmilekic if (TAILQ_EMPTY(&m->mtx_blocked)) { 53272200Sbmilekic LIST_REMOVE(m, mtx_contested); 53372200Sbmilekic _release_lock_quick(m); 53472200Sbmilekic if ((opts & MTX_QUIET) == 0) 53572200Sbmilekic CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 53672200Sbmilekic } else 53772200Sbmilekic atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 53872200Sbmilekic 53972376Sjake pri = PRI_MAX; 54072200Sbmilekic LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 54172376Sjake int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level; 54272200Sbmilekic if (cp < pri) 54372200Sbmilekic pri = cp; 54472200Sbmilekic } 54572200Sbmilekic 54672376Sjake if (pri > p->p_pri.pri_native) 54772376Sjake pri = p->p_pri.pri_native; 54872200Sbmilekic SET_PRIO(p, pri); 54972200Sbmilekic 55072200Sbmilekic if ((opts & MTX_QUIET) == 0) 55172200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 55272200Sbmilekic m, p1); 55372200Sbmilekic 55472200Sbmilekic p1->p_blocked = NULL; 55572200Sbmilekic p1->p_stat = SRUN; 55672200Sbmilekic setrunqueue(p1); 55772200Sbmilekic 55872376Sjake if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) { 55967352Sjhb#ifdef notyet 56072994Sjhb if (p->p_ithd != NULL) { 56172994Sjhb struct ithd *it = p->p_ithd; 56267352Sjhb 56372200Sbmilekic if (it->it_interrupted) { 56472200Sbmilekic if ((opts & MTX_QUIET) == 0) 56572200Sbmilekic CTR2(KTR_LOCK, 56672994Sjhb "_mtx_unlock_sleep: %p interrupted %p", 56772200Sbmilekic it, it->it_interrupted); 56872200Sbmilekic intr_thd_fixup(it); 56967352Sjhb } 57072200Sbmilekic } 57167352Sjhb#endif 57272200Sbmilekic setrunqueue(p); 57372200Sbmilekic if ((opts & MTX_QUIET) == 0) 57472200Sbmilekic CTR2(KTR_LOCK, 57572200Sbmilekic "_mtx_unlock_sleep: %p switching out lock=%p", m, 57672200Sbmilekic (void *)m->mtx_lock); 57772200Sbmilekic 57872200Sbmilekic mi_switch(); 57972200Sbmilekic if ((opts & MTX_QUIET) == 0) 58072200Sbmilekic CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 58172200Sbmilekic m, (void *)m->mtx_lock); 58267352Sjhb } 58372200Sbmilekic 58472200Sbmilekic mtx_unlock_spin(&sched_lock); 58572200Sbmilekic 58672200Sbmilekic return; 58767352Sjhb} 58867352Sjhb 58972200Sbmilekic/* 59072200Sbmilekic * All the unlocking of MTX_SPIN locks is done inline. 59172200Sbmilekic * See the _rel_spin_lock() macro for the details. 59272200Sbmilekic */ 59372200Sbmilekic 59472200Sbmilekic/* 59572994Sjhb * The backing function for the INVARIANTS-enabled mtx_assert() 59672200Sbmilekic */ 59772996Sjhb#ifdef INVARIANT_SUPPORT 59871352Sjasonevoid 59971360Sjasone_mtx_assert(struct mtx *m, int what, const char *file, int line) 60071352Sjasone{ 60173033Sjake switch (what) { 60271352Sjasone case MA_OWNED: 60371352Sjasone case MA_OWNED | MA_RECURSED: 60471352Sjasone case MA_OWNED | MA_NOTRECURSED: 60573033Sjake if (!mtx_owned(m)) 60671352Sjasone panic("mutex %s not owned at %s:%d", 60773033Sjake m->mtx_description, file, line); 60873033Sjake if (mtx_recursed(m)) { 60973033Sjake if ((what & MA_NOTRECURSED) != 0) 61071352Sjasone panic("mutex %s recursed at %s:%d", 61173033Sjake m->mtx_description, file, line); 61273033Sjake } else if ((what & MA_RECURSED) != 0) { 61371352Sjasone panic("mutex %s unrecursed at %s:%d", 61473033Sjake m->mtx_description, file, line); 61571352Sjasone } 61671352Sjasone break; 61771352Sjasone case MA_NOTOWNED: 61873033Sjake if (mtx_owned(m)) 61971352Sjasone panic("mutex %s owned at %s:%d", 62073033Sjake m->mtx_description, file, line); 62171352Sjasone break; 62271352Sjasone default: 62371360Sjasone panic("unknown mtx_assert at %s:%d", file, line); 62471352Sjasone } 62571352Sjasone} 62671352Sjasone#endif 62771352Sjasone 62872200Sbmilekic/* 62972200Sbmilekic * The MUTEX_DEBUG-enabled mtx_validate() 63072200Sbmilekic */ 63167352Sjhb#define MV_DESTROY 0 /* validate before destory */ 63267352Sjhb#define MV_INIT 1 /* validate before init */ 63367352Sjhb 63467352Sjhb#ifdef MUTEX_DEBUG 63567352Sjhb 63667352Sjhbint mtx_validate __P((struct mtx *, int)); 63767352Sjhb 63867352Sjhbint 63967352Sjhbmtx_validate(struct mtx *m, int when) 64067352Sjhb{ 64167352Sjhb struct mtx *mp; 64267352Sjhb int i; 64367352Sjhb int retval = 0; 64467352Sjhb 64571320Sjasone#ifdef WITNESS 64671320Sjasone if (witness_cold) 64771320Sjasone return 0; 64871320Sjasone#endif 64967352Sjhb if (m == &all_mtx || cold) 65067352Sjhb return 0; 65167352Sjhb 65272200Sbmilekic mtx_lock(&all_mtx); 65367352Sjhb/* 65467352Sjhb * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 65567352Sjhb * we can re-enable the kernacc() checks. 65667352Sjhb */ 65767352Sjhb#ifndef __alpha__ 65867352Sjhb MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), 65967352Sjhb VM_PROT_READ) == 1); 66067352Sjhb#endif 66167352Sjhb MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); 66267352Sjhb for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 66367352Sjhb#ifndef __alpha__ 66467352Sjhb if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), 66567352Sjhb VM_PROT_READ) != 1) { 66667352Sjhb panic("mtx_validate: mp=%p mp->mtx_next=%p", 66767352Sjhb mp, mp->mtx_next); 66867352Sjhb } 66967352Sjhb#endif 67067352Sjhb i++; 67167352Sjhb if (i > mtx_cur_cnt) { 67267352Sjhb panic("mtx_validate: too many in chain, known=%d\n", 67367352Sjhb mtx_cur_cnt); 67467352Sjhb } 67567352Sjhb } 67667352Sjhb MPASS(i == mtx_cur_cnt); 67767352Sjhb switch (when) { 67867352Sjhb case MV_DESTROY: 67967352Sjhb for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 68067352Sjhb if (mp == m) 68167352Sjhb break; 68267352Sjhb MPASS(mp == m); 68367352Sjhb break; 68467352Sjhb case MV_INIT: 68567352Sjhb for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 68667352Sjhb if (mp == m) { 68767352Sjhb /* 68867352Sjhb * Not good. This mutex already exists. 68967352Sjhb */ 69067352Sjhb printf("re-initing existing mutex %s\n", 69167352Sjhb m->mtx_description); 69267352Sjhb MPASS(m->mtx_lock == MTX_UNOWNED); 69367352Sjhb retval = 1; 69467352Sjhb } 69567352Sjhb } 69672200Sbmilekic mtx_unlock(&all_mtx); 69767352Sjhb return (retval); 69867352Sjhb} 69967352Sjhb#endif 70067352Sjhb 70172200Sbmilekic/* 70272200Sbmilekic * Mutex initialization routine; initialize lock `m' of type contained in 70372200Sbmilekic * `opts' with options contained in `opts' and description `description.' 70472200Sbmilekic * Place on "all_mtx" queue. 70572200Sbmilekic */ 70667352Sjhbvoid 70772200Sbmilekicmtx_init(struct mtx *m, const char *description, int opts) 70867352Sjhb{ 70972200Sbmilekic 71072200Sbmilekic if ((opts & MTX_QUIET) == 0) 71172200Sbmilekic CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description); 71272200Sbmilekic 71367352Sjhb#ifdef MUTEX_DEBUG 71472200Sbmilekic /* Diagnostic and error correction */ 71572200Sbmilekic if (mtx_validate(m, MV_INIT)) 71667352Sjhb return; 71769429Sjhb#endif 71867352Sjhb 71967352Sjhb bzero((void *)m, sizeof *m); 72067352Sjhb TAILQ_INIT(&m->mtx_blocked); 72172200Sbmilekic 72269429Sjhb#ifdef WITNESS 72371320Sjasone if (!witness_cold) { 72471560Sjhb m->mtx_debug = malloc(sizeof(struct mtx_debug), 72572200Sbmilekic M_WITNESS, M_NOWAIT | M_ZERO); 72671560Sjhb MPASS(m->mtx_debug != NULL); 72771320Sjasone } 72871560Sjhb#endif 72971320Sjasone 73072200Sbmilekic m->mtx_description = description; 73172200Sbmilekic m->mtx_flags = opts; 73267352Sjhb m->mtx_lock = MTX_UNOWNED; 73372200Sbmilekic 73467352Sjhb /* Put on all mutex queue */ 73572200Sbmilekic mtx_lock(&all_mtx); 73667352Sjhb m->mtx_next = &all_mtx; 73767352Sjhb m->mtx_prev = all_mtx.mtx_prev; 73867352Sjhb m->mtx_prev->mtx_next = m; 73967352Sjhb all_mtx.mtx_prev = m; 74067352Sjhb if (++mtx_cur_cnt > mtx_max_cnt) 74167352Sjhb mtx_max_cnt = mtx_cur_cnt; 74272200Sbmilekic mtx_unlock(&all_mtx); 74372200Sbmilekic 74471320Sjasone#ifdef WITNESS 74571320Sjasone if (!witness_cold) 74672200Sbmilekic witness_init(m, opts); 74771320Sjasone#endif 74867352Sjhb} 74967352Sjhb 75072200Sbmilekic/* 75172200Sbmilekic * Remove lock `m' from all_mtx queue. 75272200Sbmilekic */ 75367352Sjhbvoid 75467352Sjhbmtx_destroy(struct mtx *m) 75567352Sjhb{ 75667352Sjhb 75771320Sjasone#ifdef WITNESS 75871320Sjasone KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", 75971320Sjasone __FUNCTION__)); 76071320Sjasone#endif 76172200Sbmilekic 76271560Sjhb CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description); 76372200Sbmilekic 76467352Sjhb#ifdef MUTEX_DEBUG 76567352Sjhb if (m->mtx_next == NULL) 76667352Sjhb panic("mtx_destroy: %p (%s) already destroyed", 76767352Sjhb m, m->mtx_description); 76867352Sjhb 76967352Sjhb if (!mtx_owned(m)) { 77067352Sjhb MPASS(m->mtx_lock == MTX_UNOWNED); 77167352Sjhb } else { 77271228Sbmilekic MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 77367352Sjhb } 77472200Sbmilekic 77572200Sbmilekic /* diagnostic */ 77672200Sbmilekic mtx_validate(m, MV_DESTROY); 77767352Sjhb#endif 77867352Sjhb 77967352Sjhb#ifdef WITNESS 78067352Sjhb if (m->mtx_witness) 78167352Sjhb witness_destroy(m); 78267352Sjhb#endif /* WITNESS */ 78367352Sjhb 78467352Sjhb /* Remove from the all mutex queue */ 78572200Sbmilekic mtx_lock(&all_mtx); 78667352Sjhb m->mtx_next->mtx_prev = m->mtx_prev; 78767352Sjhb m->mtx_prev->mtx_next = m->mtx_next; 78872200Sbmilekic 78967352Sjhb#ifdef MUTEX_DEBUG 79067352Sjhb m->mtx_next = m->mtx_prev = NULL; 79169429Sjhb#endif 79272200Sbmilekic 79369429Sjhb#ifdef WITNESS 79472200Sbmilekic free(m->mtx_debug, M_WITNESS); 79571560Sjhb m->mtx_debug = NULL; 79667352Sjhb#endif 79772200Sbmilekic 79867352Sjhb mtx_cur_cnt--; 79972200Sbmilekic mtx_unlock(&all_mtx); 80067352Sjhb} 80167352Sjhb 80272200Sbmilekic 80371560Sjhb/* 80472200Sbmilekic * The WITNESS-enabled diagnostic code. 80571560Sjhb */ 80671560Sjhb#ifdef WITNESS 80771320Sjasonestatic void 80871320Sjasonewitness_fixup(void *dummy __unused) 80971320Sjasone{ 81071320Sjasone struct mtx *mp; 81171320Sjasone 81271560Sjhb /* 81371560Sjhb * We have to release Giant before initializing its witness 81471560Sjhb * structure so that WITNESS doesn't get confused. 81571560Sjhb */ 81672200Sbmilekic mtx_unlock(&Giant); 81771560Sjhb mtx_assert(&Giant, MA_NOTOWNED); 81871560Sjhb 81972200Sbmilekic mtx_lock(&all_mtx); 82072200Sbmilekic 82171320Sjasone /* Iterate through all mutexes and finish up mutex initialization. */ 82271320Sjasone for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 82371320Sjasone 82471560Sjhb mp->mtx_debug = malloc(sizeof(struct mtx_debug), 82572200Sbmilekic M_WITNESS, M_NOWAIT | M_ZERO); 82671560Sjhb MPASS(mp->mtx_debug != NULL); 82771320Sjasone 82871320Sjasone witness_init(mp, mp->mtx_flags); 82971320Sjasone } 83072200Sbmilekic mtx_unlock(&all_mtx); 83171320Sjasone 83271320Sjasone /* Mark the witness code as being ready for use. */ 83371320Sjasone atomic_store_rel_int(&witness_cold, 0); 83471560Sjhb 83572200Sbmilekic mtx_lock(&Giant); 83671320Sjasone} 83771320SjasoneSYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) 83871320Sjasone 83965557Sjasone#define WITNESS_COUNT 200 84065557Sjasone#define WITNESS_NCHILDREN 2 84165557Sjasone 84267401Sjhbint witness_watch = 1; 84365557Sjasone 84465856Sjhbstruct witness { 84565557Sjasone struct witness *w_next; 84667404Sjhb const char *w_description; 84765624Sjasone const char *w_file; 84865557Sjasone int w_line; 84965557Sjasone struct witness *w_morechildren; 85065557Sjasone u_char w_childcnt; 85165557Sjasone u_char w_Giant_squawked:1; 85265557Sjasone u_char w_other_squawked:1; 85365557Sjasone u_char w_same_squawked:1; 85471228Sbmilekic u_char w_spin:1; /* MTX_SPIN type mutex. */ 85565557Sjasone u_int w_level; 85665557Sjasone struct witness *w_children[WITNESS_NCHILDREN]; 85765856Sjhb}; 85865557Sjasone 85965856Sjhbstruct witness_blessed { 86065557Sjasone char *b_lock1; 86165557Sjasone char *b_lock2; 86265856Sjhb}; 86365557Sjasone 86467676Sjhb#ifdef DDB 86565557Sjasone/* 86667676Sjhb * When DDB is enabled and witness_ddb is set to 1, it will cause the system to 86765557Sjasone * drop into kdebug() when: 86865557Sjasone * - a lock heirarchy violation occurs 86965557Sjasone * - locks are held when going to sleep. 87065557Sjasone */ 87171560Sjhbint witness_ddb; 87267676Sjhb#ifdef WITNESS_DDB 87371560SjhbTUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb); 87467676Sjhb#else 87571560SjhbTUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb); 87665557Sjasone#endif 87767676SjhbSYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, ""); 87867676Sjhb#endif /* DDB */ 87965557Sjasone 88071560Sjhbint witness_skipspin; 88167676Sjhb#ifdef WITNESS_SKIPSPIN 88271560SjhbTUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin); 88367676Sjhb#else 88471560SjhbTUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); 88565557Sjasone#endif 88667676SjhbSYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, 88767676Sjhb ""); 88865557Sjasone 88972200Sbmilekic/* 89072200Sbmilekic * Witness-enabled globals 89172200Sbmilekic */ 89271320Sjasonestatic struct mtx w_mtx; 89365856Sjhbstatic struct witness *w_free; 89465856Sjhbstatic struct witness *w_all; 89565856Sjhbstatic int w_inited; 89665856Sjhbstatic int witness_dead; /* fatal error, probably no memory */ 89765557Sjasone 89865856Sjhbstatic struct witness w_data[WITNESS_COUNT]; 89965557Sjasone 90072200Sbmilekic/* 90172200Sbmilekic * Internal witness routine prototypes 90272200Sbmilekic */ 90372200Sbmilekicstatic struct witness *enroll(const char *description, int flag); 90472200Sbmilekicstatic int itismychild(struct witness *parent, struct witness *child); 90572200Sbmilekicstatic void removechild(struct witness *parent, struct witness *child); 90672200Sbmilekicstatic int isitmychild(struct witness *parent, struct witness *child); 90772200Sbmilekicstatic int isitmydescendant(struct witness *parent, struct witness *child); 90872200Sbmilekicstatic int dup_ok(struct witness *); 90972200Sbmilekicstatic int blessed(struct witness *, struct witness *); 91072200Sbmilekicstatic void 91172200Sbmilekic witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *); 91272200Sbmilekicstatic void witness_leveldescendents(struct witness *parent, int level); 91372200Sbmilekicstatic void witness_levelall(void); 91472200Sbmilekicstatic struct witness * witness_get(void); 91572200Sbmilekicstatic void witness_free(struct witness *m); 91665557Sjasone 91765557Sjasonestatic char *ignore_list[] = { 91865557Sjasone "witness lock", 91965557Sjasone NULL 92065557Sjasone}; 92165557Sjasone 92265557Sjasonestatic char *spin_order_list[] = { 92372224Sjhb#if defined(__i386__) && defined (SMP) 92472224Sjhb "com", 92572224Sjhb#endif 92669362Sjhb "sio", 92772224Sjhb#ifdef __i386__ 92872224Sjhb "cy", 92972224Sjhb#endif 93073238Sjulian "ng_node", 93173238Sjulian "ng_worklist", 93272836Sjhb "ithread table lock", 93372836Sjhb "ithread list lock", 93465557Sjasone "sched lock", 93568808Sjhb#ifdef __i386__ 93667676Sjhb "clk", 93768808Sjhb#endif 93868889Sjake "callout", 93965557Sjasone /* 94065557Sjasone * leaf locks 94165557Sjasone */ 94272224Sjhb#ifdef SMP 94371576Sjasone#ifdef __i386__ 94471576Sjasone "ap boot", 94571576Sjasone "imen", 94671576Sjasone#endif 94771576Sjasone "smp rendezvous", 94872224Sjhb#endif 94965557Sjasone NULL 95065557Sjasone}; 95165557Sjasone 95265557Sjasonestatic char *order_list[] = { 95372256Sjhb "Giant", "proctree", "allproc", "process lock", "uidinfo hash", 95472256Sjhb "uidinfo struct", NULL, 95565557Sjasone NULL 95665557Sjasone}; 95765557Sjasone 95865557Sjasonestatic char *dup_list[] = { 95973912Sjhb "process lock", 96065557Sjasone NULL 96165557Sjasone}; 96265557Sjasone 96365557Sjasonestatic char *sleep_list[] = { 96468862Sjake "Giant", 96565557Sjasone NULL 96665557Sjasone}; 96765557Sjasone 96865557Sjasone/* 96965557Sjasone * Pairs of locks which have been blessed 97065557Sjasone * Don't complain about order problems with blessed locks 97165557Sjasone */ 97265856Sjhbstatic struct witness_blessed blessed_list[] = { 97365557Sjasone}; 97472200Sbmilekicstatic int blessed_count = 97572200Sbmilekic sizeof(blessed_list) / sizeof(struct witness_blessed); 97665557Sjasone 97771352Sjasonestatic void 97865856Sjhbwitness_init(struct mtx *m, int flag) 97965557Sjasone{ 98065557Sjasone m->mtx_witness = enroll(m->mtx_description, flag); 98165557Sjasone} 98265557Sjasone 98371352Sjasonestatic void 98465856Sjhbwitness_destroy(struct mtx *m) 98565557Sjasone{ 98665856Sjhb struct mtx *m1; 98765557Sjasone struct proc *p; 98872393Sbmilekic p = curproc; 98972224Sjhb LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) { 99065557Sjasone if (m1 == m) { 99165557Sjasone LIST_REMOVE(m, mtx_held); 99265557Sjasone break; 99365557Sjasone } 99465557Sjasone } 99565557Sjasone return; 99665557Sjasone 99765557Sjasone} 99865557Sjasone 99971352Sjasonestatic void 100071352Sjasonewitness_display(void(*prnt)(const char *fmt, ...)) 100171352Sjasone{ 100271352Sjasone struct witness *w, *w1; 100372224Sjhb int level, found; 100471352Sjasone 100571352Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 100671352Sjasone witness_levelall(); 100771352Sjasone 100872224Sjhb /* 100972224Sjhb * First, handle sleep mutexes which have been acquired at least 101072224Sjhb * once. 101172224Sjhb */ 101272224Sjhb prnt("Sleep mutexes:\n"); 101371352Sjasone for (w = w_all; w; w = w->w_next) { 101472224Sjhb if (w->w_file == NULL || w->w_spin) 101571352Sjasone continue; 101671352Sjasone for (w1 = w_all; w1; w1 = w1->w_next) { 101771352Sjasone if (isitmychild(w1, w)) 101871352Sjasone break; 101971352Sjasone } 102071352Sjasone if (w1 != NULL) 102171352Sjasone continue; 102271352Sjasone /* 102371352Sjasone * This lock has no anscestors, display its descendants. 102471352Sjasone */ 102571352Sjasone witness_displaydescendants(prnt, w); 102671352Sjasone } 102772224Sjhb 102872224Sjhb /* 102972224Sjhb * Now do spin mutexes which have been acquired at least once. 103072224Sjhb */ 103172224Sjhb prnt("\nSpin mutexes:\n"); 103272224Sjhb level = 0; 103372224Sjhb while (level < sizeof(spin_order_list) / sizeof(char *)) { 103472224Sjhb found = 0; 103572224Sjhb for (w = w_all; w; w = w->w_next) { 103672224Sjhb if (w->w_file == NULL || !w->w_spin) 103772224Sjhb continue; 103872224Sjhb if (w->w_level == 1 << level) { 103972224Sjhb witness_displaydescendants(prnt, w); 104072224Sjhb level++; 104172224Sjhb found = 1; 104272224Sjhb } 104372224Sjhb } 104472224Sjhb if (found == 0) 104572224Sjhb level++; 104672224Sjhb } 104772224Sjhb 104872224Sjhb /* 104972224Sjhb * Finally, any mutexes which have not been acquired yet. 105072224Sjhb */ 105172224Sjhb prnt("\nMutexes which were never acquired:\n"); 105271352Sjasone for (w = w_all; w; w = w->w_next) { 105371352Sjasone if (w->w_file != NULL) 105471352Sjasone continue; 105571352Sjasone prnt("%s\n", w->w_description); 105671352Sjasone } 105771352Sjasone} 105871352Sjasone 105965557Sjasonevoid 106065856Sjhbwitness_enter(struct mtx *m, int flags, const char *file, int line) 106165557Sjasone{ 106265856Sjhb struct witness *w, *w1; 106365856Sjhb struct mtx *m1; 106465557Sjasone struct proc *p; 106565557Sjasone int i; 106667676Sjhb#ifdef DDB 106767676Sjhb int go_into_ddb = 0; 106867676Sjhb#endif /* DDB */ 106965557Sjasone 107071352Sjasone if (witness_cold || m->mtx_witness == NULL || panicstr) 107171320Sjasone return; 107265557Sjasone w = m->mtx_witness; 107372393Sbmilekic p = curproc; 107465557Sjasone 107565557Sjasone if (flags & MTX_SPIN) { 107671560Sjhb if ((m->mtx_flags & MTX_SPIN) == 0) 107765651Sjasone panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" 107865651Sjasone " %s:%d", m->mtx_description, file, line); 107971228Sbmilekic if (mtx_recursed(m)) { 108071560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 108171228Sbmilekic panic("mutex_enter: recursion on non-recursive" 108271228Sbmilekic " mutex %s @ %s:%d", m->mtx_description, 108371228Sbmilekic file, line); 108465557Sjasone return; 108571228Sbmilekic } 108672200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 108770861Sjake i = PCPU_GET(witness_spin_check); 108865557Sjasone if (i != 0 && w->w_level < i) { 108972200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 109065651Sjasone panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" 109165651Sjasone " %s:%d already holding %s:%x", 109265557Sjasone m->mtx_description, w->w_level, file, line, 109365557Sjasone spin_order_list[ffs(i)-1], i); 109465557Sjasone } 109565557Sjasone PCPU_SET(witness_spin_check, i | w->w_level); 109672200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 109769361Sjhb w->w_file = file; 109869361Sjhb w->w_line = line; 109969361Sjhb m->mtx_line = line; 110069361Sjhb m->mtx_file = file; 110165557Sjasone return; 110265557Sjasone } 110371560Sjhb if ((m->mtx_flags & MTX_SPIN) != 0) 110465557Sjasone panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 110565557Sjasone m->mtx_description, file, line); 110665557Sjasone 110771228Sbmilekic if (mtx_recursed(m)) { 110871560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 110971228Sbmilekic panic("mutex_enter: recursion on non-recursive" 111071228Sbmilekic " mutex %s @ %s:%d", m->mtx_description, 111171228Sbmilekic file, line); 111265557Sjasone return; 111371228Sbmilekic } 111465557Sjasone if (witness_dead) 111565557Sjasone goto out; 111669998Sjhb if (cold) 111765557Sjasone goto out; 111865557Sjasone 111965557Sjasone if (!mtx_legal2block()) 112072200Sbmilekic panic("blockable mtx_lock() of %s when not legal @ %s:%d", 112165557Sjasone m->mtx_description, file, line); 112265557Sjasone /* 112365557Sjasone * Is this the first mutex acquired 112465557Sjasone */ 112565557Sjasone if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) 112665557Sjasone goto out; 112765557Sjasone 112865557Sjasone if ((w1 = m1->mtx_witness) == w) { 112965557Sjasone if (w->w_same_squawked || dup_ok(w)) 113065557Sjasone goto out; 113165557Sjasone w->w_same_squawked = 1; 113265557Sjasone printf("acquring duplicate lock of same type: \"%s\"\n", 113365557Sjasone m->mtx_description); 113465557Sjasone printf(" 1st @ %s:%d\n", w->w_file, w->w_line); 113565557Sjasone printf(" 2nd @ %s:%d\n", file, line); 113667676Sjhb#ifdef DDB 113767676Sjhb go_into_ddb = 1; 113867676Sjhb#endif /* DDB */ 113965557Sjasone goto out; 114065557Sjasone } 114165557Sjasone MPASS(!mtx_owned(&w_mtx)); 114272200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 114365557Sjasone /* 114465557Sjasone * If we have a known higher number just say ok 114565557Sjasone */ 114665557Sjasone if (witness_watch > 1 && w->w_level > w1->w_level) { 114772200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 114865557Sjasone goto out; 114965557Sjasone } 115065557Sjasone if (isitmydescendant(m1->mtx_witness, w)) { 115172200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 115265557Sjasone goto out; 115365557Sjasone } 115465557Sjasone for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { 115565557Sjasone 115667352Sjhb MPASS(i < 200); 115765557Sjasone w1 = m1->mtx_witness; 115865557Sjasone if (isitmydescendant(w, w1)) { 115972200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 116065557Sjasone if (blessed(w, w1)) 116165557Sjasone goto out; 116265557Sjasone if (m1 == &Giant) { 116365557Sjasone if (w1->w_Giant_squawked) 116465557Sjasone goto out; 116565557Sjasone else 116665557Sjasone w1->w_Giant_squawked = 1; 116765557Sjasone } else { 116865557Sjasone if (w1->w_other_squawked) 116965557Sjasone goto out; 117065557Sjasone else 117165557Sjasone w1->w_other_squawked = 1; 117265557Sjasone } 117365557Sjasone printf("lock order reversal\n"); 117465557Sjasone printf(" 1st %s last acquired @ %s:%d\n", 117565557Sjasone w->w_description, w->w_file, w->w_line); 117665557Sjasone printf(" 2nd %p %s @ %s:%d\n", 117765557Sjasone m1, w1->w_description, w1->w_file, w1->w_line); 117865557Sjasone printf(" 3rd %p %s @ %s:%d\n", 117965557Sjasone m, w->w_description, file, line); 118067676Sjhb#ifdef DDB 118167676Sjhb go_into_ddb = 1; 118267676Sjhb#endif /* DDB */ 118365557Sjasone goto out; 118465557Sjasone } 118565557Sjasone } 118665557Sjasone m1 = LIST_FIRST(&p->p_heldmtx); 118765557Sjasone if (!itismychild(m1->mtx_witness, w)) 118872200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 118965557Sjasone 119065557Sjasoneout: 119167676Sjhb#ifdef DDB 119267676Sjhb if (witness_ddb && go_into_ddb) 119367676Sjhb Debugger("witness_enter"); 119467676Sjhb#endif /* DDB */ 119565557Sjasone w->w_file = file; 119665557Sjasone w->w_line = line; 119765557Sjasone m->mtx_line = line; 119865557Sjasone m->mtx_file = file; 119965557Sjasone 120065557Sjasone /* 120168582Sjhb * If this pays off it likely means that a mutex being witnessed 120265557Sjasone * is acquired in hardclock. Put it in the ignore list. It is 120365557Sjasone * likely not the mutex this assert fails on. 120465557Sjasone */ 120567352Sjhb MPASS(m->mtx_held.le_prev == NULL); 120665557Sjasone LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 120765557Sjasone} 120865557Sjasone 120965557Sjasonevoid 121065856Sjhbwitness_try_enter(struct mtx *m, int flags, const char *file, int line) 121165557Sjasone{ 121265557Sjasone struct proc *p; 121365856Sjhb struct witness *w = m->mtx_witness; 121465557Sjasone 121571320Sjasone if (witness_cold) 121671320Sjasone return; 121769998Sjhb if (panicstr) 121869998Sjhb return; 121965557Sjasone if (flags & MTX_SPIN) { 122071560Sjhb if ((m->mtx_flags & MTX_SPIN) == 0) 122165557Sjasone panic("mutex_try_enter: " 122265557Sjasone "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", 122365557Sjasone m->mtx_description, file, line); 122471228Sbmilekic if (mtx_recursed(m)) { 122571560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 122671228Sbmilekic panic("mutex_try_enter: recursion on" 122771228Sbmilekic " non-recursive mutex %s @ %s:%d", 122871228Sbmilekic m->mtx_description, file, line); 122965557Sjasone return; 123071228Sbmilekic } 123172200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 123270861Sjake PCPU_SET(witness_spin_check, 123370861Sjake PCPU_GET(witness_spin_check) | w->w_level); 123472200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 123569361Sjhb w->w_file = file; 123669361Sjhb w->w_line = line; 123769361Sjhb m->mtx_line = line; 123869361Sjhb m->mtx_file = file; 123965557Sjasone return; 124065557Sjasone } 124165557Sjasone 124271560Sjhb if ((m->mtx_flags & MTX_SPIN) != 0) 124365557Sjasone panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 124465557Sjasone m->mtx_description, file, line); 124565557Sjasone 124671228Sbmilekic if (mtx_recursed(m)) { 124771560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 124871228Sbmilekic panic("mutex_try_enter: recursion on non-recursive" 124971228Sbmilekic " mutex %s @ %s:%d", m->mtx_description, file, 125071228Sbmilekic line); 125165557Sjasone return; 125271228Sbmilekic } 125365557Sjasone w->w_file = file; 125465557Sjasone w->w_line = line; 125565557Sjasone m->mtx_line = line; 125665557Sjasone m->mtx_file = file; 125772393Sbmilekic p = curproc; 125867352Sjhb MPASS(m->mtx_held.le_prev == NULL); 125965557Sjasone LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 126065557Sjasone} 126165557Sjasone 126265557Sjasonevoid 126371352Sjasonewitness_exit(struct mtx *m, int flags, const char *file, int line) 126465557Sjasone{ 126571352Sjasone struct witness *w; 126665557Sjasone 126771352Sjasone if (witness_cold || m->mtx_witness == NULL || panicstr) 126871352Sjasone return; 126971352Sjasone w = m->mtx_witness; 127065557Sjasone 127171352Sjasone if (flags & MTX_SPIN) { 127271560Sjhb if ((m->mtx_flags & MTX_SPIN) == 0) 127371352Sjasone panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" 127471352Sjasone " %s:%d", m->mtx_description, file, line); 127571352Sjasone if (mtx_recursed(m)) { 127671560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 127771352Sjasone panic("mutex_exit: recursion on non-recursive" 127871352Sjasone " mutex %s @ %s:%d", m->mtx_description, 127971352Sjasone file, line); 128071352Sjasone return; 128165557Sjasone } 128272200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 128371352Sjasone PCPU_SET(witness_spin_check, 128471352Sjasone PCPU_GET(witness_spin_check) & ~w->w_level); 128572200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 128671352Sjasone return; 128765557Sjasone } 128871560Sjhb if ((m->mtx_flags & MTX_SPIN) != 0) 128971352Sjasone panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 129071352Sjasone m->mtx_description, file, line); 129171352Sjasone 129271352Sjasone if (mtx_recursed(m)) { 129371560Sjhb if ((m->mtx_flags & MTX_RECURSE) == 0) 129471352Sjasone panic("mutex_exit: recursion on non-recursive" 129571352Sjasone " mutex %s @ %s:%d", m->mtx_description, 129671352Sjasone file, line); 129771352Sjasone return; 129865557Sjasone } 129971352Sjasone 130071352Sjasone if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) 130172200Sbmilekic panic("switchable mtx_unlock() of %s when not legal @ %s:%d", 130271352Sjasone m->mtx_description, file, line); 130371352Sjasone LIST_REMOVE(m, mtx_held); 130471352Sjasone m->mtx_held.le_prev = NULL; 130565557Sjasone} 130665557Sjasone 130765557Sjasoneint 130865856Sjhbwitness_sleep(int check_only, struct mtx *mtx, const char *file, int line) 130965557Sjasone{ 131065856Sjhb struct mtx *m; 131165557Sjasone struct proc *p; 131265557Sjasone char **sleep; 131365557Sjasone int n = 0; 131465557Sjasone 131571320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 131672393Sbmilekic p = curproc; 131772224Sjhb LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { 131865557Sjasone if (m == mtx) 131965557Sjasone continue; 132065557Sjasone for (sleep = sleep_list; *sleep!= NULL; sleep++) 132165557Sjasone if (strcmp(m->mtx_description, *sleep) == 0) 132265557Sjasone goto next; 132372224Sjhb if (n == 0) 132472224Sjhb printf("Whee!\n"); 132565557Sjasone printf("%s:%d: %s with \"%s\" locked from %s:%d\n", 132665557Sjasone file, line, check_only ? "could sleep" : "sleeping", 132765557Sjasone m->mtx_description, 132865557Sjasone m->mtx_witness->w_file, m->mtx_witness->w_line); 132965557Sjasone n++; 133065557Sjasone next: 133165557Sjasone } 133267676Sjhb#ifdef DDB 133367676Sjhb if (witness_ddb && n) 133467676Sjhb Debugger("witness_sleep"); 133567676Sjhb#endif /* DDB */ 133665557Sjasone return (n); 133765557Sjasone} 133865557Sjasone 133965856Sjhbstatic struct witness * 134067404Sjhbenroll(const char *description, int flag) 134165557Sjasone{ 134265557Sjasone int i; 134365856Sjhb struct witness *w, *w1; 134465557Sjasone char **ignore; 134565557Sjasone char **order; 134665557Sjasone 134765557Sjasone if (!witness_watch) 134865557Sjasone return (NULL); 134965557Sjasone for (ignore = ignore_list; *ignore != NULL; ignore++) 135065557Sjasone if (strcmp(description, *ignore) == 0) 135165557Sjasone return (NULL); 135265557Sjasone 135365557Sjasone if (w_inited == 0) { 135471320Sjasone mtx_init(&w_mtx, "witness lock", MTX_SPIN); 135565557Sjasone for (i = 0; i < WITNESS_COUNT; i++) { 135665557Sjasone w = &w_data[i]; 135765557Sjasone witness_free(w); 135865557Sjasone } 135965557Sjasone w_inited = 1; 136065557Sjasone for (order = order_list; *order != NULL; order++) { 136165557Sjasone w = enroll(*order, MTX_DEF); 136265557Sjasone w->w_file = "order list"; 136365557Sjasone for (order++; *order != NULL; order++) { 136465557Sjasone w1 = enroll(*order, MTX_DEF); 136565557Sjasone w1->w_file = "order list"; 136665557Sjasone itismychild(w, w1); 136765557Sjasone w = w1; 136865557Sjasone } 136965557Sjasone } 137065557Sjasone } 137165557Sjasone if ((flag & MTX_SPIN) && witness_skipspin) 137265557Sjasone return (NULL); 137372200Sbmilekic mtx_lock_spin_flags(&w_mtx, MTX_QUIET); 137465557Sjasone for (w = w_all; w; w = w->w_next) { 137565557Sjasone if (strcmp(description, w->w_description) == 0) { 137672200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 137765557Sjasone return (w); 137865557Sjasone } 137965557Sjasone } 138065557Sjasone if ((w = witness_get()) == NULL) 138165557Sjasone return (NULL); 138265557Sjasone w->w_next = w_all; 138365557Sjasone w_all = w; 138465557Sjasone w->w_description = description; 138572200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 138665557Sjasone if (flag & MTX_SPIN) { 138765557Sjasone w->w_spin = 1; 138865557Sjasone 138965557Sjasone i = 1; 139065557Sjasone for (order = spin_order_list; *order != NULL; order++) { 139165557Sjasone if (strcmp(description, *order) == 0) 139265557Sjasone break; 139365557Sjasone i <<= 1; 139465557Sjasone } 139565557Sjasone if (*order == NULL) 139665557Sjasone panic("spin lock %s not in order list", description); 139765557Sjasone w->w_level = i; 139871560Sjhb } 139971228Sbmilekic 140065557Sjasone return (w); 140165557Sjasone} 140265557Sjasone 140365557Sjasonestatic int 140465856Sjhbitismychild(struct witness *parent, struct witness *child) 140565557Sjasone{ 140665557Sjasone static int recursed; 140765557Sjasone 140865557Sjasone /* 140965557Sjasone * Insert "child" after "parent" 141065557Sjasone */ 141165557Sjasone while (parent->w_morechildren) 141265557Sjasone parent = parent->w_morechildren; 141365557Sjasone 141465557Sjasone if (parent->w_childcnt == WITNESS_NCHILDREN) { 141565557Sjasone if ((parent->w_morechildren = witness_get()) == NULL) 141665557Sjasone return (1); 141765557Sjasone parent = parent->w_morechildren; 141865557Sjasone } 141967352Sjhb MPASS(child != NULL); 142065557Sjasone parent->w_children[parent->w_childcnt++] = child; 142165557Sjasone /* 142265557Sjasone * now prune whole tree 142365557Sjasone */ 142465557Sjasone if (recursed) 142565557Sjasone return (0); 142665557Sjasone recursed = 1; 142765557Sjasone for (child = w_all; child != NULL; child = child->w_next) { 142865557Sjasone for (parent = w_all; parent != NULL; 142965557Sjasone parent = parent->w_next) { 143065557Sjasone if (!isitmychild(parent, child)) 143165557Sjasone continue; 143265557Sjasone removechild(parent, child); 143365557Sjasone if (isitmydescendant(parent, child)) 143465557Sjasone continue; 143565557Sjasone itismychild(parent, child); 143665557Sjasone } 143765557Sjasone } 143865557Sjasone recursed = 0; 143965557Sjasone witness_levelall(); 144065557Sjasone return (0); 144165557Sjasone} 144265557Sjasone 144365557Sjasonestatic void 144465856Sjhbremovechild(struct witness *parent, struct witness *child) 144565557Sjasone{ 144665856Sjhb struct witness *w, *w1; 144765557Sjasone int i; 144865557Sjasone 144965557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) 145065557Sjasone for (i = 0; i < w->w_childcnt; i++) 145165557Sjasone if (w->w_children[i] == child) 145265557Sjasone goto found; 145365557Sjasone return; 145465557Sjasonefound: 145565557Sjasone for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) 145665557Sjasone continue; 145765557Sjasone w->w_children[i] = w1->w_children[--w1->w_childcnt]; 145867352Sjhb MPASS(w->w_children[i] != NULL); 145965557Sjasone 146065557Sjasone if (w1->w_childcnt != 0) 146165557Sjasone return; 146265557Sjasone 146365557Sjasone if (w1 == parent) 146465557Sjasone return; 146565557Sjasone for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) 146665557Sjasone continue; 146765557Sjasone w->w_morechildren = 0; 146865557Sjasone witness_free(w1); 146965557Sjasone} 147065557Sjasone 147165557Sjasonestatic int 147265856Sjhbisitmychild(struct witness *parent, struct witness *child) 147365557Sjasone{ 147465856Sjhb struct witness *w; 147565557Sjasone int i; 147665557Sjasone 147765557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) { 147865557Sjasone for (i = 0; i < w->w_childcnt; i++) { 147965557Sjasone if (w->w_children[i] == child) 148065557Sjasone return (1); 148165557Sjasone } 148265557Sjasone } 148365557Sjasone return (0); 148465557Sjasone} 148565557Sjasone 148665557Sjasonestatic int 148765856Sjhbisitmydescendant(struct witness *parent, struct witness *child) 148865557Sjasone{ 148965856Sjhb struct witness *w; 149065557Sjasone int i; 149165557Sjasone int j; 149265557Sjasone 149365557Sjasone for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { 149467352Sjhb MPASS(j < 1000); 149565557Sjasone for (i = 0; i < w->w_childcnt; i++) { 149665557Sjasone if (w->w_children[i] == child) 149765557Sjasone return (1); 149865557Sjasone } 149965557Sjasone for (i = 0; i < w->w_childcnt; i++) { 150065557Sjasone if (isitmydescendant(w->w_children[i], child)) 150165557Sjasone return (1); 150265557Sjasone } 150365557Sjasone } 150465557Sjasone return (0); 150565557Sjasone} 150665557Sjasone 150765557Sjasonevoid 150865557Sjasonewitness_levelall (void) 150965557Sjasone{ 151065856Sjhb struct witness *w, *w1; 151165557Sjasone 151265557Sjasone for (w = w_all; w; w = w->w_next) 151371228Sbmilekic if (!(w->w_spin)) 151465557Sjasone w->w_level = 0; 151565557Sjasone for (w = w_all; w; w = w->w_next) { 151665557Sjasone if (w->w_spin) 151765557Sjasone continue; 151865557Sjasone for (w1 = w_all; w1; w1 = w1->w_next) { 151965557Sjasone if (isitmychild(w1, w)) 152065557Sjasone break; 152165557Sjasone } 152265557Sjasone if (w1 != NULL) 152365557Sjasone continue; 152465557Sjasone witness_leveldescendents(w, 0); 152565557Sjasone } 152665557Sjasone} 152765557Sjasone 152865557Sjasonestatic void 152965856Sjhbwitness_leveldescendents(struct witness *parent, int level) 153065557Sjasone{ 153165557Sjasone int i; 153265856Sjhb struct witness *w; 153365557Sjasone 153465557Sjasone if (parent->w_level < level) 153565557Sjasone parent->w_level = level; 153665557Sjasone level++; 153765557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) 153865557Sjasone for (i = 0; i < w->w_childcnt; i++) 153965557Sjasone witness_leveldescendents(w->w_children[i], level); 154065557Sjasone} 154165557Sjasone 154265557Sjasonestatic void 154365856Sjhbwitness_displaydescendants(void(*prnt)(const char *fmt, ...), 154465856Sjhb struct witness *parent) 154565557Sjasone{ 154665856Sjhb struct witness *w; 154765557Sjasone int i; 154872224Sjhb int level; 154965557Sjasone 155072224Sjhb level = parent->w_spin ? ffs(parent->w_level) : parent->w_level; 155172224Sjhb 155265557Sjasone prnt("%d", level); 155365557Sjasone if (level < 10) 155465557Sjasone prnt(" "); 155565557Sjasone for (i = 0; i < level; i++) 155665557Sjasone prnt(" "); 155765557Sjasone prnt("%s", parent->w_description); 155872224Sjhb if (parent->w_file != NULL) 155972224Sjhb prnt(" -- last acquired @ %s:%d\n", parent->w_file, 156072224Sjhb parent->w_line); 156165557Sjasone 156265557Sjasone for (w = parent; w != NULL; w = w->w_morechildren) 156365557Sjasone for (i = 0; i < w->w_childcnt; i++) 156465557Sjasone witness_displaydescendants(prnt, w->w_children[i]); 156565557Sjasone } 156665557Sjasone 156765557Sjasonestatic int 156865856Sjhbdup_ok(struct witness *w) 156965557Sjasone{ 157065557Sjasone char **dup; 157165557Sjasone 157265557Sjasone for (dup = dup_list; *dup!= NULL; dup++) 157365557Sjasone if (strcmp(w->w_description, *dup) == 0) 157465557Sjasone return (1); 157565557Sjasone return (0); 157665557Sjasone} 157765557Sjasone 157865557Sjasonestatic int 157965856Sjhbblessed(struct witness *w1, struct witness *w2) 158065557Sjasone{ 158165557Sjasone int i; 158265856Sjhb struct witness_blessed *b; 158365557Sjasone 158465557Sjasone for (i = 0; i < blessed_count; i++) { 158565557Sjasone b = &blessed_list[i]; 158665557Sjasone if (strcmp(w1->w_description, b->b_lock1) == 0) { 158765557Sjasone if (strcmp(w2->w_description, b->b_lock2) == 0) 158865557Sjasone return (1); 158965557Sjasone continue; 159065557Sjasone } 159165557Sjasone if (strcmp(w1->w_description, b->b_lock2) == 0) 159265557Sjasone if (strcmp(w2->w_description, b->b_lock1) == 0) 159365557Sjasone return (1); 159465557Sjasone } 159565557Sjasone return (0); 159665557Sjasone} 159765557Sjasone 159865856Sjhbstatic struct witness * 159965557Sjasonewitness_get() 160065557Sjasone{ 160165856Sjhb struct witness *w; 160265557Sjasone 160365557Sjasone if ((w = w_free) == NULL) { 160465557Sjasone witness_dead = 1; 160572200Sbmilekic mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); 160665557Sjasone printf("witness exhausted\n"); 160765557Sjasone return (NULL); 160865557Sjasone } 160965557Sjasone w_free = w->w_next; 161065856Sjhb bzero(w, sizeof(*w)); 161165557Sjasone return (w); 161265557Sjasone} 161365557Sjasone 161465557Sjasonestatic void 161565856Sjhbwitness_free(struct witness *w) 161665557Sjasone{ 161765557Sjasone w->w_next = w_free; 161865557Sjasone w_free = w; 161965557Sjasone} 162065557Sjasone 162169881Sjakeint 162265557Sjasonewitness_list(struct proc *p) 162365557Sjasone{ 162465856Sjhb struct mtx *m; 162569881Sjake int nheld; 162665557Sjasone 162771320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 162869881Sjake nheld = 0; 162972224Sjhb LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { 163065557Sjasone printf("\t\"%s\" (%p) locked at %s:%d\n", 163165557Sjasone m->mtx_description, m, 163265557Sjasone m->mtx_witness->w_file, m->mtx_witness->w_line); 163369881Sjake nheld++; 163465557Sjasone } 163569881Sjake 163669881Sjake return (nheld); 163765557Sjasone} 163865557Sjasone 163971709Sjhb#ifdef DDB 164071709Sjhb 164172224SjhbDB_SHOW_COMMAND(mutexes, db_witness_list) 164271709Sjhb{ 164371709Sjhb 164472393Sbmilekic witness_list(curproc); 164571709Sjhb} 164671709Sjhb 164772224SjhbDB_SHOW_COMMAND(witness, db_witness_display) 164872224Sjhb{ 164972224Sjhb 165072224Sjhb witness_display(db_printf); 165172224Sjhb} 165271709Sjhb#endif 165371709Sjhb 165465557Sjasonevoid 165565856Sjhbwitness_save(struct mtx *m, const char **filep, int *linep) 165665557Sjasone{ 165771320Sjasone 165871320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 165971352Sjasone if (m->mtx_witness == NULL) 166071352Sjasone return; 166171352Sjasone 166265557Sjasone *filep = m->mtx_witness->w_file; 166365557Sjasone *linep = m->mtx_witness->w_line; 166465557Sjasone} 166565557Sjasone 166665557Sjasonevoid 166765856Sjhbwitness_restore(struct mtx *m, const char *file, int line) 166865557Sjasone{ 166971320Sjasone 167071320Sjasone KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 167171352Sjasone if (m->mtx_witness == NULL) 167271352Sjasone return; 167371352Sjasone 167465557Sjasone m->mtx_witness->w_file = file; 167565557Sjasone m->mtx_witness->w_line = line; 167665557Sjasone} 167765557Sjasone 167869429Sjhb#endif /* WITNESS */ 1679