30c30
< * $FreeBSD: head/sys/kern/subr_witness.c 74900 2001-03-28 02:40:47Z jhb $
---
> * $FreeBSD: head/sys/kern/subr_witness.c 74912 2001-03-28 09:03:24Z jhb $
34,35c34,36
< * Machine independent bits of mutex implementation and implementation of
< * `witness' structure & related debugging routines.
---
> * Implementation of the `witness' lock verifier. Originally implemented for
> * mutexes in BSD/OS. Extended to handle generic lock objects and lock
> * classes in FreeBSD.
63a65,66
> #include <sys/ktr.h>
> #include <sys/lock.h>
64a68
> #include <sys/mutex.h>
68,69d71
< #include <sys/vmmeter.h>
< #include <sys/ktr.h>
71,75d72
< #include <machine/atomic.h>
< #include <machine/bus.h>
< #include <machine/clock.h>
< #include <machine/cpu.h>
<
78,82c75,76
< #include <vm/vm.h>
< #include <vm/vm_extern.h>
<
< #include <sys/mutex.h>
<
---
> #define WITNESS_COUNT 200
> #define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4)
84c78,80
< * The WITNESS-enabled mutex debug structure.
---
> * XXX: This is somewhat bogus, as we assume here that at most 1024 processes
> * will hold LOCK_NCHILDREN * 2 locks. We handle failure ok, and we should
> * probably be safe for the most part, but it's still a SWAG.
86,92c82
< #ifdef WITNESS
< struct mtx_debug {
< struct witness *mtxd_witness;
< LIST_ENTRY(mtx) mtxd_held;
< const char *mtxd_file;
< int mtxd_line;
< };
---
> #define LOCK_CHILDCOUNT (MAXCPU + 1024) * 2
94,98c84
< #define mtx_held mtx_debug->mtxd_held
< #define mtx_file mtx_debug->mtxd_file
< #define mtx_line mtx_debug->mtxd_line
< #define mtx_witness mtx_debug->mtxd_witness
< #endif /* WITNESS */
---
> #define WITNESS_NCHILDREN 6
100,103c86
< /*
< * Internal utility macros.
< */
< #define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED)
---
> struct witness_child_list_entry;
105,106c88,101
< #define mtx_owner(m) (mtx_unowned((m)) ? NULL \
< : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK))
---
> struct witness {
> const char *w_name;
> struct lock_class *w_class;
> STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */
> STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */
> struct witness_child_list_entry *w_children; /* Great evilness... */
> const char *w_file;
> int w_line;
> u_int w_level;
> u_int w_refcount;
> u_char w_Giant_squawked:1;
> u_char w_other_squawked:1;
> u_char w_same_squawked:1;
> };
108c103,107
< #define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri)
---
> struct witness_child_list_entry {
> struct witness_child_list_entry *wcl_next;
> struct witness *wcl_children[WITNESS_NCHILDREN];
> u_int wcl_count;
> };
110,113c109
< /*
< * Early WITNESS-enabled declarations.
< */
< #ifdef WITNESS
---
> STAILQ_HEAD(witness_list, witness);
115,123c111,114
< /*
< * Internal WITNESS routines which must be prototyped early.
< *
< * XXX: When/if witness code is cleaned up, it would be wise to place all
< * witness prototyping early in this file.
< */
< static void witness_init(struct mtx *, int flag);
< static void witness_destroy(struct mtx *);
< static void witness_display(void(*)(const char *fmt, ...));
---
> struct witness_blessed {
> const char *b_lock1;
> const char *b_lock2;
> };
125c116,119
< MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure");
---
> struct witness_order_list_entry {
> const char *w_name;
> struct lock_class *w_class;
> };
127,128c121,141
< /* All mutexes in system (used for debug/panic) */
< static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
---
> static struct witness *enroll(const char *description,
> struct lock_class *lock_class);
> static int itismychild(struct witness *parent, struct witness *child);
> static void removechild(struct witness *parent, struct witness *child);
> static int isitmychild(struct witness *parent, struct witness *child);
> static int isitmydescendant(struct witness *parent, struct witness *child);
> static int dup_ok(struct witness *);
> static int blessed(struct witness *, struct witness *);
> static void witness_display_list(void(*prnt)(const char *fmt, ...),
> struct witness_list *list);
> static void witness_displaydescendants(void(*)(const char *fmt, ...),
> struct witness *);
> static void witness_leveldescendents(struct witness *parent, int level);
> static void witness_levelall(void);
> static struct witness *witness_get(void);
> static void witness_free(struct witness *m);
> static struct witness_child_list_entry *witness_child_get(void);
> static void witness_child_free(struct witness_child_list_entry *wcl);
> static struct lock_list_entry *witness_lock_list_get(void);
> static void witness_lock_list_free(struct lock_list_entry *lle);
> static void witness_display(void(*)(const char *fmt, ...));
130,133c143
< /*
< * This global is set to 0 once it becomes safe to use the witness code.
< */
< static int witness_cold = 1;
---
> MALLOC_DEFINE(M_WITNESS, "witness", "witness structure");
135c145,147
< #else /* WITNESS */
---
> static int witness_watch;
> TUNABLE_INT_DECL("debug.witness_watch", 1, witness_watch);
> SYSCTL_INT(_debug, OID_AUTO, witness_watch, CTLFLAG_RD, &witness_watch, 0, "");
137,512d148
< /* XXX XXX XXX
< * flag++ is sleazoid way of shuting up warning
< */
< #define witness_init(m, flag) flag++
< #define witness_destroy(m)
< #define witness_try_enter(m, t, f, l)
< #endif /* WITNESS */
<
< /*
< * All mutex locks in system are kept on the all_mtx list.
< */
< static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head",
< TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
< { NULL, NULL }, &all_mtx, &all_mtx,
< #ifdef WITNESS
< &all_mtx_debug
< #else
< NULL
< #endif
< };
<
< /*
< * Global variables for book keeping.
< */
< static int mtx_cur_cnt;
< static int mtx_max_cnt;
<
< /*
< * Couple of strings for KTR_LOCK tracing in order to avoid duplicates.
< */
< char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d";
<
< /*
< * Prototypes for non-exported routines.
< *
< * NOTE: Prototypes for witness routines are placed at the bottom of the file.
< */
< static void propagate_priority(struct proc *);
<
< static void
< propagate_priority(struct proc *p)
< {
< int pri = p->p_pri.pri_level;
< struct mtx *m = p->p_blocked;
<
< mtx_assert(&sched_lock, MA_OWNED);
< for (;;) {
< struct proc *p1;
<
< p = mtx_owner(m);
<
< if (p == NULL) {
< /*
< * This really isn't quite right. Really
< * ought to bump priority of process that
< * next acquires the mutex.
< */
< MPASS(m->mtx_lock == MTX_CONTESTED);
< return;
< }
<
< MPASS(p->p_magic == P_MAGIC);
< KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex"));
< if (p->p_pri.pri_level <= pri)
< return;
<
< /*
< * Bump this process' priority.
< */
< SET_PRIO(p, pri);
<
< /*
< * If lock holder is actually running, just bump priority.
< */
< if (p->p_oncpu != NOCPU) {
< MPASS(p->p_stat == SRUN || p->p_stat == SZOMB);
< return;
< }
<
< #ifndef SMP
< /*
< * For UP, we check to see if p is curproc (this shouldn't
< * ever happen however as it would mean we are in a deadlock.)
< */
< KASSERT(p != curproc, ("Deadlock detected"));
< #endif
<
< /*
< * If on run queue move to new run queue, and
< * quit.
< */
< if (p->p_stat == SRUN) {
< MPASS(p->p_blocked == NULL);
< remrunqueue(p);
< setrunqueue(p);
< return;
< }
<
< /*
< * If we aren't blocked on a mutex, we should be.
< */
< KASSERT(p->p_stat == SMTX, (
< "process %d(%s):%d holds %s but isn't blocked on a mutex\n",
< p->p_pid, p->p_comm, p->p_stat,
< m->mtx_description));
<
< /*
< * Pick up the mutex that p is blocked on.
< */
< m = p->p_blocked;
< MPASS(m != NULL);
<
< /*
< * Check if the proc needs to be moved up on
< * the blocked chain
< */
< if (p == TAILQ_FIRST(&m->mtx_blocked)) {
< continue;
< }
<
< p1 = TAILQ_PREV(p, procqueue, p_procq);
< if (p1->p_pri.pri_level <= pri) {
< continue;
< }
<
< /*
< * Remove proc from blocked chain and determine where
< * it should be moved up to. Since we know that p1 has
< * a lower priority than p, we know that at least one
< * process in the chain has a lower priority and that
< * p1 will thus not be NULL after the loop.
< */
< TAILQ_REMOVE(&m->mtx_blocked, p, p_procq);
< TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) {
< MPASS(p1->p_magic == P_MAGIC);
< if (p1->p_pri.pri_level > pri)
< break;
< }
<
< MPASS(p1 != NULL);
< TAILQ_INSERT_BEFORE(p1, p, p_procq);
< CTR4(KTR_LOCK,
< "propagate_priority: p %p moved before %p on [%p] %s",
< p, p1, m, m->mtx_description);
< }
< }
<
< /*
< * Function versions of the inlined __mtx_* macros. These are used by
< * modules and can also be called from assembly language if needed.
< */
< void
< _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_lock_flags(m, opts, file, line);
< }
<
< void
< _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_unlock_flags(m, opts, file, line);
< }
<
< void
< _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_lock_spin_flags(m, opts, file, line);
< }
<
< void
< _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_unlock_spin_flags(m, opts, file, line);
< }
<
< /*
< * The important part of mtx_trylock{,_flags}()
< * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that
< * if we're called, it's because we know we don't already own this lock.
< */
< int
< _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
< {
< int rval;
<
< MPASS(curproc != NULL);
<
< /*
< * _mtx_trylock does not accept MTX_NOSWITCH option.
< */
< KASSERT((opts & MTX_NOSWITCH) == 0,
< ("mtx_trylock() called with invalid option flag(s) %d", opts));
<
< rval = _obtain_lock(m, curproc);
<
< #ifdef WITNESS
< if (rval && m->mtx_witness != NULL) {
< /*
< * We do not handle recursion in _mtx_trylock; see the
< * note at the top of the routine.
< */
< KASSERT(!mtx_recursed(m),
< ("mtx_trylock() called on a recursed mutex"));
< witness_try_enter(m, (opts | m->mtx_flags), file, line);
< }
< #endif /* WITNESS */
<
< if ((opts & MTX_QUIET) == 0)
< CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d",
< m->mtx_description, m, rval, file, line);
<
< return rval;
< }
<
< /*
< * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
< *
< * We call this if the lock is either contested (i.e. we need to go to
< * sleep waiting for it), or if we need to recurse on it.
< */
< void
< _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
< {
< struct proc *p = curproc;
<
< if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
< m->mtx_recurse++;
< atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
< return;
< }
<
< if ((opts & MTX_QUIET) == 0)
< CTR4(KTR_LOCK,
< "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
< m->mtx_description, (void *)m->mtx_lock, file, line);
<
< while (!_obtain_lock(m, p)) {
< uintptr_t v;
< struct proc *p1;
<
< mtx_lock_spin(&sched_lock);
< /*
< * Check if the lock has been released while spinning for
< * the sched_lock.
< */
< if ((v = m->mtx_lock) == MTX_UNOWNED) {
< mtx_unlock_spin(&sched_lock);
< continue;
< }
<
< /*
< * The mutex was marked contested on release. This means that
< * there are processes blocked on it.
< */
< if (v == MTX_CONTESTED) {
< p1 = TAILQ_FIRST(&m->mtx_blocked);
< MPASS(p1 != NULL);
< m->mtx_lock = (uintptr_t)p | MTX_CONTESTED;
<
< if (p1->p_pri.pri_level < p->p_pri.pri_level)
< SET_PRIO(p, p1->p_pri.pri_level);
< mtx_unlock_spin(&sched_lock);
< return;
< }
<
< /*
< * If the mutex isn't already contested and a failure occurs
< * setting the contested bit, the mutex was either released
< * or the state of the MTX_RECURSED bit changed.
< */
< if ((v & MTX_CONTESTED) == 0 &&
< !atomic_cmpset_ptr(&m->mtx_lock, (void *)v,
< (void *)(v | MTX_CONTESTED))) {
< mtx_unlock_spin(&sched_lock);
< continue;
< }
<
< /*
< * We deffinately must sleep for this lock.
< */
< mtx_assert(m, MA_NOTOWNED);
<
< #ifdef notyet
< /*
< * If we're borrowing an interrupted thread's VM context, we
< * must clean up before going to sleep.
< */
< if (p->p_ithd != NULL) {
< struct ithd *it = p->p_ithd;
<
< if (it->it_interrupted) {
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK,
< "_mtx_lock_sleep: %p interrupted %p",
< it, it->it_interrupted);
< intr_thd_fixup(it);
< }
< }
< #endif
<
< /*
< * Put us on the list of threads blocked on this mutex.
< */
< if (TAILQ_EMPTY(&m->mtx_blocked)) {
< p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK);
< LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested);
< TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
< } else {
< TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq)
< if (p1->p_pri.pri_level > p->p_pri.pri_level)
< break;
< if (p1)
< TAILQ_INSERT_BEFORE(p1, p, p_procq);
< else
< TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
< }
<
< /*
< * Save who we're blocked on.
< */
< p->p_blocked = m;
< p->p_mtxname = m->mtx_description;
< p->p_stat = SMTX;
< propagate_priority(p);
<
< if ((opts & MTX_QUIET) == 0)
< CTR3(KTR_LOCK,
< "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m,
< m->mtx_description);
<
< mi_switch();
<
< if ((opts & MTX_QUIET) == 0)
< CTR3(KTR_LOCK,
< "_mtx_lock_sleep: p %p free from blocked on [%p] %s",
< p, m, m->mtx_description);
<
< mtx_unlock_spin(&sched_lock);
< }
<
< return;
< }
<
< /*
< * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
< *
< * This is only called if we need to actually spin for the lock. Recursion
< * is handled inline.
< */
< void
< _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file,
< int line)
< {
< int i = 0;
<
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
<
< for (;;) {
< if (_obtain_lock(m, curproc))
< break;
<
< while (m->mtx_lock != MTX_UNOWNED) {
< if (i++ < 1000000)
< continue;
< if (i++ < 6000000)
< DELAY(1);
514,529d149
< else if (!db_active)
< #else
< else
< #endif
< panic("spin lock %s held by %p for > 5 seconds",
< m->mtx_description, (void *)m->mtx_lock);
< }
< }
<
< m->mtx_savecrit = mtx_crit;
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
<
< return;
< }
<
531,897d150
< * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
< *
< * We are only called here if the lock is recursed or contested (i.e. we
< * need to wake up a blocked thread).
< */
< void
< _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
< {
< struct proc *p, *p1;
< struct mtx *m1;
< int pri;
<
< p = curproc;
<
< if (mtx_recursed(m)) {
< if (--(m->mtx_recurse) == 0)
< atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
< return;
< }
<
< mtx_lock_spin(&sched_lock);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
<
< p1 = TAILQ_FIRST(&m->mtx_blocked);
< MPASS(p->p_magic == P_MAGIC);
< MPASS(p1->p_magic == P_MAGIC);
<
< TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq);
<
< if (TAILQ_EMPTY(&m->mtx_blocked)) {
< LIST_REMOVE(m, mtx_contested);
< _release_lock_quick(m);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m);
< } else
< atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED);
<
< pri = PRI_MAX;
< LIST_FOREACH(m1, &p->p_contested, mtx_contested) {
< int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level;
< if (cp < pri)
< pri = cp;
< }
<
< if (pri > p->p_pri.pri_native)
< pri = p->p_pri.pri_native;
< SET_PRIO(p, pri);
<
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p",
< m, p1);
<
< p1->p_blocked = NULL;
< p1->p_stat = SRUN;
< setrunqueue(p1);
<
< if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) {
< #ifdef notyet
< if (p->p_ithd != NULL) {
< struct ithd *it = p->p_ithd;
<
< if (it->it_interrupted) {
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK,
< "_mtx_unlock_sleep: %p interrupted %p",
< it, it->it_interrupted);
< intr_thd_fixup(it);
< }
< }
< #endif
< setrunqueue(p);
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK,
< "_mtx_unlock_sleep: %p switching out lock=%p", m,
< (void *)m->mtx_lock);
<
< mi_switch();
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p",
< m, (void *)m->mtx_lock);
< }
<
< mtx_unlock_spin(&sched_lock);
<
< return;
< }
<
< /*
< * All the unlocking of MTX_SPIN locks is done inline.
< * See the _rel_spin_lock() macro for the details.
< */
<
< /*
< * The backing function for the INVARIANTS-enabled mtx_assert()
< */
< #ifdef INVARIANT_SUPPORT
< void
< _mtx_assert(struct mtx *m, int what, const char *file, int line)
< {
< switch (what) {
< case MA_OWNED:
< case MA_OWNED | MA_RECURSED:
< case MA_OWNED | MA_NOTRECURSED:
< if (!mtx_owned(m))
< panic("mutex %s not owned at %s:%d",
< m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((what & MA_NOTRECURSED) != 0)
< panic("mutex %s recursed at %s:%d",
< m->mtx_description, file, line);
< } else if ((what & MA_RECURSED) != 0) {
< panic("mutex %s unrecursed at %s:%d",
< m->mtx_description, file, line);
< }
< break;
< case MA_NOTOWNED:
< if (mtx_owned(m))
< panic("mutex %s owned at %s:%d",
< m->mtx_description, file, line);
< break;
< default:
< panic("unknown mtx_assert at %s:%d", file, line);
< }
< }
< #endif
<
< /*
< * The MUTEX_DEBUG-enabled mtx_validate()
< */
< #define MV_DESTROY 0 /* validate before destory */
< #define MV_INIT 1 /* validate before init */
<
< #ifdef MUTEX_DEBUG
<
< int mtx_validate __P((struct mtx *, int));
<
< int
< mtx_validate(struct mtx *m, int when)
< {
< struct mtx *mp;
< int i;
< int retval = 0;
<
< #ifdef WITNESS
< if (witness_cold)
< return 0;
< #endif
< if (m == &all_mtx || cold)
< return 0;
<
< mtx_lock(&all_mtx);
< /*
< * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly
< * we can re-enable the kernacc() checks.
< */
< #ifndef __alpha__
< MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) == 1);
< #endif
< MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx);
< for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
< #ifndef __alpha__
< if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) != 1) {
< panic("mtx_validate: mp=%p mp->mtx_next=%p",
< mp, mp->mtx_next);
< }
< #endif
< i++;
< if (i > mtx_cur_cnt) {
< panic("mtx_validate: too many in chain, known=%d\n",
< mtx_cur_cnt);
< }
< }
< MPASS(i == mtx_cur_cnt);
< switch (when) {
< case MV_DESTROY:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m)
< break;
< MPASS(mp == m);
< break;
< case MV_INIT:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m) {
< /*
< * Not good. This mutex already exists.
< */
< printf("re-initing existing mutex %s\n",
< m->mtx_description);
< MPASS(m->mtx_lock == MTX_UNOWNED);
< retval = 1;
< }
< }
< mtx_unlock(&all_mtx);
< return (retval);
< }
< #endif
<
< /*
< * Mutex initialization routine; initialize lock `m' of type contained in
< * `opts' with options contained in `opts' and description `description.'
< * Place on "all_mtx" queue.
< */
< void
< mtx_init(struct mtx *m, const char *description, int opts)
< {
<
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description);
<
< #ifdef MUTEX_DEBUG
< /* Diagnostic and error correction */
< if (mtx_validate(m, MV_INIT))
< return;
< #endif
<
< bzero((void *)m, sizeof *m);
< TAILQ_INIT(&m->mtx_blocked);
<
< #ifdef WITNESS
< if (!witness_cold) {
< m->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(m->mtx_debug != NULL);
< }
< #endif
<
< m->mtx_description = description;
< m->mtx_flags = opts;
< m->mtx_lock = MTX_UNOWNED;
<
< /* Put on all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next = &all_mtx;
< m->mtx_prev = all_mtx.mtx_prev;
< m->mtx_prev->mtx_next = m;
< all_mtx.mtx_prev = m;
< if (++mtx_cur_cnt > mtx_max_cnt)
< mtx_max_cnt = mtx_cur_cnt;
< mtx_unlock(&all_mtx);
<
< #ifdef WITNESS
< if (!witness_cold)
< witness_init(m, opts);
< #endif
< }
<
< /*
< * Remove lock `m' from all_mtx queue.
< */
< void
< mtx_destroy(struct mtx *m)
< {
<
< #ifdef WITNESS
< KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
< __FUNCTION__));
< #endif
<
< CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
<
< #ifdef MUTEX_DEBUG
< if (m->mtx_next == NULL)
< panic("mtx_destroy: %p (%s) already destroyed",
< m, m->mtx_description);
<
< if (!mtx_owned(m)) {
< MPASS(m->mtx_lock == MTX_UNOWNED);
< } else {
< MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
< }
<
< /* diagnostic */
< mtx_validate(m, MV_DESTROY);
< #endif
<
< #ifdef WITNESS
< if (m->mtx_witness)
< witness_destroy(m);
< #endif /* WITNESS */
<
< /* Remove from the all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next->mtx_prev = m->mtx_prev;
< m->mtx_prev->mtx_next = m->mtx_next;
<
< #ifdef MUTEX_DEBUG
< m->mtx_next = m->mtx_prev = NULL;
< #endif
<
< #ifdef WITNESS
< free(m->mtx_debug, M_WITNESS);
< m->mtx_debug = NULL;
< #endif
<
< mtx_cur_cnt--;
< mtx_unlock(&all_mtx);
< }
<
<
< /*
< * The WITNESS-enabled diagnostic code.
< */
< #ifdef WITNESS
< static void
< witness_fixup(void *dummy __unused)
< {
< struct mtx *mp;
<
< /*
< * We have to release Giant before initializing its witness
< * structure so that WITNESS doesn't get confused.
< */
< mtx_unlock(&Giant);
< mtx_assert(&Giant, MA_NOTOWNED);
<
< mtx_lock(&all_mtx);
<
< /* Iterate through all mutexes and finish up mutex initialization. */
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
<
< mp->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(mp->mtx_debug != NULL);
<
< witness_init(mp, mp->mtx_flags);
< }
< mtx_unlock(&all_mtx);
<
< /* Mark the witness code as being ready for use. */
< atomic_store_rel_int(&witness_cold, 0);
<
< mtx_lock(&Giant);
< }
< SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
<
< #define WITNESS_COUNT 200
< #define WITNESS_NCHILDREN 2
<
< int witness_watch = 1;
<
< struct witness {
< struct witness *w_next;
< const char *w_description;
< const char *w_file;
< int w_line;
< struct witness *w_morechildren;
< u_char w_childcnt;
< u_char w_Giant_squawked:1;
< u_char w_other_squawked:1;
< u_char w_same_squawked:1;
< u_char w_spin:1; /* MTX_SPIN type mutex. */
< u_int w_level;
< struct witness *w_children[WITNESS_NCHILDREN];
< };
<
< struct witness_blessed {
< char *b_lock1;
< char *b_lock2;
< };
<
< #ifdef DDB
< /*
921,928c174,181
< /*
< * Witness-enabled globals
< */
< static struct mtx w_mtx;
< static struct witness *w_free;
< static struct witness *w_all;
< static int w_inited;
< static int witness_dead; /* fatal error, probably no memory */
---
> static struct mtx w_mtx;
> static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free);
> static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all);
> static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin);
> static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep);
> static struct witness_child_list_entry *w_child_free = NULL;
> static struct lock_list_entry *w_lock_list_free = NULL;
> static int witness_dead; /* fatal error, probably no memory */
930c183,185
< static struct witness w_data[WITNESS_COUNT];
---
> static struct witness w_data[WITNESS_COUNT];
> static struct witness_child_list_entry w_childdata[WITNESS_CHILDCOUNT];
> static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT];
932,954c187,194
< /*
< * Internal witness routine prototypes
< */
< static struct witness *enroll(const char *description, int flag);
< static int itismychild(struct witness *parent, struct witness *child);
< static void removechild(struct witness *parent, struct witness *child);
< static int isitmychild(struct witness *parent, struct witness *child);
< static int isitmydescendant(struct witness *parent, struct witness *child);
< static int dup_ok(struct witness *);
< static int blessed(struct witness *, struct witness *);
< static void
< witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *);
< static void witness_leveldescendents(struct witness *parent, int level);
< static void witness_levelall(void);
< static struct witness * witness_get(void);
< static void witness_free(struct witness *m);
<
< static char *ignore_list[] = {
< "witness lock",
< NULL
< };
<
< static char *spin_order_list[] = {
---
> static struct witness_order_list_entry order_lists[] = {
> { "Giant", &lock_class_mtx_sleep },
> { "proctree", &lock_class_sx },
> { "allproc", &lock_class_sx },
> { "process lock", &lock_class_mtx_sleep },
> { "uidinfo hash", &lock_class_mtx_sleep },
> { "uidinfo struct", &lock_class_mtx_sleep },
> { NULL, NULL },
956c196
< "com",
---
> { "com", &lock_class_mtx_spin },
958c198
< "sio",
---
> { "sio", &lock_class_mtx_spin },
960c200
< "cy",
---
> { "cy", &lock_class_mtx_spin },
962,966c202,206
< "ng_node",
< "ng_worklist",
< "ithread table lock",
< "ithread list lock",
< "sched lock",
---
> { "ng_node", &lock_class_mtx_spin },
> { "ng_worklist", &lock_class_mtx_spin },
> { "ithread table lock", &lock_class_mtx_spin },
> { "ithread list lock", &lock_class_mtx_spin },
> { "sched lock", &lock_class_mtx_spin },
968c208
< "clk",
---
> { "clk", &lock_class_mtx_spin },
970c210
< "callout",
---
> { "callout", &lock_class_mtx_spin },
976,977c216,217
< "ap boot",
< "imen",
---
> { "ap boot", &lock_class_mtx_spin },
> { "imen", &lock_class_mtx_spin },
979c219
< "smp rendezvous",
---
> { "smp rendezvous", &lock_class_mtx_spin },
981c221,222
< NULL
---
> { NULL, NULL },
> { NULL, NULL }
984,990c225
< static char *order_list[] = {
< "Giant", "proctree", "allproc", "process lock", "uidinfo hash",
< "uidinfo struct", NULL,
< NULL
< };
<
< static char *dup_list[] = {
---
> static const char *dup_list[] = {
995,999d229
< static char *sleep_list[] = {
< "Giant",
< NULL
< };
<
1009,1013c239,242
< static void
< witness_init(struct mtx *m, int flag)
< {
< m->mtx_witness = enroll(m->mtx_description, flag);
< }
---
> /*
> * List of all locks in the system.
> */
> STAILQ_HEAD(, lock_object) all_locks = STAILQ_HEAD_INITIALIZER(all_locks);
1014a244,271
> static struct mtx all_mtx = {
> { &lock_class_mtx_sleep, /* mtx_object.lo_class */
> "All locks list", /* mtx_object.lo_name */
> NULL, /* mtx_object.lo_file */
> 0, /* mtx_object.lo_line */
> LO_INITIALIZED, /* mtx_object.lo_flags */
> { NULL }, /* mtx_object.lo_list */
> NULL }, /* mtx_object.lo_witness */
> MTX_UNOWNED, 0, /* mtx_lock, mtx_recurse */
> 0, /* mtx_savecrit */
> TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
> { NULL, NULL } /* mtx_contested */
> };
>
> /*
> * This global is set to 0 once it becomes safe to use the witness code.
> */
> static int witness_cold = 1;
>
> /*
> * Global variables for book keeping.
> */
> static int lock_cur_cnt;
> static int lock_max_cnt;
>
> /*
> * The WITNESS-enabled diagnostic code.
> */
1016c273
< witness_destroy(struct mtx *m)
---
> witness_initialize(void *dummy __unused)
1018,1024c275,304
< struct mtx *m1;
< struct proc *p;
< p = curproc;
< LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) {
< if (m1 == m) {
< LIST_REMOVE(m, mtx_held);
< break;
---
> struct lock_object *lock;
> struct witness_order_list_entry *order;
> struct witness *w, *w1;
> int i;
>
> /*
> * We have to release Giant before initializing its witness
> * structure so that WITNESS doesn't get confused.
> */
> mtx_unlock(&Giant);
> mtx_assert(&Giant, MA_NOTOWNED);
>
> STAILQ_INSERT_HEAD(&all_locks, &all_mtx.mtx_object, lo_list);
> mtx_init(&w_mtx, "witness lock", MTX_SPIN | MTX_QUIET | MTX_NOWITNESS);
> for (i = 0; i < WITNESS_COUNT; i++)
> witness_free(&w_data[i]);
> for (i = 0; i < WITNESS_CHILDCOUNT; i++)
> witness_child_free(&w_childdata[i]);
> for (i = 0; i < LOCK_CHILDCOUNT; i++)
> witness_lock_list_free(&w_locklistdata[i]);
>
> /* First add in all the specified order lists. */
> for (order = order_lists; order->w_name != NULL; order++) {
> w = enroll(order->w_name, order->w_class);
> w->w_file = "order list";
> for (order++; order->w_name != NULL; order++) {
> w1 = enroll(order->w_name, order->w_class);
> w1->w_file = "order list";
> itismychild(w, w1);
> w = w1;
1027d306
< return;
1028a308,322
> /* Iterate through all locks and add them to witness. */
> mtx_lock(&all_mtx);
> STAILQ_FOREACH(lock, &all_locks, lo_list) {
> if (lock->lo_flags & LO_WITNESS)
> lock->lo_witness = enroll(lock->lo_name,
> lock->lo_class);
> else
> lock->lo_witness = NULL;
> }
> mtx_unlock(&all_mtx);
>
> /* Mark the witness code as being ready for use. */
> atomic_store_rel_int(&witness_cold, 0);
>
> mtx_lock(&Giant);
1029a324
> SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL)
1030a326,382
> void
> witness_init(struct lock_object *lock)
> {
> struct lock_class *class;
>
> class = lock->lo_class;
> if (lock->lo_flags & LO_INITIALIZED)
> panic("%s: lock (%s) %s is already initialized!\n", __func__,
> class->lc_name, lock->lo_name);
>
> if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
> (class->lc_flags & LC_RECURSABLE) == 0)
> panic("%s: lock (%s) %s can not be recursable!\n", __func__,
> class->lc_name, lock->lo_name);
>
> if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
> (class->lc_flags & LC_SLEEPABLE) == 0)
> panic("%s: lock (%s) %s can not be sleepable!\n", __func__,
> class->lc_name, lock->lo_name);
>
> mtx_lock(&all_mtx);
> STAILQ_INSERT_TAIL(&all_locks, lock, lo_list);
> lock->lo_flags |= LO_INITIALIZED;
> lock_cur_cnt++;
> if (lock_cur_cnt > lock_max_cnt)
> lock_max_cnt = lock_cur_cnt;
> mtx_unlock(&all_mtx);
> if (!witness_cold && !witness_dead &&
> (lock->lo_flags & LO_WITNESS) != 0)
> lock->lo_witness = enroll(lock->lo_name, class);
> else
> lock->lo_witness = NULL;
> }
>
> void
> witness_destroy(struct lock_object *lock)
> {
>
> if (witness_cold)
> panic("lock (%s) %s destroyed while witness_cold",
> lock->lo_class->lc_name, lock->lo_name);
>
> if ((lock->lo_flags & LO_INITIALIZED) == 0)
> panic("%s: lock (%s) %s is not initialized!\n", __func__,
> lock->lo_class->lc_name, lock->lo_name);
>
> if (lock->lo_flags & LO_LOCKED)
> panic("lock (%s) %s destroyed while held",
> lock->lo_class->lc_name, lock->lo_name);
>
> mtx_lock(&all_mtx);
> lock_cur_cnt--;
> STAILQ_REMOVE(&all_locks, lock, lock_object, lo_list);
> lock->lo_flags &= LO_INITIALIZED;
> mtx_unlock(&all_mtx);
> }
>
1032c384,385
< witness_display(void(*prnt)(const char *fmt, ...))
---
> witness_display_list(void(*prnt)(const char *fmt, ...),
> struct witness_list *list)
1035c388
< int level, found;
---
> int found;
1037,1046c390,391
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< witness_levelall();
<
< /*
< * First, handle sleep mutexes which have been acquired at least
< * once.
< */
< prnt("Sleep mutexes:\n");
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || w->w_spin)
---
> STAILQ_FOREACH(w, list, w_typelist) {
> if (w->w_file == NULL)
1048,1049c393,396
< for (w1 = w_all; w1; w1 = w1->w_next) {
< if (isitmychild(w1, w))
---
> found = 0;
> STAILQ_FOREACH(w1, list, w_typelist) {
> if (isitmychild(w1, w)) {
> found++;
1050a398
> }
1052c400
< if (w1 != NULL)
---
> if (found)
1058a407
> }
1059a409,416
> static void
> witness_display(void(*prnt)(const char *fmt, ...))
> {
> struct witness *w;
>
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> witness_levelall();
>
1060a418,424
> * First, handle sleep mutexes which have been acquired at least
> * once.
> */
> prnt("Sleep locks:\n");
> witness_display_list(prnt, &w_sleep);
>
> /*
1063,1078c427,428
< prnt("\nSpin mutexes:\n");
< level = 0;
< while (level < sizeof(spin_order_list) / sizeof(char *)) {
< found = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || !w->w_spin)
< continue;
< if (w->w_level == 1 << level) {
< witness_displaydescendants(prnt, w);
< level++;
< found = 1;
< }
< }
< if (found == 0)
< level++;
< }
---
> prnt("\nSpin locks:\n");
> witness_display_list(prnt, &w_spin);
1083,1084c433,434
< prnt("\nMutexes which were never acquired:\n");
< for (w = w_all; w; w = w->w_next) {
---
> prnt("\nLocks which were never acquired:\n");
> STAILQ_FOREACH(w, &w_all, w_list) {
1087c437
< prnt("%s\n", w->w_description);
---
> prnt("%s\n", w->w_name);
1092c442
< witness_enter(struct mtx *m, int flags, const char *file, int line)
---
> witness_lock(struct lock_object *lock, int flags, const char *file, int line)
1093a444,446
> struct lock_list_entry **lock_list, *lle;
> struct lock_object *lock1, *lock2;
> struct lock_class *class;
1095d447
< struct mtx *m1;
1097c449
< int i;
---
> int i, j;
1102c454,455
< if (witness_cold || m->mtx_witness == NULL || panicstr)
---
> if (witness_cold || witness_dead || lock->lo_witness == NULL ||
> panicstr)
1104c457,458
< w = m->mtx_witness;
---
> w = lock->lo_witness;
> class = lock->lo_class;
1107,1139c461,463
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< i = PCPU_GET(witness_spin_check);
< if (i != 0 && w->w_level < i) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< panic("mutex_enter(%s:%x, MTX_SPIN) out of order @"
< " %s:%d already holding %s:%x",
< m->mtx_description, w->w_level, file, line,
< spin_order_list[ffs(i)-1], i);
< }
< PCPU_SET(witness_spin_check, i | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< p->p_spinlocks++;
< MPASS(p->p_spinlocks > 0);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
< }
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
---
> if ((lock->lo_flags & LO_LOCKED) == 0)
> panic("%s: lock (%s) %s is not locked @ %s:%d", __func__,
> class->lc_name, lock->lo_name, file, line);
1141,1145c465,470
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
---
> if ((lock->lo_flags & LO_RECURSED) != 0) {
> if ((lock->lo_flags & LO_RECURSABLE) == 0)
> panic(
> "%s: recursed on non-recursive lock (%s) %s @ %s:%d",
> __func__, class->lc_name, lock->lo_name, file,
> line);
1148c473,482
< if (witness_dead)
---
>
> lock_list = PCPU_PTR(spinlocks);
> if (class->lc_flags & LC_SLEEPLOCK) {
> if (*lock_list != NULL)
> panic("blockable sleep lock (%s) %s @ %s:%d",
> class->lc_name, lock->lo_name, file, line);
> lock_list = &p->p_sleeplocks;
> }
>
> if (flags & LOP_TRYLOCK)
1150,1151d483
< if (cold)
< goto out;
1153,1155d484
< if (p->p_spinlocks != 0)
< panic("blockable mtx_lock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
1157c486,487
< * Is this the first mutex acquired
---
> * Is this the first lock acquired? If so, then no order checking
> * is needed.
1159c489
< if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL)
---
> if (*lock_list == NULL)
1162c492,499
< if ((w1 = m1->mtx_witness) == w) {
---
> /*
> * Check for duplicate locks of the same type. Note that we only
> * have to check for this on the last lock we just acquired. Any
> * other cases will be caught as lock order violations.
> */
> lock1 = (*lock_list)->ll_children[(*lock_list)->ll_count - 1];
> w1 = lock1->lo_witness;
> if (w1 == w) {
1167c504
< m->mtx_description);
---
> lock->lo_name);
1176c513
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
---
> mtx_lock_spin(&w_mtx);
1181c518
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> mtx_unlock_spin(&w_mtx);
1184,1185c521,522
< if (isitmydescendant(m1->mtx_witness, w)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> if (isitmydescendant(w1, w)) {
> mtx_unlock_spin(&w_mtx);
1188c525,526
< for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) {
---
> for (j = 0, lle = *lock_list; lle != NULL; lle = lle->ll_next) {
> for (i = lle->ll_count - 1; i >= 0; i--, j++) {
1190,1193c528,547
< MPASS(i < 200);
< w1 = m1->mtx_witness;
< if (isitmydescendant(w, w1)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> MPASS(j < WITNESS_COUNT);
> lock1 = lle->ll_children[i];
> w1 = lock1->lo_witness;
>
> /*
> * If this lock doesn't undergo witness checking,
> * then skip it.
> */
> if (w1 == NULL) {
> KASSERT((lock1->lo_flags & LO_WITNESS) == 0,
> ("lock missing witness structure"));
> continue;
> }
> if (!isitmydescendant(w, w1))
> continue;
> /*
> * We have a lock order violation, check to see if it
> * is allowed or has already been yelled about.
> */
> mtx_unlock_spin(&w_mtx);
1196c550
< if (m1 == &Giant) {
---
> if (lock1 == &Giant.mtx_object) {
1206a561,563
> /*
> * Ok, yell about it.
> */
1208,1209c565,590
< printf(" 1st %s last acquired @ %s:%d\n",
< w->w_description, w->w_file, w->w_line);
---
> /*
> * Try to locate an earlier lock with
> * witness w in our list.
> */
> do {
> lock2 = lle->ll_children[i];
> MPASS(lock2 != NULL);
> if (lock2->lo_witness == w)
> break;
> i--;
> if (i == 0 && lle->ll_next != NULL) {
> lle = lle->ll_next;
> i = lle->ll_count - 1;
> MPASS(i != 0);
> }
> } while (i >= 0);
> if (i < 0)
> /*
> * We are very likely bogus in this case.
> */
> printf(" 1st %s last acquired @ %s:%d\n",
> w->w_name, w->w_file, w->w_line);
> else
> printf(" 1st %p %s @ %s:%d\n", lock2,
> lock2->lo_name, lock2->lo_file,
> lock2->lo_line);
1211c592,593
< m1, w1->w_description, w1->w_file, w1->w_line);
---
> lock1, lock1->lo_name, lock1->lo_file,
> lock1->lo_line);
1213c595
< m, w->w_description, file, line);
---
> lock, lock->lo_name, file, line);
1220,1222c602,604
< m1 = LIST_FIRST(&p->p_heldmtx);
< if (!itismychild(m1->mtx_witness, w))
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> lock1 = (*lock_list)->ll_children[(*lock_list)->ll_count - 1];
> if (!itismychild(lock1->lo_witness, w))
> mtx_unlock_spin(&w_mtx);
1231,1262c613,619
< m->mtx_line = line;
< m->mtx_file = file;
<
< /*
< * If this pays off it likely means that a mutex being witnessed
< * is acquired in hardclock. Put it in the ignore list. It is
< * likely not the mutex this assert fails on.
< */
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
< }
<
< void
< witness_try_enter(struct mtx *m, int flags, const char *file, int line)
< {
< struct proc *p;
< struct witness *w = m->mtx_witness;
<
< if (witness_cold)
< return;
< if (panicstr)
< return;
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_try_enter: "
< "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
< m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on"
< " non-recursive mutex %s @ %s:%d",
< m->mtx_description, file, line);
---
> lock->lo_line = line;
> lock->lo_file = file;
>
> lle = *lock_list;
> if (lle == NULL || lle->ll_count == LOCK_CHILDCOUNT) {
> *lock_list = witness_lock_list_get();
> if (*lock_list == NULL)
1264,1273c621,622
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
---
> (*lock_list)->ll_next = lle;
> lle = *lock_list;
1275,1293c624
<
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description, file,
< line);
< return;
< }
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< p = curproc;
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
---
> lle->ll_children[lle->ll_count++] = lock;
1297c628
< witness_exit(struct mtx *m, int flags, const char *file, int line)
---
> witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
1299c630,631
< struct witness *w;
---
> struct lock_list_entry **lock_list, *lle;
> struct lock_class *class;
1300a633
> int i, j;
1302c635,636
< if (witness_cold || m->mtx_witness == NULL || panicstr)
---
> if (witness_cold || witness_dead || lock->lo_witness == NULL ||
> panicstr)
1304d637
< w = m->mtx_witness;
1305a639
> class = lock->lo_class;
1307,1323c641,645
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) & ~w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< MPASS(p->p_spinlocks > 0);
< p->p_spinlocks--;
---
> if (lock->lo_flags & LO_RECURSED) {
> if ((lock->lo_flags & LO_LOCKED) == 0)
> panic("%s: recursed lock (%s) %s is not locked @ %s:%d",
> __func__, class->lc_name, lock->lo_name, file,
> line);
1326,1328d647
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
1330,1336c649,661
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
---
> /*
> * We don't need to protect this PCPU_GET() here against preemption
> * because if we hold any spinlocks then we are already protected,
> * and if we don't we will get NULL if we hold no spinlocks even if
> * we switch CPU's while reading it.
> */
> if (class->lc_flags & LC_SLEEPLOCK) {
> if ((flags & LOP_NOSWITCH) == 0 && PCPU_GET(spinlocks) != NULL)
> panic("switchable sleep unlock (%s) %s @ %s:%d",
> class->lc_name, lock->lo_name, file, line);
> lock_list = &p->p_sleeplocks;
> } else
> lock_list = PCPU_PTR(spinlocks);
1338,1342c663,676
< if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
< panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
< LIST_REMOVE(m, mtx_held);
< m->mtx_held.le_prev = NULL;
---
> for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next)
> for (i = 0; i < (*lock_list)->ll_count; i++)
> if ((*lock_list)->ll_children[i] == lock) {
> (*lock_list)->ll_count--;
> for (j = i; j < (*lock_list)->ll_count; j++)
> (*lock_list)->ll_children[j] =
> (*lock_list)->ll_children[j + 1];
> if ((*lock_list)->ll_count == 0) {
> lle = *lock_list;
> *lock_list = lle->ll_next;
> witness_lock_list_free(lle);
> }
> return;
> }
1344a679,683
> /*
> * Warn if any held locks are not sleepable. Note that Giant and the lock
> * passed in are both special cases since they are both released during the
> * sleep process and aren't actually held while the process is asleep.
> */
1346c685,686
< witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
---
> witness_sleep(int check_only, struct lock_object *lock, const char *file,
> int line)
1348c688,689
< struct mtx *m;
---
> struct lock_list_entry **lock_list, *lle;
> struct lock_object *lock1;
1350,1351c691,692
< char **sleep;
< int n = 0;
---
> critical_t savecrit;
> int i, n;
1353c694,701
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
---
> if (witness_dead || panicstr)
> return (0);
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> n = 0;
> /*
> * Preemption bad because we need PCPU_PTR(spinlocks) to not change.
> */
> savecrit = critical_enter();
1355,1368c703,718
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< if (m == mtx)
< continue;
< for (sleep = sleep_list; *sleep!= NULL; sleep++)
< if (strcmp(m->mtx_description, *sleep) == 0)
< goto next;
< if (n == 0)
< printf("Whee!\n");
< printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
< file, line, check_only ? "could sleep" : "sleeping",
< m->mtx_description,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< n++;
< next:
---
> lock_list = &p->p_sleeplocks;
> again:
> for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
> for (i = lle->ll_count - 1; i >= 0; i--) {
> lock1 = lle->ll_children[i];
> if (lock1 == lock || lock1 == &Giant.mtx_object ||
> (lock1->lo_flags & LO_SLEEPABLE))
> continue;
> n++;
> printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
> file, line, check_only ? "could sleep" : "sleeping",
> lock1->lo_name, lock1->lo_file, lock1->lo_line);
> }
> if (lock_list == &p->p_sleeplocks) {
> lock_list = PCPU_PTR(spinlocks);
> goto again;
1373a724
> critical_exit(savecrit);
1378c729
< enroll(const char *description, int flag)
---
> enroll(const char *description, struct lock_class *lock_class)
1380,1383c731
< int i;
< struct witness *w, *w1;
< char **ignore;
< char **order;
---
> struct witness *w;
1387,1389d734
< for (ignore = ignore_list; *ignore != NULL; ignore++)
< if (strcmp(description, *ignore) == 0)
< return (NULL);
1391,1409c736
< if (w_inited == 0) {
< mtx_init(&w_mtx, "witness lock", MTX_SPIN);
< for (i = 0; i < WITNESS_COUNT; i++) {
< w = &w_data[i];
< witness_free(w);
< }
< w_inited = 1;
< for (order = order_list; *order != NULL; order++) {
< w = enroll(*order, MTX_DEF);
< w->w_file = "order list";
< for (order++; *order != NULL; order++) {
< w1 = enroll(*order, MTX_DEF);
< w1->w_file = "order list";
< itismychild(w, w1);
< w = w1;
< }
< }
< }
< if ((flag & MTX_SPIN) && witness_skipspin)
---
> if ((lock_class->lc_flags & LC_SPINLOCK) && witness_skipspin)
1411,1414c738,746
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< for (w = w_all; w; w = w->w_next) {
< if (strcmp(description, w->w_description) == 0) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> mtx_lock_spin(&w_mtx);
> STAILQ_FOREACH(w, &w_all, w_list) {
> if (strcmp(description, w->w_name) == 0) {
> mtx_unlock_spin(&w_mtx);
> if (lock_class != w->w_class)
> panic(
> "lock (%s) %s does not match earlier (%s) lock",
> description, lock_class->lc_name,
> w->w_class->lc_name);
1417a750,755
> /*
> * This isn't quite right, as witness_cold is still 0 while we
> * enroll all the locks initialized before witness_initialize().
> */
> if ((lock_class->lc_flags & LC_SPINLOCK) && !witness_cold)
> panic("spin lock %s not in order list", description);
1420,1436c758,768
< w->w_next = w_all;
< w_all = w;
< w->w_description = description;
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< if (flag & MTX_SPIN) {
< w->w_spin = 1;
<
< i = 1;
< for (order = spin_order_list; *order != NULL; order++) {
< if (strcmp(description, *order) == 0)
< break;
< i <<= 1;
< }
< if (*order == NULL)
< panic("spin lock %s not in order list", description);
< w->w_level = i;
< }
---
> w->w_name = description;
> w->w_class = lock_class;
> STAILQ_INSERT_HEAD(&w_all, w, w_list);
> if (lock_class->lc_flags & LC_SPINLOCK)
> STAILQ_INSERT_HEAD(&w_spin, w, w_typelist);
> else if (lock_class->lc_flags & LC_SLEEPLOCK)
> STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist);
> else
> panic("lock class %s is not sleep or spin",
> lock_class->lc_name);
> mtx_unlock_spin(&w_mtx);
1444a777,778
> struct witness_child_list_entry **wcl;
> struct witness_list *list;
1445a780,787
> MPASS(child != NULL && parent != NULL);
> if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) !=
> (child->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)))
> panic(
> "%s: parent (%s) and child (%s) are not the same lock type",
> __func__, parent->w_class->lc_name,
> child->w_class->lc_name);
>
1449,1450c791,793
< while (parent->w_morechildren)
< parent = parent->w_morechildren;
---
> wcl = &parent->w_children;
> while (*wcl != NULL && (*wcl)->wcl_count == WITNESS_NCHILDREN)
> wcl = &(*wcl)->wcl_next;
1452,1453c795,797
< if (parent->w_childcnt == WITNESS_NCHILDREN) {
< if ((parent->w_morechildren = witness_get()) == NULL)
---
> if (*wcl == NULL) {
> *wcl = witness_child_get();
> if (*wcl == NULL)
1455d798
< parent = parent->w_morechildren;
1457,1458c800,802
< MPASS(child != NULL);
< parent->w_children[parent->w_childcnt++] = child;
---
>
> (*wcl)->wcl_children[(*wcl)->wcl_count++] = child;
>
1460c804,806
< * now prune whole tree
---
> * Now prune whole tree. We look for cases where a lock is now
> * both a descendant and a direct child of a given lock. In that
> * case, we want to remove the direct child link from the tree.
1465,1467c811,816
< for (child = w_all; child != NULL; child = child->w_next) {
< for (parent = w_all; parent != NULL;
< parent = parent->w_next) {
---
> if (parent->w_class->lc_flags & LC_SLEEPLOCK)
> list = &w_sleep;
> else
> list = &w_spin;
> STAILQ_FOREACH(child, list, w_typelist) {
> STAILQ_FOREACH(parent, list, w_typelist) {
1484c833
< struct witness *w, *w1;
---
> struct witness_child_list_entry **wcl, *wcl1;
1487,1489c836,838
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< if (w->w_children[i] == child)
---
> for (wcl = &parent->w_children; *wcl != NULL; wcl = &(*wcl)->wcl_next)
> for (i = 0; i < (*wcl)->wcl_count; i++)
> if ((*wcl)->wcl_children[i] == child)
1493,1496c842,846
< for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren)
< continue;
< w->w_children[i] = w1->w_children[--w1->w_childcnt];
< MPASS(w->w_children[i] != NULL);
---
> (*wcl)->wcl_count--;
> if ((*wcl)->wcl_count > i)
> (*wcl)->wcl_children[i] =
> (*wcl)->wcl_children[(*wcl)->wcl_count];
> MPASS((*wcl)->wcl_children[i] != NULL);
1498c848
< if (w1->w_childcnt != 0)
---
> if ((*wcl)->wcl_count != 0)
1501,1506c851,853
< if (w1 == parent)
< return;
< for (w = parent; w->w_morechildren != w1; w = w->w_morechildren)
< continue;
< w->w_morechildren = 0;
< witness_free(w1);
---
> wcl1 = *wcl;
> *wcl = wcl1->wcl_next;
> witness_child_free(wcl1);
1512c859
< struct witness *w;
---
> struct witness_child_list_entry *wcl;
1515,1517c862,864
< for (w = parent; w != NULL; w = w->w_morechildren) {
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
---
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
> for (i = 0; i < wcl->wcl_count; i++) {
> if (wcl->wcl_children[i] == child)
1527,1529c874,875
< struct witness *w;
< int i;
< int j;
---
> struct witness_child_list_entry *wcl;
> int i, j;
1531c877,880
< for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) {
---
> if (isitmychild(parent, child))
> return (1);
> j = 0;
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
1533,1534c882,883
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
---
> for (i = 0; i < wcl->wcl_count; i++) {
> if (isitmydescendant(wcl->wcl_children[i], child))
1537,1540c886
< for (i = 0; i < w->w_childcnt; i++) {
< if (isitmydescendant(w->w_children[i], child))
< return (1);
< }
---
> j++;
1547a894
> struct witness_list *list;
1550,1556c897,916
< for (w = w_all; w; w = w->w_next)
< if (!(w->w_spin))
< w->w_level = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_spin)
< continue;
< for (w1 = w_all; w1; w1 = w1->w_next) {
---
> /*
> * First clear all levels.
> */
> STAILQ_FOREACH(w, &w_all, w_list) {
> w->w_level = 0;
> }
>
> /*
> * Look for locks with no parent and level all their descendants.
> */
> STAILQ_FOREACH(w, &w_all, w_list) {
> /*
> * This is just an optimization, technically we could get
> * away just walking the all list each time.
> */
> if (w->w_class->lc_flags & LC_SLEEPLOCK)
> list = &w_sleep;
> else
> list = &w_spin;
> STAILQ_FOREACH(w1, list, w_typelist) {
1558c918
< break;
---
> goto skip;
1560,1561d919
< if (w1 != NULL)
< continue;
1562a921
> skip:
1568a928
> struct witness_child_list_entry *wcl;
1570d929
< struct witness *w;
1575,1577c934,936
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_leveldescendents(w->w_children[i], level);
---
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
> for (i = 0; i < wcl->wcl_count; i++)
> witness_leveldescendents(wcl->wcl_children[i], level);
1584,1586c943,944
< struct witness *w;
< int i;
< int level;
---
> struct witness_child_list_entry *wcl;
> int i, level;
1588c946
< level = parent->w_spin ? ffs(parent->w_level) : parent->w_level;
---
> level = parent->w_level;
1590,1592c948
< prnt("%d", level);
< if (level < 10)
< prnt(" ");
---
> prnt("%-2d", level);
1595c951
< prnt("%s", parent->w_description);
---
> prnt("%s", parent->w_name);
1600,1603c956,960
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_displaydescendants(prnt, w->w_children[i]);
< }
---
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
> for (i = 0; i < wcl->wcl_count; i++)
> witness_displaydescendants(prnt,
> wcl->wcl_children[i]);
> }
1608c965
< char **dup;
---
> const char **dup;
1610,1611c967,968
< for (dup = dup_list; *dup!= NULL; dup++)
< if (strcmp(w->w_description, *dup) == 0)
---
> for (dup = dup_list; *dup != NULL; dup++)
> if (strcmp(w->w_name, *dup) == 0)
1624,1625c981,982
< if (strcmp(w1->w_description, b->b_lock1) == 0) {
< if (strcmp(w2->w_description, b->b_lock2) == 0)
---
> if (strcmp(w1->w_name, b->b_lock1) == 0) {
> if (strcmp(w2->w_name, b->b_lock2) == 0)
1629,1630c986,987
< if (strcmp(w1->w_description, b->b_lock2) == 0)
< if (strcmp(w2->w_description, b->b_lock1) == 0)
---
> if (strcmp(w1->w_name, b->b_lock2) == 0)
> if (strcmp(w2->w_name, b->b_lock1) == 0)
1637c994
< witness_get()
---
> witness_get(void)
1641c998
< if ((w = w_free) == NULL) {
---
> if (STAILQ_EMPTY(&w_free)) {
1643,1644c1000,1001
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< printf("witness exhausted\n");
---
> mtx_unlock_spin(&w_mtx);
> printf("%s: witness exhausted\n", __func__);
1647c1004,1005
< w_free = w->w_next;
---
> w = STAILQ_FIRST(&w_free);
> STAILQ_REMOVE_HEAD(&w_free, w_list);
1655,1656c1013,1014
< w->w_next = w_free;
< w_free = w;
---
>
> STAILQ_INSERT_HEAD(&w_free, w, w_list);
1659,1660c1017,1018
< int
< witness_list(struct proc *p)
---
> static struct witness_child_list_entry *
> witness_child_get(void)
1662,1663c1020
< struct mtx *m;
< int nheld;
---
> struct witness_child_list_entry *wcl;
1665,1671c1022,1027
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< nheld = 0;
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< printf("\t\"%s\" (%p) locked at %s:%d\n",
< m->mtx_description, m,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< nheld++;
---
> wcl = w_child_free;
> if (wcl == NULL) {
> witness_dead = 1;
> mtx_unlock_spin(&w_mtx);
> printf("%s: witness exhausted\n", __func__);
> return (NULL);
1672a1029,1032
> w_child_free = wcl->wcl_next;
> bzero(wcl, sizeof(*wcl));
> return (wcl);
> }
1674c1034,1039
< return (nheld);
---
> static void
> witness_child_free(struct witness_child_list_entry *wcl)
> {
>
> wcl->wcl_next = w_child_free;
> w_child_free = wcl;
1677c1042,1045
< #ifdef DDB
---
> static struct lock_list_entry *
> witness_lock_list_get(void)
> {
> struct lock_list_entry *lle;
1679c1047,1062
< DB_SHOW_COMMAND(mutexes, db_witness_list)
---
> mtx_lock_spin(&w_mtx);
> lle = w_lock_list_free;
> if (lle == NULL) {
> witness_dead = 1;
> mtx_unlock_spin(&w_mtx);
> printf("%s: witness exhausted\n", __func__);
> return (NULL);
> }
> w_lock_list_free = lle->ll_next;
> mtx_unlock_spin(&w_mtx);
> bzero(lle, sizeof(*lle));
> return (lle);
> }
>
> static void
> witness_lock_list_free(struct lock_list_entry *lle)
1682c1065,1068
< witness_list(curproc);
---
> mtx_lock_spin(&w_mtx);
> lle->ll_next = w_lock_list_free;
> w_lock_list_free = lle;
> mtx_unlock_spin(&w_mtx);
1685c1071,1075
< DB_SHOW_COMMAND(witness, db_witness_display)
---
> /*
> * Calling this on p != curproc is bad unless we are in ddb.
> */
> int
> witness_list(struct proc *p)
1686a1077,1080
> struct lock_list_entry **lock_list, *lle;
> struct lock_object *lock;
> critical_t savecrit;
> int i, nheld;
1688c1082,1111
< witness_display(db_printf);
---
> KASSERT(p == curproc || db_active,
> ("%s: p != curproc and we aren't in the debugger", __func__));
> KASSERT(!witness_cold, ("%s: witness_cold", __func__));
> nheld = 0;
> /*
> * Preemption bad because we need PCPU_PTR(spinlocks) to not change.
> */
> savecrit = critical_enter();
> lock_list = &p->p_sleeplocks;
> again:
> for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
> for (i = lle->ll_count - 1; i >= 0; i--) {
> lock = lle->ll_children[i];
> printf("\t(%s) %s (%p) locked at %s:%d\n",
> lock->lo_class->lc_name, lock->lo_name, lock,
> lock->lo_file, lock->lo_line);
> nheld++;
> }
> /*
> * We only handle spinlocks if p == curproc. This is somewhat broken
> * if p is currently executing on some other CPU and holds spin locks
> * as we won't display those locks.
> */
> if (lock_list == &p->p_sleeplocks && p == curproc) {
> lock_list = PCPU_PTR(spinlocks);
> goto again;
> }
> critical_exit(savecrit);
>
> return (nheld);
1690d1112
< #endif
1693c1115
< witness_save(struct mtx *m, const char **filep, int *linep)
---
> witness_save(struct lock_object *lock, const char **filep, int *linep)
1696,1697c1118,1119
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
---
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> if (lock->lo_witness == NULL)
1700,1701c1122,1123
< *filep = m->mtx_witness->w_file;
< *linep = m->mtx_witness->w_line;
---
> *filep = lock->lo_file;
> *linep = lock->lo_line;
1705c1127
< witness_restore(struct mtx *m, const char *file, int line)
---
> witness_restore(struct lock_object *lock, const char *file, int line)
1708,1709c1130,1131
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
---
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> if (lock->lo_witness == NULL)
1712,1713c1134,1137
< m->mtx_witness->w_file = file;
< m->mtx_witness->w_line = line;
---
> lock->lo_witness->w_file = file;
> lock->lo_witness->w_line = line;
> lock->lo_file = file;
> lock->lo_line = line;
1716c1140,1153
< #endif /* WITNESS */
---
> #ifdef DDB
>
> DB_SHOW_COMMAND(mutexes, db_witness_list)
> {
>
> witness_list(curproc);
> }
>
> DB_SHOW_COMMAND(witness, db_witness_display)
> {
>
> witness_display(db_printf);
> }
> #endif
< * $FreeBSD: head/sys/kern/subr_witness.c 74900 2001-03-28 02:40:47Z jhb $
---
> * $FreeBSD: head/sys/kern/subr_witness.c 74912 2001-03-28 09:03:24Z jhb $
34,35c34,36
< * Machine independent bits of mutex implementation and implementation of
< * `witness' structure & related debugging routines.
---
> * Implementation of the `witness' lock verifier. Originally implemented for
> * mutexes in BSD/OS. Extended to handle generic lock objects and lock
> * classes in FreeBSD.
63a65,66
> #include <sys/ktr.h>
> #include <sys/lock.h>
64a68
> #include <sys/mutex.h>
68,69d71
< #include <sys/vmmeter.h>
< #include <sys/ktr.h>
71,75d72
< #include <machine/atomic.h>
< #include <machine/bus.h>
< #include <machine/clock.h>
< #include <machine/cpu.h>
<
78,82c75,76
< #include <vm/vm.h>
< #include <vm/vm_extern.h>
<
< #include <sys/mutex.h>
<
---
> #define WITNESS_COUNT 200
> #define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4)
84c78,80
< * The WITNESS-enabled mutex debug structure.
---
> * XXX: This is somewhat bogus, as we assume here that at most 1024 processes
> * will hold LOCK_NCHILDREN * 2 locks. We handle failure ok, and we should
> * probably be safe for the most part, but it's still a SWAG.
86,92c82
< #ifdef WITNESS
< struct mtx_debug {
< struct witness *mtxd_witness;
< LIST_ENTRY(mtx) mtxd_held;
< const char *mtxd_file;
< int mtxd_line;
< };
---
> #define LOCK_CHILDCOUNT (MAXCPU + 1024) * 2
94,98c84
< #define mtx_held mtx_debug->mtxd_held
< #define mtx_file mtx_debug->mtxd_file
< #define mtx_line mtx_debug->mtxd_line
< #define mtx_witness mtx_debug->mtxd_witness
< #endif /* WITNESS */
---
> #define WITNESS_NCHILDREN 6
100,103c86
< /*
< * Internal utility macros.
< */
< #define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED)
---
> struct witness_child_list_entry;
105,106c88,101
< #define mtx_owner(m) (mtx_unowned((m)) ? NULL \
< : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK))
---
> struct witness {
> const char *w_name;
> struct lock_class *w_class;
> STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */
> STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */
> struct witness_child_list_entry *w_children; /* Great evilness... */
> const char *w_file;
> int w_line;
> u_int w_level;
> u_int w_refcount;
> u_char w_Giant_squawked:1;
> u_char w_other_squawked:1;
> u_char w_same_squawked:1;
> };
108c103,107
< #define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri)
---
> struct witness_child_list_entry {
> struct witness_child_list_entry *wcl_next;
> struct witness *wcl_children[WITNESS_NCHILDREN];
> u_int wcl_count;
> };
110,113c109
< /*
< * Early WITNESS-enabled declarations.
< */
< #ifdef WITNESS
---
> STAILQ_HEAD(witness_list, witness);
115,123c111,114
< /*
< * Internal WITNESS routines which must be prototyped early.
< *
< * XXX: When/if witness code is cleaned up, it would be wise to place all
< * witness prototyping early in this file.
< */
< static void witness_init(struct mtx *, int flag);
< static void witness_destroy(struct mtx *);
< static void witness_display(void(*)(const char *fmt, ...));
---
> struct witness_blessed {
> const char *b_lock1;
> const char *b_lock2;
> };
125c116,119
< MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure");
---
> struct witness_order_list_entry {
> const char *w_name;
> struct lock_class *w_class;
> };
127,128c121,141
< /* All mutexes in system (used for debug/panic) */
< static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
---
> static struct witness *enroll(const char *description,
> struct lock_class *lock_class);
> static int itismychild(struct witness *parent, struct witness *child);
> static void removechild(struct witness *parent, struct witness *child);
> static int isitmychild(struct witness *parent, struct witness *child);
> static int isitmydescendant(struct witness *parent, struct witness *child);
> static int dup_ok(struct witness *);
> static int blessed(struct witness *, struct witness *);
> static void witness_display_list(void(*prnt)(const char *fmt, ...),
> struct witness_list *list);
> static void witness_displaydescendants(void(*)(const char *fmt, ...),
> struct witness *);
> static void witness_leveldescendents(struct witness *parent, int level);
> static void witness_levelall(void);
> static struct witness *witness_get(void);
> static void witness_free(struct witness *m);
> static struct witness_child_list_entry *witness_child_get(void);
> static void witness_child_free(struct witness_child_list_entry *wcl);
> static struct lock_list_entry *witness_lock_list_get(void);
> static void witness_lock_list_free(struct lock_list_entry *lle);
> static void witness_display(void(*)(const char *fmt, ...));
130,133c143
< /*
< * This global is set to 0 once it becomes safe to use the witness code.
< */
< static int witness_cold = 1;
---
> MALLOC_DEFINE(M_WITNESS, "witness", "witness structure");
135c145,147
< #else /* WITNESS */
---
> static int witness_watch;
> TUNABLE_INT_DECL("debug.witness_watch", 1, witness_watch);
> SYSCTL_INT(_debug, OID_AUTO, witness_watch, CTLFLAG_RD, &witness_watch, 0, "");
137,512d148
< /* XXX XXX XXX
< * flag++ is sleazoid way of shuting up warning
< */
< #define witness_init(m, flag) flag++
< #define witness_destroy(m)
< #define witness_try_enter(m, t, f, l)
< #endif /* WITNESS */
<
< /*
< * All mutex locks in system are kept on the all_mtx list.
< */
< static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head",
< TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
< { NULL, NULL }, &all_mtx, &all_mtx,
< #ifdef WITNESS
< &all_mtx_debug
< #else
< NULL
< #endif
< };
<
< /*
< * Global variables for book keeping.
< */
< static int mtx_cur_cnt;
< static int mtx_max_cnt;
<
< /*
< * Couple of strings for KTR_LOCK tracing in order to avoid duplicates.
< */
< char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d";
<
< /*
< * Prototypes for non-exported routines.
< *
< * NOTE: Prototypes for witness routines are placed at the bottom of the file.
< */
< static void propagate_priority(struct proc *);
<
< static void
< propagate_priority(struct proc *p)
< {
< int pri = p->p_pri.pri_level;
< struct mtx *m = p->p_blocked;
<
< mtx_assert(&sched_lock, MA_OWNED);
< for (;;) {
< struct proc *p1;
<
< p = mtx_owner(m);
<
< if (p == NULL) {
< /*
< * This really isn't quite right. Really
< * ought to bump priority of process that
< * next acquires the mutex.
< */
< MPASS(m->mtx_lock == MTX_CONTESTED);
< return;
< }
<
< MPASS(p->p_magic == P_MAGIC);
< KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex"));
< if (p->p_pri.pri_level <= pri)
< return;
<
< /*
< * Bump this process' priority.
< */
< SET_PRIO(p, pri);
<
< /*
< * If lock holder is actually running, just bump priority.
< */
< if (p->p_oncpu != NOCPU) {
< MPASS(p->p_stat == SRUN || p->p_stat == SZOMB);
< return;
< }
<
< #ifndef SMP
< /*
< * For UP, we check to see if p is curproc (this shouldn't
< * ever happen however as it would mean we are in a deadlock.)
< */
< KASSERT(p != curproc, ("Deadlock detected"));
< #endif
<
< /*
< * If on run queue move to new run queue, and
< * quit.
< */
< if (p->p_stat == SRUN) {
< MPASS(p->p_blocked == NULL);
< remrunqueue(p);
< setrunqueue(p);
< return;
< }
<
< /*
< * If we aren't blocked on a mutex, we should be.
< */
< KASSERT(p->p_stat == SMTX, (
< "process %d(%s):%d holds %s but isn't blocked on a mutex\n",
< p->p_pid, p->p_comm, p->p_stat,
< m->mtx_description));
<
< /*
< * Pick up the mutex that p is blocked on.
< */
< m = p->p_blocked;
< MPASS(m != NULL);
<
< /*
< * Check if the proc needs to be moved up on
< * the blocked chain
< */
< if (p == TAILQ_FIRST(&m->mtx_blocked)) {
< continue;
< }
<
< p1 = TAILQ_PREV(p, procqueue, p_procq);
< if (p1->p_pri.pri_level <= pri) {
< continue;
< }
<
< /*
< * Remove proc from blocked chain and determine where
< * it should be moved up to. Since we know that p1 has
< * a lower priority than p, we know that at least one
< * process in the chain has a lower priority and that
< * p1 will thus not be NULL after the loop.
< */
< TAILQ_REMOVE(&m->mtx_blocked, p, p_procq);
< TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) {
< MPASS(p1->p_magic == P_MAGIC);
< if (p1->p_pri.pri_level > pri)
< break;
< }
<
< MPASS(p1 != NULL);
< TAILQ_INSERT_BEFORE(p1, p, p_procq);
< CTR4(KTR_LOCK,
< "propagate_priority: p %p moved before %p on [%p] %s",
< p, p1, m, m->mtx_description);
< }
< }
<
< /*
< * Function versions of the inlined __mtx_* macros. These are used by
< * modules and can also be called from assembly language if needed.
< */
< void
< _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_lock_flags(m, opts, file, line);
< }
<
< void
< _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_unlock_flags(m, opts, file, line);
< }
<
< void
< _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_lock_spin_flags(m, opts, file, line);
< }
<
< void
< _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
< {
<
< __mtx_unlock_spin_flags(m, opts, file, line);
< }
<
< /*
< * The important part of mtx_trylock{,_flags}()
< * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that
< * if we're called, it's because we know we don't already own this lock.
< */
< int
< _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
< {
< int rval;
<
< MPASS(curproc != NULL);
<
< /*
< * _mtx_trylock does not accept MTX_NOSWITCH option.
< */
< KASSERT((opts & MTX_NOSWITCH) == 0,
< ("mtx_trylock() called with invalid option flag(s) %d", opts));
<
< rval = _obtain_lock(m, curproc);
<
< #ifdef WITNESS
< if (rval && m->mtx_witness != NULL) {
< /*
< * We do not handle recursion in _mtx_trylock; see the
< * note at the top of the routine.
< */
< KASSERT(!mtx_recursed(m),
< ("mtx_trylock() called on a recursed mutex"));
< witness_try_enter(m, (opts | m->mtx_flags), file, line);
< }
< #endif /* WITNESS */
<
< if ((opts & MTX_QUIET) == 0)
< CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d",
< m->mtx_description, m, rval, file, line);
<
< return rval;
< }
<
< /*
< * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
< *
< * We call this if the lock is either contested (i.e. we need to go to
< * sleep waiting for it), or if we need to recurse on it.
< */
< void
< _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
< {
< struct proc *p = curproc;
<
< if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
< m->mtx_recurse++;
< atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
< return;
< }
<
< if ((opts & MTX_QUIET) == 0)
< CTR4(KTR_LOCK,
< "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
< m->mtx_description, (void *)m->mtx_lock, file, line);
<
< while (!_obtain_lock(m, p)) {
< uintptr_t v;
< struct proc *p1;
<
< mtx_lock_spin(&sched_lock);
< /*
< * Check if the lock has been released while spinning for
< * the sched_lock.
< */
< if ((v = m->mtx_lock) == MTX_UNOWNED) {
< mtx_unlock_spin(&sched_lock);
< continue;
< }
<
< /*
< * The mutex was marked contested on release. This means that
< * there are processes blocked on it.
< */
< if (v == MTX_CONTESTED) {
< p1 = TAILQ_FIRST(&m->mtx_blocked);
< MPASS(p1 != NULL);
< m->mtx_lock = (uintptr_t)p | MTX_CONTESTED;
<
< if (p1->p_pri.pri_level < p->p_pri.pri_level)
< SET_PRIO(p, p1->p_pri.pri_level);
< mtx_unlock_spin(&sched_lock);
< return;
< }
<
< /*
< * If the mutex isn't already contested and a failure occurs
< * setting the contested bit, the mutex was either released
< * or the state of the MTX_RECURSED bit changed.
< */
< if ((v & MTX_CONTESTED) == 0 &&
< !atomic_cmpset_ptr(&m->mtx_lock, (void *)v,
< (void *)(v | MTX_CONTESTED))) {
< mtx_unlock_spin(&sched_lock);
< continue;
< }
<
< /*
< * We deffinately must sleep for this lock.
< */
< mtx_assert(m, MA_NOTOWNED);
<
< #ifdef notyet
< /*
< * If we're borrowing an interrupted thread's VM context, we
< * must clean up before going to sleep.
< */
< if (p->p_ithd != NULL) {
< struct ithd *it = p->p_ithd;
<
< if (it->it_interrupted) {
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK,
< "_mtx_lock_sleep: %p interrupted %p",
< it, it->it_interrupted);
< intr_thd_fixup(it);
< }
< }
< #endif
<
< /*
< * Put us on the list of threads blocked on this mutex.
< */
< if (TAILQ_EMPTY(&m->mtx_blocked)) {
< p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK);
< LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested);
< TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
< } else {
< TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq)
< if (p1->p_pri.pri_level > p->p_pri.pri_level)
< break;
< if (p1)
< TAILQ_INSERT_BEFORE(p1, p, p_procq);
< else
< TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
< }
<
< /*
< * Save who we're blocked on.
< */
< p->p_blocked = m;
< p->p_mtxname = m->mtx_description;
< p->p_stat = SMTX;
< propagate_priority(p);
<
< if ((opts & MTX_QUIET) == 0)
< CTR3(KTR_LOCK,
< "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m,
< m->mtx_description);
<
< mi_switch();
<
< if ((opts & MTX_QUIET) == 0)
< CTR3(KTR_LOCK,
< "_mtx_lock_sleep: p %p free from blocked on [%p] %s",
< p, m, m->mtx_description);
<
< mtx_unlock_spin(&sched_lock);
< }
<
< return;
< }
<
< /*
< * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
< *
< * This is only called if we need to actually spin for the lock. Recursion
< * is handled inline.
< */
< void
< _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file,
< int line)
< {
< int i = 0;
<
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
<
< for (;;) {
< if (_obtain_lock(m, curproc))
< break;
<
< while (m->mtx_lock != MTX_UNOWNED) {
< if (i++ < 1000000)
< continue;
< if (i++ < 6000000)
< DELAY(1);
514,529d149
< else if (!db_active)
< #else
< else
< #endif
< panic("spin lock %s held by %p for > 5 seconds",
< m->mtx_description, (void *)m->mtx_lock);
< }
< }
<
< m->mtx_savecrit = mtx_crit;
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
<
< return;
< }
<
531,897d150
< * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
< *
< * We are only called here if the lock is recursed or contested (i.e. we
< * need to wake up a blocked thread).
< */
< void
< _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
< {
< struct proc *p, *p1;
< struct mtx *m1;
< int pri;
<
< p = curproc;
<
< if (mtx_recursed(m)) {
< if (--(m->mtx_recurse) == 0)
< atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
< return;
< }
<
< mtx_lock_spin(&sched_lock);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
<
< p1 = TAILQ_FIRST(&m->mtx_blocked);
< MPASS(p->p_magic == P_MAGIC);
< MPASS(p1->p_magic == P_MAGIC);
<
< TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq);
<
< if (TAILQ_EMPTY(&m->mtx_blocked)) {
< LIST_REMOVE(m, mtx_contested);
< _release_lock_quick(m);
< if ((opts & MTX_QUIET) == 0)
< CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m);
< } else
< atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED);
<
< pri = PRI_MAX;
< LIST_FOREACH(m1, &p->p_contested, mtx_contested) {
< int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level;
< if (cp < pri)
< pri = cp;
< }
<
< if (pri > p->p_pri.pri_native)
< pri = p->p_pri.pri_native;
< SET_PRIO(p, pri);
<
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p",
< m, p1);
<
< p1->p_blocked = NULL;
< p1->p_stat = SRUN;
< setrunqueue(p1);
<
< if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) {
< #ifdef notyet
< if (p->p_ithd != NULL) {
< struct ithd *it = p->p_ithd;
<
< if (it->it_interrupted) {
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK,
< "_mtx_unlock_sleep: %p interrupted %p",
< it, it->it_interrupted);
< intr_thd_fixup(it);
< }
< }
< #endif
< setrunqueue(p);
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK,
< "_mtx_unlock_sleep: %p switching out lock=%p", m,
< (void *)m->mtx_lock);
<
< mi_switch();
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p",
< m, (void *)m->mtx_lock);
< }
<
< mtx_unlock_spin(&sched_lock);
<
< return;
< }
<
< /*
< * All the unlocking of MTX_SPIN locks is done inline.
< * See the _rel_spin_lock() macro for the details.
< */
<
< /*
< * The backing function for the INVARIANTS-enabled mtx_assert()
< */
< #ifdef INVARIANT_SUPPORT
< void
< _mtx_assert(struct mtx *m, int what, const char *file, int line)
< {
< switch (what) {
< case MA_OWNED:
< case MA_OWNED | MA_RECURSED:
< case MA_OWNED | MA_NOTRECURSED:
< if (!mtx_owned(m))
< panic("mutex %s not owned at %s:%d",
< m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((what & MA_NOTRECURSED) != 0)
< panic("mutex %s recursed at %s:%d",
< m->mtx_description, file, line);
< } else if ((what & MA_RECURSED) != 0) {
< panic("mutex %s unrecursed at %s:%d",
< m->mtx_description, file, line);
< }
< break;
< case MA_NOTOWNED:
< if (mtx_owned(m))
< panic("mutex %s owned at %s:%d",
< m->mtx_description, file, line);
< break;
< default:
< panic("unknown mtx_assert at %s:%d", file, line);
< }
< }
< #endif
<
< /*
< * The MUTEX_DEBUG-enabled mtx_validate()
< */
< #define MV_DESTROY 0 /* validate before destory */
< #define MV_INIT 1 /* validate before init */
<
< #ifdef MUTEX_DEBUG
<
< int mtx_validate __P((struct mtx *, int));
<
< int
< mtx_validate(struct mtx *m, int when)
< {
< struct mtx *mp;
< int i;
< int retval = 0;
<
< #ifdef WITNESS
< if (witness_cold)
< return 0;
< #endif
< if (m == &all_mtx || cold)
< return 0;
<
< mtx_lock(&all_mtx);
< /*
< * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly
< * we can re-enable the kernacc() checks.
< */
< #ifndef __alpha__
< MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) == 1);
< #endif
< MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx);
< for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
< #ifndef __alpha__
< if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) != 1) {
< panic("mtx_validate: mp=%p mp->mtx_next=%p",
< mp, mp->mtx_next);
< }
< #endif
< i++;
< if (i > mtx_cur_cnt) {
< panic("mtx_validate: too many in chain, known=%d\n",
< mtx_cur_cnt);
< }
< }
< MPASS(i == mtx_cur_cnt);
< switch (when) {
< case MV_DESTROY:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m)
< break;
< MPASS(mp == m);
< break;
< case MV_INIT:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m) {
< /*
< * Not good. This mutex already exists.
< */
< printf("re-initing existing mutex %s\n",
< m->mtx_description);
< MPASS(m->mtx_lock == MTX_UNOWNED);
< retval = 1;
< }
< }
< mtx_unlock(&all_mtx);
< return (retval);
< }
< #endif
<
< /*
< * Mutex initialization routine; initialize lock `m' of type contained in
< * `opts' with options contained in `opts' and description `description.'
< * Place on "all_mtx" queue.
< */
< void
< mtx_init(struct mtx *m, const char *description, int opts)
< {
<
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description);
<
< #ifdef MUTEX_DEBUG
< /* Diagnostic and error correction */
< if (mtx_validate(m, MV_INIT))
< return;
< #endif
<
< bzero((void *)m, sizeof *m);
< TAILQ_INIT(&m->mtx_blocked);
<
< #ifdef WITNESS
< if (!witness_cold) {
< m->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(m->mtx_debug != NULL);
< }
< #endif
<
< m->mtx_description = description;
< m->mtx_flags = opts;
< m->mtx_lock = MTX_UNOWNED;
<
< /* Put on all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next = &all_mtx;
< m->mtx_prev = all_mtx.mtx_prev;
< m->mtx_prev->mtx_next = m;
< all_mtx.mtx_prev = m;
< if (++mtx_cur_cnt > mtx_max_cnt)
< mtx_max_cnt = mtx_cur_cnt;
< mtx_unlock(&all_mtx);
<
< #ifdef WITNESS
< if (!witness_cold)
< witness_init(m, opts);
< #endif
< }
<
< /*
< * Remove lock `m' from all_mtx queue.
< */
< void
< mtx_destroy(struct mtx *m)
< {
<
< #ifdef WITNESS
< KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
< __FUNCTION__));
< #endif
<
< CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
<
< #ifdef MUTEX_DEBUG
< if (m->mtx_next == NULL)
< panic("mtx_destroy: %p (%s) already destroyed",
< m, m->mtx_description);
<
< if (!mtx_owned(m)) {
< MPASS(m->mtx_lock == MTX_UNOWNED);
< } else {
< MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
< }
<
< /* diagnostic */
< mtx_validate(m, MV_DESTROY);
< #endif
<
< #ifdef WITNESS
< if (m->mtx_witness)
< witness_destroy(m);
< #endif /* WITNESS */
<
< /* Remove from the all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next->mtx_prev = m->mtx_prev;
< m->mtx_prev->mtx_next = m->mtx_next;
<
< #ifdef MUTEX_DEBUG
< m->mtx_next = m->mtx_prev = NULL;
< #endif
<
< #ifdef WITNESS
< free(m->mtx_debug, M_WITNESS);
< m->mtx_debug = NULL;
< #endif
<
< mtx_cur_cnt--;
< mtx_unlock(&all_mtx);
< }
<
<
< /*
< * The WITNESS-enabled diagnostic code.
< */
< #ifdef WITNESS
< static void
< witness_fixup(void *dummy __unused)
< {
< struct mtx *mp;
<
< /*
< * We have to release Giant before initializing its witness
< * structure so that WITNESS doesn't get confused.
< */
< mtx_unlock(&Giant);
< mtx_assert(&Giant, MA_NOTOWNED);
<
< mtx_lock(&all_mtx);
<
< /* Iterate through all mutexes and finish up mutex initialization. */
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
<
< mp->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(mp->mtx_debug != NULL);
<
< witness_init(mp, mp->mtx_flags);
< }
< mtx_unlock(&all_mtx);
<
< /* Mark the witness code as being ready for use. */
< atomic_store_rel_int(&witness_cold, 0);
<
< mtx_lock(&Giant);
< }
< SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
<
< #define WITNESS_COUNT 200
< #define WITNESS_NCHILDREN 2
<
< int witness_watch = 1;
<
< struct witness {
< struct witness *w_next;
< const char *w_description;
< const char *w_file;
< int w_line;
< struct witness *w_morechildren;
< u_char w_childcnt;
< u_char w_Giant_squawked:1;
< u_char w_other_squawked:1;
< u_char w_same_squawked:1;
< u_char w_spin:1; /* MTX_SPIN type mutex. */
< u_int w_level;
< struct witness *w_children[WITNESS_NCHILDREN];
< };
<
< struct witness_blessed {
< char *b_lock1;
< char *b_lock2;
< };
<
< #ifdef DDB
< /*
921,928c174,181
< /*
< * Witness-enabled globals
< */
< static struct mtx w_mtx;
< static struct witness *w_free;
< static struct witness *w_all;
< static int w_inited;
< static int witness_dead; /* fatal error, probably no memory */
---
> static struct mtx w_mtx;
> static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free);
> static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all);
> static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin);
> static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep);
> static struct witness_child_list_entry *w_child_free = NULL;
> static struct lock_list_entry *w_lock_list_free = NULL;
> static int witness_dead; /* fatal error, probably no memory */
930c183,185
< static struct witness w_data[WITNESS_COUNT];
---
> static struct witness w_data[WITNESS_COUNT];
> static struct witness_child_list_entry w_childdata[WITNESS_CHILDCOUNT];
> static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT];
932,954c187,194
< /*
< * Internal witness routine prototypes
< */
< static struct witness *enroll(const char *description, int flag);
< static int itismychild(struct witness *parent, struct witness *child);
< static void removechild(struct witness *parent, struct witness *child);
< static int isitmychild(struct witness *parent, struct witness *child);
< static int isitmydescendant(struct witness *parent, struct witness *child);
< static int dup_ok(struct witness *);
< static int blessed(struct witness *, struct witness *);
< static void
< witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *);
< static void witness_leveldescendents(struct witness *parent, int level);
< static void witness_levelall(void);
< static struct witness * witness_get(void);
< static void witness_free(struct witness *m);
<
< static char *ignore_list[] = {
< "witness lock",
< NULL
< };
<
< static char *spin_order_list[] = {
---
> static struct witness_order_list_entry order_lists[] = {
> { "Giant", &lock_class_mtx_sleep },
> { "proctree", &lock_class_sx },
> { "allproc", &lock_class_sx },
> { "process lock", &lock_class_mtx_sleep },
> { "uidinfo hash", &lock_class_mtx_sleep },
> { "uidinfo struct", &lock_class_mtx_sleep },
> { NULL, NULL },
956c196
< "com",
---
> { "com", &lock_class_mtx_spin },
958c198
< "sio",
---
> { "sio", &lock_class_mtx_spin },
960c200
< "cy",
---
> { "cy", &lock_class_mtx_spin },
962,966c202,206
< "ng_node",
< "ng_worklist",
< "ithread table lock",
< "ithread list lock",
< "sched lock",
---
> { "ng_node", &lock_class_mtx_spin },
> { "ng_worklist", &lock_class_mtx_spin },
> { "ithread table lock", &lock_class_mtx_spin },
> { "ithread list lock", &lock_class_mtx_spin },
> { "sched lock", &lock_class_mtx_spin },
968c208
< "clk",
---
> { "clk", &lock_class_mtx_spin },
970c210
< "callout",
---
> { "callout", &lock_class_mtx_spin },
976,977c216,217
< "ap boot",
< "imen",
---
> { "ap boot", &lock_class_mtx_spin },
> { "imen", &lock_class_mtx_spin },
979c219
< "smp rendezvous",
---
> { "smp rendezvous", &lock_class_mtx_spin },
981c221,222
< NULL
---
> { NULL, NULL },
> { NULL, NULL }
984,990c225
< static char *order_list[] = {
< "Giant", "proctree", "allproc", "process lock", "uidinfo hash",
< "uidinfo struct", NULL,
< NULL
< };
<
< static char *dup_list[] = {
---
> static const char *dup_list[] = {
995,999d229
< static char *sleep_list[] = {
< "Giant",
< NULL
< };
<
1009,1013c239,242
< static void
< witness_init(struct mtx *m, int flag)
< {
< m->mtx_witness = enroll(m->mtx_description, flag);
< }
---
> /*
> * List of all locks in the system.
> */
> STAILQ_HEAD(, lock_object) all_locks = STAILQ_HEAD_INITIALIZER(all_locks);
1014a244,271
> static struct mtx all_mtx = {
> { &lock_class_mtx_sleep, /* mtx_object.lo_class */
> "All locks list", /* mtx_object.lo_name */
> NULL, /* mtx_object.lo_file */
> 0, /* mtx_object.lo_line */
> LO_INITIALIZED, /* mtx_object.lo_flags */
> { NULL }, /* mtx_object.lo_list */
> NULL }, /* mtx_object.lo_witness */
> MTX_UNOWNED, 0, /* mtx_lock, mtx_recurse */
> 0, /* mtx_savecrit */
> TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
> { NULL, NULL } /* mtx_contested */
> };
>
> /*
> * This global is set to 0 once it becomes safe to use the witness code.
> */
> static int witness_cold = 1;
>
> /*
> * Global variables for book keeping.
> */
> static int lock_cur_cnt;
> static int lock_max_cnt;
>
> /*
> * The WITNESS-enabled diagnostic code.
> */
1016c273
< witness_destroy(struct mtx *m)
---
> witness_initialize(void *dummy __unused)
1018,1024c275,304
< struct mtx *m1;
< struct proc *p;
< p = curproc;
< LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) {
< if (m1 == m) {
< LIST_REMOVE(m, mtx_held);
< break;
---
> struct lock_object *lock;
> struct witness_order_list_entry *order;
> struct witness *w, *w1;
> int i;
>
> /*
> * We have to release Giant before initializing its witness
> * structure so that WITNESS doesn't get confused.
> */
> mtx_unlock(&Giant);
> mtx_assert(&Giant, MA_NOTOWNED);
>
> STAILQ_INSERT_HEAD(&all_locks, &all_mtx.mtx_object, lo_list);
> mtx_init(&w_mtx, "witness lock", MTX_SPIN | MTX_QUIET | MTX_NOWITNESS);
> for (i = 0; i < WITNESS_COUNT; i++)
> witness_free(&w_data[i]);
> for (i = 0; i < WITNESS_CHILDCOUNT; i++)
> witness_child_free(&w_childdata[i]);
> for (i = 0; i < LOCK_CHILDCOUNT; i++)
> witness_lock_list_free(&w_locklistdata[i]);
>
> /* First add in all the specified order lists. */
> for (order = order_lists; order->w_name != NULL; order++) {
> w = enroll(order->w_name, order->w_class);
> w->w_file = "order list";
> for (order++; order->w_name != NULL; order++) {
> w1 = enroll(order->w_name, order->w_class);
> w1->w_file = "order list";
> itismychild(w, w1);
> w = w1;
1027d306
< return;
1028a308,322
> /* Iterate through all locks and add them to witness. */
> mtx_lock(&all_mtx);
> STAILQ_FOREACH(lock, &all_locks, lo_list) {
> if (lock->lo_flags & LO_WITNESS)
> lock->lo_witness = enroll(lock->lo_name,
> lock->lo_class);
> else
> lock->lo_witness = NULL;
> }
> mtx_unlock(&all_mtx);
>
> /* Mark the witness code as being ready for use. */
> atomic_store_rel_int(&witness_cold, 0);
>
> mtx_lock(&Giant);
1029a324
> SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL)
1030a326,382
> void
> witness_init(struct lock_object *lock)
> {
> struct lock_class *class;
>
> class = lock->lo_class;
> if (lock->lo_flags & LO_INITIALIZED)
> panic("%s: lock (%s) %s is already initialized!\n", __func__,
> class->lc_name, lock->lo_name);
>
> if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
> (class->lc_flags & LC_RECURSABLE) == 0)
> panic("%s: lock (%s) %s can not be recursable!\n", __func__,
> class->lc_name, lock->lo_name);
>
> if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
> (class->lc_flags & LC_SLEEPABLE) == 0)
> panic("%s: lock (%s) %s can not be sleepable!\n", __func__,
> class->lc_name, lock->lo_name);
>
> mtx_lock(&all_mtx);
> STAILQ_INSERT_TAIL(&all_locks, lock, lo_list);
> lock->lo_flags |= LO_INITIALIZED;
> lock_cur_cnt++;
> if (lock_cur_cnt > lock_max_cnt)
> lock_max_cnt = lock_cur_cnt;
> mtx_unlock(&all_mtx);
> if (!witness_cold && !witness_dead &&
> (lock->lo_flags & LO_WITNESS) != 0)
> lock->lo_witness = enroll(lock->lo_name, class);
> else
> lock->lo_witness = NULL;
> }
>
> void
> witness_destroy(struct lock_object *lock)
> {
>
> if (witness_cold)
> panic("lock (%s) %s destroyed while witness_cold",
> lock->lo_class->lc_name, lock->lo_name);
>
> if ((lock->lo_flags & LO_INITIALIZED) == 0)
> panic("%s: lock (%s) %s is not initialized!\n", __func__,
> lock->lo_class->lc_name, lock->lo_name);
>
> if (lock->lo_flags & LO_LOCKED)
> panic("lock (%s) %s destroyed while held",
> lock->lo_class->lc_name, lock->lo_name);
>
> mtx_lock(&all_mtx);
> lock_cur_cnt--;
> STAILQ_REMOVE(&all_locks, lock, lock_object, lo_list);
> lock->lo_flags &= LO_INITIALIZED;
> mtx_unlock(&all_mtx);
> }
>
1032c384,385
< witness_display(void(*prnt)(const char *fmt, ...))
---
> witness_display_list(void(*prnt)(const char *fmt, ...),
> struct witness_list *list)
1035c388
< int level, found;
---
> int found;
1037,1046c390,391
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< witness_levelall();
<
< /*
< * First, handle sleep mutexes which have been acquired at least
< * once.
< */
< prnt("Sleep mutexes:\n");
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || w->w_spin)
---
> STAILQ_FOREACH(w, list, w_typelist) {
> if (w->w_file == NULL)
1048,1049c393,396
< for (w1 = w_all; w1; w1 = w1->w_next) {
< if (isitmychild(w1, w))
---
> found = 0;
> STAILQ_FOREACH(w1, list, w_typelist) {
> if (isitmychild(w1, w)) {
> found++;
1050a398
> }
1052c400
< if (w1 != NULL)
---
> if (found)
1058a407
> }
1059a409,416
> static void
> witness_display(void(*prnt)(const char *fmt, ...))
> {
> struct witness *w;
>
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> witness_levelall();
>
1060a418,424
> * First, handle sleep mutexes which have been acquired at least
> * once.
> */
> prnt("Sleep locks:\n");
> witness_display_list(prnt, &w_sleep);
>
> /*
1063,1078c427,428
< prnt("\nSpin mutexes:\n");
< level = 0;
< while (level < sizeof(spin_order_list) / sizeof(char *)) {
< found = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || !w->w_spin)
< continue;
< if (w->w_level == 1 << level) {
< witness_displaydescendants(prnt, w);
< level++;
< found = 1;
< }
< }
< if (found == 0)
< level++;
< }
---
> prnt("\nSpin locks:\n");
> witness_display_list(prnt, &w_spin);
1083,1084c433,434
< prnt("\nMutexes which were never acquired:\n");
< for (w = w_all; w; w = w->w_next) {
---
> prnt("\nLocks which were never acquired:\n");
> STAILQ_FOREACH(w, &w_all, w_list) {
1087c437
< prnt("%s\n", w->w_description);
---
> prnt("%s\n", w->w_name);
1092c442
< witness_enter(struct mtx *m, int flags, const char *file, int line)
---
> witness_lock(struct lock_object *lock, int flags, const char *file, int line)
1093a444,446
> struct lock_list_entry **lock_list, *lle;
> struct lock_object *lock1, *lock2;
> struct lock_class *class;
1095d447
< struct mtx *m1;
1097c449
< int i;
---
> int i, j;
1102c454,455
< if (witness_cold || m->mtx_witness == NULL || panicstr)
---
> if (witness_cold || witness_dead || lock->lo_witness == NULL ||
> panicstr)
1104c457,458
< w = m->mtx_witness;
---
> w = lock->lo_witness;
> class = lock->lo_class;
1107,1139c461,463
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< i = PCPU_GET(witness_spin_check);
< if (i != 0 && w->w_level < i) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< panic("mutex_enter(%s:%x, MTX_SPIN) out of order @"
< " %s:%d already holding %s:%x",
< m->mtx_description, w->w_level, file, line,
< spin_order_list[ffs(i)-1], i);
< }
< PCPU_SET(witness_spin_check, i | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< p->p_spinlocks++;
< MPASS(p->p_spinlocks > 0);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
< }
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
---
> if ((lock->lo_flags & LO_LOCKED) == 0)
> panic("%s: lock (%s) %s is not locked @ %s:%d", __func__,
> class->lc_name, lock->lo_name, file, line);
1141,1145c465,470
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
---
> if ((lock->lo_flags & LO_RECURSED) != 0) {
> if ((lock->lo_flags & LO_RECURSABLE) == 0)
> panic(
> "%s: recursed on non-recursive lock (%s) %s @ %s:%d",
> __func__, class->lc_name, lock->lo_name, file,
> line);
1148c473,482
< if (witness_dead)
---
>
> lock_list = PCPU_PTR(spinlocks);
> if (class->lc_flags & LC_SLEEPLOCK) {
> if (*lock_list != NULL)
> panic("blockable sleep lock (%s) %s @ %s:%d",
> class->lc_name, lock->lo_name, file, line);
> lock_list = &p->p_sleeplocks;
> }
>
> if (flags & LOP_TRYLOCK)
1150,1151d483
< if (cold)
< goto out;
1153,1155d484
< if (p->p_spinlocks != 0)
< panic("blockable mtx_lock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
1157c486,487
< * Is this the first mutex acquired
---
> * Is this the first lock acquired? If so, then no order checking
> * is needed.
1159c489
< if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL)
---
> if (*lock_list == NULL)
1162c492,499
< if ((w1 = m1->mtx_witness) == w) {
---
> /*
> * Check for duplicate locks of the same type. Note that we only
> * have to check for this on the last lock we just acquired. Any
> * other cases will be caught as lock order violations.
> */
> lock1 = (*lock_list)->ll_children[(*lock_list)->ll_count - 1];
> w1 = lock1->lo_witness;
> if (w1 == w) {
1167c504
< m->mtx_description);
---
> lock->lo_name);
1176c513
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
---
> mtx_lock_spin(&w_mtx);
1181c518
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> mtx_unlock_spin(&w_mtx);
1184,1185c521,522
< if (isitmydescendant(m1->mtx_witness, w)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> if (isitmydescendant(w1, w)) {
> mtx_unlock_spin(&w_mtx);
1188c525,526
< for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) {
---
> for (j = 0, lle = *lock_list; lle != NULL; lle = lle->ll_next) {
> for (i = lle->ll_count - 1; i >= 0; i--, j++) {
1190,1193c528,547
< MPASS(i < 200);
< w1 = m1->mtx_witness;
< if (isitmydescendant(w, w1)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> MPASS(j < WITNESS_COUNT);
> lock1 = lle->ll_children[i];
> w1 = lock1->lo_witness;
>
> /*
> * If this lock doesn't undergo witness checking,
> * then skip it.
> */
> if (w1 == NULL) {
> KASSERT((lock1->lo_flags & LO_WITNESS) == 0,
> ("lock missing witness structure"));
> continue;
> }
> if (!isitmydescendant(w, w1))
> continue;
> /*
> * We have a lock order violation, check to see if it
> * is allowed or has already been yelled about.
> */
> mtx_unlock_spin(&w_mtx);
1196c550
< if (m1 == &Giant) {
---
> if (lock1 == &Giant.mtx_object) {
1206a561,563
> /*
> * Ok, yell about it.
> */
1208,1209c565,590
< printf(" 1st %s last acquired @ %s:%d\n",
< w->w_description, w->w_file, w->w_line);
---
> /*
> * Try to locate an earlier lock with
> * witness w in our list.
> */
> do {
> lock2 = lle->ll_children[i];
> MPASS(lock2 != NULL);
> if (lock2->lo_witness == w)
> break;
> i--;
> if (i == 0 && lle->ll_next != NULL) {
> lle = lle->ll_next;
> i = lle->ll_count - 1;
> MPASS(i != 0);
> }
> } while (i >= 0);
> if (i < 0)
> /*
> * We are very likely bogus in this case.
> */
> printf(" 1st %s last acquired @ %s:%d\n",
> w->w_name, w->w_file, w->w_line);
> else
> printf(" 1st %p %s @ %s:%d\n", lock2,
> lock2->lo_name, lock2->lo_file,
> lock2->lo_line);
1211c592,593
< m1, w1->w_description, w1->w_file, w1->w_line);
---
> lock1, lock1->lo_name, lock1->lo_file,
> lock1->lo_line);
1213c595
< m, w->w_description, file, line);
---
> lock, lock->lo_name, file, line);
1220,1222c602,604
< m1 = LIST_FIRST(&p->p_heldmtx);
< if (!itismychild(m1->mtx_witness, w))
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> lock1 = (*lock_list)->ll_children[(*lock_list)->ll_count - 1];
> if (!itismychild(lock1->lo_witness, w))
> mtx_unlock_spin(&w_mtx);
1231,1262c613,619
< m->mtx_line = line;
< m->mtx_file = file;
<
< /*
< * If this pays off it likely means that a mutex being witnessed
< * is acquired in hardclock. Put it in the ignore list. It is
< * likely not the mutex this assert fails on.
< */
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
< }
<
< void
< witness_try_enter(struct mtx *m, int flags, const char *file, int line)
< {
< struct proc *p;
< struct witness *w = m->mtx_witness;
<
< if (witness_cold)
< return;
< if (panicstr)
< return;
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_try_enter: "
< "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
< m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on"
< " non-recursive mutex %s @ %s:%d",
< m->mtx_description, file, line);
---
> lock->lo_line = line;
> lock->lo_file = file;
>
> lle = *lock_list;
> if (lle == NULL || lle->ll_count == LOCK_CHILDCOUNT) {
> *lock_list = witness_lock_list_get();
> if (*lock_list == NULL)
1264,1273c621,622
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
---
> (*lock_list)->ll_next = lle;
> lle = *lock_list;
1275,1293c624
<
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description, file,
< line);
< return;
< }
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< p = curproc;
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
---
> lle->ll_children[lle->ll_count++] = lock;
1297c628
< witness_exit(struct mtx *m, int flags, const char *file, int line)
---
> witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
1299c630,631
< struct witness *w;
---
> struct lock_list_entry **lock_list, *lle;
> struct lock_class *class;
1300a633
> int i, j;
1302c635,636
< if (witness_cold || m->mtx_witness == NULL || panicstr)
---
> if (witness_cold || witness_dead || lock->lo_witness == NULL ||
> panicstr)
1304d637
< w = m->mtx_witness;
1305a639
> class = lock->lo_class;
1307,1323c641,645
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) & ~w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< MPASS(p->p_spinlocks > 0);
< p->p_spinlocks--;
---
> if (lock->lo_flags & LO_RECURSED) {
> if ((lock->lo_flags & LO_LOCKED) == 0)
> panic("%s: recursed lock (%s) %s is not locked @ %s:%d",
> __func__, class->lc_name, lock->lo_name, file,
> line);
1326,1328d647
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
1330,1336c649,661
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
---
> /*
> * We don't need to protect this PCPU_GET() here against preemption
> * because if we hold any spinlocks then we are already protected,
> * and if we don't we will get NULL if we hold no spinlocks even if
> * we switch CPU's while reading it.
> */
> if (class->lc_flags & LC_SLEEPLOCK) {
> if ((flags & LOP_NOSWITCH) == 0 && PCPU_GET(spinlocks) != NULL)
> panic("switchable sleep unlock (%s) %s @ %s:%d",
> class->lc_name, lock->lo_name, file, line);
> lock_list = &p->p_sleeplocks;
> } else
> lock_list = PCPU_PTR(spinlocks);
1338,1342c663,676
< if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
< panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
< LIST_REMOVE(m, mtx_held);
< m->mtx_held.le_prev = NULL;
---
> for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next)
> for (i = 0; i < (*lock_list)->ll_count; i++)
> if ((*lock_list)->ll_children[i] == lock) {
> (*lock_list)->ll_count--;
> for (j = i; j < (*lock_list)->ll_count; j++)
> (*lock_list)->ll_children[j] =
> (*lock_list)->ll_children[j + 1];
> if ((*lock_list)->ll_count == 0) {
> lle = *lock_list;
> *lock_list = lle->ll_next;
> witness_lock_list_free(lle);
> }
> return;
> }
1344a679,683
> /*
> * Warn if any held locks are not sleepable. Note that Giant and the lock
> * passed in are both special cases since they are both released during the
> * sleep process and aren't actually held while the process is asleep.
> */
1346c685,686
< witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
---
> witness_sleep(int check_only, struct lock_object *lock, const char *file,
> int line)
1348c688,689
< struct mtx *m;
---
> struct lock_list_entry **lock_list, *lle;
> struct lock_object *lock1;
1350,1351c691,692
< char **sleep;
< int n = 0;
---
> critical_t savecrit;
> int i, n;
1353c694,701
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
---
> if (witness_dead || panicstr)
> return (0);
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> n = 0;
> /*
> * Preemption bad because we need PCPU_PTR(spinlocks) to not change.
> */
> savecrit = critical_enter();
1355,1368c703,718
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< if (m == mtx)
< continue;
< for (sleep = sleep_list; *sleep!= NULL; sleep++)
< if (strcmp(m->mtx_description, *sleep) == 0)
< goto next;
< if (n == 0)
< printf("Whee!\n");
< printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
< file, line, check_only ? "could sleep" : "sleeping",
< m->mtx_description,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< n++;
< next:
---
> lock_list = &p->p_sleeplocks;
> again:
> for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
> for (i = lle->ll_count - 1; i >= 0; i--) {
> lock1 = lle->ll_children[i];
> if (lock1 == lock || lock1 == &Giant.mtx_object ||
> (lock1->lo_flags & LO_SLEEPABLE))
> continue;
> n++;
> printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
> file, line, check_only ? "could sleep" : "sleeping",
> lock1->lo_name, lock1->lo_file, lock1->lo_line);
> }
> if (lock_list == &p->p_sleeplocks) {
> lock_list = PCPU_PTR(spinlocks);
> goto again;
1373a724
> critical_exit(savecrit);
1378c729
< enroll(const char *description, int flag)
---
> enroll(const char *description, struct lock_class *lock_class)
1380,1383c731
< int i;
< struct witness *w, *w1;
< char **ignore;
< char **order;
---
> struct witness *w;
1387,1389d734
< for (ignore = ignore_list; *ignore != NULL; ignore++)
< if (strcmp(description, *ignore) == 0)
< return (NULL);
1391,1409c736
< if (w_inited == 0) {
< mtx_init(&w_mtx, "witness lock", MTX_SPIN);
< for (i = 0; i < WITNESS_COUNT; i++) {
< w = &w_data[i];
< witness_free(w);
< }
< w_inited = 1;
< for (order = order_list; *order != NULL; order++) {
< w = enroll(*order, MTX_DEF);
< w->w_file = "order list";
< for (order++; *order != NULL; order++) {
< w1 = enroll(*order, MTX_DEF);
< w1->w_file = "order list";
< itismychild(w, w1);
< w = w1;
< }
< }
< }
< if ((flag & MTX_SPIN) && witness_skipspin)
---
> if ((lock_class->lc_flags & LC_SPINLOCK) && witness_skipspin)
1411,1414c738,746
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< for (w = w_all; w; w = w->w_next) {
< if (strcmp(description, w->w_description) == 0) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
---
> mtx_lock_spin(&w_mtx);
> STAILQ_FOREACH(w, &w_all, w_list) {
> if (strcmp(description, w->w_name) == 0) {
> mtx_unlock_spin(&w_mtx);
> if (lock_class != w->w_class)
> panic(
> "lock (%s) %s does not match earlier (%s) lock",
> description, lock_class->lc_name,
> w->w_class->lc_name);
1417a750,755
> /*
> * This isn't quite right, as witness_cold is still 0 while we
> * enroll all the locks initialized before witness_initialize().
> */
> if ((lock_class->lc_flags & LC_SPINLOCK) && !witness_cold)
> panic("spin lock %s not in order list", description);
1420,1436c758,768
< w->w_next = w_all;
< w_all = w;
< w->w_description = description;
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< if (flag & MTX_SPIN) {
< w->w_spin = 1;
<
< i = 1;
< for (order = spin_order_list; *order != NULL; order++) {
< if (strcmp(description, *order) == 0)
< break;
< i <<= 1;
< }
< if (*order == NULL)
< panic("spin lock %s not in order list", description);
< w->w_level = i;
< }
---
> w->w_name = description;
> w->w_class = lock_class;
> STAILQ_INSERT_HEAD(&w_all, w, w_list);
> if (lock_class->lc_flags & LC_SPINLOCK)
> STAILQ_INSERT_HEAD(&w_spin, w, w_typelist);
> else if (lock_class->lc_flags & LC_SLEEPLOCK)
> STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist);
> else
> panic("lock class %s is not sleep or spin",
> lock_class->lc_name);
> mtx_unlock_spin(&w_mtx);
1444a777,778
> struct witness_child_list_entry **wcl;
> struct witness_list *list;
1445a780,787
> MPASS(child != NULL && parent != NULL);
> if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) !=
> (child->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)))
> panic(
> "%s: parent (%s) and child (%s) are not the same lock type",
> __func__, parent->w_class->lc_name,
> child->w_class->lc_name);
>
1449,1450c791,793
< while (parent->w_morechildren)
< parent = parent->w_morechildren;
---
> wcl = &parent->w_children;
> while (*wcl != NULL && (*wcl)->wcl_count == WITNESS_NCHILDREN)
> wcl = &(*wcl)->wcl_next;
1452,1453c795,797
< if (parent->w_childcnt == WITNESS_NCHILDREN) {
< if ((parent->w_morechildren = witness_get()) == NULL)
---
> if (*wcl == NULL) {
> *wcl = witness_child_get();
> if (*wcl == NULL)
1455d798
< parent = parent->w_morechildren;
1457,1458c800,802
< MPASS(child != NULL);
< parent->w_children[parent->w_childcnt++] = child;
---
>
> (*wcl)->wcl_children[(*wcl)->wcl_count++] = child;
>
1460c804,806
< * now prune whole tree
---
> * Now prune whole tree. We look for cases where a lock is now
> * both a descendant and a direct child of a given lock. In that
> * case, we want to remove the direct child link from the tree.
1465,1467c811,816
< for (child = w_all; child != NULL; child = child->w_next) {
< for (parent = w_all; parent != NULL;
< parent = parent->w_next) {
---
> if (parent->w_class->lc_flags & LC_SLEEPLOCK)
> list = &w_sleep;
> else
> list = &w_spin;
> STAILQ_FOREACH(child, list, w_typelist) {
> STAILQ_FOREACH(parent, list, w_typelist) {
1484c833
< struct witness *w, *w1;
---
> struct witness_child_list_entry **wcl, *wcl1;
1487,1489c836,838
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< if (w->w_children[i] == child)
---
> for (wcl = &parent->w_children; *wcl != NULL; wcl = &(*wcl)->wcl_next)
> for (i = 0; i < (*wcl)->wcl_count; i++)
> if ((*wcl)->wcl_children[i] == child)
1493,1496c842,846
< for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren)
< continue;
< w->w_children[i] = w1->w_children[--w1->w_childcnt];
< MPASS(w->w_children[i] != NULL);
---
> (*wcl)->wcl_count--;
> if ((*wcl)->wcl_count > i)
> (*wcl)->wcl_children[i] =
> (*wcl)->wcl_children[(*wcl)->wcl_count];
> MPASS((*wcl)->wcl_children[i] != NULL);
1498c848
< if (w1->w_childcnt != 0)
---
> if ((*wcl)->wcl_count != 0)
1501,1506c851,853
< if (w1 == parent)
< return;
< for (w = parent; w->w_morechildren != w1; w = w->w_morechildren)
< continue;
< w->w_morechildren = 0;
< witness_free(w1);
---
> wcl1 = *wcl;
> *wcl = wcl1->wcl_next;
> witness_child_free(wcl1);
1512c859
< struct witness *w;
---
> struct witness_child_list_entry *wcl;
1515,1517c862,864
< for (w = parent; w != NULL; w = w->w_morechildren) {
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
---
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
> for (i = 0; i < wcl->wcl_count; i++) {
> if (wcl->wcl_children[i] == child)
1527,1529c874,875
< struct witness *w;
< int i;
< int j;
---
> struct witness_child_list_entry *wcl;
> int i, j;
1531c877,880
< for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) {
---
> if (isitmychild(parent, child))
> return (1);
> j = 0;
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) {
1533,1534c882,883
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
---
> for (i = 0; i < wcl->wcl_count; i++) {
> if (isitmydescendant(wcl->wcl_children[i], child))
1537,1540c886
< for (i = 0; i < w->w_childcnt; i++) {
< if (isitmydescendant(w->w_children[i], child))
< return (1);
< }
---
> j++;
1547a894
> struct witness_list *list;
1550,1556c897,916
< for (w = w_all; w; w = w->w_next)
< if (!(w->w_spin))
< w->w_level = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_spin)
< continue;
< for (w1 = w_all; w1; w1 = w1->w_next) {
---
> /*
> * First clear all levels.
> */
> STAILQ_FOREACH(w, &w_all, w_list) {
> w->w_level = 0;
> }
>
> /*
> * Look for locks with no parent and level all their descendants.
> */
> STAILQ_FOREACH(w, &w_all, w_list) {
> /*
> * This is just an optimization, technically we could get
> * away just walking the all list each time.
> */
> if (w->w_class->lc_flags & LC_SLEEPLOCK)
> list = &w_sleep;
> else
> list = &w_spin;
> STAILQ_FOREACH(w1, list, w_typelist) {
1558c918
< break;
---
> goto skip;
1560,1561d919
< if (w1 != NULL)
< continue;
1562a921
> skip:
1568a928
> struct witness_child_list_entry *wcl;
1570d929
< struct witness *w;
1575,1577c934,936
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_leveldescendents(w->w_children[i], level);
---
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
> for (i = 0; i < wcl->wcl_count; i++)
> witness_leveldescendents(wcl->wcl_children[i], level);
1584,1586c943,944
< struct witness *w;
< int i;
< int level;
---
> struct witness_child_list_entry *wcl;
> int i, level;
1588c946
< level = parent->w_spin ? ffs(parent->w_level) : parent->w_level;
---
> level = parent->w_level;
1590,1592c948
< prnt("%d", level);
< if (level < 10)
< prnt(" ");
---
> prnt("%-2d", level);
1595c951
< prnt("%s", parent->w_description);
---
> prnt("%s", parent->w_name);
1600,1603c956,960
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_displaydescendants(prnt, w->w_children[i]);
< }
---
> for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next)
> for (i = 0; i < wcl->wcl_count; i++)
> witness_displaydescendants(prnt,
> wcl->wcl_children[i]);
> }
1608c965
< char **dup;
---
> const char **dup;
1610,1611c967,968
< for (dup = dup_list; *dup!= NULL; dup++)
< if (strcmp(w->w_description, *dup) == 0)
---
> for (dup = dup_list; *dup != NULL; dup++)
> if (strcmp(w->w_name, *dup) == 0)
1624,1625c981,982
< if (strcmp(w1->w_description, b->b_lock1) == 0) {
< if (strcmp(w2->w_description, b->b_lock2) == 0)
---
> if (strcmp(w1->w_name, b->b_lock1) == 0) {
> if (strcmp(w2->w_name, b->b_lock2) == 0)
1629,1630c986,987
< if (strcmp(w1->w_description, b->b_lock2) == 0)
< if (strcmp(w2->w_description, b->b_lock1) == 0)
---
> if (strcmp(w1->w_name, b->b_lock2) == 0)
> if (strcmp(w2->w_name, b->b_lock1) == 0)
1637c994
< witness_get()
---
> witness_get(void)
1641c998
< if ((w = w_free) == NULL) {
---
> if (STAILQ_EMPTY(&w_free)) {
1643,1644c1000,1001
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< printf("witness exhausted\n");
---
> mtx_unlock_spin(&w_mtx);
> printf("%s: witness exhausted\n", __func__);
1647c1004,1005
< w_free = w->w_next;
---
> w = STAILQ_FIRST(&w_free);
> STAILQ_REMOVE_HEAD(&w_free, w_list);
1655,1656c1013,1014
< w->w_next = w_free;
< w_free = w;
---
>
> STAILQ_INSERT_HEAD(&w_free, w, w_list);
1659,1660c1017,1018
< int
< witness_list(struct proc *p)
---
> static struct witness_child_list_entry *
> witness_child_get(void)
1662,1663c1020
< struct mtx *m;
< int nheld;
---
> struct witness_child_list_entry *wcl;
1665,1671c1022,1027
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< nheld = 0;
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< printf("\t\"%s\" (%p) locked at %s:%d\n",
< m->mtx_description, m,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< nheld++;
---
> wcl = w_child_free;
> if (wcl == NULL) {
> witness_dead = 1;
> mtx_unlock_spin(&w_mtx);
> printf("%s: witness exhausted\n", __func__);
> return (NULL);
1672a1029,1032
> w_child_free = wcl->wcl_next;
> bzero(wcl, sizeof(*wcl));
> return (wcl);
> }
1674c1034,1039
< return (nheld);
---
> static void
> witness_child_free(struct witness_child_list_entry *wcl)
> {
>
> wcl->wcl_next = w_child_free;
> w_child_free = wcl;
1677c1042,1045
< #ifdef DDB
---
> static struct lock_list_entry *
> witness_lock_list_get(void)
> {
> struct lock_list_entry *lle;
1679c1047,1062
< DB_SHOW_COMMAND(mutexes, db_witness_list)
---
> mtx_lock_spin(&w_mtx);
> lle = w_lock_list_free;
> if (lle == NULL) {
> witness_dead = 1;
> mtx_unlock_spin(&w_mtx);
> printf("%s: witness exhausted\n", __func__);
> return (NULL);
> }
> w_lock_list_free = lle->ll_next;
> mtx_unlock_spin(&w_mtx);
> bzero(lle, sizeof(*lle));
> return (lle);
> }
>
> static void
> witness_lock_list_free(struct lock_list_entry *lle)
1682c1065,1068
< witness_list(curproc);
---
> mtx_lock_spin(&w_mtx);
> lle->ll_next = w_lock_list_free;
> w_lock_list_free = lle;
> mtx_unlock_spin(&w_mtx);
1685c1071,1075
< DB_SHOW_COMMAND(witness, db_witness_display)
---
> /*
> * Calling this on p != curproc is bad unless we are in ddb.
> */
> int
> witness_list(struct proc *p)
1686a1077,1080
> struct lock_list_entry **lock_list, *lle;
> struct lock_object *lock;
> critical_t savecrit;
> int i, nheld;
1688c1082,1111
< witness_display(db_printf);
---
> KASSERT(p == curproc || db_active,
> ("%s: p != curproc and we aren't in the debugger", __func__));
> KASSERT(!witness_cold, ("%s: witness_cold", __func__));
> nheld = 0;
> /*
> * Preemption bad because we need PCPU_PTR(spinlocks) to not change.
> */
> savecrit = critical_enter();
> lock_list = &p->p_sleeplocks;
> again:
> for (lle = *lock_list; lle != NULL; lle = lle->ll_next)
> for (i = lle->ll_count - 1; i >= 0; i--) {
> lock = lle->ll_children[i];
> printf("\t(%s) %s (%p) locked at %s:%d\n",
> lock->lo_class->lc_name, lock->lo_name, lock,
> lock->lo_file, lock->lo_line);
> nheld++;
> }
> /*
> * We only handle spinlocks if p == curproc. This is somewhat broken
> * if p is currently executing on some other CPU and holds spin locks
> * as we won't display those locks.
> */
> if (lock_list == &p->p_sleeplocks && p == curproc) {
> lock_list = PCPU_PTR(spinlocks);
> goto again;
> }
> critical_exit(savecrit);
>
> return (nheld);
1690d1112
< #endif
1693c1115
< witness_save(struct mtx *m, const char **filep, int *linep)
---
> witness_save(struct lock_object *lock, const char **filep, int *linep)
1696,1697c1118,1119
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
---
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> if (lock->lo_witness == NULL)
1700,1701c1122,1123
< *filep = m->mtx_witness->w_file;
< *linep = m->mtx_witness->w_line;
---
> *filep = lock->lo_file;
> *linep = lock->lo_line;
1705c1127
< witness_restore(struct mtx *m, const char *file, int line)
---
> witness_restore(struct lock_object *lock, const char *file, int line)
1708,1709c1130,1131
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
---
> KASSERT(!witness_cold, ("%s: witness_cold\n", __func__));
> if (lock->lo_witness == NULL)
1712,1713c1134,1137
< m->mtx_witness->w_file = file;
< m->mtx_witness->w_line = line;
---
> lock->lo_witness->w_file = file;
> lock->lo_witness->w_line = line;
> lock->lo_file = file;
> lock->lo_line = line;
1716c1140,1153
< #endif /* WITNESS */
---
> #ifdef DDB
>
> DB_SHOW_COMMAND(mutexes, db_witness_list)
> {
>
> witness_list(curproc);
> }
>
> DB_SHOW_COMMAND(witness, db_witness_display)
> {
>
> witness_display(db_printf);
> }
> #endif