30c30
< * $FreeBSD: head/sys/kern/kern_mutex.c 74900 2001-03-28 02:40:47Z jhb $
---
> * $FreeBSD: head/sys/kern/kern_mutex.c 74912 2001-03-28 09:03:24Z jhb $
59d58
< #include "opt_witness.h"
64a64,65
> #include <sys/lock.h>
> #include <sys/mutex.h>
81,82d81
< #include <sys/mutex.h>
<
84,100d82
< * The WITNESS-enabled mutex debug structure.
< */
< #ifdef WITNESS
< struct mtx_debug {
< struct witness *mtxd_witness;
< LIST_ENTRY(mtx) mtxd_held;
< const char *mtxd_file;
< int mtxd_line;
< };
<
< #define mtx_held mtx_debug->mtxd_held
< #define mtx_file mtx_debug->mtxd_file
< #define mtx_line mtx_debug->mtxd_line
< #define mtx_witness mtx_debug->mtxd_witness
< #endif /* WITNESS */
<
< /*
111c93
< * Early WITNESS-enabled declarations.
---
> * Lock classes for sleep and spin mutexes.
113c95,102
< #ifdef WITNESS
---
> struct lock_class lock_class_mtx_sleep = {
> "sleep mutex",
> LC_SLEEPLOCK | LC_RECURSABLE
> };
> struct lock_class lock_class_mtx_spin = {
> "spin mutex",
> LC_SPINLOCK | LC_RECURSABLE
> };
116,172d104
< * Internal WITNESS routines which must be prototyped early.
< *
< * XXX: When/if witness code is cleaned up, it would be wise to place all
< * witness prototyping early in this file.
< */
< static void witness_init(struct mtx *, int flag);
< static void witness_destroy(struct mtx *);
< static void witness_display(void(*)(const char *fmt, ...));
<
< MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure");
<
< /* All mutexes in system (used for debug/panic) */
< static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
<
< /*
< * This global is set to 0 once it becomes safe to use the witness code.
< */
< static int witness_cold = 1;
<
< #else /* WITNESS */
<
< /* XXX XXX XXX
< * flag++ is sleazoid way of shuting up warning
< */
< #define witness_init(m, flag) flag++
< #define witness_destroy(m)
< #define witness_try_enter(m, t, f, l)
< #endif /* WITNESS */
<
< /*
< * All mutex locks in system are kept on the all_mtx list.
< */
< static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head",
< TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
< { NULL, NULL }, &all_mtx, &all_mtx,
< #ifdef WITNESS
< &all_mtx_debug
< #else
< NULL
< #endif
< };
<
< /*
< * Global variables for book keeping.
< */
< static int mtx_cur_cnt;
< static int mtx_max_cnt;
<
< /*
< * Couple of strings for KTR_LOCK tracing in order to avoid duplicates.
< */
< char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d";
<
< /*
174,175d105
< *
< * NOTE: Prototypes for witness routines are placed at the bottom of the file.
244c174
< m->mtx_description));
---
> m->mtx_object.lo_name));
283c213
< p, p1, m, m->mtx_description);
---
> p, p1, m, m->mtx_object.lo_name);
339,340c269,270
< #ifdef WITNESS
< if (rval && m->mtx_witness != NULL) {
---
> LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line);
> if (rval) {
347c277,278
< witness_try_enter(m, (opts | m->mtx_flags), file, line);
---
> mtx_update_flags(m, 1);
> WITNESS_LOCK(&m->mtx_object, opts | LOP_TRYLOCK, file, line);
349d279
< #endif /* WITNESS */
351,355c281
< if ((opts & MTX_QUIET) == 0)
< CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d",
< m->mtx_description, m, rval, file, line);
<
< return rval;
---
> return (rval);
372c298
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
377c303
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
380c306
< m->mtx_description, (void *)m->mtx_lock, file, line);
---
> m->mtx_object.lo_name, (void *)m->mtx_lock, file, line);
437c363
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
467c393
< p->p_mtxname = m->mtx_description;
---
> p->p_mtxname = m->mtx_object.lo_name;
471c397
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
474c400
< m->mtx_description);
---
> m->mtx_object.lo_name);
478c404
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
481c407
< p, m, m->mtx_description);
---
> p, m, m->mtx_object.lo_name);
501c427
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
519c445
< m->mtx_description, (void *)m->mtx_lock);
---
> m->mtx_object.lo_name, (void *)m->mtx_lock);
524c450
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
548c474
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
554c480
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
566c492
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
582c508
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
596c522
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
605c531
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
611c537
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
625a552
> #ifdef WITNESS
626a554,586
> * Update the lock object flags before calling witness. Note that when we
> * lock a mutex, this is called after getting the lock, but when unlocking
> * a mutex, this function is called before releasing the lock.
> */
> void
> _mtx_update_flags(struct mtx *m, int locking)
> {
>
> mtx_assert(m, MA_OWNED);
> if (locking) {
> m->mtx_object.lo_flags |= LO_LOCKED;
> if (mtx_recursed(m))
> m->mtx_object.lo_flags |= LO_RECURSED;
> else
> /* XXX: we shouldn't need this in theory. */
> m->mtx_object.lo_flags &= ~LO_RECURSED;
> } else {
> switch (m->mtx_recurse) {
> case 0:
> /* XXX: we shouldn't need the LO_RECURSED in theory. */
> m->mtx_object.lo_flags &= ~(LO_LOCKED | LO_RECURSED);
> break;
> case 1:
> m->mtx_object.lo_flags &= ~(LO_RECURSED);
> break;
> default:
> break;
> }
> }
> }
> #endif
>
> /*
639c599
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
643c603
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
646c606
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
652c612
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
661a622,624
> *
> * Most of these checks have been moved off into the LO_INITIALIZED flag
> * maintained by the witness code.
663,665d625
< #define MV_DESTROY 0 /* validate before destory */
< #define MV_INIT 1 /* validate before init */
<
668c628
< int mtx_validate __P((struct mtx *, int));
---
> void mtx_validate __P((struct mtx *));
670,671c630,631
< int
< mtx_validate(struct mtx *m, int when)
---
> void
> mtx_validate(struct mtx *m)
673,675d632
< struct mtx *mp;
< int i;
< int retval = 0;
677,684d633
< #ifdef WITNESS
< if (witness_cold)
< return 0;
< #endif
< if (m == &all_mtx || cold)
< return 0;
<
< mtx_lock(&all_mtx);
690,691c639,640
< MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) == 1);
---
> if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE))
> panic("Can't read and write to mutex %p", m);
693,729d641
< MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx);
< for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
< #ifndef __alpha__
< if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) != 1) {
< panic("mtx_validate: mp=%p mp->mtx_next=%p",
< mp, mp->mtx_next);
< }
< #endif
< i++;
< if (i > mtx_cur_cnt) {
< panic("mtx_validate: too many in chain, known=%d\n",
< mtx_cur_cnt);
< }
< }
< MPASS(i == mtx_cur_cnt);
< switch (when) {
< case MV_DESTROY:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m)
< break;
< MPASS(mp == m);
< break;
< case MV_INIT:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m) {
< /*
< * Not good. This mutex already exists.
< */
< printf("re-initing existing mutex %s\n",
< m->mtx_description);
< MPASS(m->mtx_lock == MTX_UNOWNED);
< retval = 1;
< }
< }
< mtx_unlock(&all_mtx);
< return (retval);
736d647
< * Place on "all_mtx" queue.
740a652
> struct lock_object *lock;
742,743c654,655
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description);
---
> MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
> MTX_SLEEPABLE | MTX_NOWITNESS)) == 0);
747,748c659
< if (mtx_validate(m, MV_INIT))
< return;
---
> mtx_validate(m);
751,752c662,676
< bzero((void *)m, sizeof *m);
< TAILQ_INIT(&m->mtx_blocked);
---
> bzero(m, sizeof(*m));
> lock = &m->mtx_object;
> if (opts & MTX_SPIN)
> lock->lo_class = &lock_class_mtx_spin;
> else
> lock->lo_class = &lock_class_mtx_sleep;
> lock->lo_name = description;
> if (opts & MTX_QUIET)
> lock->lo_flags = LO_QUIET;
> if (opts & MTX_RECURSE)
> lock->lo_flags |= LO_RECURSABLE;
> if (opts & MTX_SLEEPABLE)
> lock->lo_flags |= LO_SLEEPABLE;
> if ((opts & MTX_NOWITNESS) == 0)
> lock->lo_flags |= LO_WITNESS;
754,763d677
< #ifdef WITNESS
< if (!witness_cold) {
< m->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(m->mtx_debug != NULL);
< }
< #endif
<
< m->mtx_description = description;
< m->mtx_flags = opts;
764a679
> TAILQ_INIT(&m->mtx_blocked);
766,774c681
< /* Put on all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next = &all_mtx;
< m->mtx_prev = all_mtx.mtx_prev;
< m->mtx_prev->mtx_next = m;
< all_mtx.mtx_prev = m;
< if (++mtx_cur_cnt > mtx_max_cnt)
< mtx_max_cnt = mtx_cur_cnt;
< mtx_unlock(&all_mtx);
---
> LOCK_LOG_INIT(lock, opts);
776,779c683
< #ifdef WITNESS
< if (!witness_cold)
< witness_init(m, opts);
< #endif
---
> WITNESS_INIT(lock);
783c687,690
< * Remove lock `m' from all_mtx queue.
---
> * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be
> * passed in as a flag here because if the corresponding mtx_init() was
> * called with MTX_QUIET set, then it will already be set in the mutex's
> * flags.
789,792c696
< #ifdef WITNESS
< KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
< __FUNCTION__));
< #endif
---
> LOCK_LOG_DESTROY(&m->mtx_object, 0);
794,803c698,700
< CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
<
< #ifdef MUTEX_DEBUG
< if (m->mtx_next == NULL)
< panic("mtx_destroy: %p (%s) already destroyed",
< m, m->mtx_description);
<
< if (!mtx_owned(m)) {
< MPASS(m->mtx_lock == MTX_UNOWNED);
< } else {
---
> if (!mtx_owned(m))
> MPASS(mtx_unowned(m));
> else {
805d701
< }
807,860c703,706
< /* diagnostic */
< mtx_validate(m, MV_DESTROY);
< #endif
<
< #ifdef WITNESS
< if (m->mtx_witness)
< witness_destroy(m);
< #endif /* WITNESS */
<
< /* Remove from the all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next->mtx_prev = m->mtx_prev;
< m->mtx_prev->mtx_next = m->mtx_next;
<
< #ifdef MUTEX_DEBUG
< m->mtx_next = m->mtx_prev = NULL;
< #endif
<
< #ifdef WITNESS
< free(m->mtx_debug, M_WITNESS);
< m->mtx_debug = NULL;
< #endif
<
< mtx_cur_cnt--;
< mtx_unlock(&all_mtx);
< }
<
<
< /*
< * The WITNESS-enabled diagnostic code.
< */
< #ifdef WITNESS
< static void
< witness_fixup(void *dummy __unused)
< {
< struct mtx *mp;
<
< /*
< * We have to release Giant before initializing its witness
< * structure so that WITNESS doesn't get confused.
< */
< mtx_unlock(&Giant);
< mtx_assert(&Giant, MA_NOTOWNED);
<
< mtx_lock(&all_mtx);
<
< /* Iterate through all mutexes and finish up mutex initialization. */
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
<
< mp->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(mp->mtx_debug != NULL);
<
< witness_init(mp, mp->mtx_flags);
---
> /* Tell witness this isn't locked to make it happy. */
> m->mtx_object.lo_flags &= ~LO_LOCKED;
> WITNESS_UNLOCK(&m->mtx_object, MTX_NOSWITCH, __FILE__,
> __LINE__);
862d707
< mtx_unlock(&all_mtx);
864,867c709
< /* Mark the witness code as being ready for use. */
< atomic_store_rel_int(&witness_cold, 0);
<
< mtx_lock(&Giant);
---
> WITNESS_DESTROY(&m->mtx_object);
869,1716d710
< SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
<
< #define WITNESS_COUNT 200
< #define WITNESS_NCHILDREN 2
<
< int witness_watch = 1;
<
< struct witness {
< struct witness *w_next;
< const char *w_description;
< const char *w_file;
< int w_line;
< struct witness *w_morechildren;
< u_char w_childcnt;
< u_char w_Giant_squawked:1;
< u_char w_other_squawked:1;
< u_char w_same_squawked:1;
< u_char w_spin:1; /* MTX_SPIN type mutex. */
< u_int w_level;
< struct witness *w_children[WITNESS_NCHILDREN];
< };
<
< struct witness_blessed {
< char *b_lock1;
< char *b_lock2;
< };
<
< #ifdef DDB
< /*
< * When DDB is enabled and witness_ddb is set to 1, it will cause the system to
< * drop into kdebug() when:
< * - a lock heirarchy violation occurs
< * - locks are held when going to sleep.
< */
< int witness_ddb;
< #ifdef WITNESS_DDB
< TUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb);
< #else
< TUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb);
< #endif
< SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, "");
< #endif /* DDB */
<
< int witness_skipspin;
< #ifdef WITNESS_SKIPSPIN
< TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin);
< #else
< TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin);
< #endif
< SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
< "");
<
< /*
< * Witness-enabled globals
< */
< static struct mtx w_mtx;
< static struct witness *w_free;
< static struct witness *w_all;
< static int w_inited;
< static int witness_dead; /* fatal error, probably no memory */
<
< static struct witness w_data[WITNESS_COUNT];
<
< /*
< * Internal witness routine prototypes
< */
< static struct witness *enroll(const char *description, int flag);
< static int itismychild(struct witness *parent, struct witness *child);
< static void removechild(struct witness *parent, struct witness *child);
< static int isitmychild(struct witness *parent, struct witness *child);
< static int isitmydescendant(struct witness *parent, struct witness *child);
< static int dup_ok(struct witness *);
< static int blessed(struct witness *, struct witness *);
< static void
< witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *);
< static void witness_leveldescendents(struct witness *parent, int level);
< static void witness_levelall(void);
< static struct witness * witness_get(void);
< static void witness_free(struct witness *m);
<
< static char *ignore_list[] = {
< "witness lock",
< NULL
< };
<
< static char *spin_order_list[] = {
< #if defined(__i386__) && defined (SMP)
< "com",
< #endif
< "sio",
< #ifdef __i386__
< "cy",
< #endif
< "ng_node",
< "ng_worklist",
< "ithread table lock",
< "ithread list lock",
< "sched lock",
< #ifdef __i386__
< "clk",
< #endif
< "callout",
< /*
< * leaf locks
< */
< #ifdef SMP
< #ifdef __i386__
< "ap boot",
< "imen",
< #endif
< "smp rendezvous",
< #endif
< NULL
< };
<
< static char *order_list[] = {
< "Giant", "proctree", "allproc", "process lock", "uidinfo hash",
< "uidinfo struct", NULL,
< NULL
< };
<
< static char *dup_list[] = {
< "process lock",
< NULL
< };
<
< static char *sleep_list[] = {
< "Giant",
< NULL
< };
<
< /*
< * Pairs of locks which have been blessed
< * Don't complain about order problems with blessed locks
< */
< static struct witness_blessed blessed_list[] = {
< };
< static int blessed_count =
< sizeof(blessed_list) / sizeof(struct witness_blessed);
<
< static void
< witness_init(struct mtx *m, int flag)
< {
< m->mtx_witness = enroll(m->mtx_description, flag);
< }
<
< static void
< witness_destroy(struct mtx *m)
< {
< struct mtx *m1;
< struct proc *p;
< p = curproc;
< LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) {
< if (m1 == m) {
< LIST_REMOVE(m, mtx_held);
< break;
< }
< }
< return;
<
< }
<
< static void
< witness_display(void(*prnt)(const char *fmt, ...))
< {
< struct witness *w, *w1;
< int level, found;
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< witness_levelall();
<
< /*
< * First, handle sleep mutexes which have been acquired at least
< * once.
< */
< prnt("Sleep mutexes:\n");
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || w->w_spin)
< continue;
< for (w1 = w_all; w1; w1 = w1->w_next) {
< if (isitmychild(w1, w))
< break;
< }
< if (w1 != NULL)
< continue;
< /*
< * This lock has no anscestors, display its descendants.
< */
< witness_displaydescendants(prnt, w);
< }
<
< /*
< * Now do spin mutexes which have been acquired at least once.
< */
< prnt("\nSpin mutexes:\n");
< level = 0;
< while (level < sizeof(spin_order_list) / sizeof(char *)) {
< found = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || !w->w_spin)
< continue;
< if (w->w_level == 1 << level) {
< witness_displaydescendants(prnt, w);
< level++;
< found = 1;
< }
< }
< if (found == 0)
< level++;
< }
<
< /*
< * Finally, any mutexes which have not been acquired yet.
< */
< prnt("\nMutexes which were never acquired:\n");
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file != NULL)
< continue;
< prnt("%s\n", w->w_description);
< }
< }
<
< void
< witness_enter(struct mtx *m, int flags, const char *file, int line)
< {
< struct witness *w, *w1;
< struct mtx *m1;
< struct proc *p;
< int i;
< #ifdef DDB
< int go_into_ddb = 0;
< #endif /* DDB */
<
< if (witness_cold || m->mtx_witness == NULL || panicstr)
< return;
< w = m->mtx_witness;
< p = curproc;
<
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< i = PCPU_GET(witness_spin_check);
< if (i != 0 && w->w_level < i) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< panic("mutex_enter(%s:%x, MTX_SPIN) out of order @"
< " %s:%d already holding %s:%x",
< m->mtx_description, w->w_level, file, line,
< spin_order_list[ffs(i)-1], i);
< }
< PCPU_SET(witness_spin_check, i | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< p->p_spinlocks++;
< MPASS(p->p_spinlocks > 0);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
< }
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< if (witness_dead)
< goto out;
< if (cold)
< goto out;
<
< if (p->p_spinlocks != 0)
< panic("blockable mtx_lock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
< /*
< * Is this the first mutex acquired
< */
< if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL)
< goto out;
<
< if ((w1 = m1->mtx_witness) == w) {
< if (w->w_same_squawked || dup_ok(w))
< goto out;
< w->w_same_squawked = 1;
< printf("acquring duplicate lock of same type: \"%s\"\n",
< m->mtx_description);
< printf(" 1st @ %s:%d\n", w->w_file, w->w_line);
< printf(" 2nd @ %s:%d\n", file, line);
< #ifdef DDB
< go_into_ddb = 1;
< #endif /* DDB */
< goto out;
< }
< MPASS(!mtx_owned(&w_mtx));
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< /*
< * If we have a known higher number just say ok
< */
< if (witness_watch > 1 && w->w_level > w1->w_level) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< goto out;
< }
< if (isitmydescendant(m1->mtx_witness, w)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< goto out;
< }
< for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) {
<
< MPASS(i < 200);
< w1 = m1->mtx_witness;
< if (isitmydescendant(w, w1)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< if (blessed(w, w1))
< goto out;
< if (m1 == &Giant) {
< if (w1->w_Giant_squawked)
< goto out;
< else
< w1->w_Giant_squawked = 1;
< } else {
< if (w1->w_other_squawked)
< goto out;
< else
< w1->w_other_squawked = 1;
< }
< printf("lock order reversal\n");
< printf(" 1st %s last acquired @ %s:%d\n",
< w->w_description, w->w_file, w->w_line);
< printf(" 2nd %p %s @ %s:%d\n",
< m1, w1->w_description, w1->w_file, w1->w_line);
< printf(" 3rd %p %s @ %s:%d\n",
< m, w->w_description, file, line);
< #ifdef DDB
< go_into_ddb = 1;
< #endif /* DDB */
< goto out;
< }
< }
< m1 = LIST_FIRST(&p->p_heldmtx);
< if (!itismychild(m1->mtx_witness, w))
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
<
< out:
< #ifdef DDB
< if (witness_ddb && go_into_ddb)
< Debugger("witness_enter");
< #endif /* DDB */
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
<
< /*
< * If this pays off it likely means that a mutex being witnessed
< * is acquired in hardclock. Put it in the ignore list. It is
< * likely not the mutex this assert fails on.
< */
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
< }
<
< void
< witness_try_enter(struct mtx *m, int flags, const char *file, int line)
< {
< struct proc *p;
< struct witness *w = m->mtx_witness;
<
< if (witness_cold)
< return;
< if (panicstr)
< return;
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_try_enter: "
< "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
< m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on"
< " non-recursive mutex %s @ %s:%d",
< m->mtx_description, file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
< }
<
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description, file,
< line);
< return;
< }
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< p = curproc;
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
< }
<
< void
< witness_exit(struct mtx *m, int flags, const char *file, int line)
< {
< struct witness *w;
< struct proc *p;
<
< if (witness_cold || m->mtx_witness == NULL || panicstr)
< return;
< w = m->mtx_witness;
< p = curproc;
<
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) & ~w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< MPASS(p->p_spinlocks > 0);
< p->p_spinlocks--;
< return;
< }
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
<
< if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
< panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
< LIST_REMOVE(m, mtx_held);
< m->mtx_held.le_prev = NULL;
< }
<
< int
< witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
< {
< struct mtx *m;
< struct proc *p;
< char **sleep;
< int n = 0;
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< p = curproc;
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< if (m == mtx)
< continue;
< for (sleep = sleep_list; *sleep!= NULL; sleep++)
< if (strcmp(m->mtx_description, *sleep) == 0)
< goto next;
< if (n == 0)
< printf("Whee!\n");
< printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
< file, line, check_only ? "could sleep" : "sleeping",
< m->mtx_description,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< n++;
< next:
< }
< #ifdef DDB
< if (witness_ddb && n)
< Debugger("witness_sleep");
< #endif /* DDB */
< return (n);
< }
<
< static struct witness *
< enroll(const char *description, int flag)
< {
< int i;
< struct witness *w, *w1;
< char **ignore;
< char **order;
<
< if (!witness_watch)
< return (NULL);
< for (ignore = ignore_list; *ignore != NULL; ignore++)
< if (strcmp(description, *ignore) == 0)
< return (NULL);
<
< if (w_inited == 0) {
< mtx_init(&w_mtx, "witness lock", MTX_SPIN);
< for (i = 0; i < WITNESS_COUNT; i++) {
< w = &w_data[i];
< witness_free(w);
< }
< w_inited = 1;
< for (order = order_list; *order != NULL; order++) {
< w = enroll(*order, MTX_DEF);
< w->w_file = "order list";
< for (order++; *order != NULL; order++) {
< w1 = enroll(*order, MTX_DEF);
< w1->w_file = "order list";
< itismychild(w, w1);
< w = w1;
< }
< }
< }
< if ((flag & MTX_SPIN) && witness_skipspin)
< return (NULL);
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< for (w = w_all; w; w = w->w_next) {
< if (strcmp(description, w->w_description) == 0) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< return (w);
< }
< }
< if ((w = witness_get()) == NULL)
< return (NULL);
< w->w_next = w_all;
< w_all = w;
< w->w_description = description;
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< if (flag & MTX_SPIN) {
< w->w_spin = 1;
<
< i = 1;
< for (order = spin_order_list; *order != NULL; order++) {
< if (strcmp(description, *order) == 0)
< break;
< i <<= 1;
< }
< if (*order == NULL)
< panic("spin lock %s not in order list", description);
< w->w_level = i;
< }
<
< return (w);
< }
<
< static int
< itismychild(struct witness *parent, struct witness *child)
< {
< static int recursed;
<
< /*
< * Insert "child" after "parent"
< */
< while (parent->w_morechildren)
< parent = parent->w_morechildren;
<
< if (parent->w_childcnt == WITNESS_NCHILDREN) {
< if ((parent->w_morechildren = witness_get()) == NULL)
< return (1);
< parent = parent->w_morechildren;
< }
< MPASS(child != NULL);
< parent->w_children[parent->w_childcnt++] = child;
< /*
< * now prune whole tree
< */
< if (recursed)
< return (0);
< recursed = 1;
< for (child = w_all; child != NULL; child = child->w_next) {
< for (parent = w_all; parent != NULL;
< parent = parent->w_next) {
< if (!isitmychild(parent, child))
< continue;
< removechild(parent, child);
< if (isitmydescendant(parent, child))
< continue;
< itismychild(parent, child);
< }
< }
< recursed = 0;
< witness_levelall();
< return (0);
< }
<
< static void
< removechild(struct witness *parent, struct witness *child)
< {
< struct witness *w, *w1;
< int i;
<
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< if (w->w_children[i] == child)
< goto found;
< return;
< found:
< for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren)
< continue;
< w->w_children[i] = w1->w_children[--w1->w_childcnt];
< MPASS(w->w_children[i] != NULL);
<
< if (w1->w_childcnt != 0)
< return;
<
< if (w1 == parent)
< return;
< for (w = parent; w->w_morechildren != w1; w = w->w_morechildren)
< continue;
< w->w_morechildren = 0;
< witness_free(w1);
< }
<
< static int
< isitmychild(struct witness *parent, struct witness *child)
< {
< struct witness *w;
< int i;
<
< for (w = parent; w != NULL; w = w->w_morechildren) {
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
< return (1);
< }
< }
< return (0);
< }
<
< static int
< isitmydescendant(struct witness *parent, struct witness *child)
< {
< struct witness *w;
< int i;
< int j;
<
< for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) {
< MPASS(j < 1000);
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
< return (1);
< }
< for (i = 0; i < w->w_childcnt; i++) {
< if (isitmydescendant(w->w_children[i], child))
< return (1);
< }
< }
< return (0);
< }
<
< void
< witness_levelall (void)
< {
< struct witness *w, *w1;
<
< for (w = w_all; w; w = w->w_next)
< if (!(w->w_spin))
< w->w_level = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_spin)
< continue;
< for (w1 = w_all; w1; w1 = w1->w_next) {
< if (isitmychild(w1, w))
< break;
< }
< if (w1 != NULL)
< continue;
< witness_leveldescendents(w, 0);
< }
< }
<
< static void
< witness_leveldescendents(struct witness *parent, int level)
< {
< int i;
< struct witness *w;
<
< if (parent->w_level < level)
< parent->w_level = level;
< level++;
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_leveldescendents(w->w_children[i], level);
< }
<
< static void
< witness_displaydescendants(void(*prnt)(const char *fmt, ...),
< struct witness *parent)
< {
< struct witness *w;
< int i;
< int level;
<
< level = parent->w_spin ? ffs(parent->w_level) : parent->w_level;
<
< prnt("%d", level);
< if (level < 10)
< prnt(" ");
< for (i = 0; i < level; i++)
< prnt(" ");
< prnt("%s", parent->w_description);
< if (parent->w_file != NULL)
< prnt(" -- last acquired @ %s:%d\n", parent->w_file,
< parent->w_line);
<
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_displaydescendants(prnt, w->w_children[i]);
< }
<
< static int
< dup_ok(struct witness *w)
< {
< char **dup;
<
< for (dup = dup_list; *dup!= NULL; dup++)
< if (strcmp(w->w_description, *dup) == 0)
< return (1);
< return (0);
< }
<
< static int
< blessed(struct witness *w1, struct witness *w2)
< {
< int i;
< struct witness_blessed *b;
<
< for (i = 0; i < blessed_count; i++) {
< b = &blessed_list[i];
< if (strcmp(w1->w_description, b->b_lock1) == 0) {
< if (strcmp(w2->w_description, b->b_lock2) == 0)
< return (1);
< continue;
< }
< if (strcmp(w1->w_description, b->b_lock2) == 0)
< if (strcmp(w2->w_description, b->b_lock1) == 0)
< return (1);
< }
< return (0);
< }
<
< static struct witness *
< witness_get()
< {
< struct witness *w;
<
< if ((w = w_free) == NULL) {
< witness_dead = 1;
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< printf("witness exhausted\n");
< return (NULL);
< }
< w_free = w->w_next;
< bzero(w, sizeof(*w));
< return (w);
< }
<
< static void
< witness_free(struct witness *w)
< {
< w->w_next = w_free;
< w_free = w;
< }
<
< int
< witness_list(struct proc *p)
< {
< struct mtx *m;
< int nheld;
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< nheld = 0;
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< printf("\t\"%s\" (%p) locked at %s:%d\n",
< m->mtx_description, m,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< nheld++;
< }
<
< return (nheld);
< }
<
< #ifdef DDB
<
< DB_SHOW_COMMAND(mutexes, db_witness_list)
< {
<
< witness_list(curproc);
< }
<
< DB_SHOW_COMMAND(witness, db_witness_display)
< {
<
< witness_display(db_printf);
< }
< #endif
<
< void
< witness_save(struct mtx *m, const char **filep, int *linep)
< {
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
< return;
<
< *filep = m->mtx_witness->w_file;
< *linep = m->mtx_witness->w_line;
< }
<
< void
< witness_restore(struct mtx *m, const char *file, int line)
< {
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
< return;
<
< m->mtx_witness->w_file = file;
< m->mtx_witness->w_line = line;
< }
<
< #endif /* WITNESS */
< * $FreeBSD: head/sys/kern/kern_mutex.c 74900 2001-03-28 02:40:47Z jhb $
---
> * $FreeBSD: head/sys/kern/kern_mutex.c 74912 2001-03-28 09:03:24Z jhb $
59d58
< #include "opt_witness.h"
64a64,65
> #include <sys/lock.h>
> #include <sys/mutex.h>
81,82d81
< #include <sys/mutex.h>
<
84,100d82
< * The WITNESS-enabled mutex debug structure.
< */
< #ifdef WITNESS
< struct mtx_debug {
< struct witness *mtxd_witness;
< LIST_ENTRY(mtx) mtxd_held;
< const char *mtxd_file;
< int mtxd_line;
< };
<
< #define mtx_held mtx_debug->mtxd_held
< #define mtx_file mtx_debug->mtxd_file
< #define mtx_line mtx_debug->mtxd_line
< #define mtx_witness mtx_debug->mtxd_witness
< #endif /* WITNESS */
<
< /*
111c93
< * Early WITNESS-enabled declarations.
---
> * Lock classes for sleep and spin mutexes.
113c95,102
< #ifdef WITNESS
---
> struct lock_class lock_class_mtx_sleep = {
> "sleep mutex",
> LC_SLEEPLOCK | LC_RECURSABLE
> };
> struct lock_class lock_class_mtx_spin = {
> "spin mutex",
> LC_SPINLOCK | LC_RECURSABLE
> };
116,172d104
< * Internal WITNESS routines which must be prototyped early.
< *
< * XXX: When/if witness code is cleaned up, it would be wise to place all
< * witness prototyping early in this file.
< */
< static void witness_init(struct mtx *, int flag);
< static void witness_destroy(struct mtx *);
< static void witness_display(void(*)(const char *fmt, ...));
<
< MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure");
<
< /* All mutexes in system (used for debug/panic) */
< static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
<
< /*
< * This global is set to 0 once it becomes safe to use the witness code.
< */
< static int witness_cold = 1;
<
< #else /* WITNESS */
<
< /* XXX XXX XXX
< * flag++ is sleazoid way of shuting up warning
< */
< #define witness_init(m, flag) flag++
< #define witness_destroy(m)
< #define witness_try_enter(m, t, f, l)
< #endif /* WITNESS */
<
< /*
< * All mutex locks in system are kept on the all_mtx list.
< */
< static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head",
< TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
< { NULL, NULL }, &all_mtx, &all_mtx,
< #ifdef WITNESS
< &all_mtx_debug
< #else
< NULL
< #endif
< };
<
< /*
< * Global variables for book keeping.
< */
< static int mtx_cur_cnt;
< static int mtx_max_cnt;
<
< /*
< * Couple of strings for KTR_LOCK tracing in order to avoid duplicates.
< */
< char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d";
< char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d";
< char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d";
<
< /*
174,175d105
< *
< * NOTE: Prototypes for witness routines are placed at the bottom of the file.
244c174
< m->mtx_description));
---
> m->mtx_object.lo_name));
283c213
< p, p1, m, m->mtx_description);
---
> p, p1, m, m->mtx_object.lo_name);
339,340c269,270
< #ifdef WITNESS
< if (rval && m->mtx_witness != NULL) {
---
> LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line);
> if (rval) {
347c277,278
< witness_try_enter(m, (opts | m->mtx_flags), file, line);
---
> mtx_update_flags(m, 1);
> WITNESS_LOCK(&m->mtx_object, opts | LOP_TRYLOCK, file, line);
349d279
< #endif /* WITNESS */
351,355c281
< if ((opts & MTX_QUIET) == 0)
< CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d",
< m->mtx_description, m, rval, file, line);
<
< return rval;
---
> return (rval);
372c298
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
377c303
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
380c306
< m->mtx_description, (void *)m->mtx_lock, file, line);
---
> m->mtx_object.lo_name, (void *)m->mtx_lock, file, line);
437c363
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
467c393
< p->p_mtxname = m->mtx_description;
---
> p->p_mtxname = m->mtx_object.lo_name;
471c397
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
474c400
< m->mtx_description);
---
> m->mtx_object.lo_name);
478c404
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
481c407
< p, m, m->mtx_description);
---
> p, m, m->mtx_object.lo_name);
501c427
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
519c445
< m->mtx_description, (void *)m->mtx_lock);
---
> m->mtx_object.lo_name, (void *)m->mtx_lock);
524c450
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
548c474
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
554c480
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
566c492
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
582c508
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
596c522
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
605c531
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
611c537
< if ((opts & MTX_QUIET) == 0)
---
> if (LOCK_LOG_TEST(&m->mtx_object, opts))
625a552
> #ifdef WITNESS
626a554,586
> * Update the lock object flags before calling witness. Note that when we
> * lock a mutex, this is called after getting the lock, but when unlocking
> * a mutex, this function is called before releasing the lock.
> */
> void
> _mtx_update_flags(struct mtx *m, int locking)
> {
>
> mtx_assert(m, MA_OWNED);
> if (locking) {
> m->mtx_object.lo_flags |= LO_LOCKED;
> if (mtx_recursed(m))
> m->mtx_object.lo_flags |= LO_RECURSED;
> else
> /* XXX: we shouldn't need this in theory. */
> m->mtx_object.lo_flags &= ~LO_RECURSED;
> } else {
> switch (m->mtx_recurse) {
> case 0:
> /* XXX: we shouldn't need the LO_RECURSED in theory. */
> m->mtx_object.lo_flags &= ~(LO_LOCKED | LO_RECURSED);
> break;
> case 1:
> m->mtx_object.lo_flags &= ~(LO_RECURSED);
> break;
> default:
> break;
> }
> }
> }
> #endif
>
> /*
639c599
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
643c603
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
646c606
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
652c612
< m->mtx_description, file, line);
---
> m->mtx_object.lo_name, file, line);
661a622,624
> *
> * Most of these checks have been moved off into the LO_INITIALIZED flag
> * maintained by the witness code.
663,665d625
< #define MV_DESTROY 0 /* validate before destory */
< #define MV_INIT 1 /* validate before init */
<
668c628
< int mtx_validate __P((struct mtx *, int));
---
> void mtx_validate __P((struct mtx *));
670,671c630,631
< int
< mtx_validate(struct mtx *m, int when)
---
> void
> mtx_validate(struct mtx *m)
673,675d632
< struct mtx *mp;
< int i;
< int retval = 0;
677,684d633
< #ifdef WITNESS
< if (witness_cold)
< return 0;
< #endif
< if (m == &all_mtx || cold)
< return 0;
<
< mtx_lock(&all_mtx);
690,691c639,640
< MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) == 1);
---
> if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE))
> panic("Can't read and write to mutex %p", m);
693,729d641
< MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx);
< for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
< #ifndef __alpha__
< if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t),
< VM_PROT_READ) != 1) {
< panic("mtx_validate: mp=%p mp->mtx_next=%p",
< mp, mp->mtx_next);
< }
< #endif
< i++;
< if (i > mtx_cur_cnt) {
< panic("mtx_validate: too many in chain, known=%d\n",
< mtx_cur_cnt);
< }
< }
< MPASS(i == mtx_cur_cnt);
< switch (when) {
< case MV_DESTROY:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m)
< break;
< MPASS(mp == m);
< break;
< case MV_INIT:
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next)
< if (mp == m) {
< /*
< * Not good. This mutex already exists.
< */
< printf("re-initing existing mutex %s\n",
< m->mtx_description);
< MPASS(m->mtx_lock == MTX_UNOWNED);
< retval = 1;
< }
< }
< mtx_unlock(&all_mtx);
< return (retval);
736d647
< * Place on "all_mtx" queue.
740a652
> struct lock_object *lock;
742,743c654,655
< if ((opts & MTX_QUIET) == 0)
< CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description);
---
> MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
> MTX_SLEEPABLE | MTX_NOWITNESS)) == 0);
747,748c659
< if (mtx_validate(m, MV_INIT))
< return;
---
> mtx_validate(m);
751,752c662,676
< bzero((void *)m, sizeof *m);
< TAILQ_INIT(&m->mtx_blocked);
---
> bzero(m, sizeof(*m));
> lock = &m->mtx_object;
> if (opts & MTX_SPIN)
> lock->lo_class = &lock_class_mtx_spin;
> else
> lock->lo_class = &lock_class_mtx_sleep;
> lock->lo_name = description;
> if (opts & MTX_QUIET)
> lock->lo_flags = LO_QUIET;
> if (opts & MTX_RECURSE)
> lock->lo_flags |= LO_RECURSABLE;
> if (opts & MTX_SLEEPABLE)
> lock->lo_flags |= LO_SLEEPABLE;
> if ((opts & MTX_NOWITNESS) == 0)
> lock->lo_flags |= LO_WITNESS;
754,763d677
< #ifdef WITNESS
< if (!witness_cold) {
< m->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(m->mtx_debug != NULL);
< }
< #endif
<
< m->mtx_description = description;
< m->mtx_flags = opts;
764a679
> TAILQ_INIT(&m->mtx_blocked);
766,774c681
< /* Put on all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next = &all_mtx;
< m->mtx_prev = all_mtx.mtx_prev;
< m->mtx_prev->mtx_next = m;
< all_mtx.mtx_prev = m;
< if (++mtx_cur_cnt > mtx_max_cnt)
< mtx_max_cnt = mtx_cur_cnt;
< mtx_unlock(&all_mtx);
---
> LOCK_LOG_INIT(lock, opts);
776,779c683
< #ifdef WITNESS
< if (!witness_cold)
< witness_init(m, opts);
< #endif
---
> WITNESS_INIT(lock);
783c687,690
< * Remove lock `m' from all_mtx queue.
---
> * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be
> * passed in as a flag here because if the corresponding mtx_init() was
> * called with MTX_QUIET set, then it will already be set in the mutex's
> * flags.
789,792c696
< #ifdef WITNESS
< KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
< __FUNCTION__));
< #endif
---
> LOCK_LOG_DESTROY(&m->mtx_object, 0);
794,803c698,700
< CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
<
< #ifdef MUTEX_DEBUG
< if (m->mtx_next == NULL)
< panic("mtx_destroy: %p (%s) already destroyed",
< m, m->mtx_description);
<
< if (!mtx_owned(m)) {
< MPASS(m->mtx_lock == MTX_UNOWNED);
< } else {
---
> if (!mtx_owned(m))
> MPASS(mtx_unowned(m));
> else {
805d701
< }
807,860c703,706
< /* diagnostic */
< mtx_validate(m, MV_DESTROY);
< #endif
<
< #ifdef WITNESS
< if (m->mtx_witness)
< witness_destroy(m);
< #endif /* WITNESS */
<
< /* Remove from the all mutex queue */
< mtx_lock(&all_mtx);
< m->mtx_next->mtx_prev = m->mtx_prev;
< m->mtx_prev->mtx_next = m->mtx_next;
<
< #ifdef MUTEX_DEBUG
< m->mtx_next = m->mtx_prev = NULL;
< #endif
<
< #ifdef WITNESS
< free(m->mtx_debug, M_WITNESS);
< m->mtx_debug = NULL;
< #endif
<
< mtx_cur_cnt--;
< mtx_unlock(&all_mtx);
< }
<
<
< /*
< * The WITNESS-enabled diagnostic code.
< */
< #ifdef WITNESS
< static void
< witness_fixup(void *dummy __unused)
< {
< struct mtx *mp;
<
< /*
< * We have to release Giant before initializing its witness
< * structure so that WITNESS doesn't get confused.
< */
< mtx_unlock(&Giant);
< mtx_assert(&Giant, MA_NOTOWNED);
<
< mtx_lock(&all_mtx);
<
< /* Iterate through all mutexes and finish up mutex initialization. */
< for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
<
< mp->mtx_debug = malloc(sizeof(struct mtx_debug),
< M_WITNESS, M_NOWAIT | M_ZERO);
< MPASS(mp->mtx_debug != NULL);
<
< witness_init(mp, mp->mtx_flags);
---
> /* Tell witness this isn't locked to make it happy. */
> m->mtx_object.lo_flags &= ~LO_LOCKED;
> WITNESS_UNLOCK(&m->mtx_object, MTX_NOSWITCH, __FILE__,
> __LINE__);
862d707
< mtx_unlock(&all_mtx);
864,867c709
< /* Mark the witness code as being ready for use. */
< atomic_store_rel_int(&witness_cold, 0);
<
< mtx_lock(&Giant);
---
> WITNESS_DESTROY(&m->mtx_object);
869,1716d710
< SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
<
< #define WITNESS_COUNT 200
< #define WITNESS_NCHILDREN 2
<
< int witness_watch = 1;
<
< struct witness {
< struct witness *w_next;
< const char *w_description;
< const char *w_file;
< int w_line;
< struct witness *w_morechildren;
< u_char w_childcnt;
< u_char w_Giant_squawked:1;
< u_char w_other_squawked:1;
< u_char w_same_squawked:1;
< u_char w_spin:1; /* MTX_SPIN type mutex. */
< u_int w_level;
< struct witness *w_children[WITNESS_NCHILDREN];
< };
<
< struct witness_blessed {
< char *b_lock1;
< char *b_lock2;
< };
<
< #ifdef DDB
< /*
< * When DDB is enabled and witness_ddb is set to 1, it will cause the system to
< * drop into kdebug() when:
< * - a lock heirarchy violation occurs
< * - locks are held when going to sleep.
< */
< int witness_ddb;
< #ifdef WITNESS_DDB
< TUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb);
< #else
< TUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb);
< #endif
< SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, "");
< #endif /* DDB */
<
< int witness_skipspin;
< #ifdef WITNESS_SKIPSPIN
< TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin);
< #else
< TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin);
< #endif
< SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
< "");
<
< /*
< * Witness-enabled globals
< */
< static struct mtx w_mtx;
< static struct witness *w_free;
< static struct witness *w_all;
< static int w_inited;
< static int witness_dead; /* fatal error, probably no memory */
<
< static struct witness w_data[WITNESS_COUNT];
<
< /*
< * Internal witness routine prototypes
< */
< static struct witness *enroll(const char *description, int flag);
< static int itismychild(struct witness *parent, struct witness *child);
< static void removechild(struct witness *parent, struct witness *child);
< static int isitmychild(struct witness *parent, struct witness *child);
< static int isitmydescendant(struct witness *parent, struct witness *child);
< static int dup_ok(struct witness *);
< static int blessed(struct witness *, struct witness *);
< static void
< witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *);
< static void witness_leveldescendents(struct witness *parent, int level);
< static void witness_levelall(void);
< static struct witness * witness_get(void);
< static void witness_free(struct witness *m);
<
< static char *ignore_list[] = {
< "witness lock",
< NULL
< };
<
< static char *spin_order_list[] = {
< #if defined(__i386__) && defined (SMP)
< "com",
< #endif
< "sio",
< #ifdef __i386__
< "cy",
< #endif
< "ng_node",
< "ng_worklist",
< "ithread table lock",
< "ithread list lock",
< "sched lock",
< #ifdef __i386__
< "clk",
< #endif
< "callout",
< /*
< * leaf locks
< */
< #ifdef SMP
< #ifdef __i386__
< "ap boot",
< "imen",
< #endif
< "smp rendezvous",
< #endif
< NULL
< };
<
< static char *order_list[] = {
< "Giant", "proctree", "allproc", "process lock", "uidinfo hash",
< "uidinfo struct", NULL,
< NULL
< };
<
< static char *dup_list[] = {
< "process lock",
< NULL
< };
<
< static char *sleep_list[] = {
< "Giant",
< NULL
< };
<
< /*
< * Pairs of locks which have been blessed
< * Don't complain about order problems with blessed locks
< */
< static struct witness_blessed blessed_list[] = {
< };
< static int blessed_count =
< sizeof(blessed_list) / sizeof(struct witness_blessed);
<
< static void
< witness_init(struct mtx *m, int flag)
< {
< m->mtx_witness = enroll(m->mtx_description, flag);
< }
<
< static void
< witness_destroy(struct mtx *m)
< {
< struct mtx *m1;
< struct proc *p;
< p = curproc;
< LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) {
< if (m1 == m) {
< LIST_REMOVE(m, mtx_held);
< break;
< }
< }
< return;
<
< }
<
< static void
< witness_display(void(*prnt)(const char *fmt, ...))
< {
< struct witness *w, *w1;
< int level, found;
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< witness_levelall();
<
< /*
< * First, handle sleep mutexes which have been acquired at least
< * once.
< */
< prnt("Sleep mutexes:\n");
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || w->w_spin)
< continue;
< for (w1 = w_all; w1; w1 = w1->w_next) {
< if (isitmychild(w1, w))
< break;
< }
< if (w1 != NULL)
< continue;
< /*
< * This lock has no anscestors, display its descendants.
< */
< witness_displaydescendants(prnt, w);
< }
<
< /*
< * Now do spin mutexes which have been acquired at least once.
< */
< prnt("\nSpin mutexes:\n");
< level = 0;
< while (level < sizeof(spin_order_list) / sizeof(char *)) {
< found = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file == NULL || !w->w_spin)
< continue;
< if (w->w_level == 1 << level) {
< witness_displaydescendants(prnt, w);
< level++;
< found = 1;
< }
< }
< if (found == 0)
< level++;
< }
<
< /*
< * Finally, any mutexes which have not been acquired yet.
< */
< prnt("\nMutexes which were never acquired:\n");
< for (w = w_all; w; w = w->w_next) {
< if (w->w_file != NULL)
< continue;
< prnt("%s\n", w->w_description);
< }
< }
<
< void
< witness_enter(struct mtx *m, int flags, const char *file, int line)
< {
< struct witness *w, *w1;
< struct mtx *m1;
< struct proc *p;
< int i;
< #ifdef DDB
< int go_into_ddb = 0;
< #endif /* DDB */
<
< if (witness_cold || m->mtx_witness == NULL || panicstr)
< return;
< w = m->mtx_witness;
< p = curproc;
<
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< i = PCPU_GET(witness_spin_check);
< if (i != 0 && w->w_level < i) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< panic("mutex_enter(%s:%x, MTX_SPIN) out of order @"
< " %s:%d already holding %s:%x",
< m->mtx_description, w->w_level, file, line,
< spin_order_list[ffs(i)-1], i);
< }
< PCPU_SET(witness_spin_check, i | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< p->p_spinlocks++;
< MPASS(p->p_spinlocks > 0);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
< }
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< if (witness_dead)
< goto out;
< if (cold)
< goto out;
<
< if (p->p_spinlocks != 0)
< panic("blockable mtx_lock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
< /*
< * Is this the first mutex acquired
< */
< if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL)
< goto out;
<
< if ((w1 = m1->mtx_witness) == w) {
< if (w->w_same_squawked || dup_ok(w))
< goto out;
< w->w_same_squawked = 1;
< printf("acquring duplicate lock of same type: \"%s\"\n",
< m->mtx_description);
< printf(" 1st @ %s:%d\n", w->w_file, w->w_line);
< printf(" 2nd @ %s:%d\n", file, line);
< #ifdef DDB
< go_into_ddb = 1;
< #endif /* DDB */
< goto out;
< }
< MPASS(!mtx_owned(&w_mtx));
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< /*
< * If we have a known higher number just say ok
< */
< if (witness_watch > 1 && w->w_level > w1->w_level) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< goto out;
< }
< if (isitmydescendant(m1->mtx_witness, w)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< goto out;
< }
< for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) {
<
< MPASS(i < 200);
< w1 = m1->mtx_witness;
< if (isitmydescendant(w, w1)) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< if (blessed(w, w1))
< goto out;
< if (m1 == &Giant) {
< if (w1->w_Giant_squawked)
< goto out;
< else
< w1->w_Giant_squawked = 1;
< } else {
< if (w1->w_other_squawked)
< goto out;
< else
< w1->w_other_squawked = 1;
< }
< printf("lock order reversal\n");
< printf(" 1st %s last acquired @ %s:%d\n",
< w->w_description, w->w_file, w->w_line);
< printf(" 2nd %p %s @ %s:%d\n",
< m1, w1->w_description, w1->w_file, w1->w_line);
< printf(" 3rd %p %s @ %s:%d\n",
< m, w->w_description, file, line);
< #ifdef DDB
< go_into_ddb = 1;
< #endif /* DDB */
< goto out;
< }
< }
< m1 = LIST_FIRST(&p->p_heldmtx);
< if (!itismychild(m1->mtx_witness, w))
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
<
< out:
< #ifdef DDB
< if (witness_ddb && go_into_ddb)
< Debugger("witness_enter");
< #endif /* DDB */
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
<
< /*
< * If this pays off it likely means that a mutex being witnessed
< * is acquired in hardclock. Put it in the ignore list. It is
< * likely not the mutex this assert fails on.
< */
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
< }
<
< void
< witness_try_enter(struct mtx *m, int flags, const char *file, int line)
< {
< struct proc *p;
< struct witness *w = m->mtx_witness;
<
< if (witness_cold)
< return;
< if (panicstr)
< return;
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_try_enter: "
< "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
< m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on"
< " non-recursive mutex %s @ %s:%d",
< m->mtx_description, file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) | w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< return;
< }
<
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_try_enter: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description, file,
< line);
< return;
< }
< w->w_file = file;
< w->w_line = line;
< m->mtx_line = line;
< m->mtx_file = file;
< p = curproc;
< MPASS(m->mtx_held.le_prev == NULL);
< LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held);
< }
<
< void
< witness_exit(struct mtx *m, int flags, const char *file, int line)
< {
< struct witness *w;
< struct proc *p;
<
< if (witness_cold || m->mtx_witness == NULL || panicstr)
< return;
< w = m->mtx_witness;
< p = curproc;
<
< if (flags & MTX_SPIN) {
< if ((m->mtx_flags & MTX_SPIN) == 0)
< panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
< " %s:%d", m->mtx_description, file, line);
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< PCPU_SET(witness_spin_check,
< PCPU_GET(witness_spin_check) & ~w->w_level);
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< MPASS(p->p_spinlocks > 0);
< p->p_spinlocks--;
< return;
< }
< if ((m->mtx_flags & MTX_SPIN) != 0)
< panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
< m->mtx_description, file, line);
<
< if (mtx_recursed(m)) {
< if ((m->mtx_flags & MTX_RECURSE) == 0)
< panic("mutex_exit: recursion on non-recursive"
< " mutex %s @ %s:%d", m->mtx_description,
< file, line);
< return;
< }
<
< if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
< panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
< m->mtx_description, file, line);
< LIST_REMOVE(m, mtx_held);
< m->mtx_held.le_prev = NULL;
< }
<
< int
< witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
< {
< struct mtx *m;
< struct proc *p;
< char **sleep;
< int n = 0;
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< p = curproc;
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< if (m == mtx)
< continue;
< for (sleep = sleep_list; *sleep!= NULL; sleep++)
< if (strcmp(m->mtx_description, *sleep) == 0)
< goto next;
< if (n == 0)
< printf("Whee!\n");
< printf("%s:%d: %s with \"%s\" locked from %s:%d\n",
< file, line, check_only ? "could sleep" : "sleeping",
< m->mtx_description,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< n++;
< next:
< }
< #ifdef DDB
< if (witness_ddb && n)
< Debugger("witness_sleep");
< #endif /* DDB */
< return (n);
< }
<
< static struct witness *
< enroll(const char *description, int flag)
< {
< int i;
< struct witness *w, *w1;
< char **ignore;
< char **order;
<
< if (!witness_watch)
< return (NULL);
< for (ignore = ignore_list; *ignore != NULL; ignore++)
< if (strcmp(description, *ignore) == 0)
< return (NULL);
<
< if (w_inited == 0) {
< mtx_init(&w_mtx, "witness lock", MTX_SPIN);
< for (i = 0; i < WITNESS_COUNT; i++) {
< w = &w_data[i];
< witness_free(w);
< }
< w_inited = 1;
< for (order = order_list; *order != NULL; order++) {
< w = enroll(*order, MTX_DEF);
< w->w_file = "order list";
< for (order++; *order != NULL; order++) {
< w1 = enroll(*order, MTX_DEF);
< w1->w_file = "order list";
< itismychild(w, w1);
< w = w1;
< }
< }
< }
< if ((flag & MTX_SPIN) && witness_skipspin)
< return (NULL);
< mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
< for (w = w_all; w; w = w->w_next) {
< if (strcmp(description, w->w_description) == 0) {
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< return (w);
< }
< }
< if ((w = witness_get()) == NULL)
< return (NULL);
< w->w_next = w_all;
< w_all = w;
< w->w_description = description;
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< if (flag & MTX_SPIN) {
< w->w_spin = 1;
<
< i = 1;
< for (order = spin_order_list; *order != NULL; order++) {
< if (strcmp(description, *order) == 0)
< break;
< i <<= 1;
< }
< if (*order == NULL)
< panic("spin lock %s not in order list", description);
< w->w_level = i;
< }
<
< return (w);
< }
<
< static int
< itismychild(struct witness *parent, struct witness *child)
< {
< static int recursed;
<
< /*
< * Insert "child" after "parent"
< */
< while (parent->w_morechildren)
< parent = parent->w_morechildren;
<
< if (parent->w_childcnt == WITNESS_NCHILDREN) {
< if ((parent->w_morechildren = witness_get()) == NULL)
< return (1);
< parent = parent->w_morechildren;
< }
< MPASS(child != NULL);
< parent->w_children[parent->w_childcnt++] = child;
< /*
< * now prune whole tree
< */
< if (recursed)
< return (0);
< recursed = 1;
< for (child = w_all; child != NULL; child = child->w_next) {
< for (parent = w_all; parent != NULL;
< parent = parent->w_next) {
< if (!isitmychild(parent, child))
< continue;
< removechild(parent, child);
< if (isitmydescendant(parent, child))
< continue;
< itismychild(parent, child);
< }
< }
< recursed = 0;
< witness_levelall();
< return (0);
< }
<
< static void
< removechild(struct witness *parent, struct witness *child)
< {
< struct witness *w, *w1;
< int i;
<
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< if (w->w_children[i] == child)
< goto found;
< return;
< found:
< for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren)
< continue;
< w->w_children[i] = w1->w_children[--w1->w_childcnt];
< MPASS(w->w_children[i] != NULL);
<
< if (w1->w_childcnt != 0)
< return;
<
< if (w1 == parent)
< return;
< for (w = parent; w->w_morechildren != w1; w = w->w_morechildren)
< continue;
< w->w_morechildren = 0;
< witness_free(w1);
< }
<
< static int
< isitmychild(struct witness *parent, struct witness *child)
< {
< struct witness *w;
< int i;
<
< for (w = parent; w != NULL; w = w->w_morechildren) {
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
< return (1);
< }
< }
< return (0);
< }
<
< static int
< isitmydescendant(struct witness *parent, struct witness *child)
< {
< struct witness *w;
< int i;
< int j;
<
< for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) {
< MPASS(j < 1000);
< for (i = 0; i < w->w_childcnt; i++) {
< if (w->w_children[i] == child)
< return (1);
< }
< for (i = 0; i < w->w_childcnt; i++) {
< if (isitmydescendant(w->w_children[i], child))
< return (1);
< }
< }
< return (0);
< }
<
< void
< witness_levelall (void)
< {
< struct witness *w, *w1;
<
< for (w = w_all; w; w = w->w_next)
< if (!(w->w_spin))
< w->w_level = 0;
< for (w = w_all; w; w = w->w_next) {
< if (w->w_spin)
< continue;
< for (w1 = w_all; w1; w1 = w1->w_next) {
< if (isitmychild(w1, w))
< break;
< }
< if (w1 != NULL)
< continue;
< witness_leveldescendents(w, 0);
< }
< }
<
< static void
< witness_leveldescendents(struct witness *parent, int level)
< {
< int i;
< struct witness *w;
<
< if (parent->w_level < level)
< parent->w_level = level;
< level++;
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_leveldescendents(w->w_children[i], level);
< }
<
< static void
< witness_displaydescendants(void(*prnt)(const char *fmt, ...),
< struct witness *parent)
< {
< struct witness *w;
< int i;
< int level;
<
< level = parent->w_spin ? ffs(parent->w_level) : parent->w_level;
<
< prnt("%d", level);
< if (level < 10)
< prnt(" ");
< for (i = 0; i < level; i++)
< prnt(" ");
< prnt("%s", parent->w_description);
< if (parent->w_file != NULL)
< prnt(" -- last acquired @ %s:%d\n", parent->w_file,
< parent->w_line);
<
< for (w = parent; w != NULL; w = w->w_morechildren)
< for (i = 0; i < w->w_childcnt; i++)
< witness_displaydescendants(prnt, w->w_children[i]);
< }
<
< static int
< dup_ok(struct witness *w)
< {
< char **dup;
<
< for (dup = dup_list; *dup!= NULL; dup++)
< if (strcmp(w->w_description, *dup) == 0)
< return (1);
< return (0);
< }
<
< static int
< blessed(struct witness *w1, struct witness *w2)
< {
< int i;
< struct witness_blessed *b;
<
< for (i = 0; i < blessed_count; i++) {
< b = &blessed_list[i];
< if (strcmp(w1->w_description, b->b_lock1) == 0) {
< if (strcmp(w2->w_description, b->b_lock2) == 0)
< return (1);
< continue;
< }
< if (strcmp(w1->w_description, b->b_lock2) == 0)
< if (strcmp(w2->w_description, b->b_lock1) == 0)
< return (1);
< }
< return (0);
< }
<
< static struct witness *
< witness_get()
< {
< struct witness *w;
<
< if ((w = w_free) == NULL) {
< witness_dead = 1;
< mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
< printf("witness exhausted\n");
< return (NULL);
< }
< w_free = w->w_next;
< bzero(w, sizeof(*w));
< return (w);
< }
<
< static void
< witness_free(struct witness *w)
< {
< w->w_next = w_free;
< w_free = w;
< }
<
< int
< witness_list(struct proc *p)
< {
< struct mtx *m;
< int nheld;
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< nheld = 0;
< LIST_FOREACH(m, &p->p_heldmtx, mtx_held) {
< printf("\t\"%s\" (%p) locked at %s:%d\n",
< m->mtx_description, m,
< m->mtx_witness->w_file, m->mtx_witness->w_line);
< nheld++;
< }
<
< return (nheld);
< }
<
< #ifdef DDB
<
< DB_SHOW_COMMAND(mutexes, db_witness_list)
< {
<
< witness_list(curproc);
< }
<
< DB_SHOW_COMMAND(witness, db_witness_display)
< {
<
< witness_display(db_printf);
< }
< #endif
<
< void
< witness_save(struct mtx *m, const char **filep, int *linep)
< {
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
< return;
<
< *filep = m->mtx_witness->w_file;
< *linep = m->mtx_witness->w_line;
< }
<
< void
< witness_restore(struct mtx *m, const char *file, int line)
< {
<
< KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
< if (m->mtx_witness == NULL)
< return;
<
< m->mtx_witness->w_file = file;
< m->mtx_witness->w_line = line;
< }
<
< #endif /* WITNESS */