subr_witness.c revision 162285
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Implementation of the `witness' lock verifier. Originally implemented for 34 * mutexes in BSD/OS. Extended to handle generic lock objects and lock 35 * classes in FreeBSD. 36 */ 37 38/* 39 * Main Entry: witness 40 * Pronunciation: 'wit-n&s 41 * Function: noun 42 * Etymology: Middle English witnesse, from Old English witnes knowledge, 43 * testimony, witness, from 2wit 44 * Date: before 12th century 45 * 1 : attestation of a fact or event : TESTIMONY 46 * 2 : one that gives evidence; specifically : one who testifies in 47 * a cause or before a judicial tribunal 48 * 3 : one asked to be present at a transaction so as to be able to 49 * testify to its having taken place 50 * 4 : one who has personal knowledge of something 51 * 5 a : something serving as evidence or proof : SIGN 52 * b : public affirmation by word or example of usually 53 * religious faith or conviction <the heroic witness to divine 54 * life -- Pilot> 55 * 6 capitalized : a member of the Jehovah's Witnesses 56 */ 57 58/* 59 * Special rules concerning Giant and lock orders: 60 * 61 * 1) Giant must be acquired before any other mutexes. Stated another way, 62 * no other mutex may be held when Giant is acquired. 63 * 64 * 2) Giant must be released when blocking on a sleepable lock. 65 * 66 * This rule is less obvious, but is a result of Giant providing the same 67 * semantics as spl(). Basically, when a thread sleeps, it must release 68 * Giant. When a thread blocks on a sleepable lock, it sleeps. Hence rule 69 * 2). 70 * 71 * 3) Giant may be acquired before or after sleepable locks. 72 * 73 * This rule is also not quite as obvious. Giant may be acquired after 74 * a sleepable lock because it is a non-sleepable lock and non-sleepable 75 * locks may always be acquired while holding a sleepable lock. The second 76 * case, Giant before a sleepable lock, follows from rule 2) above. Suppose 77 * you have two threads T1 and T2 and a sleepable lock X. Suppose that T1 78 * acquires X and blocks on Giant. Then suppose that T2 acquires Giant and 79 * blocks on X. When T2 blocks on X, T2 will release Giant allowing T1 to 80 * execute. Thus, acquiring Giant both before and after a sleepable lock 81 * will not result in a lock order reversal. 82 */ 83 84#include <sys/cdefs.h> 85__FBSDID("$FreeBSD: head/sys/kern/subr_witness.c 162285 2006-09-13 15:48:15Z scottl $"); 86 87#include "opt_ddb.h" 88#include "opt_witness.h" 89 90#include <sys/param.h> 91#include <sys/bus.h> 92#include <sys/kdb.h> 93#include <sys/kernel.h> 94#include <sys/ktr.h> 95#include <sys/lock.h> 96#include <sys/malloc.h> 97#include <sys/mutex.h> 98#include <sys/proc.h> 99#include <sys/sysctl.h> 100#include <sys/systm.h> 101 102#include <ddb/ddb.h> 103 104#include <machine/stdarg.h> 105 106/* Note that these traces do not work with KTR_ALQ. */ 107#if 0 108#define KTR_WITNESS KTR_SUBSYS 109#else 110#define KTR_WITNESS 0 111#endif 112 113/* Easier to stay with the old names. */ 114#define lo_list lo_witness_data.lod_list 115#define lo_witness lo_witness_data.lod_witness 116 117/* Define this to check for blessed mutexes */ 118#undef BLESSING 119 120#define WITNESS_COUNT 1024 121#define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4) 122/* 123 * XXX: This is somewhat bogus, as we assume here that at most 1024 threads 124 * will hold LOCK_NCHILDREN * 2 locks. We handle failure ok, and we should 125 * probably be safe for the most part, but it's still a SWAG. 126 */ 127#define LOCK_CHILDCOUNT (MAXCPU + 1024) * 2 128 129#define WITNESS_NCHILDREN 6 130 131struct witness_child_list_entry; 132 133struct witness { 134 const char *w_name; 135 struct lock_class *w_class; 136 STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */ 137 STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */ 138 struct witness_child_list_entry *w_children; /* Great evilness... */ 139 const char *w_file; 140 int w_line; 141 u_int w_level; 142 u_int w_refcount; 143 u_char w_Giant_squawked:1; 144 u_char w_other_squawked:1; 145 u_char w_same_squawked:1; 146 u_char w_displayed:1; 147}; 148 149struct witness_child_list_entry { 150 struct witness_child_list_entry *wcl_next; 151 struct witness *wcl_children[WITNESS_NCHILDREN]; 152 u_int wcl_count; 153}; 154 155STAILQ_HEAD(witness_list, witness); 156 157#ifdef BLESSING 158struct witness_blessed { 159 const char *b_lock1; 160 const char *b_lock2; 161}; 162#endif 163 164struct witness_order_list_entry { 165 const char *w_name; 166 struct lock_class *w_class; 167}; 168 169#ifdef BLESSING 170static int blessed(struct witness *, struct witness *); 171#endif 172static int depart(struct witness *w); 173static struct witness *enroll(const char *description, 174 struct lock_class *lock_class); 175static int insertchild(struct witness *parent, struct witness *child); 176static int isitmychild(struct witness *parent, struct witness *child); 177static int isitmydescendant(struct witness *parent, struct witness *child); 178static int itismychild(struct witness *parent, struct witness *child); 179static void removechild(struct witness *parent, struct witness *child); 180static int sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS); 181static const char *fixup_filename(const char *file); 182static struct witness *witness_get(void); 183static void witness_free(struct witness *m); 184static struct witness_child_list_entry *witness_child_get(void); 185static void witness_child_free(struct witness_child_list_entry *wcl); 186static struct lock_list_entry *witness_lock_list_get(void); 187static void witness_lock_list_free(struct lock_list_entry *lle); 188static struct lock_instance *find_instance(struct lock_list_entry *lock_list, 189 struct lock_object *lock); 190static void witness_list_lock(struct lock_instance *instance); 191#ifdef DDB 192static void witness_leveldescendents(struct witness *parent, int level); 193static void witness_levelall(void); 194static void witness_displaydescendants(void(*)(const char *fmt, ...), 195 struct witness *, int indent); 196static void witness_display_list(void(*prnt)(const char *fmt, ...), 197 struct witness_list *list); 198static void witness_display(void(*)(const char *fmt, ...)); 199static void witness_list(struct thread *td); 200#endif 201 202SYSCTL_NODE(_debug, OID_AUTO, witness, CTLFLAG_RW, 0, "Witness Locking"); 203 204/* 205 * If set to 0, witness is disabled. If set to a non-zero value, witness 206 * performs full lock order checking for all locks. At runtime, this 207 * value may be set to 0 to turn off witness. witness is not allowed be 208 * turned on once it is turned off, however. 209 */ 210static int witness_watch = 1; 211TUNABLE_INT("debug.witness.watch", &witness_watch); 212SYSCTL_PROC(_debug_witness, OID_AUTO, watch, CTLFLAG_RW | CTLTYPE_INT, NULL, 0, 213 sysctl_debug_witness_watch, "I", "witness is watching lock operations"); 214 215#ifdef KDB 216/* 217 * When KDB is enabled and witness_kdb is set to 1, it will cause the system 218 * to drop into kdebug() when: 219 * - a lock hierarchy violation occurs 220 * - locks are held when going to sleep. 221 */ 222#ifdef WITNESS_KDB 223int witness_kdb = 1; 224#else 225int witness_kdb = 0; 226#endif 227TUNABLE_INT("debug.witness.kdb", &witness_kdb); 228SYSCTL_INT(_debug_witness, OID_AUTO, kdb, CTLFLAG_RW, &witness_kdb, 0, ""); 229 230/* 231 * When KDB is enabled and witness_trace is set to 1, it will cause the system 232 * to print a stack trace: 233 * - a lock hierarchy violation occurs 234 * - locks are held when going to sleep. 235 */ 236int witness_trace = 1; 237TUNABLE_INT("debug.witness.trace", &witness_trace); 238SYSCTL_INT(_debug_witness, OID_AUTO, trace, CTLFLAG_RW, &witness_trace, 0, ""); 239#endif /* KDB */ 240 241#ifdef WITNESS_SKIPSPIN 242int witness_skipspin = 1; 243#else 244int witness_skipspin = 0; 245#endif 246TUNABLE_INT("debug.witness.skipspin", &witness_skipspin); 247SYSCTL_INT(_debug_witness, OID_AUTO, skipspin, CTLFLAG_RDTUN, 248 &witness_skipspin, 0, ""); 249 250static struct mtx w_mtx; 251static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free); 252static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all); 253static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin); 254static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep); 255static struct witness_child_list_entry *w_child_free = NULL; 256static struct lock_list_entry *w_lock_list_free = NULL; 257 258static int w_free_cnt, w_spin_cnt, w_sleep_cnt, w_child_free_cnt, w_child_cnt; 259SYSCTL_INT(_debug_witness, OID_AUTO, free_cnt, CTLFLAG_RD, &w_free_cnt, 0, ""); 260SYSCTL_INT(_debug_witness, OID_AUTO, spin_cnt, CTLFLAG_RD, &w_spin_cnt, 0, ""); 261SYSCTL_INT(_debug_witness, OID_AUTO, sleep_cnt, CTLFLAG_RD, &w_sleep_cnt, 0, 262 ""); 263SYSCTL_INT(_debug_witness, OID_AUTO, child_free_cnt, CTLFLAG_RD, 264 &w_child_free_cnt, 0, ""); 265SYSCTL_INT(_debug_witness, OID_AUTO, child_cnt, CTLFLAG_RD, &w_child_cnt, 0, 266 ""); 267 268static struct witness w_data[WITNESS_COUNT]; 269static struct witness_child_list_entry w_childdata[WITNESS_CHILDCOUNT]; 270static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT]; 271 272static struct witness_order_list_entry order_lists[] = { 273 /* 274 * sx locks 275 */ 276 { "proctree", &lock_class_sx }, 277 { "allproc", &lock_class_sx }, 278 { NULL, NULL }, 279 /* 280 * Various mutexes 281 */ 282 { "Giant", &lock_class_mtx_sleep }, 283 { "filedesc structure", &lock_class_mtx_sleep }, 284 { "pipe mutex", &lock_class_mtx_sleep }, 285 { "sigio lock", &lock_class_mtx_sleep }, 286 { "process group", &lock_class_mtx_sleep }, 287 { "process lock", &lock_class_mtx_sleep }, 288 { "session", &lock_class_mtx_sleep }, 289 { "uidinfo hash", &lock_class_mtx_sleep }, 290 { "uidinfo struct", &lock_class_mtx_sleep }, 291 { "allprison", &lock_class_mtx_sleep }, 292 { NULL, NULL }, 293 /* 294 * Sockets 295 */ 296 { "filedesc structure", &lock_class_mtx_sleep }, 297 { "accept", &lock_class_mtx_sleep }, 298 { "so_snd", &lock_class_mtx_sleep }, 299 { "so_rcv", &lock_class_mtx_sleep }, 300 { "sellck", &lock_class_mtx_sleep }, 301 { NULL, NULL }, 302 /* 303 * Routing 304 */ 305 { "so_rcv", &lock_class_mtx_sleep }, 306 { "radix node head", &lock_class_mtx_sleep }, 307 { "rtentry", &lock_class_mtx_sleep }, 308 { "ifaddr", &lock_class_mtx_sleep }, 309 { NULL, NULL }, 310 /* 311 * Multicast - protocol locks before interface locks, after UDP locks. 312 */ 313 { "udpinp", &lock_class_mtx_sleep }, 314 { "in_multi_mtx", &lock_class_mtx_sleep }, 315 { "igmp_mtx", &lock_class_mtx_sleep }, 316 { "if_addr_mtx", &lock_class_mtx_sleep }, 317 { NULL, NULL }, 318 /* 319 * UNIX Domain Sockets 320 */ 321 { "unp", &lock_class_mtx_sleep }, 322 { "so_snd", &lock_class_mtx_sleep }, 323 { NULL, NULL }, 324 /* 325 * UDP/IP 326 */ 327 { "udp", &lock_class_mtx_sleep }, 328 { "udpinp", &lock_class_mtx_sleep }, 329 { "so_snd", &lock_class_mtx_sleep }, 330 { NULL, NULL }, 331 /* 332 * TCP/IP 333 */ 334 { "tcp", &lock_class_mtx_sleep }, 335 { "tcpinp", &lock_class_mtx_sleep }, 336 { "so_snd", &lock_class_mtx_sleep }, 337 { NULL, NULL }, 338 /* 339 * SLIP 340 */ 341 { "slip_mtx", &lock_class_mtx_sleep }, 342 { "slip sc_mtx", &lock_class_mtx_sleep }, 343 { NULL, NULL }, 344 /* 345 * netatalk 346 */ 347 { "ddp_list_mtx", &lock_class_mtx_sleep }, 348 { "ddp_mtx", &lock_class_mtx_sleep }, 349 { NULL, NULL }, 350 /* 351 * BPF 352 */ 353 { "bpf global lock", &lock_class_mtx_sleep }, 354 { "bpf interface lock", &lock_class_mtx_sleep }, 355 { "bpf cdev lock", &lock_class_mtx_sleep }, 356 { NULL, NULL }, 357 /* 358 * NFS server 359 */ 360 { "nfsd_mtx", &lock_class_mtx_sleep }, 361 { "so_snd", &lock_class_mtx_sleep }, 362 { NULL, NULL }, 363 /* 364 * CDEV 365 */ 366 { "system map", &lock_class_mtx_sleep }, 367 { "vm page queue mutex", &lock_class_mtx_sleep }, 368 { "vnode interlock", &lock_class_mtx_sleep }, 369 { "cdev", &lock_class_mtx_sleep }, 370 { NULL, NULL }, 371 /* 372 * spin locks 373 */ 374#ifdef SMP 375 { "ap boot", &lock_class_mtx_spin }, 376#endif 377 { "rm.mutex_mtx", &lock_class_mtx_spin }, 378 { "hptlock", &lock_class_mtx_spin }, 379 { "sio", &lock_class_mtx_spin }, 380#ifdef __i386__ 381 { "cy", &lock_class_mtx_spin }, 382#endif 383 { "scc_hwmtx", &lock_class_mtx_spin }, 384 { "uart_hwmtx", &lock_class_mtx_spin }, 385 { "zstty", &lock_class_mtx_spin }, 386 { "ng_node", &lock_class_mtx_spin }, 387 { "ng_worklist", &lock_class_mtx_spin }, 388 { "fast_taskqueue", &lock_class_mtx_spin }, 389 { "intr table", &lock_class_mtx_spin }, 390 { "sleepq chain", &lock_class_mtx_spin }, 391 { "sched lock", &lock_class_mtx_spin }, 392 { "turnstile chain", &lock_class_mtx_spin }, 393 { "td_contested", &lock_class_mtx_spin }, 394 { "callout", &lock_class_mtx_spin }, 395 { "entropy harvest mutex", &lock_class_mtx_spin }, 396 { "syscons video lock", &lock_class_mtx_spin }, 397 /* 398 * leaf locks 399 */ 400 { "allpmaps", &lock_class_mtx_spin }, 401 { "vm page queue free mutex", &lock_class_mtx_spin }, 402 { "icu", &lock_class_mtx_spin }, 403#ifdef SMP 404 { "smp rendezvous", &lock_class_mtx_spin }, 405#if defined(__i386__) || defined(__amd64__) 406 { "tlb", &lock_class_mtx_spin }, 407#endif 408#ifdef __sparc64__ 409 { "ipi", &lock_class_mtx_spin }, 410 { "rtc_mtx", &lock_class_mtx_spin }, 411#endif 412#endif 413 { "clk", &lock_class_mtx_spin }, 414 { "mutex profiling lock", &lock_class_mtx_spin }, 415 { "kse zombie lock", &lock_class_mtx_spin }, 416 { "ALD Queue", &lock_class_mtx_spin }, 417#ifdef __ia64__ 418 { "MCA spin lock", &lock_class_mtx_spin }, 419#endif 420#if defined(__i386__) || defined(__amd64__) 421 { "pcicfg", &lock_class_mtx_spin }, 422 { "NDIS thread lock", &lock_class_mtx_spin }, 423#endif 424 { "tw_osl_io_lock", &lock_class_mtx_spin }, 425 { "tw_osl_q_lock", &lock_class_mtx_spin }, 426 { "tw_cl_io_lock", &lock_class_mtx_spin }, 427 { "tw_cl_intr_lock", &lock_class_mtx_spin }, 428 { "tw_cl_gen_lock", &lock_class_mtx_spin }, 429 { NULL, NULL }, 430 { NULL, NULL } 431}; 432 433#ifdef BLESSING 434/* 435 * Pairs of locks which have been blessed 436 * Don't complain about order problems with blessed locks 437 */ 438static struct witness_blessed blessed_list[] = { 439}; 440static int blessed_count = 441 sizeof(blessed_list) / sizeof(struct witness_blessed); 442#endif 443 444/* 445 * List of locks initialized prior to witness being initialized whose 446 * enrollment is currently deferred. 447 */ 448STAILQ_HEAD(, lock_object) pending_locks = 449 STAILQ_HEAD_INITIALIZER(pending_locks); 450 451/* 452 * This global is set to 0 once it becomes safe to use the witness code. 453 */ 454static int witness_cold = 1; 455 456/* 457 * This global is set to 1 once the static lock orders have been enrolled 458 * so that a warning can be issued for any spin locks enrolled later. 459 */ 460static int witness_spin_warn = 0; 461 462/* 463 * The WITNESS-enabled diagnostic code. Note that the witness code does 464 * assume that the early boot is single-threaded at least until after this 465 * routine is completed. 466 */ 467static void 468witness_initialize(void *dummy __unused) 469{ 470 struct lock_object *lock; 471 struct witness_order_list_entry *order; 472 struct witness *w, *w1; 473 int i; 474 475 /* 476 * We have to release Giant before initializing its witness 477 * structure so that WITNESS doesn't get confused. 478 */ 479 mtx_unlock(&Giant); 480 mtx_assert(&Giant, MA_NOTOWNED); 481 482 CTR1(KTR_WITNESS, "%s: initializing witness", __func__); 483 mtx_init(&w_mtx, "witness lock", NULL, MTX_SPIN | MTX_QUIET | 484 MTX_NOWITNESS); 485 for (i = 0; i < WITNESS_COUNT; i++) 486 witness_free(&w_data[i]); 487 for (i = 0; i < WITNESS_CHILDCOUNT; i++) 488 witness_child_free(&w_childdata[i]); 489 for (i = 0; i < LOCK_CHILDCOUNT; i++) 490 witness_lock_list_free(&w_locklistdata[i]); 491 492 /* First add in all the specified order lists. */ 493 for (order = order_lists; order->w_name != NULL; order++) { 494 w = enroll(order->w_name, order->w_class); 495 if (w == NULL) 496 continue; 497 w->w_file = "order list"; 498 for (order++; order->w_name != NULL; order++) { 499 w1 = enroll(order->w_name, order->w_class); 500 if (w1 == NULL) 501 continue; 502 w1->w_file = "order list"; 503 if (!itismychild(w, w1)) 504 panic("Not enough memory for static orders!"); 505 w = w1; 506 } 507 } 508 witness_spin_warn = 1; 509 510 /* Iterate through all locks and add them to witness. */ 511 while (!STAILQ_EMPTY(&pending_locks)) { 512 lock = STAILQ_FIRST(&pending_locks); 513 STAILQ_REMOVE_HEAD(&pending_locks, lo_list); 514 KASSERT(lock->lo_flags & LO_WITNESS, 515 ("%s: lock %s is on pending list but not LO_WITNESS", 516 __func__, lock->lo_name)); 517 lock->lo_witness = enroll(lock->lo_type, LOCK_CLASS(lock)); 518 } 519 520 /* Mark the witness code as being ready for use. */ 521 witness_cold = 0; 522 523 mtx_lock(&Giant); 524} 525SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL) 526 527static int 528sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS) 529{ 530 int error, value; 531 532 value = witness_watch; 533 error = sysctl_handle_int(oidp, &value, 0, req); 534 if (error != 0 || req->newptr == NULL) 535 return (error); 536 error = suser(req->td); 537 if (error != 0) 538 return (error); 539 if (value == witness_watch) 540 return (0); 541 if (value != 0) 542 return (EINVAL); 543 witness_watch = 0; 544 return (0); 545} 546 547void 548witness_init(struct lock_object *lock) 549{ 550 struct lock_class *class; 551 552 /* Various sanity checks. */ 553 class = LOCK_CLASS(lock); 554 if ((lock->lo_flags & LO_RECURSABLE) != 0 && 555 (class->lc_flags & LC_RECURSABLE) == 0) 556 panic("%s: lock (%s) %s can not be recursable", __func__, 557 class->lc_name, lock->lo_name); 558 if ((lock->lo_flags & LO_SLEEPABLE) != 0 && 559 (class->lc_flags & LC_SLEEPABLE) == 0) 560 panic("%s: lock (%s) %s can not be sleepable", __func__, 561 class->lc_name, lock->lo_name); 562 if ((lock->lo_flags & LO_UPGRADABLE) != 0 && 563 (class->lc_flags & LC_UPGRADABLE) == 0) 564 panic("%s: lock (%s) %s can not be upgradable", __func__, 565 class->lc_name, lock->lo_name); 566 567 /* 568 * If we shouldn't watch this lock, then just clear lo_witness. 569 * Otherwise, if witness_cold is set, then it is too early to 570 * enroll this lock, so defer it to witness_initialize() by adding 571 * it to the pending_locks list. If it is not too early, then enroll 572 * the lock now. 573 */ 574 if (witness_watch == 0 || panicstr != NULL || 575 (lock->lo_flags & LO_WITNESS) == 0) 576 lock->lo_witness = NULL; 577 else if (witness_cold) { 578 STAILQ_INSERT_TAIL(&pending_locks, lock, lo_list); 579 lock->lo_flags |= LO_ENROLLPEND; 580 } else 581 lock->lo_witness = enroll(lock->lo_type, class); 582} 583 584void 585witness_destroy(struct lock_object *lock) 586{ 587 struct lock_class *class; 588 struct witness *w; 589 590 class = LOCK_CLASS(lock); 591 if (witness_cold) 592 panic("lock (%s) %s destroyed while witness_cold", 593 class->lc_name, lock->lo_name); 594 595 /* XXX: need to verify that no one holds the lock */ 596 if ((lock->lo_flags & (LO_WITNESS | LO_ENROLLPEND)) == LO_WITNESS && 597 lock->lo_witness != NULL) { 598 w = lock->lo_witness; 599 mtx_lock_spin(&w_mtx); 600 MPASS(w->w_refcount > 0); 601 w->w_refcount--; 602 603 /* 604 * Lock is already released if we have an allocation failure 605 * and depart() fails. 606 */ 607 if (w->w_refcount != 0 || depart(w)) 608 mtx_unlock_spin(&w_mtx); 609 } 610 611 /* 612 * If this lock is destroyed before witness is up and running, 613 * remove it from the pending list. 614 */ 615 if (lock->lo_flags & LO_ENROLLPEND) { 616 STAILQ_REMOVE(&pending_locks, lock, lock_object, lo_list); 617 lock->lo_flags &= ~LO_ENROLLPEND; 618 } 619} 620 621#ifdef DDB 622static void 623witness_levelall (void) 624{ 625 struct witness_list *list; 626 struct witness *w, *w1; 627 628 /* 629 * First clear all levels. 630 */ 631 STAILQ_FOREACH(w, &w_all, w_list) { 632 w->w_level = 0; 633 } 634 635 /* 636 * Look for locks with no parent and level all their descendants. 637 */ 638 STAILQ_FOREACH(w, &w_all, w_list) { 639 /* 640 * This is just an optimization, technically we could get 641 * away just walking the all list each time. 642 */ 643 if (w->w_class->lc_flags & LC_SLEEPLOCK) 644 list = &w_sleep; 645 else 646 list = &w_spin; 647 STAILQ_FOREACH(w1, list, w_typelist) { 648 if (isitmychild(w1, w)) 649 goto skip; 650 } 651 witness_leveldescendents(w, 0); 652 skip: 653 ; /* silence GCC 3.x */ 654 } 655} 656 657static void 658witness_leveldescendents(struct witness *parent, int level) 659{ 660 struct witness_child_list_entry *wcl; 661 int i; 662 663 if (parent->w_level < level) 664 parent->w_level = level; 665 level++; 666 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) 667 for (i = 0; i < wcl->wcl_count; i++) 668 witness_leveldescendents(wcl->wcl_children[i], level); 669} 670 671static void 672witness_displaydescendants(void(*prnt)(const char *fmt, ...), 673 struct witness *parent, int indent) 674{ 675 struct witness_child_list_entry *wcl; 676 int i, level; 677 678 level = parent->w_level; 679 prnt("%-2d", level); 680 for (i = 0; i < indent; i++) 681 prnt(" "); 682 if (parent->w_refcount > 0) 683 prnt("%s", parent->w_name); 684 else 685 prnt("(dead)"); 686 if (parent->w_displayed) { 687 prnt(" -- (already displayed)\n"); 688 return; 689 } 690 parent->w_displayed = 1; 691 if (parent->w_refcount > 0) { 692 if (parent->w_file != NULL) 693 prnt(" -- last acquired @ %s:%d", parent->w_file, 694 parent->w_line); 695 } 696 prnt("\n"); 697 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) 698 for (i = 0; i < wcl->wcl_count; i++) 699 witness_displaydescendants(prnt, 700 wcl->wcl_children[i], indent + 1); 701} 702 703static void 704witness_display_list(void(*prnt)(const char *fmt, ...), 705 struct witness_list *list) 706{ 707 struct witness *w; 708 709 STAILQ_FOREACH(w, list, w_typelist) { 710 if (w->w_file == NULL || w->w_level > 0) 711 continue; 712 /* 713 * This lock has no anscestors, display its descendants. 714 */ 715 witness_displaydescendants(prnt, w, 0); 716 } 717} 718 719static void 720witness_display(void(*prnt)(const char *fmt, ...)) 721{ 722 struct witness *w; 723 724 KASSERT(!witness_cold, ("%s: witness_cold", __func__)); 725 witness_levelall(); 726 727 /* Clear all the displayed flags. */ 728 STAILQ_FOREACH(w, &w_all, w_list) { 729 w->w_displayed = 0; 730 } 731 732 /* 733 * First, handle sleep locks which have been acquired at least 734 * once. 735 */ 736 prnt("Sleep locks:\n"); 737 witness_display_list(prnt, &w_sleep); 738 739 /* 740 * Now do spin locks which have been acquired at least once. 741 */ 742 prnt("\nSpin locks:\n"); 743 witness_display_list(prnt, &w_spin); 744 745 /* 746 * Finally, any locks which have not been acquired yet. 747 */ 748 prnt("\nLocks which were never acquired:\n"); 749 STAILQ_FOREACH(w, &w_all, w_list) { 750 if (w->w_file != NULL || w->w_refcount == 0) 751 continue; 752 prnt("%s\n", w->w_name); 753 } 754} 755#endif /* DDB */ 756 757/* Trim useless garbage from filenames. */ 758static const char * 759fixup_filename(const char *file) 760{ 761 762 if (file == NULL) 763 return (NULL); 764 while (strncmp(file, "../", 3) == 0) 765 file += 3; 766 return (file); 767} 768 769int 770witness_defineorder(struct lock_object *lock1, struct lock_object *lock2) 771{ 772 773 if (witness_watch == 0 || panicstr != NULL) 774 return (0); 775 776 /* Require locks that witness knows about. */ 777 if (lock1 == NULL || lock1->lo_witness == NULL || lock2 == NULL || 778 lock2->lo_witness == NULL) 779 return (EINVAL); 780 781 MPASS(!mtx_owned(&w_mtx)); 782 mtx_lock_spin(&w_mtx); 783 784 /* 785 * If we already have either an explicit or implied lock order that 786 * is the other way around, then return an error. 787 */ 788 if (isitmydescendant(lock2->lo_witness, lock1->lo_witness)) { 789 mtx_unlock_spin(&w_mtx); 790 return (EDOOFUS); 791 } 792 793 /* Try to add the new order. */ 794 CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__, 795 lock2->lo_type, lock1->lo_type); 796 if (!itismychild(lock1->lo_witness, lock2->lo_witness)) 797 return (ENOMEM); 798 mtx_unlock_spin(&w_mtx); 799 return (0); 800} 801 802void 803witness_checkorder(struct lock_object *lock, int flags, const char *file, 804 int line) 805{ 806 struct lock_list_entry **lock_list, *lle; 807 struct lock_instance *lock1, *lock2; 808 struct lock_class *class; 809 struct witness *w, *w1; 810 struct thread *td; 811 int i, j; 812 813 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL || 814 panicstr != NULL) 815 return; 816 817 /* 818 * Try locks do not block if they fail to acquire the lock, thus 819 * there is no danger of deadlocks or of switching while holding a 820 * spin lock if we acquire a lock via a try operation. This 821 * function shouldn't even be called for try locks, so panic if 822 * that happens. 823 */ 824 if (flags & LOP_TRYLOCK) 825 panic("%s should not be called for try lock operations", 826 __func__); 827 828 w = lock->lo_witness; 829 class = LOCK_CLASS(lock); 830 td = curthread; 831 file = fixup_filename(file); 832 833 if (class->lc_flags & LC_SLEEPLOCK) { 834 /* 835 * Since spin locks include a critical section, this check 836 * implicitly enforces a lock order of all sleep locks before 837 * all spin locks. 838 */ 839 if (td->td_critnest != 0 && !kdb_active) 840 panic("blockable sleep lock (%s) %s @ %s:%d", 841 class->lc_name, lock->lo_name, file, line); 842 843 /* 844 * If this is the first lock acquired then just return as 845 * no order checking is needed. 846 */ 847 if (td->td_sleeplocks == NULL) 848 return; 849 lock_list = &td->td_sleeplocks; 850 } else { 851 /* 852 * If this is the first lock, just return as no order 853 * checking is needed. We check this in both if clauses 854 * here as unifying the check would require us to use a 855 * critical section to ensure we don't migrate while doing 856 * the check. Note that if this is not the first lock, we 857 * are already in a critical section and are safe for the 858 * rest of the check. 859 */ 860 if (PCPU_GET(spinlocks) == NULL) 861 return; 862 lock_list = PCPU_PTR(spinlocks); 863 } 864 865 /* 866 * Check to see if we are recursing on a lock we already own. If 867 * so, make sure that we don't mismatch exclusive and shared lock 868 * acquires. 869 */ 870 lock1 = find_instance(*lock_list, lock); 871 if (lock1 != NULL) { 872 if ((lock1->li_flags & LI_EXCLUSIVE) != 0 && 873 (flags & LOP_EXCLUSIVE) == 0) { 874 printf("shared lock of (%s) %s @ %s:%d\n", 875 class->lc_name, lock->lo_name, file, line); 876 printf("while exclusively locked from %s:%d\n", 877 lock1->li_file, lock1->li_line); 878 panic("share->excl"); 879 } 880 if ((lock1->li_flags & LI_EXCLUSIVE) == 0 && 881 (flags & LOP_EXCLUSIVE) != 0) { 882 printf("exclusive lock of (%s) %s @ %s:%d\n", 883 class->lc_name, lock->lo_name, file, line); 884 printf("while share locked from %s:%d\n", 885 lock1->li_file, lock1->li_line); 886 panic("excl->share"); 887 } 888 return; 889 } 890 891 /* 892 * Try locks do not block if they fail to acquire the lock, thus 893 * there is no danger of deadlocks or of switching while holding a 894 * spin lock if we acquire a lock via a try operation. 895 */ 896 if (flags & LOP_TRYLOCK) 897 return; 898 899 /* 900 * Check for duplicate locks of the same type. Note that we only 901 * have to check for this on the last lock we just acquired. Any 902 * other cases will be caught as lock order violations. 903 */ 904 lock1 = &(*lock_list)->ll_children[(*lock_list)->ll_count - 1]; 905 w1 = lock1->li_lock->lo_witness; 906 if (w1 == w) { 907 if (w->w_same_squawked || (lock->lo_flags & LO_DUPOK) || 908 (flags & LOP_DUPOK)) 909 return; 910 w->w_same_squawked = 1; 911 printf("acquiring duplicate lock of same type: \"%s\"\n", 912 lock->lo_type); 913 printf(" 1st %s @ %s:%d\n", lock1->li_lock->lo_name, 914 lock1->li_file, lock1->li_line); 915 printf(" 2nd %s @ %s:%d\n", lock->lo_name, file, line); 916#ifdef KDB 917 goto debugger; 918#else 919 return; 920#endif 921 } 922 MPASS(!mtx_owned(&w_mtx)); 923 mtx_lock_spin(&w_mtx); 924 /* 925 * If we know that the the lock we are acquiring comes after 926 * the lock we most recently acquired in the lock order tree, 927 * then there is no need for any further checks. 928 */ 929 if (isitmychild(w1, w)) { 930 mtx_unlock_spin(&w_mtx); 931 return; 932 } 933 for (j = 0, lle = *lock_list; lle != NULL; lle = lle->ll_next) { 934 for (i = lle->ll_count - 1; i >= 0; i--, j++) { 935 936 MPASS(j < WITNESS_COUNT); 937 lock1 = &lle->ll_children[i]; 938 w1 = lock1->li_lock->lo_witness; 939 940 /* 941 * If this lock doesn't undergo witness checking, 942 * then skip it. 943 */ 944 if (w1 == NULL) { 945 KASSERT((lock1->li_lock->lo_flags & LO_WITNESS) == 0, 946 ("lock missing witness structure")); 947 continue; 948 } 949 /* 950 * If we are locking Giant and this is a sleepable 951 * lock, then skip it. 952 */ 953 if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0 && 954 lock == &Giant.mtx_object) 955 continue; 956 /* 957 * If we are locking a sleepable lock and this lock 958 * is Giant, then skip it. 959 */ 960 if ((lock->lo_flags & LO_SLEEPABLE) != 0 && 961 lock1->li_lock == &Giant.mtx_object) 962 continue; 963 /* 964 * If we are locking a sleepable lock and this lock 965 * isn't sleepable, we want to treat it as a lock 966 * order violation to enfore a general lock order of 967 * sleepable locks before non-sleepable locks. 968 */ 969 if (((lock->lo_flags & LO_SLEEPABLE) != 0 && 970 (lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0)) 971 goto reversal; 972 /* 973 * If we are locking Giant and this is a non-sleepable 974 * lock, then treat it as a reversal. 975 */ 976 if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0 && 977 lock == &Giant.mtx_object) 978 goto reversal; 979 /* 980 * Check the lock order hierarchy for a reveresal. 981 */ 982 if (!isitmydescendant(w, w1)) 983 continue; 984 reversal: 985 /* 986 * We have a lock order violation, check to see if it 987 * is allowed or has already been yelled about. 988 */ 989 mtx_unlock_spin(&w_mtx); 990#ifdef BLESSING 991 /* 992 * If the lock order is blessed, just bail. We don't 993 * look for other lock order violations though, which 994 * may be a bug. 995 */ 996 if (blessed(w, w1)) 997 return; 998#endif 999 if (lock1->li_lock == &Giant.mtx_object) { 1000 if (w1->w_Giant_squawked) 1001 return; 1002 else 1003 w1->w_Giant_squawked = 1; 1004 } else { 1005 if (w1->w_other_squawked) 1006 return; 1007 else 1008 w1->w_other_squawked = 1; 1009 } 1010 /* 1011 * Ok, yell about it. 1012 */ 1013 if (((lock->lo_flags & LO_SLEEPABLE) != 0 && 1014 (lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0)) 1015 printf( 1016 "lock order reversal: (sleepable after non-sleepable)\n"); 1017 else if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0 1018 && lock == &Giant.mtx_object) 1019 printf( 1020 "lock order reversal: (Giant after non-sleepable)\n"); 1021 else 1022 printf("lock order reversal:\n"); 1023 /* 1024 * Try to locate an earlier lock with 1025 * witness w in our list. 1026 */ 1027 do { 1028 lock2 = &lle->ll_children[i]; 1029 MPASS(lock2->li_lock != NULL); 1030 if (lock2->li_lock->lo_witness == w) 1031 break; 1032 if (i == 0 && lle->ll_next != NULL) { 1033 lle = lle->ll_next; 1034 i = lle->ll_count - 1; 1035 MPASS(i >= 0 && i < LOCK_NCHILDREN); 1036 } else 1037 i--; 1038 } while (i >= 0); 1039 if (i < 0) { 1040 printf(" 1st %p %s (%s) @ %s:%d\n", 1041 lock1->li_lock, lock1->li_lock->lo_name, 1042 lock1->li_lock->lo_type, lock1->li_file, 1043 lock1->li_line); 1044 printf(" 2nd %p %s (%s) @ %s:%d\n", lock, 1045 lock->lo_name, lock->lo_type, file, line); 1046 } else { 1047 printf(" 1st %p %s (%s) @ %s:%d\n", 1048 lock2->li_lock, lock2->li_lock->lo_name, 1049 lock2->li_lock->lo_type, lock2->li_file, 1050 lock2->li_line); 1051 printf(" 2nd %p %s (%s) @ %s:%d\n", 1052 lock1->li_lock, lock1->li_lock->lo_name, 1053 lock1->li_lock->lo_type, lock1->li_file, 1054 lock1->li_line); 1055 printf(" 3rd %p %s (%s) @ %s:%d\n", lock, 1056 lock->lo_name, lock->lo_type, file, line); 1057 } 1058#ifdef KDB 1059 goto debugger; 1060#else 1061 return; 1062#endif 1063 } 1064 } 1065 lock1 = &(*lock_list)->ll_children[(*lock_list)->ll_count - 1]; 1066 /* 1067 * If requested, build a new lock order. However, don't build a new 1068 * relationship between a sleepable lock and Giant if it is in the 1069 * wrong direction. The correct lock order is that sleepable locks 1070 * always come before Giant. 1071 */ 1072 if (flags & LOP_NEWORDER && 1073 !(lock1->li_lock == &Giant.mtx_object && 1074 (lock->lo_flags & LO_SLEEPABLE) != 0)) { 1075 CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__, 1076 lock->lo_type, lock1->li_lock->lo_type); 1077 if (!itismychild(lock1->li_lock->lo_witness, w)) 1078 /* Witness is dead. */ 1079 return; 1080 } 1081 mtx_unlock_spin(&w_mtx); 1082 return; 1083 1084#ifdef KDB 1085debugger: 1086 if (witness_trace) 1087 kdb_backtrace(); 1088 if (witness_kdb) 1089 kdb_enter(__func__); 1090#endif 1091} 1092 1093void 1094witness_lock(struct lock_object *lock, int flags, const char *file, int line) 1095{ 1096 struct lock_list_entry **lock_list, *lle; 1097 struct lock_instance *instance; 1098 struct witness *w; 1099 struct thread *td; 1100 1101 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL || 1102 panicstr != NULL) 1103 return; 1104 w = lock->lo_witness; 1105 td = curthread; 1106 file = fixup_filename(file); 1107 1108 /* Determine lock list for this lock. */ 1109 if (LOCK_CLASS(lock)->lc_flags & LC_SLEEPLOCK) 1110 lock_list = &td->td_sleeplocks; 1111 else 1112 lock_list = PCPU_PTR(spinlocks); 1113 1114 /* Check to see if we are recursing on a lock we already own. */ 1115 instance = find_instance(*lock_list, lock); 1116 if (instance != NULL) { 1117 instance->li_flags++; 1118 CTR4(KTR_WITNESS, "%s: pid %d recursed on %s r=%d", __func__, 1119 td->td_proc->p_pid, lock->lo_name, 1120 instance->li_flags & LI_RECURSEMASK); 1121 instance->li_file = file; 1122 instance->li_line = line; 1123 return; 1124 } 1125 1126 /* Update per-witness last file and line acquire. */ 1127 w->w_file = file; 1128 w->w_line = line; 1129 1130 /* Find the next open lock instance in the list and fill it. */ 1131 lle = *lock_list; 1132 if (lle == NULL || lle->ll_count == LOCK_NCHILDREN) { 1133 lle = witness_lock_list_get(); 1134 if (lle == NULL) 1135 return; 1136 lle->ll_next = *lock_list; 1137 CTR3(KTR_WITNESS, "%s: pid %d added lle %p", __func__, 1138 td->td_proc->p_pid, lle); 1139 *lock_list = lle; 1140 } 1141 instance = &lle->ll_children[lle->ll_count++]; 1142 instance->li_lock = lock; 1143 instance->li_line = line; 1144 instance->li_file = file; 1145 if ((flags & LOP_EXCLUSIVE) != 0) 1146 instance->li_flags = LI_EXCLUSIVE; 1147 else 1148 instance->li_flags = 0; 1149 CTR4(KTR_WITNESS, "%s: pid %d added %s as lle[%d]", __func__, 1150 td->td_proc->p_pid, lock->lo_name, lle->ll_count - 1); 1151} 1152 1153void 1154witness_upgrade(struct lock_object *lock, int flags, const char *file, int line) 1155{ 1156 struct lock_instance *instance; 1157 struct lock_class *class; 1158 1159 KASSERT(!witness_cold, ("%s: witness_cold", __func__)); 1160 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) 1161 return; 1162 class = LOCK_CLASS(lock); 1163 file = fixup_filename(file); 1164 if ((lock->lo_flags & LO_UPGRADABLE) == 0) 1165 panic("upgrade of non-upgradable lock (%s) %s @ %s:%d", 1166 class->lc_name, lock->lo_name, file, line); 1167 if ((flags & LOP_TRYLOCK) == 0) 1168 panic("non-try upgrade of lock (%s) %s @ %s:%d", class->lc_name, 1169 lock->lo_name, file, line); 1170 if ((class->lc_flags & LC_SLEEPLOCK) == 0) 1171 panic("upgrade of non-sleep lock (%s) %s @ %s:%d", 1172 class->lc_name, lock->lo_name, file, line); 1173 instance = find_instance(curthread->td_sleeplocks, lock); 1174 if (instance == NULL) 1175 panic("upgrade of unlocked lock (%s) %s @ %s:%d", 1176 class->lc_name, lock->lo_name, file, line); 1177 if ((instance->li_flags & LI_EXCLUSIVE) != 0) 1178 panic("upgrade of exclusive lock (%s) %s @ %s:%d", 1179 class->lc_name, lock->lo_name, file, line); 1180 if ((instance->li_flags & LI_RECURSEMASK) != 0) 1181 panic("upgrade of recursed lock (%s) %s r=%d @ %s:%d", 1182 class->lc_name, lock->lo_name, 1183 instance->li_flags & LI_RECURSEMASK, file, line); 1184 instance->li_flags |= LI_EXCLUSIVE; 1185} 1186 1187void 1188witness_downgrade(struct lock_object *lock, int flags, const char *file, 1189 int line) 1190{ 1191 struct lock_instance *instance; 1192 struct lock_class *class; 1193 1194 KASSERT(!witness_cold, ("%s: witness_cold", __func__)); 1195 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) 1196 return; 1197 class = LOCK_CLASS(lock); 1198 file = fixup_filename(file); 1199 if ((lock->lo_flags & LO_UPGRADABLE) == 0) 1200 panic("downgrade of non-upgradable lock (%s) %s @ %s:%d", 1201 class->lc_name, lock->lo_name, file, line); 1202 if ((class->lc_flags & LC_SLEEPLOCK) == 0) 1203 panic("downgrade of non-sleep lock (%s) %s @ %s:%d", 1204 class->lc_name, lock->lo_name, file, line); 1205 instance = find_instance(curthread->td_sleeplocks, lock); 1206 if (instance == NULL) 1207 panic("downgrade of unlocked lock (%s) %s @ %s:%d", 1208 class->lc_name, lock->lo_name, file, line); 1209 if ((instance->li_flags & LI_EXCLUSIVE) == 0) 1210 panic("downgrade of shared lock (%s) %s @ %s:%d", 1211 class->lc_name, lock->lo_name, file, line); 1212 if ((instance->li_flags & LI_RECURSEMASK) != 0) 1213 panic("downgrade of recursed lock (%s) %s r=%d @ %s:%d", 1214 class->lc_name, lock->lo_name, 1215 instance->li_flags & LI_RECURSEMASK, file, line); 1216 instance->li_flags &= ~LI_EXCLUSIVE; 1217} 1218 1219void 1220witness_unlock(struct lock_object *lock, int flags, const char *file, int line) 1221{ 1222 struct lock_list_entry **lock_list, *lle; 1223 struct lock_instance *instance; 1224 struct lock_class *class; 1225 struct thread *td; 1226 register_t s; 1227 int i, j; 1228 1229 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL || 1230 panicstr != NULL) 1231 return; 1232 td = curthread; 1233 class = LOCK_CLASS(lock); 1234 file = fixup_filename(file); 1235 1236 /* Find lock instance associated with this lock. */ 1237 if (class->lc_flags & LC_SLEEPLOCK) 1238 lock_list = &td->td_sleeplocks; 1239 else 1240 lock_list = PCPU_PTR(spinlocks); 1241 for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next) 1242 for (i = 0; i < (*lock_list)->ll_count; i++) { 1243 instance = &(*lock_list)->ll_children[i]; 1244 if (instance->li_lock == lock) 1245 goto found; 1246 } 1247 panic("lock (%s) %s not locked @ %s:%d", class->lc_name, lock->lo_name, 1248 file, line); 1249found: 1250 1251 /* First, check for shared/exclusive mismatches. */ 1252 if ((instance->li_flags & LI_EXCLUSIVE) != 0 && 1253 (flags & LOP_EXCLUSIVE) == 0) { 1254 printf("shared unlock of (%s) %s @ %s:%d\n", class->lc_name, 1255 lock->lo_name, file, line); 1256 printf("while exclusively locked from %s:%d\n", 1257 instance->li_file, instance->li_line); 1258 panic("excl->ushare"); 1259 } 1260 if ((instance->li_flags & LI_EXCLUSIVE) == 0 && 1261 (flags & LOP_EXCLUSIVE) != 0) { 1262 printf("exclusive unlock of (%s) %s @ %s:%d\n", class->lc_name, 1263 lock->lo_name, file, line); 1264 printf("while share locked from %s:%d\n", instance->li_file, 1265 instance->li_line); 1266 panic("share->uexcl"); 1267 } 1268 1269 /* If we are recursed, unrecurse. */ 1270 if ((instance->li_flags & LI_RECURSEMASK) > 0) { 1271 CTR4(KTR_WITNESS, "%s: pid %d unrecursed on %s r=%d", __func__, 1272 td->td_proc->p_pid, instance->li_lock->lo_name, 1273 instance->li_flags); 1274 instance->li_flags--; 1275 return; 1276 } 1277 1278 /* Otherwise, remove this item from the list. */ 1279 s = intr_disable(); 1280 CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__, 1281 td->td_proc->p_pid, instance->li_lock->lo_name, 1282 (*lock_list)->ll_count - 1); 1283 for (j = i; j < (*lock_list)->ll_count - 1; j++) 1284 (*lock_list)->ll_children[j] = 1285 (*lock_list)->ll_children[j + 1]; 1286 (*lock_list)->ll_count--; 1287 intr_restore(s); 1288 1289 /* If this lock list entry is now empty, free it. */ 1290 if ((*lock_list)->ll_count == 0) { 1291 lle = *lock_list; 1292 *lock_list = lle->ll_next; 1293 CTR3(KTR_WITNESS, "%s: pid %d removed lle %p", __func__, 1294 td->td_proc->p_pid, lle); 1295 witness_lock_list_free(lle); 1296 } 1297} 1298 1299/* 1300 * Warn if any locks other than 'lock' are held. Flags can be passed in to 1301 * exempt Giant and sleepable locks from the checks as well. If any 1302 * non-exempt locks are held, then a supplied message is printed to the 1303 * console along with a list of the offending locks. If indicated in the 1304 * flags then a failure results in a panic as well. 1305 */ 1306int 1307witness_warn(int flags, struct lock_object *lock, const char *fmt, ...) 1308{ 1309 struct lock_list_entry *lle; 1310 struct lock_instance *lock1; 1311 struct thread *td; 1312 va_list ap; 1313 int i, n; 1314 1315 if (witness_cold || witness_watch == 0 || panicstr != NULL) 1316 return (0); 1317 n = 0; 1318 td = curthread; 1319 for (lle = td->td_sleeplocks; lle != NULL; lle = lle->ll_next) 1320 for (i = lle->ll_count - 1; i >= 0; i--) { 1321 lock1 = &lle->ll_children[i]; 1322 if (lock1->li_lock == lock) 1323 continue; 1324 if (flags & WARN_GIANTOK && 1325 lock1->li_lock == &Giant.mtx_object) 1326 continue; 1327 if (flags & WARN_SLEEPOK && 1328 (lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0) 1329 continue; 1330 if (n == 0) { 1331 va_start(ap, fmt); 1332 vprintf(fmt, ap); 1333 va_end(ap); 1334 printf(" with the following"); 1335 if (flags & WARN_SLEEPOK) 1336 printf(" non-sleepable"); 1337 printf(" locks held:\n"); 1338 } 1339 n++; 1340 witness_list_lock(lock1); 1341 } 1342 if (PCPU_GET(spinlocks) != NULL) { 1343 /* 1344 * Since we already hold a spinlock preemption is 1345 * already blocked. 1346 */ 1347 if (n == 0) { 1348 va_start(ap, fmt); 1349 vprintf(fmt, ap); 1350 va_end(ap); 1351 printf(" with the following"); 1352 if (flags & WARN_SLEEPOK) 1353 printf(" non-sleepable"); 1354 printf(" locks held:\n"); 1355 } 1356 n += witness_list_locks(PCPU_PTR(spinlocks)); 1357 } 1358 if (flags & WARN_PANIC && n) 1359 panic("witness_warn"); 1360#ifdef KDB 1361 else if (witness_kdb && n) 1362 kdb_enter(__func__); 1363 else if (witness_trace && n) 1364 kdb_backtrace(); 1365#endif 1366 return (n); 1367} 1368 1369const char * 1370witness_file(struct lock_object *lock) 1371{ 1372 struct witness *w; 1373 1374 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL) 1375 return ("?"); 1376 w = lock->lo_witness; 1377 return (w->w_file); 1378} 1379 1380int 1381witness_line(struct lock_object *lock) 1382{ 1383 struct witness *w; 1384 1385 if (witness_cold || witness_watch == 0 || lock->lo_witness == NULL) 1386 return (0); 1387 w = lock->lo_witness; 1388 return (w->w_line); 1389} 1390 1391static struct witness * 1392enroll(const char *description, struct lock_class *lock_class) 1393{ 1394 struct witness *w; 1395 1396 if (witness_watch == 0 || panicstr != NULL) 1397 return (NULL); 1398 if ((lock_class->lc_flags & LC_SPINLOCK) && witness_skipspin) 1399 return (NULL); 1400 mtx_lock_spin(&w_mtx); 1401 STAILQ_FOREACH(w, &w_all, w_list) { 1402 if (w->w_name == description || (w->w_refcount > 0 && 1403 strcmp(description, w->w_name) == 0)) { 1404 w->w_refcount++; 1405 mtx_unlock_spin(&w_mtx); 1406 if (lock_class != w->w_class) 1407 panic( 1408 "lock (%s) %s does not match earlier (%s) lock", 1409 description, lock_class->lc_name, 1410 w->w_class->lc_name); 1411 return (w); 1412 } 1413 } 1414 if ((w = witness_get()) == NULL) 1415 goto out; 1416 w->w_name = description; 1417 w->w_class = lock_class; 1418 w->w_refcount = 1; 1419 STAILQ_INSERT_HEAD(&w_all, w, w_list); 1420 if (lock_class->lc_flags & LC_SPINLOCK) { 1421 STAILQ_INSERT_HEAD(&w_spin, w, w_typelist); 1422 w_spin_cnt++; 1423 } else if (lock_class->lc_flags & LC_SLEEPLOCK) { 1424 STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist); 1425 w_sleep_cnt++; 1426 } else { 1427 mtx_unlock_spin(&w_mtx); 1428 panic("lock class %s is not sleep or spin", 1429 lock_class->lc_name); 1430 } 1431 mtx_unlock_spin(&w_mtx); 1432out: 1433 /* 1434 * We issue a warning for any spin locks not defined in the static 1435 * order list as a way to discourage their use (folks should really 1436 * be using non-spin mutexes most of the time). However, several 1437 * 3rd part device drivers use spin locks because that is all they 1438 * have available on Windows and Linux and they think that normal 1439 * mutexes are insufficient. 1440 */ 1441 if ((lock_class->lc_flags & LC_SPINLOCK) && witness_spin_warn) 1442 printf("WITNESS: spin lock %s not in order list\n", 1443 description); 1444 return (w); 1445} 1446 1447/* Don't let the door bang you on the way out... */ 1448static int 1449depart(struct witness *w) 1450{ 1451 struct witness_child_list_entry *wcl, *nwcl; 1452 struct witness_list *list; 1453 struct witness *parent; 1454 1455 MPASS(w->w_refcount == 0); 1456 if (w->w_class->lc_flags & LC_SLEEPLOCK) { 1457 list = &w_sleep; 1458 w_sleep_cnt--; 1459 } else { 1460 list = &w_spin; 1461 w_spin_cnt--; 1462 } 1463 /* 1464 * First, we run through the entire tree looking for any 1465 * witnesses that the outgoing witness is a child of. For 1466 * each parent that we find, we reparent all the direct 1467 * children of the outgoing witness to its parent. 1468 */ 1469 STAILQ_FOREACH(parent, list, w_typelist) { 1470 if (!isitmychild(parent, w)) 1471 continue; 1472 removechild(parent, w); 1473 } 1474 1475 /* 1476 * Now we go through and free up the child list of the 1477 * outgoing witness. 1478 */ 1479 for (wcl = w->w_children; wcl != NULL; wcl = nwcl) { 1480 nwcl = wcl->wcl_next; 1481 w_child_cnt--; 1482 witness_child_free(wcl); 1483 } 1484 1485 /* 1486 * Detach from various lists and free. 1487 */ 1488 STAILQ_REMOVE(list, w, witness, w_typelist); 1489 STAILQ_REMOVE(&w_all, w, witness, w_list); 1490 witness_free(w); 1491 1492 return (1); 1493} 1494 1495/* 1496 * Add "child" as a direct child of "parent". Returns false if 1497 * we fail due to out of memory. 1498 */ 1499static int 1500insertchild(struct witness *parent, struct witness *child) 1501{ 1502 struct witness_child_list_entry **wcl; 1503 1504 MPASS(child != NULL && parent != NULL); 1505 1506 /* 1507 * Insert "child" after "parent" 1508 */ 1509 wcl = &parent->w_children; 1510 while (*wcl != NULL && (*wcl)->wcl_count == WITNESS_NCHILDREN) 1511 wcl = &(*wcl)->wcl_next; 1512 if (*wcl == NULL) { 1513 *wcl = witness_child_get(); 1514 if (*wcl == NULL) 1515 return (0); 1516 w_child_cnt++; 1517 } 1518 (*wcl)->wcl_children[(*wcl)->wcl_count++] = child; 1519 1520 return (1); 1521} 1522 1523 1524static int 1525itismychild(struct witness *parent, struct witness *child) 1526{ 1527 struct witness_list *list; 1528 1529 MPASS(child != NULL && parent != NULL); 1530 if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) != 1531 (child->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK))) 1532 panic( 1533 "%s: parent (%s) and child (%s) are not the same lock type", 1534 __func__, parent->w_class->lc_name, 1535 child->w_class->lc_name); 1536 1537 if (!insertchild(parent, child)) 1538 return (0); 1539 1540 if (parent->w_class->lc_flags & LC_SLEEPLOCK) 1541 list = &w_sleep; 1542 else 1543 list = &w_spin; 1544 return (1); 1545} 1546 1547static void 1548removechild(struct witness *parent, struct witness *child) 1549{ 1550 struct witness_child_list_entry **wcl, *wcl1; 1551 int i; 1552 1553 for (wcl = &parent->w_children; *wcl != NULL; wcl = &(*wcl)->wcl_next) 1554 for (i = 0; i < (*wcl)->wcl_count; i++) 1555 if ((*wcl)->wcl_children[i] == child) 1556 goto found; 1557 return; 1558found: 1559 (*wcl)->wcl_count--; 1560 if ((*wcl)->wcl_count > i) 1561 (*wcl)->wcl_children[i] = 1562 (*wcl)->wcl_children[(*wcl)->wcl_count]; 1563 MPASS((*wcl)->wcl_children[i] != NULL); 1564 if ((*wcl)->wcl_count != 0) 1565 return; 1566 wcl1 = *wcl; 1567 *wcl = wcl1->wcl_next; 1568 w_child_cnt--; 1569 witness_child_free(wcl1); 1570} 1571 1572static int 1573isitmychild(struct witness *parent, struct witness *child) 1574{ 1575 struct witness_child_list_entry *wcl; 1576 int i; 1577 1578 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) { 1579 for (i = 0; i < wcl->wcl_count; i++) { 1580 if (wcl->wcl_children[i] == child) 1581 return (1); 1582 } 1583 } 1584 return (0); 1585} 1586 1587static int 1588isitmydescendant(struct witness *parent, struct witness *child) 1589{ 1590 struct witness_child_list_entry *wcl; 1591 int i, j; 1592 1593 if (isitmychild(parent, child)) 1594 return (1); 1595 j = 0; 1596 for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) { 1597 MPASS(j < 1000); 1598 for (i = 0; i < wcl->wcl_count; i++) { 1599 if (isitmydescendant(wcl->wcl_children[i], child)) 1600 return (1); 1601 } 1602 j++; 1603 } 1604 return (0); 1605} 1606 1607#ifdef BLESSING 1608static int 1609blessed(struct witness *w1, struct witness *w2) 1610{ 1611 int i; 1612 struct witness_blessed *b; 1613 1614 for (i = 0; i < blessed_count; i++) { 1615 b = &blessed_list[i]; 1616 if (strcmp(w1->w_name, b->b_lock1) == 0) { 1617 if (strcmp(w2->w_name, b->b_lock2) == 0) 1618 return (1); 1619 continue; 1620 } 1621 if (strcmp(w1->w_name, b->b_lock2) == 0) 1622 if (strcmp(w2->w_name, b->b_lock1) == 0) 1623 return (1); 1624 } 1625 return (0); 1626} 1627#endif 1628 1629static struct witness * 1630witness_get(void) 1631{ 1632 struct witness *w; 1633 1634 if (witness_watch == 0) { 1635 mtx_unlock_spin(&w_mtx); 1636 return (NULL); 1637 } 1638 if (STAILQ_EMPTY(&w_free)) { 1639 witness_watch = 0; 1640 mtx_unlock_spin(&w_mtx); 1641 printf("%s: witness exhausted\n", __func__); 1642 return (NULL); 1643 } 1644 w = STAILQ_FIRST(&w_free); 1645 STAILQ_REMOVE_HEAD(&w_free, w_list); 1646 w_free_cnt--; 1647 bzero(w, sizeof(*w)); 1648 return (w); 1649} 1650 1651static void 1652witness_free(struct witness *w) 1653{ 1654 1655 STAILQ_INSERT_HEAD(&w_free, w, w_list); 1656 w_free_cnt++; 1657} 1658 1659static struct witness_child_list_entry * 1660witness_child_get(void) 1661{ 1662 struct witness_child_list_entry *wcl; 1663 1664 if (witness_watch == 0) { 1665 mtx_unlock_spin(&w_mtx); 1666 return (NULL); 1667 } 1668 wcl = w_child_free; 1669 if (wcl == NULL) { 1670 witness_watch = 0; 1671 mtx_unlock_spin(&w_mtx); 1672 printf("%s: witness exhausted\n", __func__); 1673 return (NULL); 1674 } 1675 w_child_free = wcl->wcl_next; 1676 w_child_free_cnt--; 1677 bzero(wcl, sizeof(*wcl)); 1678 return (wcl); 1679} 1680 1681static void 1682witness_child_free(struct witness_child_list_entry *wcl) 1683{ 1684 1685 wcl->wcl_next = w_child_free; 1686 w_child_free = wcl; 1687 w_child_free_cnt++; 1688} 1689 1690static struct lock_list_entry * 1691witness_lock_list_get(void) 1692{ 1693 struct lock_list_entry *lle; 1694 1695 if (witness_watch == 0) 1696 return (NULL); 1697 mtx_lock_spin(&w_mtx); 1698 lle = w_lock_list_free; 1699 if (lle == NULL) { 1700 witness_watch = 0; 1701 mtx_unlock_spin(&w_mtx); 1702 printf("%s: witness exhausted\n", __func__); 1703 return (NULL); 1704 } 1705 w_lock_list_free = lle->ll_next; 1706 mtx_unlock_spin(&w_mtx); 1707 bzero(lle, sizeof(*lle)); 1708 return (lle); 1709} 1710 1711static void 1712witness_lock_list_free(struct lock_list_entry *lle) 1713{ 1714 1715 mtx_lock_spin(&w_mtx); 1716 lle->ll_next = w_lock_list_free; 1717 w_lock_list_free = lle; 1718 mtx_unlock_spin(&w_mtx); 1719} 1720 1721static struct lock_instance * 1722find_instance(struct lock_list_entry *lock_list, struct lock_object *lock) 1723{ 1724 struct lock_list_entry *lle; 1725 struct lock_instance *instance; 1726 int i; 1727 1728 for (lle = lock_list; lle != NULL; lle = lle->ll_next) 1729 for (i = lle->ll_count - 1; i >= 0; i--) { 1730 instance = &lle->ll_children[i]; 1731 if (instance->li_lock == lock) 1732 return (instance); 1733 } 1734 return (NULL); 1735} 1736 1737static void 1738witness_list_lock(struct lock_instance *instance) 1739{ 1740 struct lock_object *lock; 1741 1742 lock = instance->li_lock; 1743 printf("%s %s %s", (instance->li_flags & LI_EXCLUSIVE) != 0 ? 1744 "exclusive" : "shared", LOCK_CLASS(lock)->lc_name, lock->lo_name); 1745 if (lock->lo_type != lock->lo_name) 1746 printf(" (%s)", lock->lo_type); 1747 printf(" r = %d (%p) locked @ %s:%d\n", 1748 instance->li_flags & LI_RECURSEMASK, lock, instance->li_file, 1749 instance->li_line); 1750} 1751 1752#ifdef DDB 1753static int 1754witness_thread_has_locks(struct thread *td) 1755{ 1756 1757 return (td->td_sleeplocks != NULL); 1758} 1759 1760static int 1761witness_proc_has_locks(struct proc *p) 1762{ 1763 struct thread *td; 1764 1765 FOREACH_THREAD_IN_PROC(p, td) { 1766 if (witness_thread_has_locks(td)) 1767 return (1); 1768 } 1769 return (0); 1770} 1771#endif 1772 1773int 1774witness_list_locks(struct lock_list_entry **lock_list) 1775{ 1776 struct lock_list_entry *lle; 1777 int i, nheld; 1778 1779 nheld = 0; 1780 for (lle = *lock_list; lle != NULL; lle = lle->ll_next) 1781 for (i = lle->ll_count - 1; i >= 0; i--) { 1782 witness_list_lock(&lle->ll_children[i]); 1783 nheld++; 1784 } 1785 return (nheld); 1786} 1787 1788/* 1789 * This is a bit risky at best. We call this function when we have timed 1790 * out acquiring a spin lock, and we assume that the other CPU is stuck 1791 * with this lock held. So, we go groveling around in the other CPU's 1792 * per-cpu data to try to find the lock instance for this spin lock to 1793 * see when it was last acquired. 1794 */ 1795void 1796witness_display_spinlock(struct lock_object *lock, struct thread *owner) 1797{ 1798 struct lock_instance *instance; 1799 struct pcpu *pc; 1800 1801 if (owner->td_critnest == 0 || owner->td_oncpu == NOCPU) 1802 return; 1803 pc = pcpu_find(owner->td_oncpu); 1804 instance = find_instance(pc->pc_spinlocks, lock); 1805 if (instance != NULL) 1806 witness_list_lock(instance); 1807} 1808 1809void 1810witness_save(struct lock_object *lock, const char **filep, int *linep) 1811{ 1812 struct lock_list_entry *lock_list; 1813 struct lock_instance *instance; 1814 struct lock_class *class; 1815 1816 KASSERT(!witness_cold, ("%s: witness_cold", __func__)); 1817 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) 1818 return; 1819 class = LOCK_CLASS(lock); 1820 if (class->lc_flags & LC_SLEEPLOCK) 1821 lock_list = curthread->td_sleeplocks; 1822 else { 1823 if (witness_skipspin) 1824 return; 1825 lock_list = PCPU_GET(spinlocks); 1826 } 1827 instance = find_instance(lock_list, lock); 1828 if (instance == NULL) 1829 panic("%s: lock (%s) %s not locked", __func__, 1830 class->lc_name, lock->lo_name); 1831 *filep = instance->li_file; 1832 *linep = instance->li_line; 1833} 1834 1835void 1836witness_restore(struct lock_object *lock, const char *file, int line) 1837{ 1838 struct lock_list_entry *lock_list; 1839 struct lock_instance *instance; 1840 struct lock_class *class; 1841 1842 KASSERT(!witness_cold, ("%s: witness_cold", __func__)); 1843 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) 1844 return; 1845 class = LOCK_CLASS(lock); 1846 if (class->lc_flags & LC_SLEEPLOCK) 1847 lock_list = curthread->td_sleeplocks; 1848 else { 1849 if (witness_skipspin) 1850 return; 1851 lock_list = PCPU_GET(spinlocks); 1852 } 1853 instance = find_instance(lock_list, lock); 1854 if (instance == NULL) 1855 panic("%s: lock (%s) %s not locked", __func__, 1856 class->lc_name, lock->lo_name); 1857 lock->lo_witness->w_file = file; 1858 lock->lo_witness->w_line = line; 1859 instance->li_file = file; 1860 instance->li_line = line; 1861} 1862 1863void 1864witness_assert(struct lock_object *lock, int flags, const char *file, int line) 1865{ 1866#ifdef INVARIANT_SUPPORT 1867 struct lock_instance *instance; 1868 struct lock_class *class; 1869 1870 if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) 1871 return; 1872 class = LOCK_CLASS(lock); 1873 if ((class->lc_flags & LC_SLEEPLOCK) != 0) 1874 instance = find_instance(curthread->td_sleeplocks, lock); 1875 else if ((class->lc_flags & LC_SPINLOCK) != 0) 1876 instance = find_instance(PCPU_GET(spinlocks), lock); 1877 else { 1878 panic("Lock (%s) %s is not sleep or spin!", 1879 class->lc_name, lock->lo_name); 1880 } 1881 file = fixup_filename(file); 1882 switch (flags) { 1883 case LA_UNLOCKED: 1884 if (instance != NULL) 1885 panic("Lock (%s) %s locked @ %s:%d.", 1886 class->lc_name, lock->lo_name, file, line); 1887 break; 1888 case LA_LOCKED: 1889 case LA_LOCKED | LA_RECURSED: 1890 case LA_LOCKED | LA_NOTRECURSED: 1891 case LA_SLOCKED: 1892 case LA_SLOCKED | LA_RECURSED: 1893 case LA_SLOCKED | LA_NOTRECURSED: 1894 case LA_XLOCKED: 1895 case LA_XLOCKED | LA_RECURSED: 1896 case LA_XLOCKED | LA_NOTRECURSED: 1897 if (instance == NULL) { 1898 panic("Lock (%s) %s not locked @ %s:%d.", 1899 class->lc_name, lock->lo_name, file, line); 1900 break; 1901 } 1902 if ((flags & LA_XLOCKED) != 0 && 1903 (instance->li_flags & LI_EXCLUSIVE) == 0) 1904 panic("Lock (%s) %s not exclusively locked @ %s:%d.", 1905 class->lc_name, lock->lo_name, file, line); 1906 if ((flags & LA_SLOCKED) != 0 && 1907 (instance->li_flags & LI_EXCLUSIVE) != 0) 1908 panic("Lock (%s) %s exclusively locked @ %s:%d.", 1909 class->lc_name, lock->lo_name, file, line); 1910 if ((flags & LA_RECURSED) != 0 && 1911 (instance->li_flags & LI_RECURSEMASK) == 0) 1912 panic("Lock (%s) %s not recursed @ %s:%d.", 1913 class->lc_name, lock->lo_name, file, line); 1914 if ((flags & LA_NOTRECURSED) != 0 && 1915 (instance->li_flags & LI_RECURSEMASK) != 0) 1916 panic("Lock (%s) %s recursed @ %s:%d.", 1917 class->lc_name, lock->lo_name, file, line); 1918 break; 1919 default: 1920 panic("Invalid lock assertion at %s:%d.", file, line); 1921 1922 } 1923#endif /* INVARIANT_SUPPORT */ 1924} 1925 1926#ifdef DDB 1927static void 1928witness_list(struct thread *td) 1929{ 1930 1931 KASSERT(!witness_cold, ("%s: witness_cold", __func__)); 1932 KASSERT(kdb_active, ("%s: not in the debugger", __func__)); 1933 1934 if (witness_watch == 0) 1935 return; 1936 1937 witness_list_locks(&td->td_sleeplocks); 1938 1939 /* 1940 * We only handle spinlocks if td == curthread. This is somewhat broken 1941 * if td is currently executing on some other CPU and holds spin locks 1942 * as we won't display those locks. If we had a MI way of getting 1943 * the per-cpu data for a given cpu then we could use 1944 * td->td_oncpu to get the list of spinlocks for this thread 1945 * and "fix" this. 1946 * 1947 * That still wouldn't really fix this unless we locked sched_lock 1948 * or stopped the other CPU to make sure it wasn't changing the list 1949 * out from under us. It is probably best to just not try to handle 1950 * threads on other CPU's for now. 1951 */ 1952 if (td == curthread && PCPU_GET(spinlocks) != NULL) 1953 witness_list_locks(PCPU_PTR(spinlocks)); 1954} 1955 1956DB_SHOW_COMMAND(locks, db_witness_list) 1957{ 1958 struct thread *td; 1959 1960 if (have_addr) 1961 td = db_lookup_thread(addr, TRUE); 1962 else 1963 td = kdb_thread; 1964 witness_list(td); 1965} 1966 1967DB_SHOW_COMMAND(alllocks, db_witness_list_all) 1968{ 1969 struct thread *td; 1970 struct proc *p; 1971 1972 /* 1973 * It would be nice to list only threads and processes that actually 1974 * held sleep locks, but that information is currently not exported 1975 * by WITNESS. 1976 */ 1977 FOREACH_PROC_IN_SYSTEM(p) { 1978 if (!witness_proc_has_locks(p)) 1979 continue; 1980 FOREACH_THREAD_IN_PROC(p, td) { 1981 if (!witness_thread_has_locks(td)) 1982 continue; 1983 db_printf("Process %d (%s) thread %p (%d)\n", p->p_pid, 1984 p->p_comm, td, td->td_tid); 1985 witness_list(td); 1986 } 1987 } 1988} 1989 1990DB_SHOW_COMMAND(witness, db_witness_display) 1991{ 1992 1993 witness_display(db_printf); 1994} 1995#endif 1996