Deleted Added
sdiff udiff text old ( 239864 ) new ( 244105 )
full compact
subr_witness.c (239864) subr_witness.c (244105)
1/*-
2 * Copyright (c) 2008 Isilon Systems, Inc.
3 * Copyright (c) 2008 Ilya Maykov <ivmaykov@gmail.com>
4 * Copyright (c) 1998 Berkeley Software Design, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 71 unchanged lines hidden (view full) ---

80 * you have two threads T1 and T2 and a sleepable lock X. Suppose that T1
81 * acquires X and blocks on Giant. Then suppose that T2 acquires Giant and
82 * blocks on X. When T2 blocks on X, T2 will release Giant allowing T1 to
83 * execute. Thus, acquiring Giant both before and after a sleepable lock
84 * will not result in a lock order reversal.
85 */
86
87#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2008 Isilon Systems, Inc.
3 * Copyright (c) 2008 Ilya Maykov <ivmaykov@gmail.com>
4 * Copyright (c) 1998 Berkeley Software Design, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 71 unchanged lines hidden (view full) ---

80 * you have two threads T1 and T2 and a sleepable lock X. Suppose that T1
81 * acquires X and blocks on Giant. Then suppose that T2 acquires Giant and
82 * blocks on X. When T2 blocks on X, T2 will release Giant allowing T1 to
83 * execute. Thus, acquiring Giant both before and after a sleepable lock
84 * will not result in a lock order reversal.
85 */
86
87#include <sys/cdefs.h>
88__FBSDID("$FreeBSD: head/sys/kern/subr_witness.c 239864 2012-08-29 16:56:50Z marius $");
88__FBSDID("$FreeBSD: head/sys/kern/subr_witness.c 244105 2012-12-11 01:23:50Z alfred $");
89
90#include "opt_ddb.h"
91#include "opt_hwpmc_hooks.h"
92#include "opt_stack.h"
93#include "opt_witness.h"
94
95#include <sys/param.h>
96#include <sys/bus.h>

--- 720 unchanged lines hidden (view full) ---

817witness_init(struct lock_object *lock, const char *type)
818{
819 struct lock_class *class;
820
821 /* Various sanity checks. */
822 class = LOCK_CLASS(lock);
823 if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
824 (class->lc_flags & LC_RECURSABLE) == 0)
89
90#include "opt_ddb.h"
91#include "opt_hwpmc_hooks.h"
92#include "opt_stack.h"
93#include "opt_witness.h"
94
95#include <sys/param.h>
96#include <sys/bus.h>

--- 720 unchanged lines hidden (view full) ---

817witness_init(struct lock_object *lock, const char *type)
818{
819 struct lock_class *class;
820
821 /* Various sanity checks. */
822 class = LOCK_CLASS(lock);
823 if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
824 (class->lc_flags & LC_RECURSABLE) == 0)
825 panic("%s: lock (%s) %s can not be recursable", __func__,
826 class->lc_name, lock->lo_name);
825 kassert_panic("%s: lock (%s) %s can not be recursable",
826 __func__, class->lc_name, lock->lo_name);
827 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
828 (class->lc_flags & LC_SLEEPABLE) == 0)
827 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
828 (class->lc_flags & LC_SLEEPABLE) == 0)
829 panic("%s: lock (%s) %s can not be sleepable", __func__,
830 class->lc_name, lock->lo_name);
829 kassert_panic("%s: lock (%s) %s can not be sleepable",
830 __func__, class->lc_name, lock->lo_name);
831 if ((lock->lo_flags & LO_UPGRADABLE) != 0 &&
832 (class->lc_flags & LC_UPGRADABLE) == 0)
831 if ((lock->lo_flags & LO_UPGRADABLE) != 0 &&
832 (class->lc_flags & LC_UPGRADABLE) == 0)
833 panic("%s: lock (%s) %s can not be upgradable", __func__,
834 class->lc_name, lock->lo_name);
833 kassert_panic("%s: lock (%s) %s can not be upgradable",
834 __func__, class->lc_name, lock->lo_name);
835
836 /*
837 * If we shouldn't watch this lock, then just clear lo_witness.
838 * Otherwise, if witness_cold is set, then it is too early to
839 * enroll this lock, so defer it to witness_initialize() by adding
840 * it to the pending_locks list. If it is not too early, then enroll
841 * the lock now.
842 */
843 if (witness_watch < 1 || panicstr != NULL ||
844 (lock->lo_flags & LO_WITNESS) == 0)
845 lock->lo_witness = NULL;
846 else if (witness_cold) {
847 pending_locks[pending_cnt].wh_lock = lock;
848 pending_locks[pending_cnt++].wh_type = type;
849 if (pending_cnt > WITNESS_PENDLIST)
835
836 /*
837 * If we shouldn't watch this lock, then just clear lo_witness.
838 * Otherwise, if witness_cold is set, then it is too early to
839 * enroll this lock, so defer it to witness_initialize() by adding
840 * it to the pending_locks list. If it is not too early, then enroll
841 * the lock now.
842 */
843 if (witness_watch < 1 || panicstr != NULL ||
844 (lock->lo_flags & LO_WITNESS) == 0)
845 lock->lo_witness = NULL;
846 else if (witness_cold) {
847 pending_locks[pending_cnt].wh_lock = lock;
848 pending_locks[pending_cnt++].wh_type = type;
849 if (pending_cnt > WITNESS_PENDLIST)
850 panic("%s: pending locks list is too small, bump it\n",
850 panic("%s: pending locks list is too small, "
851 "increase WITNESS_PENDLIST\n",
851 __func__);
852 } else
853 lock->lo_witness = enroll(type, class);
854}
855
856void
857witness_destroy(struct lock_object *lock)
858{

--- 209 unchanged lines hidden (view full) ---

1068 if (class->lc_flags & LC_SLEEPLOCK) {
1069
1070 /*
1071 * Since spin locks include a critical section, this check
1072 * implicitly enforces a lock order of all sleep locks before
1073 * all spin locks.
1074 */
1075 if (td->td_critnest != 0 && !kdb_active)
852 __func__);
853 } else
854 lock->lo_witness = enroll(type, class);
855}
856
857void
858witness_destroy(struct lock_object *lock)
859{

--- 209 unchanged lines hidden (view full) ---

1069 if (class->lc_flags & LC_SLEEPLOCK) {
1070
1071 /*
1072 * Since spin locks include a critical section, this check
1073 * implicitly enforces a lock order of all sleep locks before
1074 * all spin locks.
1075 */
1076 if (td->td_critnest != 0 && !kdb_active)
1076 panic("blockable sleep lock (%s) %s @ %s:%d",
1077 kassert_panic("acquiring blockable sleep lock with "
1078 "spinlock or critical section held (%s) %s @ %s:%d",
1077 class->lc_name, lock->lo_name,
1078 fixup_filename(file), line);
1079
1080 /*
1081 * If this is the first lock acquired then just return as
1082 * no order checking is needed.
1083 */
1084 lock_list = td->td_sleeplocks;

--- 27 unchanged lines hidden (view full) ---

1112 if (lock1 != NULL) {
1113 if ((lock1->li_flags & LI_EXCLUSIVE) != 0 &&
1114 (flags & LOP_EXCLUSIVE) == 0) {
1115 printf("shared lock of (%s) %s @ %s:%d\n",
1116 class->lc_name, lock->lo_name,
1117 fixup_filename(file), line);
1118 printf("while exclusively locked from %s:%d\n",
1119 fixup_filename(lock1->li_file), lock1->li_line);
1079 class->lc_name, lock->lo_name,
1080 fixup_filename(file), line);
1081
1082 /*
1083 * If this is the first lock acquired then just return as
1084 * no order checking is needed.
1085 */
1086 lock_list = td->td_sleeplocks;

--- 27 unchanged lines hidden (view full) ---

1114 if (lock1 != NULL) {
1115 if ((lock1->li_flags & LI_EXCLUSIVE) != 0 &&
1116 (flags & LOP_EXCLUSIVE) == 0) {
1117 printf("shared lock of (%s) %s @ %s:%d\n",
1118 class->lc_name, lock->lo_name,
1119 fixup_filename(file), line);
1120 printf("while exclusively locked from %s:%d\n",
1121 fixup_filename(lock1->li_file), lock1->li_line);
1120 panic("share->excl");
1122 kassert_panic("share->excl");
1121 }
1122 if ((lock1->li_flags & LI_EXCLUSIVE) == 0 &&
1123 (flags & LOP_EXCLUSIVE) != 0) {
1124 printf("exclusive lock of (%s) %s @ %s:%d\n",
1125 class->lc_name, lock->lo_name,
1126 fixup_filename(file), line);
1127 printf("while share locked from %s:%d\n",
1128 fixup_filename(lock1->li_file), lock1->li_line);
1123 }
1124 if ((lock1->li_flags & LI_EXCLUSIVE) == 0 &&
1125 (flags & LOP_EXCLUSIVE) != 0) {
1126 printf("exclusive lock of (%s) %s @ %s:%d\n",
1127 class->lc_name, lock->lo_name,
1128 fixup_filename(file), line);
1129 printf("while share locked from %s:%d\n",
1130 fixup_filename(lock1->li_file), lock1->li_line);
1129 panic("excl->share");
1131 kassert_panic("excl->share");
1130 }
1131 return;
1132 }
1133
1134 /*
1135 * Find the previously acquired lock, but ignore interlocks.
1136 */
1137 plock = &lock_list->ll_children[lock_list->ll_count - 1];

--- 290 unchanged lines hidden (view full) ---

1428 struct lock_class *class;
1429
1430 KASSERT(witness_cold == 0, ("%s: witness_cold", __func__));
1431 if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL)
1432 return;
1433 class = LOCK_CLASS(lock);
1434 if (witness_watch) {
1435 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1132 }
1133 return;
1134 }
1135
1136 /*
1137 * Find the previously acquired lock, but ignore interlocks.
1138 */
1139 plock = &lock_list->ll_children[lock_list->ll_count - 1];

--- 290 unchanged lines hidden (view full) ---

1430 struct lock_class *class;
1431
1432 KASSERT(witness_cold == 0, ("%s: witness_cold", __func__));
1433 if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL)
1434 return;
1435 class = LOCK_CLASS(lock);
1436 if (witness_watch) {
1437 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1436 panic("upgrade of non-upgradable lock (%s) %s @ %s:%d",
1438 kassert_panic(
1439 "upgrade of non-upgradable lock (%s) %s @ %s:%d",
1437 class->lc_name, lock->lo_name,
1438 fixup_filename(file), line);
1439 if ((class->lc_flags & LC_SLEEPLOCK) == 0)
1440 class->lc_name, lock->lo_name,
1441 fixup_filename(file), line);
1442 if ((class->lc_flags & LC_SLEEPLOCK) == 0)
1440 panic("upgrade of non-sleep lock (%s) %s @ %s:%d",
1443 kassert_panic(
1444 "upgrade of non-sleep lock (%s) %s @ %s:%d",
1441 class->lc_name, lock->lo_name,
1442 fixup_filename(file), line);
1443 }
1444 instance = find_instance(curthread->td_sleeplocks, lock);
1445 if (instance == NULL)
1445 class->lc_name, lock->lo_name,
1446 fixup_filename(file), line);
1447 }
1448 instance = find_instance(curthread->td_sleeplocks, lock);
1449 if (instance == NULL)
1446 panic("upgrade of unlocked lock (%s) %s @ %s:%d",
1450 kassert_panic("upgrade of unlocked lock (%s) %s @ %s:%d",
1447 class->lc_name, lock->lo_name,
1448 fixup_filename(file), line);
1449 if (witness_watch) {
1450 if ((instance->li_flags & LI_EXCLUSIVE) != 0)
1451 class->lc_name, lock->lo_name,
1452 fixup_filename(file), line);
1453 if (witness_watch) {
1454 if ((instance->li_flags & LI_EXCLUSIVE) != 0)
1451 panic("upgrade of exclusive lock (%s) %s @ %s:%d",
1455 kassert_panic(
1456 "upgrade of exclusive lock (%s) %s @ %s:%d",
1452 class->lc_name, lock->lo_name,
1453 fixup_filename(file), line);
1454 if ((instance->li_flags & LI_RECURSEMASK) != 0)
1457 class->lc_name, lock->lo_name,
1458 fixup_filename(file), line);
1459 if ((instance->li_flags & LI_RECURSEMASK) != 0)
1455 panic("upgrade of recursed lock (%s) %s r=%d @ %s:%d",
1460 kassert_panic(
1461 "upgrade of recursed lock (%s) %s r=%d @ %s:%d",
1456 class->lc_name, lock->lo_name,
1457 instance->li_flags & LI_RECURSEMASK,
1458 fixup_filename(file), line);
1459 }
1460 instance->li_flags |= LI_EXCLUSIVE;
1461}
1462
1463void

--- 4 unchanged lines hidden (view full) ---

1468 struct lock_class *class;
1469
1470 KASSERT(witness_cold == 0, ("%s: witness_cold", __func__));
1471 if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL)
1472 return;
1473 class = LOCK_CLASS(lock);
1474 if (witness_watch) {
1475 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1462 class->lc_name, lock->lo_name,
1463 instance->li_flags & LI_RECURSEMASK,
1464 fixup_filename(file), line);
1465 }
1466 instance->li_flags |= LI_EXCLUSIVE;
1467}
1468
1469void

--- 4 unchanged lines hidden (view full) ---

1474 struct lock_class *class;
1475
1476 KASSERT(witness_cold == 0, ("%s: witness_cold", __func__));
1477 if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL)
1478 return;
1479 class = LOCK_CLASS(lock);
1480 if (witness_watch) {
1481 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1476 panic("downgrade of non-upgradable lock (%s) %s @ %s:%d",
1482 kassert_panic(
1483 "downgrade of non-upgradable lock (%s) %s @ %s:%d",
1477 class->lc_name, lock->lo_name,
1478 fixup_filename(file), line);
1479 if ((class->lc_flags & LC_SLEEPLOCK) == 0)
1484 class->lc_name, lock->lo_name,
1485 fixup_filename(file), line);
1486 if ((class->lc_flags & LC_SLEEPLOCK) == 0)
1480 panic("downgrade of non-sleep lock (%s) %s @ %s:%d",
1487 kassert_panic(
1488 "downgrade of non-sleep lock (%s) %s @ %s:%d",
1481 class->lc_name, lock->lo_name,
1482 fixup_filename(file), line);
1483 }
1484 instance = find_instance(curthread->td_sleeplocks, lock);
1485 if (instance == NULL)
1489 class->lc_name, lock->lo_name,
1490 fixup_filename(file), line);
1491 }
1492 instance = find_instance(curthread->td_sleeplocks, lock);
1493 if (instance == NULL)
1486 panic("downgrade of unlocked lock (%s) %s @ %s:%d",
1494 kassert_panic("downgrade of unlocked lock (%s) %s @ %s:%d",
1487 class->lc_name, lock->lo_name,
1488 fixup_filename(file), line);
1489 if (witness_watch) {
1490 if ((instance->li_flags & LI_EXCLUSIVE) == 0)
1495 class->lc_name, lock->lo_name,
1496 fixup_filename(file), line);
1497 if (witness_watch) {
1498 if ((instance->li_flags & LI_EXCLUSIVE) == 0)
1491 panic("downgrade of shared lock (%s) %s @ %s:%d",
1499 kassert_panic(
1500 "downgrade of shared lock (%s) %s @ %s:%d",
1492 class->lc_name, lock->lo_name,
1493 fixup_filename(file), line);
1494 if ((instance->li_flags & LI_RECURSEMASK) != 0)
1501 class->lc_name, lock->lo_name,
1502 fixup_filename(file), line);
1503 if ((instance->li_flags & LI_RECURSEMASK) != 0)
1495 panic("downgrade of recursed lock (%s) %s r=%d @ %s:%d",
1504 kassert_panic(
1505 "downgrade of recursed lock (%s) %s r=%d @ %s:%d",
1496 class->lc_name, lock->lo_name,
1497 instance->li_flags & LI_RECURSEMASK,
1498 fixup_filename(file), line);
1499 }
1500 instance->li_flags &= ~LI_EXCLUSIVE;
1501}
1502
1503void

--- 26 unchanged lines hidden (view full) ---

1530
1531 /*
1532 * When disabling WITNESS through witness_watch we could end up in
1533 * having registered locks in the td_sleeplocks queue.
1534 * We have to make sure we flush these queues, so just search for
1535 * eventual register locks and remove them.
1536 */
1537 if (witness_watch > 0)
1506 class->lc_name, lock->lo_name,
1507 instance->li_flags & LI_RECURSEMASK,
1508 fixup_filename(file), line);
1509 }
1510 instance->li_flags &= ~LI_EXCLUSIVE;
1511}
1512
1513void

--- 26 unchanged lines hidden (view full) ---

1540
1541 /*
1542 * When disabling WITNESS through witness_watch we could end up in
1543 * having registered locks in the td_sleeplocks queue.
1544 * We have to make sure we flush these queues, so just search for
1545 * eventual register locks and remove them.
1546 */
1547 if (witness_watch > 0)
1538 panic("lock (%s) %s not locked @ %s:%d", class->lc_name,
1548 kassert_panic("lock (%s) %s not locked @ %s:%d", class->lc_name,
1539 lock->lo_name, fixup_filename(file), line);
1540 else
1541 return;
1542found:
1543
1544 /* First, check for shared/exclusive mismatches. */
1545 if ((instance->li_flags & LI_EXCLUSIVE) != 0 && witness_watch > 0 &&
1546 (flags & LOP_EXCLUSIVE) == 0) {
1547 printf("shared unlock of (%s) %s @ %s:%d\n", class->lc_name,
1548 lock->lo_name, fixup_filename(file), line);
1549 printf("while exclusively locked from %s:%d\n",
1550 fixup_filename(instance->li_file), instance->li_line);
1549 lock->lo_name, fixup_filename(file), line);
1550 else
1551 return;
1552found:
1553
1554 /* First, check for shared/exclusive mismatches. */
1555 if ((instance->li_flags & LI_EXCLUSIVE) != 0 && witness_watch > 0 &&
1556 (flags & LOP_EXCLUSIVE) == 0) {
1557 printf("shared unlock of (%s) %s @ %s:%d\n", class->lc_name,
1558 lock->lo_name, fixup_filename(file), line);
1559 printf("while exclusively locked from %s:%d\n",
1560 fixup_filename(instance->li_file), instance->li_line);
1551 panic("excl->ushare");
1561 kassert_panic("excl->ushare");
1552 }
1553 if ((instance->li_flags & LI_EXCLUSIVE) == 0 && witness_watch > 0 &&
1554 (flags & LOP_EXCLUSIVE) != 0) {
1555 printf("exclusive unlock of (%s) %s @ %s:%d\n", class->lc_name,
1556 lock->lo_name, fixup_filename(file), line);
1557 printf("while share locked from %s:%d\n",
1558 fixup_filename(instance->li_file),
1559 instance->li_line);
1562 }
1563 if ((instance->li_flags & LI_EXCLUSIVE) == 0 && witness_watch > 0 &&
1564 (flags & LOP_EXCLUSIVE) != 0) {
1565 printf("exclusive unlock of (%s) %s @ %s:%d\n", class->lc_name,
1566 lock->lo_name, fixup_filename(file), line);
1567 printf("while share locked from %s:%d\n",
1568 fixup_filename(instance->li_file),
1569 instance->li_line);
1560 panic("share->uexcl");
1570 kassert_panic("share->uexcl");
1561 }
1562 /* If we are recursed, unrecurse. */
1563 if ((instance->li_flags & LI_RECURSEMASK) > 0) {
1564 CTR4(KTR_WITNESS, "%s: pid %d unrecursed on %s r=%d", __func__,
1565 td->td_proc->p_pid, instance->li_lock->lo_name,
1566 instance->li_flags);
1567 instance->li_flags--;
1568 return;
1569 }
1570 /* The lock is now being dropped, check for NORELEASE flag */
1571 if ((instance->li_flags & LI_NORELEASE) != 0 && witness_watch > 0) {
1572 printf("forbidden unlock of (%s) %s @ %s:%d\n", class->lc_name,
1573 lock->lo_name, fixup_filename(file), line);
1571 }
1572 /* If we are recursed, unrecurse. */
1573 if ((instance->li_flags & LI_RECURSEMASK) > 0) {
1574 CTR4(KTR_WITNESS, "%s: pid %d unrecursed on %s r=%d", __func__,
1575 td->td_proc->p_pid, instance->li_lock->lo_name,
1576 instance->li_flags);
1577 instance->li_flags--;
1578 return;
1579 }
1580 /* The lock is now being dropped, check for NORELEASE flag */
1581 if ((instance->li_flags & LI_NORELEASE) != 0 && witness_watch > 0) {
1582 printf("forbidden unlock of (%s) %s @ %s:%d\n", class->lc_name,
1583 lock->lo_name, fixup_filename(file), line);
1574 panic("lock marked norelease");
1584 kassert_panic("lock marked norelease");
1575 }
1576
1577 /* Otherwise, remove this item from the list. */
1578 s = intr_disable();
1579 CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__,
1580 td->td_proc->p_pid, instance->li_lock->lo_name,
1581 (*lock_list)->ll_count - 1);
1582 for (j = i; j < (*lock_list)->ll_count - 1; j++)

--- 38 unchanged lines hidden (view full) ---

1621 for (i = lle->ll_count - 1; i >= 0; i--) {
1622 if (n == 0)
1623 printf("Thread %p exiting with the following locks held:\n",
1624 td);
1625 n++;
1626 witness_list_lock(&lle->ll_children[i], printf);
1627
1628 }
1585 }
1586
1587 /* Otherwise, remove this item from the list. */
1588 s = intr_disable();
1589 CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__,
1590 td->td_proc->p_pid, instance->li_lock->lo_name,
1591 (*lock_list)->ll_count - 1);
1592 for (j = i; j < (*lock_list)->ll_count - 1; j++)

--- 38 unchanged lines hidden (view full) ---

1631 for (i = lle->ll_count - 1; i >= 0; i--) {
1632 if (n == 0)
1633 printf("Thread %p exiting with the following locks held:\n",
1634 td);
1635 n++;
1636 witness_list_lock(&lle->ll_children[i], printf);
1637
1638 }
1629 panic("Thread %p cannot exit while holding sleeplocks\n", td);
1639 kassert_panic(
1640 "Thread %p cannot exit while holding sleeplocks\n", td);
1630 }
1631 witness_lock_list_free(lle);
1632}
1633
1634/*
1635 * Warn if any locks other than 'lock' are held. Flags can be passed in to
1636 * exempt Giant and sleepable locks from the checks as well. If any
1637 * non-exempt locks are held, then a supplied message is printed to the

--- 64 unchanged lines hidden (view full) ---

1702 printf(" with the following");
1703 if (flags & WARN_SLEEPOK)
1704 printf(" non-sleepable");
1705 printf(" locks held:\n");
1706 n += witness_list_locks(&lock_list, printf);
1707 } else
1708 sched_unpin();
1709 if (flags & WARN_PANIC && n)
1641 }
1642 witness_lock_list_free(lle);
1643}
1644
1645/*
1646 * Warn if any locks other than 'lock' are held. Flags can be passed in to
1647 * exempt Giant and sleepable locks from the checks as well. If any
1648 * non-exempt locks are held, then a supplied message is printed to the

--- 64 unchanged lines hidden (view full) ---

1713 printf(" with the following");
1714 if (flags & WARN_SLEEPOK)
1715 printf(" non-sleepable");
1716 printf(" locks held:\n");
1717 n += witness_list_locks(&lock_list, printf);
1718 } else
1719 sched_unpin();
1720 if (flags & WARN_PANIC && n)
1710 panic("%s", __func__);
1721 kassert_panic("%s", __func__);
1711 else
1712 witness_debugger(n);
1713 return (n);
1714}
1715
1716const char *
1717witness_file(struct lock_object *lock)
1718{

--- 29 unchanged lines hidden (view full) ---

1748 if ((lock_class->lc_flags & LC_SPINLOCK)) {
1749 if (witness_skipspin)
1750 return (NULL);
1751 else
1752 typelist = &w_spin;
1753 } else if ((lock_class->lc_flags & LC_SLEEPLOCK))
1754 typelist = &w_sleep;
1755 else
1722 else
1723 witness_debugger(n);
1724 return (n);
1725}
1726
1727const char *
1728witness_file(struct lock_object *lock)
1729{

--- 29 unchanged lines hidden (view full) ---

1759 if ((lock_class->lc_flags & LC_SPINLOCK)) {
1760 if (witness_skipspin)
1761 return (NULL);
1762 else
1763 typelist = &w_spin;
1764 } else if ((lock_class->lc_flags & LC_SLEEPLOCK))
1765 typelist = &w_sleep;
1766 else
1756 panic("lock class %s is not sleep or spin",
1767 kassert_panic("lock class %s is not sleep or spin",
1757 lock_class->lc_name);
1758
1759 mtx_lock_spin(&w_mtx);
1760 w = witness_hash_get(description);
1761 if (w)
1762 goto found;
1763 if ((w = witness_get()) == NULL)
1764 return (NULL);

--- 14 unchanged lines hidden (view full) ---

1779 witness_hash_put(w);
1780 witness_increment_graph_generation();
1781 mtx_unlock_spin(&w_mtx);
1782 return (w);
1783found:
1784 w->w_refcount++;
1785 mtx_unlock_spin(&w_mtx);
1786 if (lock_class != w->w_class)
1768 lock_class->lc_name);
1769
1770 mtx_lock_spin(&w_mtx);
1771 w = witness_hash_get(description);
1772 if (w)
1773 goto found;
1774 if ((w = witness_get()) == NULL)
1775 return (NULL);

--- 14 unchanged lines hidden (view full) ---

1790 witness_hash_put(w);
1791 witness_increment_graph_generation();
1792 mtx_unlock_spin(&w_mtx);
1793 return (w);
1794found:
1795 w->w_refcount++;
1796 mtx_unlock_spin(&w_mtx);
1797 if (lock_class != w->w_class)
1787 panic(
1798 kassert_panic(
1788 "lock (%s) %s does not match earlier (%s) lock",
1789 description, lock_class->lc_name,
1790 w->w_class->lc_name);
1791 return (w);
1792}
1793
1794static void
1795depart(struct witness *w)

--- 117 unchanged lines hidden (view full) ---

1913
1914 MPASS(child != NULL && parent != NULL);
1915 if (witness_cold == 0)
1916 mtx_assert(&w_mtx, MA_OWNED);
1917
1918 if (!witness_lock_type_equal(parent, child)) {
1919 if (witness_cold == 0)
1920 mtx_unlock_spin(&w_mtx);
1799 "lock (%s) %s does not match earlier (%s) lock",
1800 description, lock_class->lc_name,
1801 w->w_class->lc_name);
1802 return (w);
1803}
1804
1805static void
1806depart(struct witness *w)

--- 117 unchanged lines hidden (view full) ---

1924
1925 MPASS(child != NULL && parent != NULL);
1926 if (witness_cold == 0)
1927 mtx_assert(&w_mtx, MA_OWNED);
1928
1929 if (!witness_lock_type_equal(parent, child)) {
1930 if (witness_cold == 0)
1931 mtx_unlock_spin(&w_mtx);
1921 panic("%s: parent \"%s\" (%s) and child \"%s\" (%s) are not "
1932 kassert_panic(
1933 "%s: parent \"%s\" (%s) and child \"%s\" (%s) are not "
1922 "the same lock type", __func__, parent->w_name,
1923 parent->w_class->lc_name, child->w_name,
1924 child->w_class->lc_name);
1925 }
1926 adopt(parent, child);
1927}
1928
1929/*

--- 257 unchanged lines hidden (view full) ---

2187 lock_list = curthread->td_sleeplocks;
2188 else {
2189 if (witness_skipspin)
2190 return;
2191 lock_list = PCPU_GET(spinlocks);
2192 }
2193 instance = find_instance(lock_list, lock);
2194 if (instance == NULL)
1934 "the same lock type", __func__, parent->w_name,
1935 parent->w_class->lc_name, child->w_name,
1936 child->w_class->lc_name);
1937 }
1938 adopt(parent, child);
1939}
1940
1941/*

--- 257 unchanged lines hidden (view full) ---

2199 lock_list = curthread->td_sleeplocks;
2200 else {
2201 if (witness_skipspin)
2202 return;
2203 lock_list = PCPU_GET(spinlocks);
2204 }
2205 instance = find_instance(lock_list, lock);
2206 if (instance == NULL)
2195 panic("%s: lock (%s) %s not locked", __func__,
2207 kassert_panic("%s: lock (%s) %s not locked", __func__,
2196 class->lc_name, lock->lo_name);
2197 *filep = instance->li_file;
2198 *linep = instance->li_line;
2199}
2200
2201void
2202witness_restore(struct lock_object *lock, const char *file, int line)
2203{

--- 16 unchanged lines hidden (view full) ---

2220 lock_list = curthread->td_sleeplocks;
2221 else {
2222 if (witness_skipspin)
2223 return;
2224 lock_list = PCPU_GET(spinlocks);
2225 }
2226 instance = find_instance(lock_list, lock);
2227 if (instance == NULL)
2208 class->lc_name, lock->lo_name);
2209 *filep = instance->li_file;
2210 *linep = instance->li_line;
2211}
2212
2213void
2214witness_restore(struct lock_object *lock, const char *file, int line)
2215{

--- 16 unchanged lines hidden (view full) ---

2232 lock_list = curthread->td_sleeplocks;
2233 else {
2234 if (witness_skipspin)
2235 return;
2236 lock_list = PCPU_GET(spinlocks);
2237 }
2238 instance = find_instance(lock_list, lock);
2239 if (instance == NULL)
2228 panic("%s: lock (%s) %s not locked", __func__,
2240 kassert_panic("%s: lock (%s) %s not locked", __func__,
2229 class->lc_name, lock->lo_name);
2230 lock->lo_witness->w_file = file;
2231 lock->lo_witness->w_line = line;
2232 instance->li_file = file;
2233 instance->li_line = line;
2234}
2235
2236void

--- 7 unchanged lines hidden (view full) ---

2244 if (lock->lo_witness == NULL || witness_watch < 1 || panicstr != NULL)
2245 return;
2246 class = LOCK_CLASS(lock);
2247 if ((class->lc_flags & LC_SLEEPLOCK) != 0)
2248 instance = find_instance(curthread->td_sleeplocks, lock);
2249 else if ((class->lc_flags & LC_SPINLOCK) != 0)
2250 instance = find_instance(PCPU_GET(spinlocks), lock);
2251 else {
2241 class->lc_name, lock->lo_name);
2242 lock->lo_witness->w_file = file;
2243 lock->lo_witness->w_line = line;
2244 instance->li_file = file;
2245 instance->li_line = line;
2246}
2247
2248void

--- 7 unchanged lines hidden (view full) ---

2256 if (lock->lo_witness == NULL || witness_watch < 1 || panicstr != NULL)
2257 return;
2258 class = LOCK_CLASS(lock);
2259 if ((class->lc_flags & LC_SLEEPLOCK) != 0)
2260 instance = find_instance(curthread->td_sleeplocks, lock);
2261 else if ((class->lc_flags & LC_SPINLOCK) != 0)
2262 instance = find_instance(PCPU_GET(spinlocks), lock);
2263 else {
2252 panic("Lock (%s) %s is not sleep or spin!",
2264 kassert_panic("Lock (%s) %s is not sleep or spin!",
2253 class->lc_name, lock->lo_name);
2254 }
2255 switch (flags) {
2256 case LA_UNLOCKED:
2257 if (instance != NULL)
2265 class->lc_name, lock->lo_name);
2266 }
2267 switch (flags) {
2268 case LA_UNLOCKED:
2269 if (instance != NULL)
2258 panic("Lock (%s) %s locked @ %s:%d.",
2270 kassert_panic("Lock (%s) %s locked @ %s:%d.",
2259 class->lc_name, lock->lo_name,
2260 fixup_filename(file), line);
2261 break;
2262 case LA_LOCKED:
2263 case LA_LOCKED | LA_RECURSED:
2264 case LA_LOCKED | LA_NOTRECURSED:
2265 case LA_SLOCKED:
2266 case LA_SLOCKED | LA_RECURSED:
2267 case LA_SLOCKED | LA_NOTRECURSED:
2268 case LA_XLOCKED:
2269 case LA_XLOCKED | LA_RECURSED:
2270 case LA_XLOCKED | LA_NOTRECURSED:
2271 if (instance == NULL) {
2271 class->lc_name, lock->lo_name,
2272 fixup_filename(file), line);
2273 break;
2274 case LA_LOCKED:
2275 case LA_LOCKED | LA_RECURSED:
2276 case LA_LOCKED | LA_NOTRECURSED:
2277 case LA_SLOCKED:
2278 case LA_SLOCKED | LA_RECURSED:
2279 case LA_SLOCKED | LA_NOTRECURSED:
2280 case LA_XLOCKED:
2281 case LA_XLOCKED | LA_RECURSED:
2282 case LA_XLOCKED | LA_NOTRECURSED:
2283 if (instance == NULL) {
2272 panic("Lock (%s) %s not locked @ %s:%d.",
2284 kassert_panic("Lock (%s) %s not locked @ %s:%d.",
2273 class->lc_name, lock->lo_name,
2274 fixup_filename(file), line);
2275 break;
2276 }
2277 if ((flags & LA_XLOCKED) != 0 &&
2278 (instance->li_flags & LI_EXCLUSIVE) == 0)
2285 class->lc_name, lock->lo_name,
2286 fixup_filename(file), line);
2287 break;
2288 }
2289 if ((flags & LA_XLOCKED) != 0 &&
2290 (instance->li_flags & LI_EXCLUSIVE) == 0)
2279 panic("Lock (%s) %s not exclusively locked @ %s:%d.",
2291 kassert_panic(
2292 "Lock (%s) %s not exclusively locked @ %s:%d.",
2280 class->lc_name, lock->lo_name,
2281 fixup_filename(file), line);
2282 if ((flags & LA_SLOCKED) != 0 &&
2283 (instance->li_flags & LI_EXCLUSIVE) != 0)
2293 class->lc_name, lock->lo_name,
2294 fixup_filename(file), line);
2295 if ((flags & LA_SLOCKED) != 0 &&
2296 (instance->li_flags & LI_EXCLUSIVE) != 0)
2284 panic("Lock (%s) %s exclusively locked @ %s:%d.",
2297 kassert_panic(
2298 "Lock (%s) %s exclusively locked @ %s:%d.",
2285 class->lc_name, lock->lo_name,
2286 fixup_filename(file), line);
2287 if ((flags & LA_RECURSED) != 0 &&
2288 (instance->li_flags & LI_RECURSEMASK) == 0)
2299 class->lc_name, lock->lo_name,
2300 fixup_filename(file), line);
2301 if ((flags & LA_RECURSED) != 0 &&
2302 (instance->li_flags & LI_RECURSEMASK) == 0)
2289 panic("Lock (%s) %s not recursed @ %s:%d.",
2303 kassert_panic("Lock (%s) %s not recursed @ %s:%d.",
2290 class->lc_name, lock->lo_name,
2291 fixup_filename(file), line);
2292 if ((flags & LA_NOTRECURSED) != 0 &&
2293 (instance->li_flags & LI_RECURSEMASK) != 0)
2304 class->lc_name, lock->lo_name,
2305 fixup_filename(file), line);
2306 if ((flags & LA_NOTRECURSED) != 0 &&
2307 (instance->li_flags & LI_RECURSEMASK) != 0)
2294 panic("Lock (%s) %s recursed @ %s:%d.",
2308 kassert_panic("Lock (%s) %s recursed @ %s:%d.",
2295 class->lc_name, lock->lo_name,
2296 fixup_filename(file), line);
2297 break;
2298 default:
2309 class->lc_name, lock->lo_name,
2310 fixup_filename(file), line);
2311 break;
2312 default:
2299 panic("Invalid lock assertion at %s:%d.",
2313 kassert_panic("Invalid lock assertion at %s:%d.",
2300 fixup_filename(file), line);
2301
2302 }
2303#endif /* INVARIANT_SUPPORT */
2304}
2305
2306static void
2307witness_setflag(struct lock_object *lock, int flag, int set)

--- 9 unchanged lines hidden (view full) ---

2317 lock_list = curthread->td_sleeplocks;
2318 else {
2319 if (witness_skipspin)
2320 return;
2321 lock_list = PCPU_GET(spinlocks);
2322 }
2323 instance = find_instance(lock_list, lock);
2324 if (instance == NULL)
2314 fixup_filename(file), line);
2315
2316 }
2317#endif /* INVARIANT_SUPPORT */
2318}
2319
2320static void
2321witness_setflag(struct lock_object *lock, int flag, int set)

--- 9 unchanged lines hidden (view full) ---

2331 lock_list = curthread->td_sleeplocks;
2332 else {
2333 if (witness_skipspin)
2334 return;
2335 lock_list = PCPU_GET(spinlocks);
2336 }
2337 instance = find_instance(lock_list, lock);
2338 if (instance == NULL)
2325 panic("%s: lock (%s) %s not locked", __func__,
2339 kassert_panic("%s: lock (%s) %s not locked", __func__,
2326 class->lc_name, lock->lo_name);
2327
2328 if (set)
2329 instance->li_flags |= flag;
2330 else
2331 instance->li_flags &= ~flag;
2332}
2333

--- 519 unchanged lines hidden --- 		2340 class->lc_name, lock->lo_name);
2341
2342 if (set)
2343 instance->li_flags |= flag;
2344 else
2345 instance->li_flags &= ~flag;
2346}
2347

--- 519 unchanged lines hidden ---