1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
| 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
|
30 * $FreeBSD: head/sys/kern/subr_witness.c 71709 2001-01-27 07:51:34Z jhb $
| 30 * $FreeBSD: head/sys/kern/subr_witness.c 72200 2001-02-09 06:11:45Z bmilekic $
|
31 */ 32 33/*
| 31 */ 32 33/*
|
| 34 * Machine independent bits of mutex implementation and implementation of 35 * `witness' structure & related debugging routines. 36 */ 37 38/*
|
34 * Main Entry: witness 35 * Pronunciation: 'wit-n&s 36 * Function: noun 37 * Etymology: Middle English witnesse, from Old English witnes knowledge, 38 * testimony, witness, from 2wit 39 * Date: before 12th century 40 * 1 : attestation of a fact or event : TESTIMONY 41 * 2 : one that gives evidence; specifically : one who testifies in 42 * a cause or before a judicial tribunal 43 * 3 : one asked to be present at a transaction so as to be able to 44 * testify to its having taken place 45 * 4 : one who has personal knowledge of something 46 * 5 a : something serving as evidence or proof : SIGN 47 * b : public affirmation by word or example of usually 48 * religious faith or conviction <the heroic witness to divine 49 * life -- Pilot> 50 * 6 capitalized : a member of the Jehovah's Witnesses 51 */ 52 53#include "opt_ddb.h" 54#include "opt_witness.h" 55
| 39 * Main Entry: witness 40 * Pronunciation: 'wit-n&s 41 * Function: noun 42 * Etymology: Middle English witnesse, from Old English witnes knowledge, 43 * testimony, witness, from 2wit 44 * Date: before 12th century 45 * 1 : attestation of a fact or event : TESTIMONY 46 * 2 : one that gives evidence; specifically : one who testifies in 47 * a cause or before a judicial tribunal 48 * 3 : one asked to be present at a transaction so as to be able to 49 * testify to its having taken place 50 * 4 : one who has personal knowledge of something 51 * 5 a : something serving as evidence or proof : SIGN 52 * b : public affirmation by word or example of usually 53 * religious faith or conviction <the heroic witness to divine 54 * life -- Pilot> 55 * 6 capitalized : a member of the Jehovah's Witnesses 56 */ 57 58#include "opt_ddb.h" 59#include "opt_witness.h" 60
|
56/* 57 * Cause non-inlined mtx_*() to be compiled. 58 * Must be defined early because other system headers may include mutex.h. 59 */ 60#define _KERN_MUTEX_C_ 61
| |
62#include <sys/param.h> 63#include <sys/bus.h> 64#include <sys/kernel.h> 65#include <sys/malloc.h> 66#include <sys/proc.h> 67#include <sys/sysctl.h> 68#include <sys/systm.h> 69#include <sys/vmmeter.h> 70#include <sys/ktr.h> 71 72#include <machine/atomic.h> 73#include <machine/bus.h> 74#include <machine/clock.h> 75#include <machine/cpu.h> 76 77#include <ddb/ddb.h> 78 79#include <vm/vm.h> 80#include <vm/vm_extern.h> 81 82#include <sys/mutex.h> 83 84/*
| 61#include <sys/param.h> 62#include <sys/bus.h> 63#include <sys/kernel.h> 64#include <sys/malloc.h> 65#include <sys/proc.h> 66#include <sys/sysctl.h> 67#include <sys/systm.h> 68#include <sys/vmmeter.h> 69#include <sys/ktr.h> 70 71#include <machine/atomic.h> 72#include <machine/bus.h> 73#include <machine/clock.h> 74#include <machine/cpu.h> 75 76#include <ddb/ddb.h> 77 78#include <vm/vm.h> 79#include <vm/vm_extern.h> 80 81#include <sys/mutex.h> 82 83/*
|
85 * Machine independent bits of the mutex implementation
| 84 * The WITNESS-enabled mutex debug structure.
|
86 */
| 85 */
|
87
| |
88#ifdef WITNESS 89struct mtx_debug { 90 struct witness *mtxd_witness; 91 LIST_ENTRY(mtx) mtxd_held; 92 const char *mtxd_file; 93 int mtxd_line; 94}; 95 96#define mtx_held mtx_debug->mtxd_held 97#define mtx_file mtx_debug->mtxd_file 98#define mtx_line mtx_debug->mtxd_line 99#define mtx_witness mtx_debug->mtxd_witness 100#endif /* WITNESS */ 101 102/*
| 86#ifdef WITNESS 87struct mtx_debug { 88 struct witness *mtxd_witness; 89 LIST_ENTRY(mtx) mtxd_held; 90 const char *mtxd_file; 91 int mtxd_line; 92}; 93 94#define mtx_held mtx_debug->mtxd_held 95#define mtx_file mtx_debug->mtxd_file 96#define mtx_line mtx_debug->mtxd_line 97#define mtx_witness mtx_debug->mtxd_witness 98#endif /* WITNESS */ 99 100/*
|
103 * Assembly macros 104 *------------------------------------------------------------------------------
| 101 * Internal utility macros.
|
105 */
| 102 */
|
| 103#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED)
|
106
| 104
|
107#define _V(x) __STRING(x)
| 105#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 106 : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK))
|
108
| 107
|
109/* 110 * Default, unoptimized mutex micro-operations 111 */
| 108#define RETIP(x) *(((uintptr_t *)(&x)) - 1) 109#define SET_PRIO(p, pri) (p)->p_priority = (pri)
|
112
| 110
|
113#ifndef _obtain_lock 114/* Actually obtain mtx_lock */ 115#define _obtain_lock(mp, tid) \ 116 atomic_cmpset_acq_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED, (tid)) 117#endif 118 119#ifndef _release_lock 120/* Actually release mtx_lock */ 121#define _release_lock(mp, tid) \ 122 atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), (void *)MTX_UNOWNED) 123#endif 124 125#ifndef _release_lock_quick 126/* Actually release mtx_lock quickly assuming that we own it */ 127#define _release_lock_quick(mp) \ 128 atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED) 129#endif 130 131#ifndef _getlock_sleep 132/* Get a sleep lock, deal with recursion inline. */ 133#define _getlock_sleep(mp, tid, type) do { \ 134 if (!_obtain_lock(mp, tid)) { \ 135 if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\ 136 mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \ 137 else { \ 138 atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED); \ 139 (mp)->mtx_recurse++; \ 140 } \ 141 } \ 142} while (0) 143#endif 144 145#ifndef _getlock_spin_block 146/* Get a spin lock, handle recursion inline (as the less common case) */ 147#define _getlock_spin_block(mp, tid, type) do { \ 148 u_int _mtx_intr = save_intr(); \ 149 disable_intr(); \ 150 if (!_obtain_lock(mp, tid)) \ 151 mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr); \ 152 else \ 153 (mp)->mtx_saveintr = _mtx_intr; \ 154} while (0) 155#endif 156 157#ifndef _getlock_norecurse
| |
158/*
| 111/*
|
159 * Get a lock without any recursion handling. Calls the hard enter function if 160 * we can't get it inline.
| 112 * Early WITNESS-enabled declarations.
|
161 */
| 113 */
|
162#define _getlock_norecurse(mp, tid, type) do { \ 163 if (!_obtain_lock(mp, tid)) \ 164 mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0); \ 165} while (0) 166#endif
| 114#ifdef WITNESS
|
167
| 115
|
168#ifndef _exitlock_norecurse
| |
169/*
| 116/*
|
170 * Release a sleep lock assuming we haven't recursed on it, recursion is handled 171 * in the hard function. 172 */ 173#define _exitlock_norecurse(mp, tid, type) do { \ 174 if (!_release_lock(mp, tid)) \ 175 mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ 176} while (0) 177#endif
| 117 * Internal WITNESS routines which must be prototyped early. 118 * 119 * XXX: When/if witness code is cleaned up, it would be wise to place all 120 * witness prototyping early in this file. 121 */ 122static void witness_init(struct mtx *, int flag); 123static void witness_destroy(struct mtx *); 124static void witness_display(void(*)(const char *fmt, ...));
|
178
| 125
|
179#ifndef _exitlock 180/* 181 * Release a sleep lock when its likely we recursed (the code to 182 * deal with simple recursion is inline). 183 */ 184#define _exitlock(mp, tid, type) do { \ 185 if (!_release_lock(mp, tid)) { \ 186 if ((mp)->mtx_lock & MTX_RECURSED) { \ 187 if (--((mp)->mtx_recurse) == 0) \ 188 atomic_clear_ptr(&(mp)->mtx_lock, \ 189 MTX_RECURSED); \ 190 } else { \ 191 mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ 192 } \ 193 } \ 194} while (0) 195#endif
| 126MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure");
|
196
| 127
|
197#ifndef _exitlock_spin 198/* Release a spin lock (with possible recursion). */ 199#define _exitlock_spin(mp) do { \ 200 if (!mtx_recursed((mp))) { \ 201 int _mtx_intr = (mp)->mtx_saveintr; \ 202 \ 203 _release_lock_quick(mp); \ 204 restore_intr(_mtx_intr); \ 205 } else { \ 206 (mp)->mtx_recurse--; \ 207 } \ 208} while (0) 209#endif 210 211#ifdef WITNESS 212static void witness_init(struct mtx *, int flag); 213static void witness_destroy(struct mtx *); 214static void witness_display(void(*)(const char *fmt, ...)); 215
| |
216/* All mutexes in system (used for debug/panic) */ 217static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
| 128/* All mutexes in system (used for debug/panic) */ 129static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
|
| 130
|
218/*
| 131/*
|
219 * Set to 0 once mutexes have been fully initialized so that witness code can be 220 * safely executed.
| 132 * This global is set to 0 once it becomes safe to use the witness code.
|
221 */ 222static int witness_cold = 1;
| 133 */ 134static int witness_cold = 1;
|
| 135
|
223#else /* WITNESS */ 224
| 136#else /* WITNESS */ 137
|
225/* 226 * flag++ is slezoid way of shutting up unused parameter warning 227 * in mtx_init()
| 138/* XXX XXX XXX 139 * flag++ is sleazoid way of shuting up warning
|
228 */ 229#define witness_init(m, flag) flag++ 230#define witness_destroy(m) 231#define witness_try_enter(m, t, f, l) 232#endif /* WITNESS */ 233
| 140 */ 141#define witness_init(m, flag) flag++ 142#define witness_destroy(m) 143#define witness_try_enter(m, t, f, l) 144#endif /* WITNESS */ 145
|
234/* All mutexes in system (used for debug/panic) */
| 146/* 147 * All mutex locks in system are kept on the all_mtx list. 148 */
|
235static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", 236 TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), 237 { NULL, NULL }, &all_mtx, &all_mtx, 238#ifdef WITNESS 239 &all_mtx_debug 240#else 241 NULL 242#endif 243 }; 244
| 149static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", 150 TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), 151 { NULL, NULL }, &all_mtx, &all_mtx, 152#ifdef WITNESS 153 &all_mtx_debug 154#else 155 NULL 156#endif 157 }; 158
|
| 159/* 160 * Global variables for book keeping. 161 */
|
245static int mtx_cur_cnt; 246static int mtx_max_cnt; 247
| 162static int mtx_cur_cnt; 163static int mtx_max_cnt; 164
|
| 165/* 166 * Prototypes for non-exported routines. 167 * 168 * NOTE: Prototypes for witness routines are placed at the bottom of the file. 169 */
|
248static void propagate_priority(struct proc *);
| 170static void propagate_priority(struct proc *);
|
249static void mtx_enter_hard(struct mtx *, int type, int saveintr); 250static void mtx_exit_hard(struct mtx *, int type);
| |
251
| 171
|
252#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 253#define mtx_owner(m) (mtx_unowned(m) ? NULL \ 254 : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) 255 256#define RETIP(x) *(((uintptr_t *)(&x)) - 1) 257#define SET_PRIO(p, pri) (p)->p_priority = (pri) 258
| |
259static void 260propagate_priority(struct proc *p) 261{ 262 int pri = p->p_priority; 263 struct mtx *m = p->p_blocked; 264 265 mtx_assert(&sched_lock, MA_OWNED); 266 for (;;) { 267 struct proc *p1; 268 269 p = mtx_owner(m); 270 271 if (p == NULL) { 272 /* 273 * This really isn't quite right. Really 274 * ought to bump priority of process that 275 * next acquires the mutex. 276 */ 277 MPASS(m->mtx_lock == MTX_CONTESTED); 278 return; 279 }
| 172static void 173propagate_priority(struct proc *p) 174{ 175 int pri = p->p_priority; 176 struct mtx *m = p->p_blocked; 177 178 mtx_assert(&sched_lock, MA_OWNED); 179 for (;;) { 180 struct proc *p1; 181 182 p = mtx_owner(m); 183 184 if (p == NULL) { 185 /* 186 * This really isn't quite right. Really 187 * ought to bump priority of process that 188 * next acquires the mutex. 189 */ 190 MPASS(m->mtx_lock == MTX_CONTESTED); 191 return; 192 }
|
| 193
|
280 MPASS(p->p_magic == P_MAGIC); 281 KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 282 if (p->p_priority <= pri) 283 return; 284 285 /* 286 * Bump this process' priority. 287 */ 288 SET_PRIO(p, pri); 289 290 /* 291 * If lock holder is actually running, just bump priority. 292 */ 293#ifdef SMP 294 /* 295 * For SMP, we can check the p_oncpu field to see if we are 296 * running. 297 */ 298 if (p->p_oncpu != 0xff) { 299 MPASS(p->p_stat == SRUN || p->p_stat == SZOMB); 300 return; 301 } 302#else 303 /* 304 * For UP, we check to see if p is curproc (this shouldn't 305 * ever happen however as it would mean we are in a deadlock.) 306 */ 307 if (p == curproc) { 308 panic("Deadlock detected"); 309 return; 310 } 311#endif 312 /* 313 * If on run queue move to new run queue, and 314 * quit. 315 */ 316 if (p->p_stat == SRUN) {
| 194 MPASS(p->p_magic == P_MAGIC); 195 KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 196 if (p->p_priority <= pri) 197 return; 198 199 /* 200 * Bump this process' priority. 201 */ 202 SET_PRIO(p, pri); 203 204 /* 205 * If lock holder is actually running, just bump priority. 206 */ 207#ifdef SMP 208 /* 209 * For SMP, we can check the p_oncpu field to see if we are 210 * running. 211 */ 212 if (p->p_oncpu != 0xff) { 213 MPASS(p->p_stat == SRUN || p->p_stat == SZOMB); 214 return; 215 } 216#else 217 /* 218 * For UP, we check to see if p is curproc (this shouldn't 219 * ever happen however as it would mean we are in a deadlock.) 220 */ 221 if (p == curproc) { 222 panic("Deadlock detected"); 223 return; 224 } 225#endif 226 /* 227 * If on run queue move to new run queue, and 228 * quit. 229 */ 230 if (p->p_stat == SRUN) {
|
317 printf("XXX: moving process %d(%s) to a new run queue\n",
| 231 printf("XXX: moving proc %d(%s) to a new run queue\n",
|
318 p->p_pid, p->p_comm); 319 MPASS(p->p_blocked == NULL); 320 remrunqueue(p); 321 setrunqueue(p); 322 return; 323 } 324 325 /* 326 * If we aren't blocked on a mutex, we should be. 327 */ 328 KASSERT(p->p_stat == SMTX, ( 329 "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 330 p->p_pid, p->p_comm, p->p_stat, 331 m->mtx_description)); 332 333 /* 334 * Pick up the mutex that p is blocked on. 335 */ 336 m = p->p_blocked; 337 MPASS(m != NULL); 338 339 printf("XXX: process %d(%s) is blocked on %s\n", p->p_pid, 340 p->p_comm, m->mtx_description);
| 232 p->p_pid, p->p_comm); 233 MPASS(p->p_blocked == NULL); 234 remrunqueue(p); 235 setrunqueue(p); 236 return; 237 } 238 239 /* 240 * If we aren't blocked on a mutex, we should be. 241 */ 242 KASSERT(p->p_stat == SMTX, ( 243 "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 244 p->p_pid, p->p_comm, p->p_stat, 245 m->mtx_description)); 246 247 /* 248 * Pick up the mutex that p is blocked on. 249 */ 250 m = p->p_blocked; 251 MPASS(m != NULL); 252 253 printf("XXX: process %d(%s) is blocked on %s\n", p->p_pid, 254 p->p_comm, m->mtx_description);
|
| 255
|
341 /* 342 * Check if the proc needs to be moved up on 343 * the blocked chain 344 */ 345 if (p == TAILQ_FIRST(&m->mtx_blocked)) { 346 printf("XXX: process at head of run queue\n"); 347 continue; 348 }
| 256 /* 257 * Check if the proc needs to be moved up on 258 * the blocked chain 259 */ 260 if (p == TAILQ_FIRST(&m->mtx_blocked)) { 261 printf("XXX: process at head of run queue\n"); 262 continue; 263 }
|
| 264
|
349 p1 = TAILQ_PREV(p, rq, p_procq); 350 if (p1->p_priority <= pri) { 351 printf(
| 265 p1 = TAILQ_PREV(p, rq, p_procq); 266 if (p1->p_priority <= pri) { 267 printf(
|
352 "XXX: previous process %d(%s) has higher priority\n",
| 268 "XXX: previous process %d(%s) has higher priority\n",
|
353 p->p_pid, p->p_comm); 354 continue; 355 } 356 357 /* 358 * Remove proc from blocked chain and determine where 359 * it should be moved up to. Since we know that p1 has 360 * a lower priority than p, we know that at least one 361 * process in the chain has a lower priority and that 362 * p1 will thus not be NULL after the loop. 363 */ 364 TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 365 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 366 MPASS(p1->p_magic == P_MAGIC); 367 if (p1->p_priority > pri) 368 break; 369 }
| 269 p->p_pid, p->p_comm); 270 continue; 271 } 272 273 /* 274 * Remove proc from blocked chain and determine where 275 * it should be moved up to. Since we know that p1 has 276 * a lower priority than p, we know that at least one 277 * process in the chain has a lower priority and that 278 * p1 will thus not be NULL after the loop. 279 */ 280 TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 281 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 282 MPASS(p1->p_magic == P_MAGIC); 283 if (p1->p_priority > pri) 284 break; 285 }
|
| 286
|
370 MPASS(p1 != NULL); 371 TAILQ_INSERT_BEFORE(p1, p, p_procq); 372 CTR4(KTR_LOCK, 373 "propagate_priority: p %p moved before %p on [%p] %s", 374 p, p1, m, m->mtx_description); 375 } 376} 377 378/*
| 287 MPASS(p1 != NULL); 288 TAILQ_INSERT_BEFORE(p1, p, p_procq); 289 CTR4(KTR_LOCK, 290 "propagate_priority: p %p moved before %p on [%p] %s", 291 p, p1, m, m->mtx_description); 292 } 293} 294 295/*
|
379 * Get lock 'm', the macro handles the easy (and most common cases) and leaves 380 * the slow stuff to the mtx_enter_hard() function. 381 * 382 * Note: since type is usually a constant much of this code is optimized out.
| 296 * The important part of mtx_trylock{,_flags}() 297 * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 298 * if we're called, it's because we know we don't already own this lock.
|
383 */
| 299 */
|
384void 385_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
| 300int 301_mtx_trylock(struct mtx *m, int opts, const char *file, int line)
|
386{
| 302{
|
387 struct mtx *mpp = mtxp;
| 303 int rval;
|
388
| 304
|
389 /* bits only valid on mtx_exit() */ 390 MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0, 391 STR_mtx_bad_type, file, line);
| 305 KASSERT(CURPROC != NULL, ("curproc is NULL in _mtx_trylock"));
|
392
| 306
|
393 if ((type) & MTX_SPIN) {
| 307 /* 308 * _mtx_trylock does not accept MTX_NOSWITCH option. 309 */ 310 MPASS((opts & MTX_NOSWITCH) == 0); 311 312 rval = _obtain_lock(m, CURTHD); 313 314#ifdef WITNESS 315 if (rval && m->mtx_witness != NULL) {
|
394 /*
| 316 /*
|
395 * Easy cases of spin locks: 396 * 397 * 1) We already own the lock and will simply recurse on it (if 398 * RLIKELY) 399 * 400 * 2) The lock is free, we just get it
| 317 * We do not handle recursion in _mtx_trylock; see the 318 * note at the top of the routine.
|
401 */
| 319 */
|
402 if ((type) & MTX_RLIKELY) { 403 /* 404 * Check for recursion, if we already have this 405 * lock we just bump the recursion count. 406 */ 407 if (mpp->mtx_lock == (uintptr_t)CURTHD) { 408 mpp->mtx_recurse++; 409 goto done; 410 } 411 } 412 413 if (((type) & MTX_TOPHALF) == 0) { 414 /* 415 * If an interrupt thread uses this we must block 416 * interrupts here. 417 */ 418 if ((type) & MTX_FIRST) { 419 ASS_IEN; 420 disable_intr(); 421 _getlock_norecurse(mpp, CURTHD, 422 (type) & MTX_HARDOPTS); 423 } else { 424 _getlock_spin_block(mpp, CURTHD, 425 (type) & MTX_HARDOPTS); 426 } 427 } else 428 _getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS); 429 } else { 430 /* Sleep locks */ 431 if ((type) & MTX_RLIKELY) 432 _getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS); 433 else 434 _getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
| 320 MPASS(!mtx_recursed(m)); 321 witness_try_enter(m, (opts | m->mtx_flags), file, line);
|
435 }
| 322 }
|
436done: 437 WITNESS_ENTER(mpp, type, file, line); 438 if (((type) & MTX_QUIET) == 0) 439 CTR5(KTR_LOCK, STR_mtx_enter_fmt, 440 mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
| 323#endif /* WITNESS */
|
441
| 324
|
| 325 if ((opts & MTX_QUIET) == 0) 326 CTR5(KTR_LOCK, "TRY_ENTER %s [%p] result=%d at %s:%d", 327 m->mtx_description, m, rval, file, line); 328 329 return rval;
|
442} 443 444/*
| 330} 331 332/*
|
445 * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
| 333 * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
|
446 *
| 334 *
|
447 * XXX DOES NOT HANDLE RECURSION
| 335 * We call this if the lock is either contested (i.e. we need to go to 336 * sleep waiting for it), or if we need to recurse on it.
|
448 */
| 337 */
|
449int 450_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
| 338void 339_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
|
451{
| 340{
|
452 struct mtx *const mpp = mtxp; 453 int rval;
| 341 struct proc *p = CURPROC;
|
454
| 342
|
455 rval = _obtain_lock(mpp, CURTHD); 456#ifdef WITNESS 457 if (rval && mpp->mtx_witness != NULL) { 458 MPASS(mpp->mtx_recurse == 0); 459 witness_try_enter(mpp, type, file, line);
| 343 if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 344 m->mtx_recurse++; 345 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 346 if ((opts & MTX_QUIET) == 0) 347 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recurse", m); 348 return;
|
460 }
| 349 }
|
461#endif /* WITNESS */ 462 if (((type) & MTX_QUIET) == 0) 463 CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, 464 mpp->mtx_description, mpp, rval, file, line);
| |
465
| 350
|
466 return rval; 467}
| 351 if ((opts & MTX_QUIET) == 0) 352 CTR3(KTR_LOCK, "mtx_lock: %p contested (lock=%p) [%p]", m, 353 (void *)m->mtx_lock, (void *)RETIP(m));
|
468
| 354
|
469/* 470 * Release lock m. 471 */ 472void 473_mtx_exit(struct mtx *mtxp, int type, const char *file, int line) 474{ 475 struct mtx *const mpp = mtxp;
| 355 /* 356 * Save our priority. Even though p_nativepri is protected by 357 * sched_lock, we don't obtain it here as it can be expensive. 358 * Since this is the only place p_nativepri is set, and since two 359 * CPUs will not be executing the same process concurrently, we know 360 * that no other CPU is going to be messing with this. Also, 361 * p_nativepri is only read when we are blocked on a mutex, so that 362 * can't be happening right now either. 363 */ 364 p->p_nativepri = p->p_priority;
|
476
| 365
|
477 MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line); 478 WITNESS_EXIT(mpp, type, file, line); 479 if (((type) & MTX_QUIET) == 0) 480 CTR5(KTR_LOCK, STR_mtx_exit_fmt, 481 mpp->mtx_description, mpp, mpp->mtx_recurse, file, line); 482 if ((type) & MTX_SPIN) { 483 if ((type) & MTX_NORECURSE) { 484 int mtx_intr = mpp->mtx_saveintr;
| 366 while (!_obtain_lock(m, p)) { 367 uintptr_t v; 368 struct proc *p1;
|
485
| 369
|
486 MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse, 487 file, line); 488 _release_lock_quick(mpp); 489 if (((type) & MTX_TOPHALF) == 0) { 490 if ((type) & MTX_FIRST) { 491 ASS_IDIS; 492 enable_intr(); 493 } else 494 restore_intr(mtx_intr); 495 } 496 } else { 497 if (((type & MTX_TOPHALF) == 0) && 498 (type & MTX_FIRST)) { 499 ASS_IDIS; 500 ASS_SIEN(mpp); 501 } 502 _exitlock_spin(mpp);
| 370 mtx_lock_spin(&sched_lock); 371 /* 372 * Check if the lock has been released while spinning for 373 * the sched_lock. 374 */ 375 if ((v = m->mtx_lock) == MTX_UNOWNED) { 376 mtx_unlock_spin(&sched_lock); 377 continue;
|
503 }
| 378 }
|
504 } else { 505 /* Handle sleep locks */ 506 if ((type) & MTX_RLIKELY) 507 _exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS); 508 else { 509 _exitlock_norecurse(mpp, CURTHD, 510 (type) & MTX_HARDOPTS); 511 } 512 } 513}
| |
514
| 379
|
515void 516mtx_enter_hard(struct mtx *m, int type, int saveintr) 517{ 518 struct proc *p = CURPROC;
| 380 /* 381 * The mutex was marked contested on release. This means that 382 * there are processes blocked on it. 383 */ 384 if (v == MTX_CONTESTED) { 385 p1 = TAILQ_FIRST(&m->mtx_blocked); 386 KASSERT(p1 != NULL, 387 ("contested mutex has no contesters")); 388 m->mtx_lock = (uintptr_t)p | MTX_CONTESTED;
|
519
| 389
|
520 KASSERT(p != NULL, ("curproc is NULL in mutex")); 521 522 switch (type) { 523 case MTX_DEF: 524 if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 525 m->mtx_recurse++; 526 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 527 if ((type & MTX_QUIET) == 0) 528 CTR1(KTR_LOCK, "mtx_enter: %p recurse", m);
| 390 if (p1->p_priority < p->p_priority) 391 SET_PRIO(p, p1->p_priority); 392 mtx_unlock_spin(&sched_lock);
|
529 return; 530 }
| 393 return; 394 }
|
531 if ((type & MTX_QUIET) == 0) 532 CTR3(KTR_LOCK, 533 "mtx_enter: %p contested (lock=%p) [%p]", 534 m, (void *)m->mtx_lock, (void *)RETIP(m));
| |
535 536 /*
| 395 396 /*
|
537 * Save our priority. Even though p_nativepri is protected 538 * by sched_lock, we don't obtain it here as it can be 539 * expensive. Since this is the only place p_nativepri is 540 * set, and since two CPUs will not be executing the same 541 * process concurrently, we know that no other CPU is going 542 * to be messing with this. Also, p_nativepri is only read 543 * when we are blocked on a mutex, so that can't be happening 544 * right now either.
| 397 * If the mutex isn't already contested and a failure occurs 398 * setting the contested bit, the mutex was either released 399 * or the state of the MTX_RECURSED bit changed.
|
545 */
| 400 */
|
546 p->p_nativepri = p->p_priority; 547 while (!_obtain_lock(m, p)) { 548 uintptr_t v; 549 struct proc *p1;
| 401 if ((v & MTX_CONTESTED) == 0 && 402 !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 403 (void *)(v | MTX_CONTESTED))) { 404 mtx_unlock_spin(&sched_lock); 405 continue; 406 }
|
550
| 407
|
551 mtx_enter(&sched_lock, MTX_SPIN | MTX_RLIKELY); 552 /* 553 * check if the lock has been released while 554 * waiting for the schedlock. 555 */ 556 if ((v = m->mtx_lock) == MTX_UNOWNED) { 557 mtx_exit(&sched_lock, MTX_SPIN); 558 continue; 559 } 560 /* 561 * The mutex was marked contested on release. This 562 * means that there are processes blocked on it. 563 */ 564 if (v == MTX_CONTESTED) { 565 p1 = TAILQ_FIRST(&m->mtx_blocked); 566 KASSERT(p1 != NULL, ("contested mutex has no contesters")); 567 KASSERT(p != NULL, ("curproc is NULL for contested mutex")); 568 m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; 569 if (p1->p_priority < p->p_priority) { 570 SET_PRIO(p, p1->p_priority); 571 } 572 mtx_exit(&sched_lock, MTX_SPIN); 573 return; 574 } 575 /* 576 * If the mutex isn't already contested and 577 * a failure occurs setting the contested bit the 578 * mutex was either release or the 579 * state of the RECURSION bit changed. 580 */ 581 if ((v & MTX_CONTESTED) == 0 && 582 !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 583 (void *)(v | MTX_CONTESTED))) { 584 mtx_exit(&sched_lock, MTX_SPIN); 585 continue; 586 }
| 408 /* 409 * We deffinately must sleep for this lock. 410 */ 411 mtx_assert(m, MA_NOTOWNED);
|
587
| 412
|
588 /* We definitely have to sleep for this lock */ 589 mtx_assert(m, MA_NOTOWNED); 590
| |
591#ifdef notyet
| 413#ifdef notyet
|
592 /* 593 * If we're borrowing an interrupted thread's VM 594 * context must clean up before going to sleep. 595 */ 596 if (p->p_flag & (P_ITHD | P_SITHD)) { 597 ithd_t *it = (ithd_t *)p;
| 414 /* 415 * If we're borrowing an interrupted thread's VM context, we 416 * must clean up before going to sleep. 417 */ 418 if (p->p_flag & (P_ITHD | P_SITHD)) { 419 ithd_t *it = (ithd_t *)p;
|
598
| 420
|
599 if (it->it_interrupted) { 600 if ((type & MTX_QUIET) == 0) 601 CTR2(KTR_LOCK, 602 "mtx_enter: 0x%x interrupted 0x%x", 603 it, it->it_interrupted); 604 intr_thd_fixup(it); 605 }
| 421 if (it->it_interrupted) { 422 if ((opts & MTX_QUIET) == 0) 423 CTR2(KTR_LOCK, 424 "mtx_lock: 0x%x interrupted 0x%x", 425 it, it->it_interrupted); 426 intr_thd_fixup(it);
|
606 }
| 427 }
|
| 428 }
|
607#endif 608
| 429#endif 430
|
609 /* Put us on the list of procs blocked on this mutex */ 610 if (TAILQ_EMPTY(&m->mtx_blocked)) { 611 p1 = (struct proc *)(m->mtx_lock & 612 MTX_FLAGMASK); 613 LIST_INSERT_HEAD(&p1->p_contested, m, 614 mtx_contested);
| 431 /* 432 * Put us on the list of threads blocked on this mutex. 433 */ 434 if (TAILQ_EMPTY(&m->mtx_blocked)) { 435 p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); 436 LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); 437 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 438 } else { 439 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 440 if (p1->p_priority > p->p_priority) 441 break; 442 if (p1) 443 TAILQ_INSERT_BEFORE(p1, p, p_procq); 444 else
|
615 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
| 445 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
|
616 } else { 617 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 618 if (p1->p_priority > p->p_priority) 619 break; 620 if (p1) 621 TAILQ_INSERT_BEFORE(p1, p, p_procq); 622 else 623 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, 624 p_procq); 625 }
| 446 }
|
626
| 447
|
627 p->p_blocked = m; /* Who we're blocked on */ 628 p->p_mtxname = m->mtx_description; 629 p->p_stat = SMTX;
| 448 /* 449 * Save who we're blocked on. 450 */ 451 p->p_blocked = m; 452 p->p_mtxname = m->mtx_description; 453 p->p_stat = SMTX;
|
630#if 0
| 454#if 0
|
631 propagate_priority(p);
| 455 propagate_priority(p);
|
632#endif
| 456#endif
|
633 if ((type & MTX_QUIET) == 0) 634 CTR3(KTR_LOCK, 635 "mtx_enter: p %p blocked on [%p] %s", 636 p, m, m->mtx_description); 637 mi_switch(); 638 if ((type & MTX_QUIET) == 0) 639 CTR3(KTR_LOCK, 640 "mtx_enter: p %p free from blocked on [%p] %s", 641 p, m, m->mtx_description); 642 mtx_exit(&sched_lock, MTX_SPIN); 643 } 644 return; 645 case MTX_SPIN: 646 case MTX_SPIN | MTX_FIRST: 647 case MTX_SPIN | MTX_TOPHALF: 648 { 649 int i = 0;
| |
650
| 457
|
651 if (m->mtx_lock == (uintptr_t)p) { 652 m->mtx_recurse++; 653 return; 654 } 655 if ((type & MTX_QUIET) == 0) 656 CTR1(KTR_LOCK, "mtx_enter: %p spinning", m); 657 for (;;) { 658 if (_obtain_lock(m, p)) 659 break; 660 while (m->mtx_lock != MTX_UNOWNED) { 661 if (i++ < 1000000) 662 continue; 663 if (i++ < 6000000) 664 DELAY (1);
| 458 if ((opts & MTX_QUIET) == 0) 459 CTR3(KTR_LOCK, 460 "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, 461 m->mtx_description); 462 463 mi_switch(); 464 465 if ((opts & MTX_QUIET) == 0) 466 CTR3(KTR_LOCK, 467 "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 468 p, m, m->mtx_description); 469 470 mtx_unlock_spin(&sched_lock); 471 } 472 473 return; 474} 475 476/* 477 * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 478 * 479 * This is only called if we need to actually spin for the lock. Recursion 480 * is handled inline. 481 */ 482void 483_mtx_lock_spin(struct mtx *m, int opts, u_int mtx_intr, const char *file, 484 int line) 485{ 486 int i = 0; 487 488 if ((opts & MTX_QUIET) == 0) 489 CTR1(KTR_LOCK, "mtx_lock_spin: %p spinning", m); 490 491 for (;;) { 492 if (_obtain_lock(m, CURPROC)) 493 break; 494 495 while (m->mtx_lock != MTX_UNOWNED) { 496 if (i++ < 1000000) 497 continue; 498 if (i++ < 6000000) 499 DELAY(1);
|
665#ifdef DDB
| 500#ifdef DDB
|
666 else if (!db_active)
| 501 else if (!db_active)
|
667#else
| 502#else
|
668 else
| 503 else
|
669#endif
| 504#endif
|
670 panic( 671 "spin lock %s held by %p for > 5 seconds", 672 m->mtx_description, 673 (void *)m->mtx_lock); 674 }
| 505 panic("spin lock %s held by %p for > 5 seconds", 506 m->mtx_description, (void *)m->mtx_lock);
|
675 }
| 507 }
|
676 677#ifdef MUTEX_DEBUG 678 if (type != MTX_SPIN) 679 m->mtx_saveintr = 0xbeefface; 680 else 681#endif 682 m->mtx_saveintr = saveintr; 683 if ((type & MTX_QUIET) == 0) 684 CTR1(KTR_LOCK, "mtx_enter: %p spin done", m); 685 return; 686 }
| |
687 }
| 508 }
|
| 509 510 m->mtx_saveintr = mtx_intr; 511 if ((opts & MTX_QUIET) == 0) 512 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 513 514 return;
|
688} 689
| 515} 516
|
| 517/* 518 * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 519 * 520 * We are only called here if the lock is recursed or contested (i.e. we 521 * need to wake up a blocked thread). 522 */
|
690void
| 523void
|
691mtx_exit_hard(struct mtx *m, int type)
| 524_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
|
692{ 693 struct proc *p, *p1; 694 struct mtx *m1; 695 int pri; 696 697 p = CURPROC;
| 525{ 526 struct proc *p, *p1; 527 struct mtx *m1; 528 int pri; 529 530 p = CURPROC;
|
698 switch (type) { 699 case MTX_DEF: 700 case MTX_DEF | MTX_NOSWITCH: 701 if (mtx_recursed(m)) { 702 if (--(m->mtx_recurse) == 0) 703 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 704 if ((type & MTX_QUIET) == 0) 705 CTR1(KTR_LOCK, "mtx_exit: %p unrecurse", m); 706 return; 707 } 708 mtx_enter(&sched_lock, MTX_SPIN); 709 if ((type & MTX_QUIET) == 0) 710 CTR1(KTR_LOCK, "mtx_exit: %p contested", m); 711 p1 = TAILQ_FIRST(&m->mtx_blocked); 712 MPASS(p->p_magic == P_MAGIC); 713 MPASS(p1->p_magic == P_MAGIC); 714 TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 715 if (TAILQ_EMPTY(&m->mtx_blocked)) { 716 LIST_REMOVE(m, mtx_contested); 717 _release_lock_quick(m); 718 if ((type & MTX_QUIET) == 0) 719 CTR1(KTR_LOCK, "mtx_exit: %p not held", m); 720 } else 721 atomic_store_rel_ptr(&m->mtx_lock, 722 (void *)MTX_CONTESTED); 723 pri = MAXPRI; 724 LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 725 int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_priority; 726 if (cp < pri) 727 pri = cp; 728 } 729 if (pri > p->p_nativepri) 730 pri = p->p_nativepri; 731 SET_PRIO(p, pri); 732 if ((type & MTX_QUIET) == 0) 733 CTR2(KTR_LOCK, 734 "mtx_exit: %p contested setrunqueue %p", m, p1); 735 p1->p_blocked = NULL; 736 p1->p_mtxname = NULL; 737 p1->p_stat = SRUN; 738 setrunqueue(p1); 739 if ((type & MTX_NOSWITCH) == 0 && p1->p_priority < pri) {
| 531 MPASS4(mtx_owned(m), "mtx_owned(mpp)", file, line); 532 533 if ((opts & MTX_QUIET) == 0) 534 CTR5(KTR_LOCK, "REL %s [%p] r=%d at %s:%d", m->mtx_description, 535 m, m->mtx_recurse, file, line); 536 537 if (mtx_recursed(m)) { 538 if (--(m->mtx_recurse) == 0) 539 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 540 if ((opts & MTX_QUIET) == 0) 541 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 542 return; 543 } 544 545 mtx_lock_spin(&sched_lock); 546 if ((opts & MTX_QUIET) == 0) 547 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 548 549 p1 = TAILQ_FIRST(&m->mtx_blocked); 550 MPASS(p->p_magic == P_MAGIC); 551 MPASS(p1->p_magic == P_MAGIC); 552 553 TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 554 555 if (TAILQ_EMPTY(&m->mtx_blocked)) { 556 LIST_REMOVE(m, mtx_contested); 557 _release_lock_quick(m); 558 if ((opts & MTX_QUIET) == 0) 559 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 560 } else 561 atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 562 563 pri = MAXPRI; 564 LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 565 int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_priority; 566 if (cp < pri) 567 pri = cp; 568 } 569 570 if (pri > p->p_nativepri) 571 pri = p->p_nativepri; 572 SET_PRIO(p, pri); 573 574 if ((opts & MTX_QUIET) == 0) 575 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 576 m, p1); 577 578 p1->p_blocked = NULL; 579 p1->p_mtxname = NULL; 580 p1->p_stat = SRUN; 581 setrunqueue(p1); 582 583 if ((opts & MTX_NOSWITCH) == 0 && p1->p_priority < pri) {
|
740#ifdef notyet
| 584#ifdef notyet
|
741 if (p->p_flag & (P_ITHD | P_SITHD)) { 742 ithd_t *it = (ithd_t *)p;
| 585 if (p->p_flag & (P_ITHD | P_SITHD)) { 586 ithd_t *it = (ithd_t *)p;
|
743
| 587
|
744 if (it->it_interrupted) { 745 if ((type & MTX_QUIET) == 0) 746 CTR2(KTR_LOCK, 747 "mtx_exit: 0x%x interruped 0x%x", 748 it, it->it_interrupted); 749 intr_thd_fixup(it); 750 }
| 588 if (it->it_interrupted) { 589 if ((opts & MTX_QUIET) == 0) 590 CTR2(KTR_LOCK, 591 "_mtx_unlock_sleep: 0x%x interrupted 0x%x", 592 it, it->it_interrupted); 593 intr_thd_fixup(it);
|
751 }
| 594 }
|
752#endif 753 setrunqueue(p); 754 if ((type & MTX_QUIET) == 0) 755 CTR2(KTR_LOCK, 756 "mtx_exit: %p switching out lock=%p", 757 m, (void *)m->mtx_lock); 758 mi_switch(); 759 if ((type & MTX_QUIET) == 0) 760 CTR2(KTR_LOCK, 761 "mtx_exit: %p resuming lock=%p", 762 m, (void *)m->mtx_lock);
| |
763 }
| 595 }
|
764 mtx_exit(&sched_lock, MTX_SPIN); 765 break; 766 case MTX_SPIN: 767 case MTX_SPIN | MTX_FIRST: 768 if (mtx_recursed(m)) { 769 m->mtx_recurse--; 770 return; 771 } 772 MPASS(mtx_owned(m)); 773 _release_lock_quick(m); 774 if (type & MTX_FIRST) 775 enable_intr(); /* XXX is this kosher? */ 776 else { 777 MPASS(m->mtx_saveintr != 0xbeefface); 778 restore_intr(m->mtx_saveintr); 779 } 780 break; 781 case MTX_SPIN | MTX_TOPHALF: 782 if (mtx_recursed(m)) { 783 m->mtx_recurse--; 784 return; 785 } 786 MPASS(mtx_owned(m)); 787 _release_lock_quick(m); 788 break; 789 default: 790 panic("mtx_exit_hard: unsupported type 0x%x\n", type);
| 596#endif 597 setrunqueue(p); 598 if ((opts & MTX_QUIET) == 0) 599 CTR2(KTR_LOCK, 600 "_mtx_unlock_sleep: %p switching out lock=%p", m, 601 (void *)m->mtx_lock); 602 603 mi_switch(); 604 if ((opts & MTX_QUIET) == 0) 605 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 606 m, (void *)m->mtx_lock);
|
791 }
| 607 }
|
| 608 609 mtx_unlock_spin(&sched_lock); 610 611 return;
|
792} 793
| 612} 613
|
| 614/* 615 * All the unlocking of MTX_SPIN locks is done inline. 616 * See the _rel_spin_lock() macro for the details. 617 */ 618 619/* 620 * The INVARIANTS-enabled mtx_assert() 621 */
|
794#ifdef INVARIANTS 795void 796_mtx_assert(struct mtx *m, int what, const char *file, int line) 797{ 798 switch ((what)) { 799 case MA_OWNED: 800 case MA_OWNED | MA_RECURSED: 801 case MA_OWNED | MA_NOTRECURSED: 802 if (!mtx_owned((m))) 803 panic("mutex %s not owned at %s:%d", 804 (m)->mtx_description, file, line); 805 if (mtx_recursed((m))) { 806 if (((what) & MA_NOTRECURSED) != 0) 807 panic("mutex %s recursed at %s:%d", 808 (m)->mtx_description, file, line); 809 } else if (((what) & MA_RECURSED) != 0) { 810 panic("mutex %s unrecursed at %s:%d", 811 (m)->mtx_description, file, line); 812 } 813 break; 814 case MA_NOTOWNED: 815 if (mtx_owned((m))) 816 panic("mutex %s owned at %s:%d", 817 (m)->mtx_description, file, line); 818 break; 819 default: 820 panic("unknown mtx_assert at %s:%d", file, line); 821 } 822} 823#endif 824
| 622#ifdef INVARIANTS 623void 624_mtx_assert(struct mtx *m, int what, const char *file, int line) 625{ 626 switch ((what)) { 627 case MA_OWNED: 628 case MA_OWNED | MA_RECURSED: 629 case MA_OWNED | MA_NOTRECURSED: 630 if (!mtx_owned((m))) 631 panic("mutex %s not owned at %s:%d", 632 (m)->mtx_description, file, line); 633 if (mtx_recursed((m))) { 634 if (((what) & MA_NOTRECURSED) != 0) 635 panic("mutex %s recursed at %s:%d", 636 (m)->mtx_description, file, line); 637 } else if (((what) & MA_RECURSED) != 0) { 638 panic("mutex %s unrecursed at %s:%d", 639 (m)->mtx_description, file, line); 640 } 641 break; 642 case MA_NOTOWNED: 643 if (mtx_owned((m))) 644 panic("mutex %s owned at %s:%d", 645 (m)->mtx_description, file, line); 646 break; 647 default: 648 panic("unknown mtx_assert at %s:%d", file, line); 649 } 650} 651#endif 652
|
| 653/* 654 * The MUTEX_DEBUG-enabled mtx_validate() 655 */
|
825#define MV_DESTROY 0 /* validate before destory */ 826#define MV_INIT 1 /* validate before init */ 827 828#ifdef MUTEX_DEBUG 829 830int mtx_validate __P((struct mtx *, int)); 831 832int 833mtx_validate(struct mtx *m, int when) 834{ 835 struct mtx *mp; 836 int i; 837 int retval = 0; 838 839#ifdef WITNESS 840 if (witness_cold) 841 return 0; 842#endif 843 if (m == &all_mtx || cold) 844 return 0; 845
| 656#define MV_DESTROY 0 /* validate before destory */ 657#define MV_INIT 1 /* validate before init */ 658 659#ifdef MUTEX_DEBUG 660 661int mtx_validate __P((struct mtx *, int)); 662 663int 664mtx_validate(struct mtx *m, int when) 665{ 666 struct mtx *mp; 667 int i; 668 int retval = 0; 669 670#ifdef WITNESS 671 if (witness_cold) 672 return 0; 673#endif 674 if (m == &all_mtx || cold) 675 return 0; 676
|
846 mtx_enter(&all_mtx, MTX_DEF);
| 677 mtx_lock(&all_mtx);
|
847/* 848 * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 849 * we can re-enable the kernacc() checks. 850 */ 851#ifndef __alpha__ 852 MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), 853 VM_PROT_READ) == 1); 854#endif 855 MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); 856 for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 857#ifndef __alpha__ 858 if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), 859 VM_PROT_READ) != 1) { 860 panic("mtx_validate: mp=%p mp->mtx_next=%p", 861 mp, mp->mtx_next); 862 } 863#endif 864 i++; 865 if (i > mtx_cur_cnt) { 866 panic("mtx_validate: too many in chain, known=%d\n", 867 mtx_cur_cnt); 868 } 869 } 870 MPASS(i == mtx_cur_cnt); 871 switch (when) { 872 case MV_DESTROY: 873 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 874 if (mp == m) 875 break; 876 MPASS(mp == m); 877 break; 878 case MV_INIT: 879 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 880 if (mp == m) { 881 /* 882 * Not good. This mutex already exists. 883 */ 884 printf("re-initing existing mutex %s\n", 885 m->mtx_description); 886 MPASS(m->mtx_lock == MTX_UNOWNED); 887 retval = 1; 888 } 889 }
| 678/* 679 * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 680 * we can re-enable the kernacc() checks. 681 */ 682#ifndef __alpha__ 683 MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), 684 VM_PROT_READ) == 1); 685#endif 686 MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); 687 for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 688#ifndef __alpha__ 689 if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), 690 VM_PROT_READ) != 1) { 691 panic("mtx_validate: mp=%p mp->mtx_next=%p", 692 mp, mp->mtx_next); 693 } 694#endif 695 i++; 696 if (i > mtx_cur_cnt) { 697 panic("mtx_validate: too many in chain, known=%d\n", 698 mtx_cur_cnt); 699 } 700 } 701 MPASS(i == mtx_cur_cnt); 702 switch (when) { 703 case MV_DESTROY: 704 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 705 if (mp == m) 706 break; 707 MPASS(mp == m); 708 break; 709 case MV_INIT: 710 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) 711 if (mp == m) { 712 /* 713 * Not good. This mutex already exists. 714 */ 715 printf("re-initing existing mutex %s\n", 716 m->mtx_description); 717 MPASS(m->mtx_lock == MTX_UNOWNED); 718 retval = 1; 719 } 720 }
|
890 mtx_exit(&all_mtx, MTX_DEF);
| 721 mtx_unlock(&all_mtx);
|
891 return (retval); 892} 893#endif 894
| 722 return (retval); 723} 724#endif 725
|
| 726/* 727 * Mutex initialization routine; initialize lock `m' of type contained in 728 * `opts' with options contained in `opts' and description `description.' 729 * Place on "all_mtx" queue. 730 */
|
895void
| 731void
|
896mtx_init(struct mtx *m, const char *t, int flag)
| 732mtx_init(struct mtx *m, const char *description, int opts)
|
897{
| 733{
|
898 if ((flag & MTX_QUIET) == 0) 899 CTR2(KTR_LOCK, "mtx_init %p (%s)", m, t);
| 734 735 if ((opts & MTX_QUIET) == 0) 736 CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description); 737
|
900#ifdef MUTEX_DEBUG
| 738#ifdef MUTEX_DEBUG
|
901 if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */
| 739 /* Diagnostic and error correction */ 740 if (mtx_validate(m, MV_INIT))
|
902 return; 903#endif 904 905 bzero((void *)m, sizeof *m); 906 TAILQ_INIT(&m->mtx_blocked);
| 741 return; 742#endif 743 744 bzero((void *)m, sizeof *m); 745 TAILQ_INIT(&m->mtx_blocked);
|
| 746
|
907#ifdef WITNESS 908 if (!witness_cold) {
| 747#ifdef WITNESS 748 if (!witness_cold) {
|
909 /* XXX - should not use DEVBUF */
| |
910 m->mtx_debug = malloc(sizeof(struct mtx_debug),
| 749 m->mtx_debug = malloc(sizeof(struct mtx_debug),
|
911 M_DEVBUF, M_NOWAIT | M_ZERO);
| 750 M_WITNESS, M_NOWAIT | M_ZERO);
|
912 MPASS(m->mtx_debug != NULL); 913 } 914#endif
| 751 MPASS(m->mtx_debug != NULL); 752 } 753#endif
|
915 m->mtx_description = t;
| |
916
| 754
|
917 m->mtx_flags = flag;
| 755 m->mtx_description = description; 756 m->mtx_flags = opts;
|
918 m->mtx_lock = MTX_UNOWNED;
| 757 m->mtx_lock = MTX_UNOWNED;
|
| 758
|
919 /* Put on all mutex queue */
| 759 /* Put on all mutex queue */
|
920 mtx_enter(&all_mtx, MTX_DEF);
| 760 mtx_lock(&all_mtx);
|
921 m->mtx_next = &all_mtx; 922 m->mtx_prev = all_mtx.mtx_prev; 923 m->mtx_prev->mtx_next = m; 924 all_mtx.mtx_prev = m; 925 if (++mtx_cur_cnt > mtx_max_cnt) 926 mtx_max_cnt = mtx_cur_cnt;
| 761 m->mtx_next = &all_mtx; 762 m->mtx_prev = all_mtx.mtx_prev; 763 m->mtx_prev->mtx_next = m; 764 all_mtx.mtx_prev = m; 765 if (++mtx_cur_cnt > mtx_max_cnt) 766 mtx_max_cnt = mtx_cur_cnt;
|
927 mtx_exit(&all_mtx, MTX_DEF);
| 767 mtx_unlock(&all_mtx); 768
|
928#ifdef WITNESS 929 if (!witness_cold)
| 769#ifdef WITNESS 770 if (!witness_cold)
|
930 witness_init(m, flag);
| 771 witness_init(m, opts);
|
931#endif 932} 933
| 772#endif 773} 774
|
| 775/* 776 * Remove lock `m' from all_mtx queue. 777 */
|
934void 935mtx_destroy(struct mtx *m) 936{ 937 938#ifdef WITNESS 939 KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", 940 __FUNCTION__)); 941#endif
| 778void 779mtx_destroy(struct mtx *m) 780{ 781 782#ifdef WITNESS 783 KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", 784 __FUNCTION__)); 785#endif
|
| 786
|
942 CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
| 787 CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
|
| 788
|
943#ifdef MUTEX_DEBUG 944 if (m->mtx_next == NULL) 945 panic("mtx_destroy: %p (%s) already destroyed", 946 m, m->mtx_description); 947 948 if (!mtx_owned(m)) { 949 MPASS(m->mtx_lock == MTX_UNOWNED); 950 } else { 951 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 952 }
| 789#ifdef MUTEX_DEBUG 790 if (m->mtx_next == NULL) 791 panic("mtx_destroy: %p (%s) already destroyed", 792 m, m->mtx_description); 793 794 if (!mtx_owned(m)) { 795 MPASS(m->mtx_lock == MTX_UNOWNED); 796 } else { 797 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 798 }
|
953 mtx_validate(m, MV_DESTROY); /* diagnostic */
| 799 800 /* diagnostic */ 801 mtx_validate(m, MV_DESTROY);
|
954#endif 955 956#ifdef WITNESS 957 if (m->mtx_witness) 958 witness_destroy(m); 959#endif /* WITNESS */ 960 961 /* Remove from the all mutex queue */
| 802#endif 803 804#ifdef WITNESS 805 if (m->mtx_witness) 806 witness_destroy(m); 807#endif /* WITNESS */ 808 809 /* Remove from the all mutex queue */
|
962 mtx_enter(&all_mtx, MTX_DEF);
| 810 mtx_lock(&all_mtx);
|
963 m->mtx_next->mtx_prev = m->mtx_prev; 964 m->mtx_prev->mtx_next = m->mtx_next;
| 811 m->mtx_next->mtx_prev = m->mtx_prev; 812 m->mtx_prev->mtx_next = m->mtx_next;
|
| 813
|
965#ifdef MUTEX_DEBUG 966 m->mtx_next = m->mtx_prev = NULL; 967#endif
| 814#ifdef MUTEX_DEBUG 815 m->mtx_next = m->mtx_prev = NULL; 816#endif
|
| 817
|
968#ifdef WITNESS
| 818#ifdef WITNESS
|
969 free(m->mtx_debug, M_DEVBUF);
| 819 free(m->mtx_debug, M_WITNESS);
|
970 m->mtx_debug = NULL; 971#endif
| 820 m->mtx_debug = NULL; 821#endif
|
| 822
|
972 mtx_cur_cnt--;
| 823 mtx_cur_cnt--;
|
973 mtx_exit(&all_mtx, MTX_DEF);
| 824 mtx_unlock(&all_mtx);
|
974} 975
| 825} 826
|
| 827
|
976/*
| 828/*
|
977 * The non-inlined versions of the mtx_*() functions are always built (above), 978 * but the witness code depends on the WITNESS kernel option being specified.
| 829 * The WITNESS-enabled diagnostic code.
|
979 */
| 830 */
|
980
| |
981#ifdef WITNESS 982static void 983witness_fixup(void *dummy __unused) 984{ 985 struct mtx *mp; 986 987 /* 988 * We have to release Giant before initializing its witness 989 * structure so that WITNESS doesn't get confused. 990 */
| 831#ifdef WITNESS 832static void 833witness_fixup(void *dummy __unused) 834{ 835 struct mtx *mp; 836 837 /* 838 * We have to release Giant before initializing its witness 839 * structure so that WITNESS doesn't get confused. 840 */
|
991 mtx_exit(&Giant, MTX_DEF);
| 841 mtx_unlock(&Giant);
|
992 mtx_assert(&Giant, MA_NOTOWNED);
| 842 mtx_assert(&Giant, MA_NOTOWNED);
|
993 mtx_enter(&all_mtx, MTX_DEF);
| |
994
| 843
|
| 844 mtx_lock(&all_mtx); 845
|
995 /* Iterate through all mutexes and finish up mutex initialization. */ 996 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 997
| 846 /* Iterate through all mutexes and finish up mutex initialization. */ 847 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { 848
|
998 /* XXX - should not use DEVBUF */
| |
999 mp->mtx_debug = malloc(sizeof(struct mtx_debug),
| 849 mp->mtx_debug = malloc(sizeof(struct mtx_debug),
|
1000 M_DEVBUF, M_NOWAIT | M_ZERO);
| 850 M_WITNESS, M_NOWAIT | M_ZERO);
|
1001 MPASS(mp->mtx_debug != NULL); 1002 1003 witness_init(mp, mp->mtx_flags); 1004 }
| 851 MPASS(mp->mtx_debug != NULL); 852 853 witness_init(mp, mp->mtx_flags); 854 }
|
1005 mtx_exit(&all_mtx, MTX_DEF);
| 855 mtx_unlock(&all_mtx);
|
1006 1007 /* Mark the witness code as being ready for use. */ 1008 atomic_store_rel_int(&witness_cold, 0); 1009
| 856 857 /* Mark the witness code as being ready for use. */ 858 atomic_store_rel_int(&witness_cold, 0); 859
|
1010 mtx_enter(&Giant, MTX_DEF);
| 860 mtx_lock(&Giant);
|
1011} 1012SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) 1013 1014#define WITNESS_COUNT 200 1015#define WITNESS_NCHILDREN 2 1016 1017int witness_watch = 1; 1018 1019struct witness { 1020 struct witness *w_next; 1021 const char *w_description; 1022 const char *w_file; 1023 int w_line; 1024 struct witness *w_morechildren; 1025 u_char w_childcnt; 1026 u_char w_Giant_squawked:1; 1027 u_char w_other_squawked:1; 1028 u_char w_same_squawked:1; 1029 u_char w_spin:1; /* MTX_SPIN type mutex. */ 1030 u_int w_level; 1031 struct witness *w_children[WITNESS_NCHILDREN]; 1032}; 1033 1034struct witness_blessed { 1035 char *b_lock1; 1036 char *b_lock2; 1037}; 1038 1039#ifdef DDB 1040/* 1041 * When DDB is enabled and witness_ddb is set to 1, it will cause the system to 1042 * drop into kdebug() when: 1043 * - a lock heirarchy violation occurs 1044 * - locks are held when going to sleep. 1045 */ 1046int witness_ddb; 1047#ifdef WITNESS_DDB 1048TUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb); 1049#else 1050TUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb); 1051#endif 1052SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, ""); 1053#endif /* DDB */ 1054 1055int witness_skipspin; 1056#ifdef WITNESS_SKIPSPIN 1057TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin); 1058#else 1059TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); 1060#endif 1061SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, 1062 ""); 1063
| 861} 862SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) 863 864#define WITNESS_COUNT 200 865#define WITNESS_NCHILDREN 2 866 867int witness_watch = 1; 868 869struct witness { 870 struct witness *w_next; 871 const char *w_description; 872 const char *w_file; 873 int w_line; 874 struct witness *w_morechildren; 875 u_char w_childcnt; 876 u_char w_Giant_squawked:1; 877 u_char w_other_squawked:1; 878 u_char w_same_squawked:1; 879 u_char w_spin:1; /* MTX_SPIN type mutex. */ 880 u_int w_level; 881 struct witness *w_children[WITNESS_NCHILDREN]; 882}; 883 884struct witness_blessed { 885 char *b_lock1; 886 char *b_lock2; 887}; 888 889#ifdef DDB 890/* 891 * When DDB is enabled and witness_ddb is set to 1, it will cause the system to 892 * drop into kdebug() when: 893 * - a lock heirarchy violation occurs 894 * - locks are held when going to sleep. 895 */ 896int witness_ddb; 897#ifdef WITNESS_DDB 898TUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb); 899#else 900TUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb); 901#endif 902SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, ""); 903#endif /* DDB */ 904 905int witness_skipspin; 906#ifdef WITNESS_SKIPSPIN 907TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin); 908#else 909TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); 910#endif 911SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, 912 ""); 913
|
| 914/* 915 * Witness-enabled globals 916 */
|
1064static struct mtx w_mtx; 1065static struct witness *w_free; 1066static struct witness *w_all; 1067static int w_inited; 1068static int witness_dead; /* fatal error, probably no memory */ 1069 1070static struct witness w_data[WITNESS_COUNT]; 1071
| 917static struct mtx w_mtx; 918static struct witness *w_free; 919static struct witness *w_all; 920static int w_inited; 921static int witness_dead; /* fatal error, probably no memory */ 922 923static struct witness w_data[WITNESS_COUNT]; 924
|
1072static struct witness *enroll __P((const char *description, int flag)); 1073static int itismychild __P((struct witness *parent, struct witness *child)); 1074static void removechild __P((struct witness *parent, struct witness *child)); 1075static int isitmychild __P((struct witness *parent, struct witness *child)); 1076static int isitmydescendant __P((struct witness *parent, struct witness *child)); 1077static int dup_ok __P((struct witness *)); 1078static int blessed __P((struct witness *, struct witness *)); 1079static void witness_displaydescendants 1080 __P((void(*)(const char *fmt, ...), struct witness *)); 1081static void witness_leveldescendents __P((struct witness *parent, int level)); 1082static void witness_levelall __P((void)); 1083static struct witness * witness_get __P((void)); 1084static void witness_free __P((struct witness *m));
| 925/* 926 * Internal witness routine prototypes 927 */ 928static struct witness *enroll(const char *description, int flag); 929static int itismychild(struct witness *parent, struct witness *child); 930static void removechild(struct witness *parent, struct witness *child); 931static int isitmychild(struct witness *parent, struct witness *child); 932static int isitmydescendant(struct witness *parent, struct witness *child); 933static int dup_ok(struct witness *); 934static int blessed(struct witness *, struct witness *); 935static void 936 witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *); 937static void witness_leveldescendents(struct witness *parent, int level); 938static void witness_levelall(void); 939static struct witness * witness_get(void); 940static void witness_free(struct witness *m);
|
1085
| 941
|
1086
| |
1087static char *ignore_list[] = { 1088 "witness lock", 1089 NULL 1090}; 1091 1092static char *spin_order_list[] = { 1093 "sio", 1094 "sched lock", 1095#ifdef __i386__ 1096 "clk", 1097#endif 1098 "callout", 1099 /* 1100 * leaf locks 1101 */ 1102#ifdef __i386__ 1103 "ap boot", 1104 "imen", 1105#endif 1106 "com", 1107 "smp rendezvous", 1108 NULL 1109}; 1110 1111static char *order_list[] = { 1112 "Giant", "uidinfo hash", "uidinfo struct", NULL, 1113 "Giant", "proctree", "allproc", "process lock", NULL, 1114 NULL 1115}; 1116 1117static char *dup_list[] = { 1118 NULL 1119}; 1120 1121static char *sleep_list[] = { 1122 "Giant", 1123 NULL 1124}; 1125 1126/* 1127 * Pairs of locks which have been blessed 1128 * Don't complain about order problems with blessed locks 1129 */ 1130static struct witness_blessed blessed_list[] = { 1131};
| 942static char *ignore_list[] = { 943 "witness lock", 944 NULL 945}; 946 947static char *spin_order_list[] = { 948 "sio", 949 "sched lock", 950#ifdef __i386__ 951 "clk", 952#endif 953 "callout", 954 /* 955 * leaf locks 956 */ 957#ifdef __i386__ 958 "ap boot", 959 "imen", 960#endif 961 "com", 962 "smp rendezvous", 963 NULL 964}; 965 966static char *order_list[] = { 967 "Giant", "uidinfo hash", "uidinfo struct", NULL, 968 "Giant", "proctree", "allproc", "process lock", NULL, 969 NULL 970}; 971 972static char *dup_list[] = { 973 NULL 974}; 975 976static char *sleep_list[] = { 977 "Giant", 978 NULL 979}; 980 981/* 982 * Pairs of locks which have been blessed 983 * Don't complain about order problems with blessed locks 984 */ 985static struct witness_blessed blessed_list[] = { 986};
|
1132static int blessed_count = sizeof(blessed_list) / sizeof(struct witness_blessed);
| 987static int blessed_count = 988 sizeof(blessed_list) / sizeof(struct witness_blessed);
|
1133 1134static void 1135witness_init(struct mtx *m, int flag) 1136{ 1137 m->mtx_witness = enroll(m->mtx_description, flag); 1138} 1139 1140static void 1141witness_destroy(struct mtx *m) 1142{ 1143 struct mtx *m1; 1144 struct proc *p; 1145 p = CURPROC; 1146 for ((m1 = LIST_FIRST(&p->p_heldmtx)); m1 != NULL; 1147 m1 = LIST_NEXT(m1, mtx_held)) { 1148 if (m1 == m) { 1149 LIST_REMOVE(m, mtx_held); 1150 break; 1151 } 1152 } 1153 return; 1154 1155} 1156 1157static void 1158witness_display(void(*prnt)(const char *fmt, ...)) 1159{ 1160 struct witness *w, *w1; 1161 1162 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1163 witness_levelall(); 1164 1165 for (w = w_all; w; w = w->w_next) { 1166 if (w->w_file == NULL) 1167 continue; 1168 for (w1 = w_all; w1; w1 = w1->w_next) { 1169 if (isitmychild(w1, w)) 1170 break; 1171 } 1172 if (w1 != NULL) 1173 continue; 1174 /* 1175 * This lock has no anscestors, display its descendants. 1176 */ 1177 witness_displaydescendants(prnt, w); 1178 } 1179 prnt("\nMutex which were never acquired\n"); 1180 for (w = w_all; w; w = w->w_next) { 1181 if (w->w_file != NULL) 1182 continue; 1183 prnt("%s\n", w->w_description); 1184 } 1185} 1186 1187void 1188witness_enter(struct mtx *m, int flags, const char *file, int line) 1189{ 1190 struct witness *w, *w1; 1191 struct mtx *m1; 1192 struct proc *p; 1193 int i; 1194#ifdef DDB 1195 int go_into_ddb = 0; 1196#endif /* DDB */ 1197 1198 if (witness_cold || m->mtx_witness == NULL || panicstr) 1199 return; 1200 w = m->mtx_witness; 1201 p = CURPROC; 1202 1203 if (flags & MTX_SPIN) { 1204 if ((m->mtx_flags & MTX_SPIN) == 0) 1205 panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" 1206 " %s:%d", m->mtx_description, file, line); 1207 if (mtx_recursed(m)) { 1208 if ((m->mtx_flags & MTX_RECURSE) == 0) 1209 panic("mutex_enter: recursion on non-recursive" 1210 " mutex %s @ %s:%d", m->mtx_description, 1211 file, line); 1212 return; 1213 }
| 989 990static void 991witness_init(struct mtx *m, int flag) 992{ 993 m->mtx_witness = enroll(m->mtx_description, flag); 994} 995 996static void 997witness_destroy(struct mtx *m) 998{ 999 struct mtx *m1; 1000 struct proc *p; 1001 p = CURPROC; 1002 for ((m1 = LIST_FIRST(&p->p_heldmtx)); m1 != NULL; 1003 m1 = LIST_NEXT(m1, mtx_held)) { 1004 if (m1 == m) { 1005 LIST_REMOVE(m, mtx_held); 1006 break; 1007 } 1008 } 1009 return; 1010 1011} 1012 1013static void 1014witness_display(void(*prnt)(const char *fmt, ...)) 1015{ 1016 struct witness *w, *w1; 1017 1018 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1019 witness_levelall(); 1020 1021 for (w = w_all; w; w = w->w_next) { 1022 if (w->w_file == NULL) 1023 continue; 1024 for (w1 = w_all; w1; w1 = w1->w_next) { 1025 if (isitmychild(w1, w)) 1026 break; 1027 } 1028 if (w1 != NULL) 1029 continue; 1030 /* 1031 * This lock has no anscestors, display its descendants. 1032 */ 1033 witness_displaydescendants(prnt, w); 1034 } 1035 prnt("\nMutex which were never acquired\n"); 1036 for (w = w_all; w; w = w->w_next) { 1037 if (w->w_file != NULL) 1038 continue; 1039 prnt("%s\n", w->w_description); 1040 } 1041} 1042 1043void 1044witness_enter(struct mtx *m, int flags, const char *file, int line) 1045{ 1046 struct witness *w, *w1; 1047 struct mtx *m1; 1048 struct proc *p; 1049 int i; 1050#ifdef DDB 1051 int go_into_ddb = 0; 1052#endif /* DDB */ 1053 1054 if (witness_cold || m->mtx_witness == NULL || panicstr) 1055 return; 1056 w = m->mtx_witness; 1057 p = CURPROC; 1058 1059 if (flags & MTX_SPIN) { 1060 if ((m->mtx_flags & MTX_SPIN) == 0) 1061 panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" 1062 " %s:%d", m->mtx_description, file, line); 1063 if (mtx_recursed(m)) { 1064 if ((m->mtx_flags & MTX_RECURSE) == 0) 1065 panic("mutex_enter: recursion on non-recursive" 1066 " mutex %s @ %s:%d", m->mtx_description, 1067 file, line); 1068 return; 1069 }
|
1214 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1070 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
|
1215 i = PCPU_GET(witness_spin_check); 1216 if (i != 0 && w->w_level < i) {
| 1071 i = PCPU_GET(witness_spin_check); 1072 if (i != 0 && w->w_level < i) {
|
1217 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1073 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1218 panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" 1219 " %s:%d already holding %s:%x", 1220 m->mtx_description, w->w_level, file, line, 1221 spin_order_list[ffs(i)-1], i); 1222 } 1223 PCPU_SET(witness_spin_check, i | w->w_level);
| 1074 panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" 1075 " %s:%d already holding %s:%x", 1076 m->mtx_description, w->w_level, file, line, 1077 spin_order_list[ffs(i)-1], i); 1078 } 1079 PCPU_SET(witness_spin_check, i | w->w_level);
|
1224 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1080 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1225 w->w_file = file; 1226 w->w_line = line; 1227 m->mtx_line = line; 1228 m->mtx_file = file; 1229 return; 1230 } 1231 if ((m->mtx_flags & MTX_SPIN) != 0) 1232 panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 1233 m->mtx_description, file, line); 1234 1235 if (mtx_recursed(m)) { 1236 if ((m->mtx_flags & MTX_RECURSE) == 0) 1237 panic("mutex_enter: recursion on non-recursive" 1238 " mutex %s @ %s:%d", m->mtx_description, 1239 file, line); 1240 return; 1241 } 1242 if (witness_dead) 1243 goto out; 1244 if (cold) 1245 goto out; 1246 1247 if (!mtx_legal2block())
| 1081 w->w_file = file; 1082 w->w_line = line; 1083 m->mtx_line = line; 1084 m->mtx_file = file; 1085 return; 1086 } 1087 if ((m->mtx_flags & MTX_SPIN) != 0) 1088 panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 1089 m->mtx_description, file, line); 1090 1091 if (mtx_recursed(m)) { 1092 if ((m->mtx_flags & MTX_RECURSE) == 0) 1093 panic("mutex_enter: recursion on non-recursive" 1094 " mutex %s @ %s:%d", m->mtx_description, 1095 file, line); 1096 return; 1097 } 1098 if (witness_dead) 1099 goto out; 1100 if (cold) 1101 goto out; 1102 1103 if (!mtx_legal2block())
|
1248 panic("blockable mtx_enter() of %s when not legal @ %s:%d",
| 1104 panic("blockable mtx_lock() of %s when not legal @ %s:%d",
|
1249 m->mtx_description, file, line); 1250 /* 1251 * Is this the first mutex acquired 1252 */ 1253 if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) 1254 goto out; 1255 1256 if ((w1 = m1->mtx_witness) == w) { 1257 if (w->w_same_squawked || dup_ok(w)) 1258 goto out; 1259 w->w_same_squawked = 1; 1260 printf("acquring duplicate lock of same type: \"%s\"\n", 1261 m->mtx_description); 1262 printf(" 1st @ %s:%d\n", w->w_file, w->w_line); 1263 printf(" 2nd @ %s:%d\n", file, line); 1264#ifdef DDB 1265 go_into_ddb = 1; 1266#endif /* DDB */ 1267 goto out; 1268 } 1269 MPASS(!mtx_owned(&w_mtx));
| 1105 m->mtx_description, file, line); 1106 /* 1107 * Is this the first mutex acquired 1108 */ 1109 if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) 1110 goto out; 1111 1112 if ((w1 = m1->mtx_witness) == w) { 1113 if (w->w_same_squawked || dup_ok(w)) 1114 goto out; 1115 w->w_same_squawked = 1; 1116 printf("acquring duplicate lock of same type: \"%s\"\n", 1117 m->mtx_description); 1118 printf(" 1st @ %s:%d\n", w->w_file, w->w_line); 1119 printf(" 2nd @ %s:%d\n", file, line); 1120#ifdef DDB 1121 go_into_ddb = 1; 1122#endif /* DDB */ 1123 goto out; 1124 } 1125 MPASS(!mtx_owned(&w_mtx));
|
1270 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1126 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
|
1271 /* 1272 * If we have a known higher number just say ok 1273 */ 1274 if (witness_watch > 1 && w->w_level > w1->w_level) {
| 1127 /* 1128 * If we have a known higher number just say ok 1129 */ 1130 if (witness_watch > 1 && w->w_level > w1->w_level) {
|
1275 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1131 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1276 goto out; 1277 } 1278 if (isitmydescendant(m1->mtx_witness, w)) {
| 1132 goto out; 1133 } 1134 if (isitmydescendant(m1->mtx_witness, w)) {
|
1279 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1135 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1280 goto out; 1281 } 1282 for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { 1283 1284 MPASS(i < 200); 1285 w1 = m1->mtx_witness; 1286 if (isitmydescendant(w, w1)) {
| 1136 goto out; 1137 } 1138 for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { 1139 1140 MPASS(i < 200); 1141 w1 = m1->mtx_witness; 1142 if (isitmydescendant(w, w1)) {
|
1287 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1143 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1288 if (blessed(w, w1)) 1289 goto out; 1290 if (m1 == &Giant) { 1291 if (w1->w_Giant_squawked) 1292 goto out; 1293 else 1294 w1->w_Giant_squawked = 1; 1295 } else { 1296 if (w1->w_other_squawked) 1297 goto out; 1298 else 1299 w1->w_other_squawked = 1; 1300 } 1301 printf("lock order reversal\n"); 1302 printf(" 1st %s last acquired @ %s:%d\n", 1303 w->w_description, w->w_file, w->w_line); 1304 printf(" 2nd %p %s @ %s:%d\n", 1305 m1, w1->w_description, w1->w_file, w1->w_line); 1306 printf(" 3rd %p %s @ %s:%d\n", 1307 m, w->w_description, file, line); 1308#ifdef DDB 1309 go_into_ddb = 1; 1310#endif /* DDB */ 1311 goto out; 1312 } 1313 } 1314 m1 = LIST_FIRST(&p->p_heldmtx); 1315 if (!itismychild(m1->mtx_witness, w))
| 1144 if (blessed(w, w1)) 1145 goto out; 1146 if (m1 == &Giant) { 1147 if (w1->w_Giant_squawked) 1148 goto out; 1149 else 1150 w1->w_Giant_squawked = 1; 1151 } else { 1152 if (w1->w_other_squawked) 1153 goto out; 1154 else 1155 w1->w_other_squawked = 1; 1156 } 1157 printf("lock order reversal\n"); 1158 printf(" 1st %s last acquired @ %s:%d\n", 1159 w->w_description, w->w_file, w->w_line); 1160 printf(" 2nd %p %s @ %s:%d\n", 1161 m1, w1->w_description, w1->w_file, w1->w_line); 1162 printf(" 3rd %p %s @ %s:%d\n", 1163 m, w->w_description, file, line); 1164#ifdef DDB 1165 go_into_ddb = 1; 1166#endif /* DDB */ 1167 goto out; 1168 } 1169 } 1170 m1 = LIST_FIRST(&p->p_heldmtx); 1171 if (!itismychild(m1->mtx_witness, w))
|
1316 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1172 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1317 1318out: 1319#ifdef DDB 1320 if (witness_ddb && go_into_ddb) 1321 Debugger("witness_enter"); 1322#endif /* DDB */ 1323 w->w_file = file; 1324 w->w_line = line; 1325 m->mtx_line = line; 1326 m->mtx_file = file; 1327 1328 /* 1329 * If this pays off it likely means that a mutex being witnessed 1330 * is acquired in hardclock. Put it in the ignore list. It is 1331 * likely not the mutex this assert fails on. 1332 */ 1333 MPASS(m->mtx_held.le_prev == NULL); 1334 LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 1335} 1336 1337void 1338witness_try_enter(struct mtx *m, int flags, const char *file, int line) 1339{ 1340 struct proc *p; 1341 struct witness *w = m->mtx_witness; 1342 1343 if (witness_cold) 1344 return; 1345 if (panicstr) 1346 return; 1347 if (flags & MTX_SPIN) { 1348 if ((m->mtx_flags & MTX_SPIN) == 0) 1349 panic("mutex_try_enter: " 1350 "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", 1351 m->mtx_description, file, line); 1352 if (mtx_recursed(m)) { 1353 if ((m->mtx_flags & MTX_RECURSE) == 0) 1354 panic("mutex_try_enter: recursion on" 1355 " non-recursive mutex %s @ %s:%d", 1356 m->mtx_description, file, line); 1357 return; 1358 }
| 1173 1174out: 1175#ifdef DDB 1176 if (witness_ddb && go_into_ddb) 1177 Debugger("witness_enter"); 1178#endif /* DDB */ 1179 w->w_file = file; 1180 w->w_line = line; 1181 m->mtx_line = line; 1182 m->mtx_file = file; 1183 1184 /* 1185 * If this pays off it likely means that a mutex being witnessed 1186 * is acquired in hardclock. Put it in the ignore list. It is 1187 * likely not the mutex this assert fails on. 1188 */ 1189 MPASS(m->mtx_held.le_prev == NULL); 1190 LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 1191} 1192 1193void 1194witness_try_enter(struct mtx *m, int flags, const char *file, int line) 1195{ 1196 struct proc *p; 1197 struct witness *w = m->mtx_witness; 1198 1199 if (witness_cold) 1200 return; 1201 if (panicstr) 1202 return; 1203 if (flags & MTX_SPIN) { 1204 if ((m->mtx_flags & MTX_SPIN) == 0) 1205 panic("mutex_try_enter: " 1206 "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", 1207 m->mtx_description, file, line); 1208 if (mtx_recursed(m)) { 1209 if ((m->mtx_flags & MTX_RECURSE) == 0) 1210 panic("mutex_try_enter: recursion on" 1211 " non-recursive mutex %s @ %s:%d", 1212 m->mtx_description, file, line); 1213 return; 1214 }
|
1359 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1215 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
|
1360 PCPU_SET(witness_spin_check, 1361 PCPU_GET(witness_spin_check) | w->w_level);
| 1216 PCPU_SET(witness_spin_check, 1217 PCPU_GET(witness_spin_check) | w->w_level);
|
1362 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1218 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1363 w->w_file = file; 1364 w->w_line = line; 1365 m->mtx_line = line; 1366 m->mtx_file = file; 1367 return; 1368 } 1369 1370 if ((m->mtx_flags & MTX_SPIN) != 0) 1371 panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 1372 m->mtx_description, file, line); 1373 1374 if (mtx_recursed(m)) { 1375 if ((m->mtx_flags & MTX_RECURSE) == 0) 1376 panic("mutex_try_enter: recursion on non-recursive" 1377 " mutex %s @ %s:%d", m->mtx_description, file, 1378 line); 1379 return; 1380 } 1381 w->w_file = file; 1382 w->w_line = line; 1383 m->mtx_line = line; 1384 m->mtx_file = file; 1385 p = CURPROC; 1386 MPASS(m->mtx_held.le_prev == NULL); 1387 LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 1388} 1389 1390void 1391witness_exit(struct mtx *m, int flags, const char *file, int line) 1392{ 1393 struct witness *w; 1394 1395 if (witness_cold || m->mtx_witness == NULL || panicstr) 1396 return; 1397 w = m->mtx_witness; 1398 1399 if (flags & MTX_SPIN) { 1400 if ((m->mtx_flags & MTX_SPIN) == 0) 1401 panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" 1402 " %s:%d", m->mtx_description, file, line); 1403 if (mtx_recursed(m)) { 1404 if ((m->mtx_flags & MTX_RECURSE) == 0) 1405 panic("mutex_exit: recursion on non-recursive" 1406 " mutex %s @ %s:%d", m->mtx_description, 1407 file, line); 1408 return; 1409 }
| 1219 w->w_file = file; 1220 w->w_line = line; 1221 m->mtx_line = line; 1222 m->mtx_file = file; 1223 return; 1224 } 1225 1226 if ((m->mtx_flags & MTX_SPIN) != 0) 1227 panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 1228 m->mtx_description, file, line); 1229 1230 if (mtx_recursed(m)) { 1231 if ((m->mtx_flags & MTX_RECURSE) == 0) 1232 panic("mutex_try_enter: recursion on non-recursive" 1233 " mutex %s @ %s:%d", m->mtx_description, file, 1234 line); 1235 return; 1236 } 1237 w->w_file = file; 1238 w->w_line = line; 1239 m->mtx_line = line; 1240 m->mtx_file = file; 1241 p = CURPROC; 1242 MPASS(m->mtx_held.le_prev == NULL); 1243 LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); 1244} 1245 1246void 1247witness_exit(struct mtx *m, int flags, const char *file, int line) 1248{ 1249 struct witness *w; 1250 1251 if (witness_cold || m->mtx_witness == NULL || panicstr) 1252 return; 1253 w = m->mtx_witness; 1254 1255 if (flags & MTX_SPIN) { 1256 if ((m->mtx_flags & MTX_SPIN) == 0) 1257 panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" 1258 " %s:%d", m->mtx_description, file, line); 1259 if (mtx_recursed(m)) { 1260 if ((m->mtx_flags & MTX_RECURSE) == 0) 1261 panic("mutex_exit: recursion on non-recursive" 1262 " mutex %s @ %s:%d", m->mtx_description, 1263 file, line); 1264 return; 1265 }
|
1410 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1266 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
|
1411 PCPU_SET(witness_spin_check, 1412 PCPU_GET(witness_spin_check) & ~w->w_level);
| 1267 PCPU_SET(witness_spin_check, 1268 PCPU_GET(witness_spin_check) & ~w->w_level);
|
1413 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1269 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1414 return; 1415 } 1416 if ((m->mtx_flags & MTX_SPIN) != 0) 1417 panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 1418 m->mtx_description, file, line); 1419 1420 if (mtx_recursed(m)) { 1421 if ((m->mtx_flags & MTX_RECURSE) == 0) 1422 panic("mutex_exit: recursion on non-recursive" 1423 " mutex %s @ %s:%d", m->mtx_description, 1424 file, line); 1425 return; 1426 } 1427 1428 if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
| 1270 return; 1271 } 1272 if ((m->mtx_flags & MTX_SPIN) != 0) 1273 panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", 1274 m->mtx_description, file, line); 1275 1276 if (mtx_recursed(m)) { 1277 if ((m->mtx_flags & MTX_RECURSE) == 0) 1278 panic("mutex_exit: recursion on non-recursive" 1279 " mutex %s @ %s:%d", m->mtx_description, 1280 file, line); 1281 return; 1282 } 1283 1284 if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
|
1429 panic("switchable mtx_exit() of %s when not legal @ %s:%d",
| 1285 panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
|
1430 m->mtx_description, file, line); 1431 LIST_REMOVE(m, mtx_held); 1432 m->mtx_held.le_prev = NULL; 1433} 1434 1435int 1436witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) 1437{ 1438 struct mtx *m; 1439 struct proc *p; 1440 char **sleep; 1441 int n = 0; 1442 1443 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1444 p = CURPROC; 1445 for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; 1446 m = LIST_NEXT(m, mtx_held)) { 1447 if (m == mtx) 1448 continue; 1449 for (sleep = sleep_list; *sleep!= NULL; sleep++) 1450 if (strcmp(m->mtx_description, *sleep) == 0) 1451 goto next; 1452 printf("%s:%d: %s with \"%s\" locked from %s:%d\n", 1453 file, line, check_only ? "could sleep" : "sleeping", 1454 m->mtx_description, 1455 m->mtx_witness->w_file, m->mtx_witness->w_line); 1456 n++; 1457 next: 1458 } 1459#ifdef DDB 1460 if (witness_ddb && n) 1461 Debugger("witness_sleep"); 1462#endif /* DDB */ 1463 return (n); 1464} 1465 1466static struct witness * 1467enroll(const char *description, int flag) 1468{ 1469 int i; 1470 struct witness *w, *w1; 1471 char **ignore; 1472 char **order; 1473 1474 if (!witness_watch) 1475 return (NULL); 1476 for (ignore = ignore_list; *ignore != NULL; ignore++) 1477 if (strcmp(description, *ignore) == 0) 1478 return (NULL); 1479 1480 if (w_inited == 0) { 1481 mtx_init(&w_mtx, "witness lock", MTX_SPIN); 1482 for (i = 0; i < WITNESS_COUNT; i++) { 1483 w = &w_data[i]; 1484 witness_free(w); 1485 } 1486 w_inited = 1; 1487 for (order = order_list; *order != NULL; order++) { 1488 w = enroll(*order, MTX_DEF); 1489 w->w_file = "order list"; 1490 for (order++; *order != NULL; order++) { 1491 w1 = enroll(*order, MTX_DEF); 1492 w1->w_file = "order list"; 1493 itismychild(w, w1); 1494 w = w1; 1495 } 1496 } 1497 } 1498 if ((flag & MTX_SPIN) && witness_skipspin) 1499 return (NULL);
| 1286 m->mtx_description, file, line); 1287 LIST_REMOVE(m, mtx_held); 1288 m->mtx_held.le_prev = NULL; 1289} 1290 1291int 1292witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) 1293{ 1294 struct mtx *m; 1295 struct proc *p; 1296 char **sleep; 1297 int n = 0; 1298 1299 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1300 p = CURPROC; 1301 for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; 1302 m = LIST_NEXT(m, mtx_held)) { 1303 if (m == mtx) 1304 continue; 1305 for (sleep = sleep_list; *sleep!= NULL; sleep++) 1306 if (strcmp(m->mtx_description, *sleep) == 0) 1307 goto next; 1308 printf("%s:%d: %s with \"%s\" locked from %s:%d\n", 1309 file, line, check_only ? "could sleep" : "sleeping", 1310 m->mtx_description, 1311 m->mtx_witness->w_file, m->mtx_witness->w_line); 1312 n++; 1313 next: 1314 } 1315#ifdef DDB 1316 if (witness_ddb && n) 1317 Debugger("witness_sleep"); 1318#endif /* DDB */ 1319 return (n); 1320} 1321 1322static struct witness * 1323enroll(const char *description, int flag) 1324{ 1325 int i; 1326 struct witness *w, *w1; 1327 char **ignore; 1328 char **order; 1329 1330 if (!witness_watch) 1331 return (NULL); 1332 for (ignore = ignore_list; *ignore != NULL; ignore++) 1333 if (strcmp(description, *ignore) == 0) 1334 return (NULL); 1335 1336 if (w_inited == 0) { 1337 mtx_init(&w_mtx, "witness lock", MTX_SPIN); 1338 for (i = 0; i < WITNESS_COUNT; i++) { 1339 w = &w_data[i]; 1340 witness_free(w); 1341 } 1342 w_inited = 1; 1343 for (order = order_list; *order != NULL; order++) { 1344 w = enroll(*order, MTX_DEF); 1345 w->w_file = "order list"; 1346 for (order++; *order != NULL; order++) { 1347 w1 = enroll(*order, MTX_DEF); 1348 w1->w_file = "order list"; 1349 itismychild(w, w1); 1350 w = w1; 1351 } 1352 } 1353 } 1354 if ((flag & MTX_SPIN) && witness_skipspin) 1355 return (NULL);
|
1500 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1356 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
|
1501 for (w = w_all; w; w = w->w_next) { 1502 if (strcmp(description, w->w_description) == 0) {
| 1357 for (w = w_all; w; w = w->w_next) { 1358 if (strcmp(description, w->w_description) == 0) {
|
1503 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1359 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1504 return (w); 1505 } 1506 } 1507 if ((w = witness_get()) == NULL) 1508 return (NULL); 1509 w->w_next = w_all; 1510 w_all = w; 1511 w->w_description = description;
| 1360 return (w); 1361 } 1362 } 1363 if ((w = witness_get()) == NULL) 1364 return (NULL); 1365 w->w_next = w_all; 1366 w_all = w; 1367 w->w_description = description;
|
1512 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1368 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1513 if (flag & MTX_SPIN) { 1514 w->w_spin = 1; 1515 1516 i = 1; 1517 for (order = spin_order_list; *order != NULL; order++) { 1518 if (strcmp(description, *order) == 0) 1519 break; 1520 i <<= 1; 1521 } 1522 if (*order == NULL) 1523 panic("spin lock %s not in order list", description); 1524 w->w_level = i; 1525 } 1526 1527 return (w); 1528} 1529 1530static int 1531itismychild(struct witness *parent, struct witness *child) 1532{ 1533 static int recursed; 1534 1535 /* 1536 * Insert "child" after "parent" 1537 */ 1538 while (parent->w_morechildren) 1539 parent = parent->w_morechildren; 1540 1541 if (parent->w_childcnt == WITNESS_NCHILDREN) { 1542 if ((parent->w_morechildren = witness_get()) == NULL) 1543 return (1); 1544 parent = parent->w_morechildren; 1545 } 1546 MPASS(child != NULL); 1547 parent->w_children[parent->w_childcnt++] = child; 1548 /* 1549 * now prune whole tree 1550 */ 1551 if (recursed) 1552 return (0); 1553 recursed = 1; 1554 for (child = w_all; child != NULL; child = child->w_next) { 1555 for (parent = w_all; parent != NULL; 1556 parent = parent->w_next) { 1557 if (!isitmychild(parent, child)) 1558 continue; 1559 removechild(parent, child); 1560 if (isitmydescendant(parent, child)) 1561 continue; 1562 itismychild(parent, child); 1563 } 1564 } 1565 recursed = 0; 1566 witness_levelall(); 1567 return (0); 1568} 1569 1570static void 1571removechild(struct witness *parent, struct witness *child) 1572{ 1573 struct witness *w, *w1; 1574 int i; 1575 1576 for (w = parent; w != NULL; w = w->w_morechildren) 1577 for (i = 0; i < w->w_childcnt; i++) 1578 if (w->w_children[i] == child) 1579 goto found; 1580 return; 1581found: 1582 for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) 1583 continue; 1584 w->w_children[i] = w1->w_children[--w1->w_childcnt]; 1585 MPASS(w->w_children[i] != NULL); 1586 1587 if (w1->w_childcnt != 0) 1588 return; 1589 1590 if (w1 == parent) 1591 return; 1592 for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) 1593 continue; 1594 w->w_morechildren = 0; 1595 witness_free(w1); 1596} 1597 1598static int 1599isitmychild(struct witness *parent, struct witness *child) 1600{ 1601 struct witness *w; 1602 int i; 1603 1604 for (w = parent; w != NULL; w = w->w_morechildren) { 1605 for (i = 0; i < w->w_childcnt; i++) { 1606 if (w->w_children[i] == child) 1607 return (1); 1608 } 1609 } 1610 return (0); 1611} 1612 1613static int 1614isitmydescendant(struct witness *parent, struct witness *child) 1615{ 1616 struct witness *w; 1617 int i; 1618 int j; 1619 1620 for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { 1621 MPASS(j < 1000); 1622 for (i = 0; i < w->w_childcnt; i++) { 1623 if (w->w_children[i] == child) 1624 return (1); 1625 } 1626 for (i = 0; i < w->w_childcnt; i++) { 1627 if (isitmydescendant(w->w_children[i], child)) 1628 return (1); 1629 } 1630 } 1631 return (0); 1632} 1633 1634void 1635witness_levelall (void) 1636{ 1637 struct witness *w, *w1; 1638 1639 for (w = w_all; w; w = w->w_next) 1640 if (!(w->w_spin)) 1641 w->w_level = 0; 1642 for (w = w_all; w; w = w->w_next) { 1643 if (w->w_spin) 1644 continue; 1645 for (w1 = w_all; w1; w1 = w1->w_next) { 1646 if (isitmychild(w1, w)) 1647 break; 1648 } 1649 if (w1 != NULL) 1650 continue; 1651 witness_leveldescendents(w, 0); 1652 } 1653} 1654 1655static void 1656witness_leveldescendents(struct witness *parent, int level) 1657{ 1658 int i; 1659 struct witness *w; 1660 1661 if (parent->w_level < level) 1662 parent->w_level = level; 1663 level++; 1664 for (w = parent; w != NULL; w = w->w_morechildren) 1665 for (i = 0; i < w->w_childcnt; i++) 1666 witness_leveldescendents(w->w_children[i], level); 1667} 1668 1669static void 1670witness_displaydescendants(void(*prnt)(const char *fmt, ...), 1671 struct witness *parent) 1672{ 1673 struct witness *w; 1674 int i; 1675 int level = parent->w_level; 1676 1677 prnt("%d", level); 1678 if (level < 10) 1679 prnt(" "); 1680 for (i = 0; i < level; i++) 1681 prnt(" "); 1682 prnt("%s", parent->w_description); 1683 if (parent->w_file != NULL) { 1684 prnt(" -- last acquired @ %s", parent->w_file); 1685#ifndef W_USE_WHERE 1686 prnt(":%d", parent->w_line); 1687#endif 1688 prnt("\n"); 1689 } 1690 1691 for (w = parent; w != NULL; w = w->w_morechildren) 1692 for (i = 0; i < w->w_childcnt; i++) 1693 witness_displaydescendants(prnt, w->w_children[i]); 1694 } 1695 1696static int 1697dup_ok(struct witness *w) 1698{ 1699 char **dup; 1700 1701 for (dup = dup_list; *dup!= NULL; dup++) 1702 if (strcmp(w->w_description, *dup) == 0) 1703 return (1); 1704 return (0); 1705} 1706 1707static int 1708blessed(struct witness *w1, struct witness *w2) 1709{ 1710 int i; 1711 struct witness_blessed *b; 1712 1713 for (i = 0; i < blessed_count; i++) { 1714 b = &blessed_list[i]; 1715 if (strcmp(w1->w_description, b->b_lock1) == 0) { 1716 if (strcmp(w2->w_description, b->b_lock2) == 0) 1717 return (1); 1718 continue; 1719 } 1720 if (strcmp(w1->w_description, b->b_lock2) == 0) 1721 if (strcmp(w2->w_description, b->b_lock1) == 0) 1722 return (1); 1723 } 1724 return (0); 1725} 1726 1727static struct witness * 1728witness_get() 1729{ 1730 struct witness *w; 1731 1732 if ((w = w_free) == NULL) { 1733 witness_dead = 1;
| 1369 if (flag & MTX_SPIN) { 1370 w->w_spin = 1; 1371 1372 i = 1; 1373 for (order = spin_order_list; *order != NULL; order++) { 1374 if (strcmp(description, *order) == 0) 1375 break; 1376 i <<= 1; 1377 } 1378 if (*order == NULL) 1379 panic("spin lock %s not in order list", description); 1380 w->w_level = i; 1381 } 1382 1383 return (w); 1384} 1385 1386static int 1387itismychild(struct witness *parent, struct witness *child) 1388{ 1389 static int recursed; 1390 1391 /* 1392 * Insert "child" after "parent" 1393 */ 1394 while (parent->w_morechildren) 1395 parent = parent->w_morechildren; 1396 1397 if (parent->w_childcnt == WITNESS_NCHILDREN) { 1398 if ((parent->w_morechildren = witness_get()) == NULL) 1399 return (1); 1400 parent = parent->w_morechildren; 1401 } 1402 MPASS(child != NULL); 1403 parent->w_children[parent->w_childcnt++] = child; 1404 /* 1405 * now prune whole tree 1406 */ 1407 if (recursed) 1408 return (0); 1409 recursed = 1; 1410 for (child = w_all; child != NULL; child = child->w_next) { 1411 for (parent = w_all; parent != NULL; 1412 parent = parent->w_next) { 1413 if (!isitmychild(parent, child)) 1414 continue; 1415 removechild(parent, child); 1416 if (isitmydescendant(parent, child)) 1417 continue; 1418 itismychild(parent, child); 1419 } 1420 } 1421 recursed = 0; 1422 witness_levelall(); 1423 return (0); 1424} 1425 1426static void 1427removechild(struct witness *parent, struct witness *child) 1428{ 1429 struct witness *w, *w1; 1430 int i; 1431 1432 for (w = parent; w != NULL; w = w->w_morechildren) 1433 for (i = 0; i < w->w_childcnt; i++) 1434 if (w->w_children[i] == child) 1435 goto found; 1436 return; 1437found: 1438 for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) 1439 continue; 1440 w->w_children[i] = w1->w_children[--w1->w_childcnt]; 1441 MPASS(w->w_children[i] != NULL); 1442 1443 if (w1->w_childcnt != 0) 1444 return; 1445 1446 if (w1 == parent) 1447 return; 1448 for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) 1449 continue; 1450 w->w_morechildren = 0; 1451 witness_free(w1); 1452} 1453 1454static int 1455isitmychild(struct witness *parent, struct witness *child) 1456{ 1457 struct witness *w; 1458 int i; 1459 1460 for (w = parent; w != NULL; w = w->w_morechildren) { 1461 for (i = 0; i < w->w_childcnt; i++) { 1462 if (w->w_children[i] == child) 1463 return (1); 1464 } 1465 } 1466 return (0); 1467} 1468 1469static int 1470isitmydescendant(struct witness *parent, struct witness *child) 1471{ 1472 struct witness *w; 1473 int i; 1474 int j; 1475 1476 for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { 1477 MPASS(j < 1000); 1478 for (i = 0; i < w->w_childcnt; i++) { 1479 if (w->w_children[i] == child) 1480 return (1); 1481 } 1482 for (i = 0; i < w->w_childcnt; i++) { 1483 if (isitmydescendant(w->w_children[i], child)) 1484 return (1); 1485 } 1486 } 1487 return (0); 1488} 1489 1490void 1491witness_levelall (void) 1492{ 1493 struct witness *w, *w1; 1494 1495 for (w = w_all; w; w = w->w_next) 1496 if (!(w->w_spin)) 1497 w->w_level = 0; 1498 for (w = w_all; w; w = w->w_next) { 1499 if (w->w_spin) 1500 continue; 1501 for (w1 = w_all; w1; w1 = w1->w_next) { 1502 if (isitmychild(w1, w)) 1503 break; 1504 } 1505 if (w1 != NULL) 1506 continue; 1507 witness_leveldescendents(w, 0); 1508 } 1509} 1510 1511static void 1512witness_leveldescendents(struct witness *parent, int level) 1513{ 1514 int i; 1515 struct witness *w; 1516 1517 if (parent->w_level < level) 1518 parent->w_level = level; 1519 level++; 1520 for (w = parent; w != NULL; w = w->w_morechildren) 1521 for (i = 0; i < w->w_childcnt; i++) 1522 witness_leveldescendents(w->w_children[i], level); 1523} 1524 1525static void 1526witness_displaydescendants(void(*prnt)(const char *fmt, ...), 1527 struct witness *parent) 1528{ 1529 struct witness *w; 1530 int i; 1531 int level = parent->w_level; 1532 1533 prnt("%d", level); 1534 if (level < 10) 1535 prnt(" "); 1536 for (i = 0; i < level; i++) 1537 prnt(" "); 1538 prnt("%s", parent->w_description); 1539 if (parent->w_file != NULL) { 1540 prnt(" -- last acquired @ %s", parent->w_file); 1541#ifndef W_USE_WHERE 1542 prnt(":%d", parent->w_line); 1543#endif 1544 prnt("\n"); 1545 } 1546 1547 for (w = parent; w != NULL; w = w->w_morechildren) 1548 for (i = 0; i < w->w_childcnt; i++) 1549 witness_displaydescendants(prnt, w->w_children[i]); 1550 } 1551 1552static int 1553dup_ok(struct witness *w) 1554{ 1555 char **dup; 1556 1557 for (dup = dup_list; *dup!= NULL; dup++) 1558 if (strcmp(w->w_description, *dup) == 0) 1559 return (1); 1560 return (0); 1561} 1562 1563static int 1564blessed(struct witness *w1, struct witness *w2) 1565{ 1566 int i; 1567 struct witness_blessed *b; 1568 1569 for (i = 0; i < blessed_count; i++) { 1570 b = &blessed_list[i]; 1571 if (strcmp(w1->w_description, b->b_lock1) == 0) { 1572 if (strcmp(w2->w_description, b->b_lock2) == 0) 1573 return (1); 1574 continue; 1575 } 1576 if (strcmp(w1->w_description, b->b_lock2) == 0) 1577 if (strcmp(w2->w_description, b->b_lock1) == 0) 1578 return (1); 1579 } 1580 return (0); 1581} 1582 1583static struct witness * 1584witness_get() 1585{ 1586 struct witness *w; 1587 1588 if ((w = w_free) == NULL) { 1589 witness_dead = 1;
|
1734 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
| 1590 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
|
1735 printf("witness exhausted\n"); 1736 return (NULL); 1737 } 1738 w_free = w->w_next; 1739 bzero(w, sizeof(*w)); 1740 return (w); 1741} 1742 1743static void 1744witness_free(struct witness *w) 1745{ 1746 w->w_next = w_free; 1747 w_free = w; 1748} 1749 1750int 1751witness_list(struct proc *p) 1752{ 1753 struct mtx *m; 1754 int nheld; 1755 1756 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1757 nheld = 0; 1758 for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; 1759 m = LIST_NEXT(m, mtx_held)) { 1760 printf("\t\"%s\" (%p) locked at %s:%d\n", 1761 m->mtx_description, m, 1762 m->mtx_witness->w_file, m->mtx_witness->w_line); 1763 nheld++; 1764 } 1765 1766 return (nheld); 1767} 1768 1769#ifdef DDB 1770 1771DB_COMMAND(witness_list, db_witness_list) 1772{ 1773 1774 witness_list(CURPROC); 1775} 1776 1777#endif 1778 1779void 1780witness_save(struct mtx *m, const char **filep, int *linep) 1781{ 1782 1783 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1784 if (m->mtx_witness == NULL) 1785 return; 1786 1787 *filep = m->mtx_witness->w_file; 1788 *linep = m->mtx_witness->w_line; 1789} 1790 1791void 1792witness_restore(struct mtx *m, const char *file, int line) 1793{ 1794 1795 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1796 if (m->mtx_witness == NULL) 1797 return; 1798 1799 m->mtx_witness->w_file = file; 1800 m->mtx_witness->w_line = line; 1801} 1802 1803#endif /* WITNESS */
| 1591 printf("witness exhausted\n"); 1592 return (NULL); 1593 } 1594 w_free = w->w_next; 1595 bzero(w, sizeof(*w)); 1596 return (w); 1597} 1598 1599static void 1600witness_free(struct witness *w) 1601{ 1602 w->w_next = w_free; 1603 w_free = w; 1604} 1605 1606int 1607witness_list(struct proc *p) 1608{ 1609 struct mtx *m; 1610 int nheld; 1611 1612 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1613 nheld = 0; 1614 for ((m = LIST_FIRST(&p->p_heldmtx)); m != NULL; 1615 m = LIST_NEXT(m, mtx_held)) { 1616 printf("\t\"%s\" (%p) locked at %s:%d\n", 1617 m->mtx_description, m, 1618 m->mtx_witness->w_file, m->mtx_witness->w_line); 1619 nheld++; 1620 } 1621 1622 return (nheld); 1623} 1624 1625#ifdef DDB 1626 1627DB_COMMAND(witness_list, db_witness_list) 1628{ 1629 1630 witness_list(CURPROC); 1631} 1632 1633#endif 1634 1635void 1636witness_save(struct mtx *m, const char **filep, int *linep) 1637{ 1638 1639 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1640 if (m->mtx_witness == NULL) 1641 return; 1642 1643 *filep = m->mtx_witness->w_file; 1644 *linep = m->mtx_witness->w_line; 1645} 1646 1647void 1648witness_restore(struct mtx *m, const char *file, int line) 1649{ 1650 1651 KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); 1652 if (m->mtx_witness == NULL) 1653 return; 1654 1655 m->mtx_witness->w_file = file; 1656 m->mtx_witness->w_line = line; 1657} 1658 1659#endif /* WITNESS */
|