Deleted Added
sdiff udiff text old ( 71709 ) new ( 72200 )
full compact
kern_mutex.c (71709) kern_mutex.c (72200)
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 13 unchanged lines hidden (view full) ---

22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 13 unchanged lines hidden (view full) ---

22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 * $FreeBSD: head/sys/kern/kern_mutex.c 71709 2001-01-27 07:51:34Z jhb $
30 * $FreeBSD: head/sys/kern/kern_mutex.c 72200 2001-02-09 06:11:45Z bmilekic $
31 */
32
33/*
31 */
32
33/*
34 * Machine independent bits of mutex implementation and implementation of
35 * `witness' structure & related debugging routines.
36 */
37
38/*
34 * Main Entry: witness
35 * Pronunciation: 'wit-n&s
36 * Function: noun
37 * Etymology: Middle English witnesse, from Old English witnes knowledge,
38 * testimony, witness, from 2wit
39 * Date: before 12th century
40 * 1 : attestation of a fact or event : TESTIMONY
41 * 2 : one that gives evidence; specifically : one who testifies in

--- 6 unchanged lines hidden (view full) ---

48 * religious faith or conviction <the heroic witness to divine
49 * life -- Pilot>
50 * 6 capitalized : a member of the Jehovah's Witnesses
51 */
52
53#include "opt_ddb.h"
54#include "opt_witness.h"
55
39 * Main Entry: witness
40 * Pronunciation: 'wit-n&s
41 * Function: noun
42 * Etymology: Middle English witnesse, from Old English witnes knowledge,
43 * testimony, witness, from 2wit
44 * Date: before 12th century
45 * 1 : attestation of a fact or event : TESTIMONY
46 * 2 : one that gives evidence; specifically : one who testifies in

--- 6 unchanged lines hidden (view full) ---

53 * religious faith or conviction <the heroic witness to divine
54 * life -- Pilot>
55 * 6 capitalized : a member of the Jehovah's Witnesses
56 */
57
58#include "opt_ddb.h"
59#include "opt_witness.h"
60
56/*
57 * Cause non-inlined mtx_*() to be compiled.
58 * Must be defined early because other system headers may include mutex.h.
59 */
60#define _KERN_MUTEX_C_
61
62#include <sys/param.h>
63#include <sys/bus.h>
64#include <sys/kernel.h>
65#include <sys/malloc.h>
66#include <sys/proc.h>
67#include <sys/sysctl.h>
68#include <sys/systm.h>
69#include <sys/vmmeter.h>

--- 7 unchanged lines hidden (view full) ---

77#include <ddb/ddb.h>
78
79#include <vm/vm.h>
80#include <vm/vm_extern.h>
81
82#include <sys/mutex.h>
83
84/*
61#include <sys/param.h>
62#include <sys/bus.h>
63#include <sys/kernel.h>
64#include <sys/malloc.h>
65#include <sys/proc.h>
66#include <sys/sysctl.h>
67#include <sys/systm.h>
68#include <sys/vmmeter.h>

--- 7 unchanged lines hidden (view full) ---

76#include <ddb/ddb.h>
77
78#include <vm/vm.h>
79#include <vm/vm_extern.h>
80
81#include <sys/mutex.h>
82
83/*
85 * Machine independent bits of the mutex implementation
84 * The WITNESS-enabled mutex debug structure.
86 */
85 */
87
88#ifdef WITNESS
89struct mtx_debug {
90 struct witness *mtxd_witness;
91 LIST_ENTRY(mtx) mtxd_held;
92 const char *mtxd_file;
93 int mtxd_line;
94};
95
96#define mtx_held mtx_debug->mtxd_held
97#define mtx_file mtx_debug->mtxd_file
98#define mtx_line mtx_debug->mtxd_line
99#define mtx_witness mtx_debug->mtxd_witness
100#endif /* WITNESS */
101
102/*
86#ifdef WITNESS
87struct mtx_debug {
88 struct witness *mtxd_witness;
89 LIST_ENTRY(mtx) mtxd_held;
90 const char *mtxd_file;
91 int mtxd_line;
92};
93
94#define mtx_held mtx_debug->mtxd_held
95#define mtx_file mtx_debug->mtxd_file
96#define mtx_line mtx_debug->mtxd_line
97#define mtx_witness mtx_debug->mtxd_witness
98#endif /* WITNESS */
99
100/*
103 * Assembly macros
104 *------------------------------------------------------------------------------
101 * Internal utility macros.
105 */
102 */
103#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED)
106
104
107#define _V(x) __STRING(x)
105#define mtx_owner(m) (mtx_unowned((m)) ? NULL \
106 : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK))
108
107
109/*
110 * Default, unoptimized mutex micro-operations
111 */
108#define RETIP(x) *(((uintptr_t *)(&x)) - 1)
109#define SET_PRIO(p, pri) (p)->p_priority = (pri)
112
110
113#ifndef _obtain_lock
114/* Actually obtain mtx_lock */
115#define _obtain_lock(mp, tid) \
116 atomic_cmpset_acq_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED, (tid))
117#endif
118
119#ifndef _release_lock
120/* Actually release mtx_lock */
121#define _release_lock(mp, tid) \
122 atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), (void *)MTX_UNOWNED)
123#endif
124
125#ifndef _release_lock_quick
126/* Actually release mtx_lock quickly assuming that we own it */
127#define _release_lock_quick(mp) \
128 atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED)
129#endif
130
131#ifndef _getlock_sleep
132/* Get a sleep lock, deal with recursion inline. */
133#define _getlock_sleep(mp, tid, type) do { \
134 if (!_obtain_lock(mp, tid)) { \
135 if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\
136 mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \
137 else { \
138 atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED); \
139 (mp)->mtx_recurse++; \
140 } \
141 } \
142} while (0)
143#endif
144
145#ifndef _getlock_spin_block
146/* Get a spin lock, handle recursion inline (as the less common case) */
147#define _getlock_spin_block(mp, tid, type) do { \
148 u_int _mtx_intr = save_intr(); \
149 disable_intr(); \
150 if (!_obtain_lock(mp, tid)) \
151 mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr); \
152 else \
153 (mp)->mtx_saveintr = _mtx_intr; \
154} while (0)
155#endif
156
157#ifndef _getlock_norecurse
158/*
111/*
159 * Get a lock without any recursion handling. Calls the hard enter function if
160 * we can't get it inline.
112 * Early WITNESS-enabled declarations.
161 */
113 */
162#define _getlock_norecurse(mp, tid, type) do { \
163 if (!_obtain_lock(mp, tid)) \
164 mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0); \
165} while (0)
166#endif
114#ifdef WITNESS
167
115
168#ifndef _exitlock_norecurse
169/*
116/*
170 * Release a sleep lock assuming we haven't recursed on it, recursion is handled
171 * in the hard function.
172 */
173#define _exitlock_norecurse(mp, tid, type) do { \
174 if (!_release_lock(mp, tid)) \
175 mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \
176} while (0)
177#endif
117 * Internal WITNESS routines which must be prototyped early.
118 *
119 * XXX: When/if witness code is cleaned up, it would be wise to place all
120 * witness prototyping early in this file.
121 */
122static void witness_init(struct mtx *, int flag);
123static void witness_destroy(struct mtx *);
124static void witness_display(void(*)(const char *fmt, ...));
178
125
179#ifndef _exitlock
180/*
181 * Release a sleep lock when its likely we recursed (the code to
182 * deal with simple recursion is inline).
183 */
184#define _exitlock(mp, tid, type) do { \
185 if (!_release_lock(mp, tid)) { \
186 if ((mp)->mtx_lock & MTX_RECURSED) { \
187 if (--((mp)->mtx_recurse) == 0) \
188 atomic_clear_ptr(&(mp)->mtx_lock, \
189 MTX_RECURSED); \
190 } else { \
191 mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \
192 } \
193 } \
194} while (0)
195#endif
126MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure");
196
127
197#ifndef _exitlock_spin
198/* Release a spin lock (with possible recursion). */
199#define _exitlock_spin(mp) do { \
200 if (!mtx_recursed((mp))) { \
201 int _mtx_intr = (mp)->mtx_saveintr; \
202 \
203 _release_lock_quick(mp); \
204 restore_intr(_mtx_intr); \
205 } else { \
206 (mp)->mtx_recurse--; \
207 } \
208} while (0)
209#endif
210
211#ifdef WITNESS
212static void witness_init(struct mtx *, int flag);
213static void witness_destroy(struct mtx *);
214static void witness_display(void(*)(const char *fmt, ...));
215
216/* All mutexes in system (used for debug/panic) */
217static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
128/* All mutexes in system (used for debug/panic) */
129static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 };
130
218/*
131/*
219 * Set to 0 once mutexes have been fully initialized so that witness code can be
220 * safely executed.
132 * This global is set to 0 once it becomes safe to use the witness code.
221 */
222static int witness_cold = 1;
133 */
134static int witness_cold = 1;
135
223#else /* WITNESS */
224
136#else /* WITNESS */
137
225/*
226 * flag++ is slezoid way of shutting up unused parameter warning
227 * in mtx_init()
138/* XXX XXX XXX
139 * flag++ is sleazoid way of shuting up warning
228 */
229#define witness_init(m, flag) flag++
230#define witness_destroy(m)
231#define witness_try_enter(m, t, f, l)
232#endif /* WITNESS */
233
140 */
141#define witness_init(m, flag) flag++
142#define witness_destroy(m)
143#define witness_try_enter(m, t, f, l)
144#endif /* WITNESS */
145
234/* All mutexes in system (used for debug/panic) */
146/*
147 * All mutex locks in system are kept on the all_mtx list.
148 */
235static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head",
236 TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
237 { NULL, NULL }, &all_mtx, &all_mtx,
238#ifdef WITNESS
239 &all_mtx_debug
240#else
241 NULL
242#endif
243 };
244
149static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head",
150 TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
151 { NULL, NULL }, &all_mtx, &all_mtx,
152#ifdef WITNESS
153 &all_mtx_debug
154#else
155 NULL
156#endif
157 };
158
159/*
160 * Global variables for book keeping.
161 */
245static int mtx_cur_cnt;
246static int mtx_max_cnt;
247
162static int mtx_cur_cnt;
163static int mtx_max_cnt;
164
165/*
166 * Prototypes for non-exported routines.
167 *
168 * NOTE: Prototypes for witness routines are placed at the bottom of the file.
169 */
248static void propagate_priority(struct proc *);
170static void propagate_priority(struct proc *);
249static void mtx_enter_hard(struct mtx *, int type, int saveintr);
250static void mtx_exit_hard(struct mtx *, int type);
251
171
252#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED)
253#define mtx_owner(m) (mtx_unowned(m) ? NULL \
254 : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK))
255
256#define RETIP(x) *(((uintptr_t *)(&x)) - 1)
257#define SET_PRIO(p, pri) (p)->p_priority = (pri)
258
259static void
260propagate_priority(struct proc *p)
261{
262 int pri = p->p_priority;
263 struct mtx *m = p->p_blocked;
264
265 mtx_assert(&sched_lock, MA_OWNED);
266 for (;;) {

--- 5 unchanged lines hidden (view full) ---

272 /*
273 * This really isn't quite right. Really
274 * ought to bump priority of process that
275 * next acquires the mutex.
276 */
277 MPASS(m->mtx_lock == MTX_CONTESTED);
278 return;
279 }
172static void
173propagate_priority(struct proc *p)
174{
175 int pri = p->p_priority;
176 struct mtx *m = p->p_blocked;
177
178 mtx_assert(&sched_lock, MA_OWNED);
179 for (;;) {

--- 5 unchanged lines hidden (view full) ---

185 /*
186 * This really isn't quite right. Really
187 * ought to bump priority of process that
188 * next acquires the mutex.
189 */
190 MPASS(m->mtx_lock == MTX_CONTESTED);
191 return;
192 }
193
280 MPASS(p->p_magic == P_MAGIC);
281 KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex"));
282 if (p->p_priority <= pri)
283 return;
284
285 /*
286 * Bump this process' priority.
287 */

--- 21 unchanged lines hidden (view full) ---

309 return;
310 }
311#endif
312 /*
313 * If on run queue move to new run queue, and
314 * quit.
315 */
316 if (p->p_stat == SRUN) {
194 MPASS(p->p_magic == P_MAGIC);
195 KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex"));
196 if (p->p_priority <= pri)
197 return;
198
199 /*
200 * Bump this process' priority.
201 */

--- 21 unchanged lines hidden (view full) ---

223 return;
224 }
225#endif
226 /*
227 * If on run queue move to new run queue, and
228 * quit.
229 */
230 if (p->p_stat == SRUN) {
317 printf("XXX: moving process %d(%s) to a new run queue\n",
231 printf("XXX: moving proc %d(%s) to a new run queue\n",
318 p->p_pid, p->p_comm);
319 MPASS(p->p_blocked == NULL);
320 remrunqueue(p);
321 setrunqueue(p);
322 return;
323 }
324
325 /*

--- 7 unchanged lines hidden (view full) ---

333 /*
334 * Pick up the mutex that p is blocked on.
335 */
336 m = p->p_blocked;
337 MPASS(m != NULL);
338
339 printf("XXX: process %d(%s) is blocked on %s\n", p->p_pid,
340 p->p_comm, m->mtx_description);
232 p->p_pid, p->p_comm);
233 MPASS(p->p_blocked == NULL);
234 remrunqueue(p);
235 setrunqueue(p);
236 return;
237 }
238
239 /*

--- 7 unchanged lines hidden (view full) ---

247 /*
248 * Pick up the mutex that p is blocked on.
249 */
250 m = p->p_blocked;
251 MPASS(m != NULL);
252
253 printf("XXX: process %d(%s) is blocked on %s\n", p->p_pid,
254 p->p_comm, m->mtx_description);
255
341 /*
342 * Check if the proc needs to be moved up on
343 * the blocked chain
344 */
345 if (p == TAILQ_FIRST(&m->mtx_blocked)) {
346 printf("XXX: process at head of run queue\n");
347 continue;
348 }
256 /*
257 * Check if the proc needs to be moved up on
258 * the blocked chain
259 */
260 if (p == TAILQ_FIRST(&m->mtx_blocked)) {
261 printf("XXX: process at head of run queue\n");
262 continue;
263 }
264
349 p1 = TAILQ_PREV(p, rq, p_procq);
350 if (p1->p_priority <= pri) {
351 printf(
265 p1 = TAILQ_PREV(p, rq, p_procq);
266 if (p1->p_priority <= pri) {
267 printf(
352 "XXX: previous process %d(%s) has higher priority\n",
268 "XXX: previous process %d(%s) has higher priority\n",
353 p->p_pid, p->p_comm);
354 continue;
355 }
356
357 /*
358 * Remove proc from blocked chain and determine where
359 * it should be moved up to. Since we know that p1 has
360 * a lower priority than p, we know that at least one
361 * process in the chain has a lower priority and that
362 * p1 will thus not be NULL after the loop.
363 */
364 TAILQ_REMOVE(&m->mtx_blocked, p, p_procq);
365 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) {
366 MPASS(p1->p_magic == P_MAGIC);
367 if (p1->p_priority > pri)
368 break;
369 }
269 p->p_pid, p->p_comm);
270 continue;
271 }
272
273 /*
274 * Remove proc from blocked chain and determine where
275 * it should be moved up to. Since we know that p1 has
276 * a lower priority than p, we know that at least one
277 * process in the chain has a lower priority and that
278 * p1 will thus not be NULL after the loop.
279 */
280 TAILQ_REMOVE(&m->mtx_blocked, p, p_procq);
281 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) {
282 MPASS(p1->p_magic == P_MAGIC);
283 if (p1->p_priority > pri)
284 break;
285 }
286
370 MPASS(p1 != NULL);
371 TAILQ_INSERT_BEFORE(p1, p, p_procq);
372 CTR4(KTR_LOCK,
373 "propagate_priority: p %p moved before %p on [%p] %s",
374 p, p1, m, m->mtx_description);
375 }
376}
377
378/*
287 MPASS(p1 != NULL);
288 TAILQ_INSERT_BEFORE(p1, p, p_procq);
289 CTR4(KTR_LOCK,
290 "propagate_priority: p %p moved before %p on [%p] %s",
291 p, p1, m, m->mtx_description);
292 }
293}
294
295/*
379 * Get lock 'm', the macro handles the easy (and most common cases) and leaves
380 * the slow stuff to the mtx_enter_hard() function.
381 *
382 * Note: since type is usually a constant much of this code is optimized out.
296 * The important part of mtx_trylock{,_flags}()
297 * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that
298 * if we're called, it's because we know we don't already own this lock.
383 */
299 */
384void
385_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
300int
301_mtx_trylock(struct mtx *m, int opts, const char *file, int line)
386{
302{
387 struct mtx *mpp = mtxp;
303 int rval;
388
304
389 /* bits only valid on mtx_exit() */
390 MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
391 STR_mtx_bad_type, file, line);
305 KASSERT(CURPROC != NULL, ("curproc is NULL in _mtx_trylock"));
392
306
393 if ((type) & MTX_SPIN) {
307 /*
308 * _mtx_trylock does not accept MTX_NOSWITCH option.
309 */
310 MPASS((opts & MTX_NOSWITCH) == 0);
311
312 rval = _obtain_lock(m, CURTHD);
313
314#ifdef WITNESS
315 if (rval && m->mtx_witness != NULL) {
394 /*
316 /*
395 * Easy cases of spin locks:
396 *
397 * 1) We already own the lock and will simply recurse on it (if
398 * RLIKELY)
399 *
400 * 2) The lock is free, we just get it
317 * We do not handle recursion in _mtx_trylock; see the
318 * note at the top of the routine.
401 */
319 */
402 if ((type) & MTX_RLIKELY) {
403 /*
404 * Check for recursion, if we already have this
405 * lock we just bump the recursion count.
406 */
407 if (mpp->mtx_lock == (uintptr_t)CURTHD) {
408 mpp->mtx_recurse++;
409 goto done;
410 }
411 }
412
413 if (((type) & MTX_TOPHALF) == 0) {
414 /*
415 * If an interrupt thread uses this we must block
416 * interrupts here.
417 */
418 if ((type) & MTX_FIRST) {
419 ASS_IEN;
420 disable_intr();
421 _getlock_norecurse(mpp, CURTHD,
422 (type) & MTX_HARDOPTS);
423 } else {
424 _getlock_spin_block(mpp, CURTHD,
425 (type) & MTX_HARDOPTS);
426 }
427 } else
428 _getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
429 } else {
430 /* Sleep locks */
431 if ((type) & MTX_RLIKELY)
432 _getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
433 else
434 _getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
320 MPASS(!mtx_recursed(m));
321 witness_try_enter(m, (opts | m->mtx_flags), file, line);
435 }
322 }
436done:
437 WITNESS_ENTER(mpp, type, file, line);
438 if (((type) & MTX_QUIET) == 0)
439 CTR5(KTR_LOCK, STR_mtx_enter_fmt,
440 mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
323#endif /* WITNESS */
441
324
325 if ((opts & MTX_QUIET) == 0)
326 CTR5(KTR_LOCK, "TRY_ENTER %s [%p] result=%d at %s:%d",
327 m->mtx_description, m, rval, file, line);
328
329 return rval;
442}
443
444/*
330}
331
332/*
445 * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
333 * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
446 *
334 *
447 * XXX DOES NOT HANDLE RECURSION
335 * We call this if the lock is either contested (i.e. we need to go to
336 * sleep waiting for it), or if we need to recurse on it.
448 */
337 */
449int
450_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
338void
339_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
451{
340{
452 struct mtx *const mpp = mtxp;
453 int rval;
341 struct proc *p = CURPROC;
454
342
455 rval = _obtain_lock(mpp, CURTHD);
456#ifdef WITNESS
457 if (rval && mpp->mtx_witness != NULL) {
458 MPASS(mpp->mtx_recurse == 0);
459 witness_try_enter(mpp, type, file, line);
343 if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
344 m->mtx_recurse++;
345 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
346 if ((opts & MTX_QUIET) == 0)
347 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recurse", m);
348 return;
460 }
349 }
461#endif /* WITNESS */
462 if (((type) & MTX_QUIET) == 0)
463 CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
464 mpp->mtx_description, mpp, rval, file, line);
465
350
466 return rval;
467}
351 if ((opts & MTX_QUIET) == 0)
352 CTR3(KTR_LOCK, "mtx_lock: %p contested (lock=%p) [%p]", m,
353 (void *)m->mtx_lock, (void *)RETIP(m));
468
354
469/*
470 * Release lock m.
471 */
472void
473_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
474{
475 struct mtx *const mpp = mtxp;
355 /*
356 * Save our priority. Even though p_nativepri is protected by
357 * sched_lock, we don't obtain it here as it can be expensive.
358 * Since this is the only place p_nativepri is set, and since two
359 * CPUs will not be executing the same process concurrently, we know
360 * that no other CPU is going to be messing with this. Also,
361 * p_nativepri is only read when we are blocked on a mutex, so that
362 * can't be happening right now either.
363 */
364 p->p_nativepri = p->p_priority;
476
365
477 MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line);
478 WITNESS_EXIT(mpp, type, file, line);
479 if (((type) & MTX_QUIET) == 0)
480 CTR5(KTR_LOCK, STR_mtx_exit_fmt,
481 mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
482 if ((type) & MTX_SPIN) {
483 if ((type) & MTX_NORECURSE) {
484 int mtx_intr = mpp->mtx_saveintr;
366 while (!_obtain_lock(m, p)) {
367 uintptr_t v;
368 struct proc *p1;
485
369
486 MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse,
487 file, line);
488 _release_lock_quick(mpp);
489 if (((type) & MTX_TOPHALF) == 0) {
490 if ((type) & MTX_FIRST) {
491 ASS_IDIS;
492 enable_intr();
493 } else
494 restore_intr(mtx_intr);
495 }
496 } else {
497 if (((type & MTX_TOPHALF) == 0) &&
498 (type & MTX_FIRST)) {
499 ASS_IDIS;
500 ASS_SIEN(mpp);
501 }
502 _exitlock_spin(mpp);
370 mtx_lock_spin(&sched_lock);
371 /*
372 * Check if the lock has been released while spinning for
373 * the sched_lock.
374 */
375 if ((v = m->mtx_lock) == MTX_UNOWNED) {
376 mtx_unlock_spin(&sched_lock);
377 continue;
503 }
378 }
504 } else {
505 /* Handle sleep locks */
506 if ((type) & MTX_RLIKELY)
507 _exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
508 else {
509 _exitlock_norecurse(mpp, CURTHD,
510 (type) & MTX_HARDOPTS);
511 }
512 }
513}
514
379
515void
516mtx_enter_hard(struct mtx *m, int type, int saveintr)
517{
518 struct proc *p = CURPROC;
380 /*
381 * The mutex was marked contested on release. This means that
382 * there are processes blocked on it.
383 */
384 if (v == MTX_CONTESTED) {
385 p1 = TAILQ_FIRST(&m->mtx_blocked);
386 KASSERT(p1 != NULL,
387 ("contested mutex has no contesters"));
388 m->mtx_lock = (uintptr_t)p | MTX_CONTESTED;
519
389
520 KASSERT(p != NULL, ("curproc is NULL in mutex"));
521
522 switch (type) {
523 case MTX_DEF:
524 if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
525 m->mtx_recurse++;
526 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
527 if ((type & MTX_QUIET) == 0)
528 CTR1(KTR_LOCK, "mtx_enter: %p recurse", m);
390 if (p1->p_priority < p->p_priority)
391 SET_PRIO(p, p1->p_priority);
392 mtx_unlock_spin(&sched_lock);
529 return;
530 }
393 return;
394 }
531 if ((type & MTX_QUIET) == 0)
532 CTR3(KTR_LOCK,
533 "mtx_enter: %p contested (lock=%p) [%p]",
534 m, (void *)m->mtx_lock, (void *)RETIP(m));
535
536 /*
395
396 /*
537 * Save our priority. Even though p_nativepri is protected
538 * by sched_lock, we don't obtain it here as it can be
539 * expensive. Since this is the only place p_nativepri is
540 * set, and since two CPUs will not be executing the same
541 * process concurrently, we know that no other CPU is going
542 * to be messing with this. Also, p_nativepri is only read
543 * when we are blocked on a mutex, so that can't be happening
544 * right now either.
397 * If the mutex isn't already contested and a failure occurs
398 * setting the contested bit, the mutex was either released
399 * or the state of the MTX_RECURSED bit changed.
545 */
400 */
546 p->p_nativepri = p->p_priority;
547 while (!_obtain_lock(m, p)) {
548 uintptr_t v;
549 struct proc *p1;
401 if ((v & MTX_CONTESTED) == 0 &&
402 !atomic_cmpset_ptr(&m->mtx_lock, (void *)v,
403 (void *)(v | MTX_CONTESTED))) {
404 mtx_unlock_spin(&sched_lock);
405 continue;
406 }
550
407
551 mtx_enter(&sched_lock, MTX_SPIN | MTX_RLIKELY);
552 /*
553 * check if the lock has been released while
554 * waiting for the schedlock.
555 */
556 if ((v = m->mtx_lock) == MTX_UNOWNED) {
557 mtx_exit(&sched_lock, MTX_SPIN);
558 continue;
559 }
560 /*
561 * The mutex was marked contested on release. This
562 * means that there are processes blocked on it.
563 */
564 if (v == MTX_CONTESTED) {
565 p1 = TAILQ_FIRST(&m->mtx_blocked);
566 KASSERT(p1 != NULL, ("contested mutex has no contesters"));
567 KASSERT(p != NULL, ("curproc is NULL for contested mutex"));
568 m->mtx_lock = (uintptr_t)p | MTX_CONTESTED;
569 if (p1->p_priority < p->p_priority) {
570 SET_PRIO(p, p1->p_priority);
571 }
572 mtx_exit(&sched_lock, MTX_SPIN);
573 return;
574 }
575 /*
576 * If the mutex isn't already contested and
577 * a failure occurs setting the contested bit the
578 * mutex was either release or the
579 * state of the RECURSION bit changed.
580 */
581 if ((v & MTX_CONTESTED) == 0 &&
582 !atomic_cmpset_ptr(&m->mtx_lock, (void *)v,
583 (void *)(v | MTX_CONTESTED))) {
584 mtx_exit(&sched_lock, MTX_SPIN);
585 continue;
586 }
408 /*
409 * We deffinately must sleep for this lock.
410 */
411 mtx_assert(m, MA_NOTOWNED);
587
412
588 /* We definitely have to sleep for this lock */
589 mtx_assert(m, MA_NOTOWNED);
590
591#ifdef notyet
413#ifdef notyet
592 /*
593 * If we're borrowing an interrupted thread's VM
594 * context must clean up before going to sleep.
595 */
596 if (p->p_flag & (P_ITHD | P_SITHD)) {
597 ithd_t *it = (ithd_t *)p;
414 /*
415 * If we're borrowing an interrupted thread's VM context, we
416 * must clean up before going to sleep.
417 */
418 if (p->p_flag & (P_ITHD | P_SITHD)) {
419 ithd_t *it = (ithd_t *)p;
598
420
599 if (it->it_interrupted) {
600 if ((type & MTX_QUIET) == 0)
601 CTR2(KTR_LOCK,
602 "mtx_enter: 0x%x interrupted 0x%x",
603 it, it->it_interrupted);
604 intr_thd_fixup(it);
605 }
421 if (it->it_interrupted) {
422 if ((opts & MTX_QUIET) == 0)
423 CTR2(KTR_LOCK,
424 "mtx_lock: 0x%x interrupted 0x%x",
425 it, it->it_interrupted);
426 intr_thd_fixup(it);
606 }
427 }
428 }
607#endif
608
429#endif
430
609 /* Put us on the list of procs blocked on this mutex */
610 if (TAILQ_EMPTY(&m->mtx_blocked)) {
611 p1 = (struct proc *)(m->mtx_lock &
612 MTX_FLAGMASK);
613 LIST_INSERT_HEAD(&p1->p_contested, m,
614 mtx_contested);
431 /*
432 * Put us on the list of threads blocked on this mutex.
433 */
434 if (TAILQ_EMPTY(&m->mtx_blocked)) {
435 p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK);
436 LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested);
437 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
438 } else {
439 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq)
440 if (p1->p_priority > p->p_priority)
441 break;
442 if (p1)
443 TAILQ_INSERT_BEFORE(p1, p, p_procq);
444 else
615 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
445 TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq);
616 } else {
617 TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq)
618 if (p1->p_priority > p->p_priority)
619 break;
620 if (p1)
621 TAILQ_INSERT_BEFORE(p1, p, p_procq);
622 else
623 TAILQ_INSERT_TAIL(&m->mtx_blocked, p,
624 p_procq);
625 }
446 }
626
447
627 p->p_blocked = m; /* Who we're blocked on */
628 p->p_mtxname = m->mtx_description;
629 p->p_stat = SMTX;
448 /*
449 * Save who we're blocked on.
450 */
451 p->p_blocked = m;
452 p->p_mtxname = m->mtx_description;
453 p->p_stat = SMTX;
630#if 0
454#if 0
631 propagate_priority(p);
455 propagate_priority(p);
632#endif
456#endif
633 if ((type & MTX_QUIET) == 0)
634 CTR3(KTR_LOCK,
635 "mtx_enter: p %p blocked on [%p] %s",
636 p, m, m->mtx_description);
637 mi_switch();
638 if ((type & MTX_QUIET) == 0)
639 CTR3(KTR_LOCK,
640 "mtx_enter: p %p free from blocked on [%p] %s",
641 p, m, m->mtx_description);
642 mtx_exit(&sched_lock, MTX_SPIN);
643 }
644 return;
645 case MTX_SPIN:
646 case MTX_SPIN | MTX_FIRST:
647 case MTX_SPIN | MTX_TOPHALF:
648 {
649 int i = 0;
650
457
651 if (m->mtx_lock == (uintptr_t)p) {
652 m->mtx_recurse++;
653 return;
654 }
655 if ((type & MTX_QUIET) == 0)
656 CTR1(KTR_LOCK, "mtx_enter: %p spinning", m);
657 for (;;) {
658 if (_obtain_lock(m, p))
659 break;
660 while (m->mtx_lock != MTX_UNOWNED) {
661 if (i++ < 1000000)
662 continue;
663 if (i++ < 6000000)
664 DELAY (1);
458 if ((opts & MTX_QUIET) == 0)
459 CTR3(KTR_LOCK,
460 "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m,
461 m->mtx_description);
462
463 mi_switch();
464
465 if ((opts & MTX_QUIET) == 0)
466 CTR3(KTR_LOCK,
467 "_mtx_lock_sleep: p %p free from blocked on [%p] %s",
468 p, m, m->mtx_description);
469
470 mtx_unlock_spin(&sched_lock);
471 }
472
473 return;
474}
475
476/*
477 * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
478 *
479 * This is only called if we need to actually spin for the lock. Recursion
480 * is handled inline.
481 */
482void
483_mtx_lock_spin(struct mtx *m, int opts, u_int mtx_intr, const char *file,
484 int line)
485{
486 int i = 0;
487
488 if ((opts & MTX_QUIET) == 0)
489 CTR1(KTR_LOCK, "mtx_lock_spin: %p spinning", m);
490
491 for (;;) {
492 if (_obtain_lock(m, CURPROC))
493 break;
494
495 while (m->mtx_lock != MTX_UNOWNED) {
496 if (i++ < 1000000)
497 continue;
498 if (i++ < 6000000)
499 DELAY(1);
665#ifdef DDB
500#ifdef DDB
666 else if (!db_active)
501 else if (!db_active)
667#else
502#else
668 else
503 else
669#endif
504#endif
670 panic(
671 "spin lock %s held by %p for > 5 seconds",
672 m->mtx_description,
673 (void *)m->mtx_lock);
674 }
505 panic("spin lock %s held by %p for > 5 seconds",
506 m->mtx_description, (void *)m->mtx_lock);
675 }
507 }
676
677#ifdef MUTEX_DEBUG
678 if (type != MTX_SPIN)
679 m->mtx_saveintr = 0xbeefface;
680 else
681#endif
682 m->mtx_saveintr = saveintr;
683 if ((type & MTX_QUIET) == 0)
684 CTR1(KTR_LOCK, "mtx_enter: %p spin done", m);
685 return;
686 }
687 }
508 }
509
510 m->mtx_saveintr = mtx_intr;
511 if ((opts & MTX_QUIET) == 0)
512 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
513
514 return;
688}
689
515}
516
517/*
518 * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
519 *
520 * We are only called here if the lock is recursed or contested (i.e. we
521 * need to wake up a blocked thread).
522 */
690void
523void
691mtx_exit_hard(struct mtx *m, int type)
524_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
692{
693 struct proc *p, *p1;
694 struct mtx *m1;
695 int pri;
696
697 p = CURPROC;
525{
526 struct proc *p, *p1;
527 struct mtx *m1;
528 int pri;
529
530 p = CURPROC;
698 switch (type) {
699 case MTX_DEF:
700 case MTX_DEF | MTX_NOSWITCH:
701 if (mtx_recursed(m)) {
702 if (--(m->mtx_recurse) == 0)
703 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
704 if ((type & MTX_QUIET) == 0)
705 CTR1(KTR_LOCK, "mtx_exit: %p unrecurse", m);
706 return;
707 }
708 mtx_enter(&sched_lock, MTX_SPIN);
709 if ((type & MTX_QUIET) == 0)
710 CTR1(KTR_LOCK, "mtx_exit: %p contested", m);
711 p1 = TAILQ_FIRST(&m->mtx_blocked);
712 MPASS(p->p_magic == P_MAGIC);
713 MPASS(p1->p_magic == P_MAGIC);
714 TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq);
715 if (TAILQ_EMPTY(&m->mtx_blocked)) {
716 LIST_REMOVE(m, mtx_contested);
717 _release_lock_quick(m);
718 if ((type & MTX_QUIET) == 0)
719 CTR1(KTR_LOCK, "mtx_exit: %p not held", m);
720 } else
721 atomic_store_rel_ptr(&m->mtx_lock,
722 (void *)MTX_CONTESTED);
723 pri = MAXPRI;
724 LIST_FOREACH(m1, &p->p_contested, mtx_contested) {
725 int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_priority;
726 if (cp < pri)
727 pri = cp;
728 }
729 if (pri > p->p_nativepri)
730 pri = p->p_nativepri;
731 SET_PRIO(p, pri);
732 if ((type & MTX_QUIET) == 0)
733 CTR2(KTR_LOCK,
734 "mtx_exit: %p contested setrunqueue %p", m, p1);
735 p1->p_blocked = NULL;
736 p1->p_mtxname = NULL;
737 p1->p_stat = SRUN;
738 setrunqueue(p1);
739 if ((type & MTX_NOSWITCH) == 0 && p1->p_priority < pri) {
531 MPASS4(mtx_owned(m), "mtx_owned(mpp)", file, line);
532
533 if ((opts & MTX_QUIET) == 0)
534 CTR5(KTR_LOCK, "REL %s [%p] r=%d at %s:%d", m->mtx_description,
535 m, m->mtx_recurse, file, line);
536
537 if (mtx_recursed(m)) {
538 if (--(m->mtx_recurse) == 0)
539 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
540 if ((opts & MTX_QUIET) == 0)
541 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
542 return;
543 }
544
545 mtx_lock_spin(&sched_lock);
546 if ((opts & MTX_QUIET) == 0)
547 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
548
549 p1 = TAILQ_FIRST(&m->mtx_blocked);
550 MPASS(p->p_magic == P_MAGIC);
551 MPASS(p1->p_magic == P_MAGIC);
552
553 TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq);
554
555 if (TAILQ_EMPTY(&m->mtx_blocked)) {
556 LIST_REMOVE(m, mtx_contested);
557 _release_lock_quick(m);
558 if ((opts & MTX_QUIET) == 0)
559 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m);
560 } else
561 atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED);
562
563 pri = MAXPRI;
564 LIST_FOREACH(m1, &p->p_contested, mtx_contested) {
565 int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_priority;
566 if (cp < pri)
567 pri = cp;
568 }
569
570 if (pri > p->p_nativepri)
571 pri = p->p_nativepri;
572 SET_PRIO(p, pri);
573
574 if ((opts & MTX_QUIET) == 0)
575 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p",
576 m, p1);
577
578 p1->p_blocked = NULL;
579 p1->p_mtxname = NULL;
580 p1->p_stat = SRUN;
581 setrunqueue(p1);
582
583 if ((opts & MTX_NOSWITCH) == 0 && p1->p_priority < pri) {
740#ifdef notyet
584#ifdef notyet
741 if (p->p_flag & (P_ITHD | P_SITHD)) {
742 ithd_t *it = (ithd_t *)p;
585 if (p->p_flag & (P_ITHD | P_SITHD)) {
586 ithd_t *it = (ithd_t *)p;
743
587
744 if (it->it_interrupted) {
745 if ((type & MTX_QUIET) == 0)
746 CTR2(KTR_LOCK,
747 "mtx_exit: 0x%x interruped 0x%x",
748 it, it->it_interrupted);
749 intr_thd_fixup(it);
750 }
588 if (it->it_interrupted) {
589 if ((opts & MTX_QUIET) == 0)
590 CTR2(KTR_LOCK,
591 "_mtx_unlock_sleep: 0x%x interrupted 0x%x",
592 it, it->it_interrupted);
593 intr_thd_fixup(it);
751 }
594 }
752#endif
753 setrunqueue(p);
754 if ((type & MTX_QUIET) == 0)
755 CTR2(KTR_LOCK,
756 "mtx_exit: %p switching out lock=%p",
757 m, (void *)m->mtx_lock);
758 mi_switch();
759 if ((type & MTX_QUIET) == 0)
760 CTR2(KTR_LOCK,
761 "mtx_exit: %p resuming lock=%p",
762 m, (void *)m->mtx_lock);
763 }
595 }
764 mtx_exit(&sched_lock, MTX_SPIN);
765 break;
766 case MTX_SPIN:
767 case MTX_SPIN | MTX_FIRST:
768 if (mtx_recursed(m)) {
769 m->mtx_recurse--;
770 return;
771 }
772 MPASS(mtx_owned(m));
773 _release_lock_quick(m);
774 if (type & MTX_FIRST)
775 enable_intr(); /* XXX is this kosher? */
776 else {
777 MPASS(m->mtx_saveintr != 0xbeefface);
778 restore_intr(m->mtx_saveintr);
779 }
780 break;
781 case MTX_SPIN | MTX_TOPHALF:
782 if (mtx_recursed(m)) {
783 m->mtx_recurse--;
784 return;
785 }
786 MPASS(mtx_owned(m));
787 _release_lock_quick(m);
788 break;
789 default:
790 panic("mtx_exit_hard: unsupported type 0x%x\n", type);
596#endif
597 setrunqueue(p);
598 if ((opts & MTX_QUIET) == 0)
599 CTR2(KTR_LOCK,
600 "_mtx_unlock_sleep: %p switching out lock=%p", m,
601 (void *)m->mtx_lock);
602
603 mi_switch();
604 if ((opts & MTX_QUIET) == 0)
605 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p",
606 m, (void *)m->mtx_lock);
791 }
607 }
608
609 mtx_unlock_spin(&sched_lock);
610
611 return;
792}
793
612}
613
614/*
615 * All the unlocking of MTX_SPIN locks is done inline.
616 * See the _rel_spin_lock() macro for the details.
617 */
618
619/*
620 * The INVARIANTS-enabled mtx_assert()
621 */
794#ifdef INVARIANTS
795void
796_mtx_assert(struct mtx *m, int what, const char *file, int line)
797{
798 switch ((what)) {
799 case MA_OWNED:
800 case MA_OWNED | MA_RECURSED:
801 case MA_OWNED | MA_NOTRECURSED:

--- 15 unchanged lines hidden (view full) ---

817 (m)->mtx_description, file, line);
818 break;
819 default:
820 panic("unknown mtx_assert at %s:%d", file, line);
821 }
822}
823#endif
824
622#ifdef INVARIANTS
623void
624_mtx_assert(struct mtx *m, int what, const char *file, int line)
625{
626 switch ((what)) {
627 case MA_OWNED:
628 case MA_OWNED | MA_RECURSED:
629 case MA_OWNED | MA_NOTRECURSED:

--- 15 unchanged lines hidden (view full) ---

645 (m)->mtx_description, file, line);
646 break;
647 default:
648 panic("unknown mtx_assert at %s:%d", file, line);
649 }
650}
651#endif
652
653/*
654 * The MUTEX_DEBUG-enabled mtx_validate()
655 */
825#define MV_DESTROY 0 /* validate before destory */
826#define MV_INIT 1 /* validate before init */
827
828#ifdef MUTEX_DEBUG
829
830int mtx_validate __P((struct mtx *, int));
831
832int

--- 5 unchanged lines hidden (view full) ---

838
839#ifdef WITNESS
840 if (witness_cold)
841 return 0;
842#endif
843 if (m == &all_mtx || cold)
844 return 0;
845
656#define MV_DESTROY 0 /* validate before destory */
657#define MV_INIT 1 /* validate before init */
658
659#ifdef MUTEX_DEBUG
660
661int mtx_validate __P((struct mtx *, int));
662
663int

--- 5 unchanged lines hidden (view full) ---

669
670#ifdef WITNESS
671 if (witness_cold)
672 return 0;
673#endif
674 if (m == &all_mtx || cold)
675 return 0;
676
846 mtx_enter(&all_mtx, MTX_DEF);
677 mtx_lock(&all_mtx);
847/*
848 * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly
849 * we can re-enable the kernacc() checks.
850 */
851#ifndef __alpha__
852 MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t),
853 VM_PROT_READ) == 1);
854#endif

--- 27 unchanged lines hidden (view full) ---

882 * Not good. This mutex already exists.
883 */
884 printf("re-initing existing mutex %s\n",
885 m->mtx_description);
886 MPASS(m->mtx_lock == MTX_UNOWNED);
887 retval = 1;
888 }
889 }
678/*
679 * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly
680 * we can re-enable the kernacc() checks.
681 */
682#ifndef __alpha__
683 MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t),
684 VM_PROT_READ) == 1);
685#endif

--- 27 unchanged lines hidden (view full) ---

713 * Not good. This mutex already exists.
714 */
715 printf("re-initing existing mutex %s\n",
716 m->mtx_description);
717 MPASS(m->mtx_lock == MTX_UNOWNED);
718 retval = 1;
719 }
720 }
890 mtx_exit(&all_mtx, MTX_DEF);
721 mtx_unlock(&all_mtx);
891 return (retval);
892}
893#endif
894
722 return (retval);
723}
724#endif
725
726/*
727 * Mutex initialization routine; initialize lock `m' of type contained in
728 * `opts' with options contained in `opts' and description `description.'
729 * Place on "all_mtx" queue.
730 */
895void
731void
896mtx_init(struct mtx *m, const char *t, int flag)
732mtx_init(struct mtx *m, const char *description, int opts)
897{
733{
898 if ((flag & MTX_QUIET) == 0)
899 CTR2(KTR_LOCK, "mtx_init %p (%s)", m, t);
734
735 if ((opts & MTX_QUIET) == 0)
736 CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description);
737
900#ifdef MUTEX_DEBUG
738#ifdef MUTEX_DEBUG
901 if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */
739 /* Diagnostic and error correction */
740 if (mtx_validate(m, MV_INIT))
902 return;
903#endif
904
905 bzero((void *)m, sizeof *m);
906 TAILQ_INIT(&m->mtx_blocked);
741 return;
742#endif
743
744 bzero((void *)m, sizeof *m);
745 TAILQ_INIT(&m->mtx_blocked);
746
907#ifdef WITNESS
908 if (!witness_cold) {
747#ifdef WITNESS
748 if (!witness_cold) {
909 /* XXX - should not use DEVBUF */
910 m->mtx_debug = malloc(sizeof(struct mtx_debug),
749 m->mtx_debug = malloc(sizeof(struct mtx_debug),
911 M_DEVBUF, M_NOWAIT | M_ZERO);
750 M_WITNESS, M_NOWAIT | M_ZERO);
912 MPASS(m->mtx_debug != NULL);
913 }
914#endif
751 MPASS(m->mtx_debug != NULL);
752 }
753#endif
915 m->mtx_description = t;
916
754
917 m->mtx_flags = flag;
755 m->mtx_description = description;
756 m->mtx_flags = opts;
918 m->mtx_lock = MTX_UNOWNED;
757 m->mtx_lock = MTX_UNOWNED;
758
919 /* Put on all mutex queue */
759 /* Put on all mutex queue */
920 mtx_enter(&all_mtx, MTX_DEF);
760 mtx_lock(&all_mtx);
921 m->mtx_next = &all_mtx;
922 m->mtx_prev = all_mtx.mtx_prev;
923 m->mtx_prev->mtx_next = m;
924 all_mtx.mtx_prev = m;
925 if (++mtx_cur_cnt > mtx_max_cnt)
926 mtx_max_cnt = mtx_cur_cnt;
761 m->mtx_next = &all_mtx;
762 m->mtx_prev = all_mtx.mtx_prev;
763 m->mtx_prev->mtx_next = m;
764 all_mtx.mtx_prev = m;
765 if (++mtx_cur_cnt > mtx_max_cnt)
766 mtx_max_cnt = mtx_cur_cnt;
927 mtx_exit(&all_mtx, MTX_DEF);
767 mtx_unlock(&all_mtx);
768
928#ifdef WITNESS
929 if (!witness_cold)
769#ifdef WITNESS
770 if (!witness_cold)
930 witness_init(m, flag);
771 witness_init(m, opts);
931#endif
932}
933
772#endif
773}
774
775/*
776 * Remove lock `m' from all_mtx queue.
777 */
934void
935mtx_destroy(struct mtx *m)
936{
937
938#ifdef WITNESS
939 KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
940 __FUNCTION__));
941#endif
778void
779mtx_destroy(struct mtx *m)
780{
781
782#ifdef WITNESS
783 KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n",
784 __FUNCTION__));
785#endif
786
942 CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
787 CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description);
788
943#ifdef MUTEX_DEBUG
944 if (m->mtx_next == NULL)
945 panic("mtx_destroy: %p (%s) already destroyed",
946 m, m->mtx_description);
947
948 if (!mtx_owned(m)) {
949 MPASS(m->mtx_lock == MTX_UNOWNED);
950 } else {
951 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
952 }
789#ifdef MUTEX_DEBUG
790 if (m->mtx_next == NULL)
791 panic("mtx_destroy: %p (%s) already destroyed",
792 m, m->mtx_description);
793
794 if (!mtx_owned(m)) {
795 MPASS(m->mtx_lock == MTX_UNOWNED);
796 } else {
797 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
798 }
953 mtx_validate(m, MV_DESTROY); /* diagnostic */
799
800 /* diagnostic */
801 mtx_validate(m, MV_DESTROY);
954#endif
955
956#ifdef WITNESS
957 if (m->mtx_witness)
958 witness_destroy(m);
959#endif /* WITNESS */
960
961 /* Remove from the all mutex queue */
802#endif
803
804#ifdef WITNESS
805 if (m->mtx_witness)
806 witness_destroy(m);
807#endif /* WITNESS */
808
809 /* Remove from the all mutex queue */
962 mtx_enter(&all_mtx, MTX_DEF);
810 mtx_lock(&all_mtx);
963 m->mtx_next->mtx_prev = m->mtx_prev;
964 m->mtx_prev->mtx_next = m->mtx_next;
811 m->mtx_next->mtx_prev = m->mtx_prev;
812 m->mtx_prev->mtx_next = m->mtx_next;
813
965#ifdef MUTEX_DEBUG
966 m->mtx_next = m->mtx_prev = NULL;
967#endif
814#ifdef MUTEX_DEBUG
815 m->mtx_next = m->mtx_prev = NULL;
816#endif
817
968#ifdef WITNESS
818#ifdef WITNESS
969 free(m->mtx_debug, M_DEVBUF);
819 free(m->mtx_debug, M_WITNESS);
970 m->mtx_debug = NULL;
971#endif
820 m->mtx_debug = NULL;
821#endif
822
972 mtx_cur_cnt--;
823 mtx_cur_cnt--;
973 mtx_exit(&all_mtx, MTX_DEF);
824 mtx_unlock(&all_mtx);
974}
975
825}
826
827
976/*
828/*
977 * The non-inlined versions of the mtx_*() functions are always built (above),
978 * but the witness code depends on the WITNESS kernel option being specified.
829 * The WITNESS-enabled diagnostic code.
979 */
830 */
980
981#ifdef WITNESS
982static void
983witness_fixup(void *dummy __unused)
984{
985 struct mtx *mp;
986
987 /*
988 * We have to release Giant before initializing its witness
989 * structure so that WITNESS doesn't get confused.
990 */
831#ifdef WITNESS
832static void
833witness_fixup(void *dummy __unused)
834{
835 struct mtx *mp;
836
837 /*
838 * We have to release Giant before initializing its witness
839 * structure so that WITNESS doesn't get confused.
840 */
991 mtx_exit(&Giant, MTX_DEF);
841 mtx_unlock(&Giant);
992 mtx_assert(&Giant, MA_NOTOWNED);
842 mtx_assert(&Giant, MA_NOTOWNED);
993 mtx_enter(&all_mtx, MTX_DEF);
994
843
844 mtx_lock(&all_mtx);
845
995 /* Iterate through all mutexes and finish up mutex initialization. */
996 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
997
846 /* Iterate through all mutexes and finish up mutex initialization. */
847 for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) {
848
998 /* XXX - should not use DEVBUF */
999 mp->mtx_debug = malloc(sizeof(struct mtx_debug),
849 mp->mtx_debug = malloc(sizeof(struct mtx_debug),
1000 M_DEVBUF, M_NOWAIT | M_ZERO);
850 M_WITNESS, M_NOWAIT | M_ZERO);
1001 MPASS(mp->mtx_debug != NULL);
1002
1003 witness_init(mp, mp->mtx_flags);
1004 }
851 MPASS(mp->mtx_debug != NULL);
852
853 witness_init(mp, mp->mtx_flags);
854 }
1005 mtx_exit(&all_mtx, MTX_DEF);
855 mtx_unlock(&all_mtx);
1006
1007 /* Mark the witness code as being ready for use. */
1008 atomic_store_rel_int(&witness_cold, 0);
1009
856
857 /* Mark the witness code as being ready for use. */
858 atomic_store_rel_int(&witness_cold, 0);
859
1010 mtx_enter(&Giant, MTX_DEF);
860 mtx_lock(&Giant);
1011}
1012SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
1013
1014#define WITNESS_COUNT 200
1015#define WITNESS_NCHILDREN 2
1016
1017int witness_watch = 1;
1018

--- 37 unchanged lines hidden (view full) ---

1056#ifdef WITNESS_SKIPSPIN
1057TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin);
1058#else
1059TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin);
1060#endif
1061SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
1062 "");
1063
861}
862SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL)
863
864#define WITNESS_COUNT 200
865#define WITNESS_NCHILDREN 2
866
867int witness_watch = 1;
868

--- 37 unchanged lines hidden (view full) ---

906#ifdef WITNESS_SKIPSPIN
907TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin);
908#else
909TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin);
910#endif
911SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0,
912 "");
913
914/*
915 * Witness-enabled globals
916 */
1064static struct mtx w_mtx;
1065static struct witness *w_free;
1066static struct witness *w_all;
1067static int w_inited;
1068static int witness_dead; /* fatal error, probably no memory */
1069
1070static struct witness w_data[WITNESS_COUNT];
1071
917static struct mtx w_mtx;
918static struct witness *w_free;
919static struct witness *w_all;
920static int w_inited;
921static int witness_dead; /* fatal error, probably no memory */
922
923static struct witness w_data[WITNESS_COUNT];
924
1072static struct witness *enroll __P((const char *description, int flag));
1073static int itismychild __P((struct witness *parent, struct witness *child));
1074static void removechild __P((struct witness *parent, struct witness *child));
1075static int isitmychild __P((struct witness *parent, struct witness *child));
1076static int isitmydescendant __P((struct witness *parent, struct witness *child));
1077static int dup_ok __P((struct witness *));
1078static int blessed __P((struct witness *, struct witness *));
1079static void witness_displaydescendants
1080 __P((void(*)(const char *fmt, ...), struct witness *));
1081static void witness_leveldescendents __P((struct witness *parent, int level));
1082static void witness_levelall __P((void));
1083static struct witness * witness_get __P((void));
1084static void witness_free __P((struct witness *m));
925/*
926 * Internal witness routine prototypes
927 */
928static struct witness *enroll(const char *description, int flag);
929static int itismychild(struct witness *parent, struct witness *child);
930static void removechild(struct witness *parent, struct witness *child);
931static int isitmychild(struct witness *parent, struct witness *child);
932static int isitmydescendant(struct witness *parent, struct witness *child);
933static int dup_ok(struct witness *);
934static int blessed(struct witness *, struct witness *);
935static void
936 witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *);
937static void witness_leveldescendents(struct witness *parent, int level);
938static void witness_levelall(void);
939static struct witness * witness_get(void);
940static void witness_free(struct witness *m);
1085
941
1086
1087static char *ignore_list[] = {
1088 "witness lock",
1089 NULL
1090};
1091
1092static char *spin_order_list[] = {
1093 "sio",
1094 "sched lock",

--- 29 unchanged lines hidden (view full) ---

1124};
1125
1126/*
1127 * Pairs of locks which have been blessed
1128 * Don't complain about order problems with blessed locks
1129 */
1130static struct witness_blessed blessed_list[] = {
1131};
942static char *ignore_list[] = {
943 "witness lock",
944 NULL
945};
946
947static char *spin_order_list[] = {
948 "sio",
949 "sched lock",

--- 29 unchanged lines hidden (view full) ---

979};
980
981/*
982 * Pairs of locks which have been blessed
983 * Don't complain about order problems with blessed locks
984 */
985static struct witness_blessed blessed_list[] = {
986};
1132static int blessed_count = sizeof(blessed_list) / sizeof(struct witness_blessed);
987static int blessed_count =
988 sizeof(blessed_list) / sizeof(struct witness_blessed);
1133
1134static void
1135witness_init(struct mtx *m, int flag)
1136{
1137 m->mtx_witness = enroll(m->mtx_description, flag);
1138}
1139
1140static void

--- 65 unchanged lines hidden (view full) ---

1206 " %s:%d", m->mtx_description, file, line);
1207 if (mtx_recursed(m)) {
1208 if ((m->mtx_flags & MTX_RECURSE) == 0)
1209 panic("mutex_enter: recursion on non-recursive"
1210 " mutex %s @ %s:%d", m->mtx_description,
1211 file, line);
1212 return;
1213 }
989
990static void
991witness_init(struct mtx *m, int flag)
992{
993 m->mtx_witness = enroll(m->mtx_description, flag);
994}
995
996static void

--- 65 unchanged lines hidden (view full) ---

1062 " %s:%d", m->mtx_description, file, line);
1063 if (mtx_recursed(m)) {
1064 if ((m->mtx_flags & MTX_RECURSE) == 0)
1065 panic("mutex_enter: recursion on non-recursive"
1066 " mutex %s @ %s:%d", m->mtx_description,
1067 file, line);
1068 return;
1069 }
1214 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
1070 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
1215 i = PCPU_GET(witness_spin_check);
1216 if (i != 0 && w->w_level < i) {
1071 i = PCPU_GET(witness_spin_check);
1072 if (i != 0 && w->w_level < i) {
1217 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1073 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1218 panic("mutex_enter(%s:%x, MTX_SPIN) out of order @"
1219 " %s:%d already holding %s:%x",
1220 m->mtx_description, w->w_level, file, line,
1221 spin_order_list[ffs(i)-1], i);
1222 }
1223 PCPU_SET(witness_spin_check, i | w->w_level);
1074 panic("mutex_enter(%s:%x, MTX_SPIN) out of order @"
1075 " %s:%d already holding %s:%x",
1076 m->mtx_description, w->w_level, file, line,
1077 spin_order_list[ffs(i)-1], i);
1078 }
1079 PCPU_SET(witness_spin_check, i | w->w_level);
1224 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1080 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1225 w->w_file = file;
1226 w->w_line = line;
1227 m->mtx_line = line;
1228 m->mtx_file = file;
1229 return;
1230 }
1231 if ((m->mtx_flags & MTX_SPIN) != 0)
1232 panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",

--- 7 unchanged lines hidden (view full) ---

1240 return;
1241 }
1242 if (witness_dead)
1243 goto out;
1244 if (cold)
1245 goto out;
1246
1247 if (!mtx_legal2block())
1081 w->w_file = file;
1082 w->w_line = line;
1083 m->mtx_line = line;
1084 m->mtx_file = file;
1085 return;
1086 }
1087 if ((m->mtx_flags & MTX_SPIN) != 0)
1088 panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",

--- 7 unchanged lines hidden (view full) ---

1096 return;
1097 }
1098 if (witness_dead)
1099 goto out;
1100 if (cold)
1101 goto out;
1102
1103 if (!mtx_legal2block())
1248 panic("blockable mtx_enter() of %s when not legal @ %s:%d",
1104 panic("blockable mtx_lock() of %s when not legal @ %s:%d",
1249 m->mtx_description, file, line);
1250 /*
1251 * Is this the first mutex acquired
1252 */
1253 if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL)
1254 goto out;
1255
1256 if ((w1 = m1->mtx_witness) == w) {

--- 5 unchanged lines hidden (view full) ---

1262 printf(" 1st @ %s:%d\n", w->w_file, w->w_line);
1263 printf(" 2nd @ %s:%d\n", file, line);
1264#ifdef DDB
1265 go_into_ddb = 1;
1266#endif /* DDB */
1267 goto out;
1268 }
1269 MPASS(!mtx_owned(&w_mtx));
1105 m->mtx_description, file, line);
1106 /*
1107 * Is this the first mutex acquired
1108 */
1109 if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL)
1110 goto out;
1111
1112 if ((w1 = m1->mtx_witness) == w) {

--- 5 unchanged lines hidden (view full) ---

1118 printf(" 1st @ %s:%d\n", w->w_file, w->w_line);
1119 printf(" 2nd @ %s:%d\n", file, line);
1120#ifdef DDB
1121 go_into_ddb = 1;
1122#endif /* DDB */
1123 goto out;
1124 }
1125 MPASS(!mtx_owned(&w_mtx));
1270 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
1126 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
1271 /*
1272 * If we have a known higher number just say ok
1273 */
1274 if (witness_watch > 1 && w->w_level > w1->w_level) {
1127 /*
1128 * If we have a known higher number just say ok
1129 */
1130 if (witness_watch > 1 && w->w_level > w1->w_level) {
1275 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1131 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1276 goto out;
1277 }
1278 if (isitmydescendant(m1->mtx_witness, w)) {
1132 goto out;
1133 }
1134 if (isitmydescendant(m1->mtx_witness, w)) {
1279 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1135 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1280 goto out;
1281 }
1282 for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) {
1283
1284 MPASS(i < 200);
1285 w1 = m1->mtx_witness;
1286 if (isitmydescendant(w, w1)) {
1136 goto out;
1137 }
1138 for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) {
1139
1140 MPASS(i < 200);
1141 w1 = m1->mtx_witness;
1142 if (isitmydescendant(w, w1)) {
1287 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1143 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1288 if (blessed(w, w1))
1289 goto out;
1290 if (m1 == &Giant) {
1291 if (w1->w_Giant_squawked)
1292 goto out;
1293 else
1294 w1->w_Giant_squawked = 1;
1295 } else {

--- 12 unchanged lines hidden (view full) ---

1308#ifdef DDB
1309 go_into_ddb = 1;
1310#endif /* DDB */
1311 goto out;
1312 }
1313 }
1314 m1 = LIST_FIRST(&p->p_heldmtx);
1315 if (!itismychild(m1->mtx_witness, w))
1144 if (blessed(w, w1))
1145 goto out;
1146 if (m1 == &Giant) {
1147 if (w1->w_Giant_squawked)
1148 goto out;
1149 else
1150 w1->w_Giant_squawked = 1;
1151 } else {

--- 12 unchanged lines hidden (view full) ---

1164#ifdef DDB
1165 go_into_ddb = 1;
1166#endif /* DDB */
1167 goto out;
1168 }
1169 }
1170 m1 = LIST_FIRST(&p->p_heldmtx);
1171 if (!itismychild(m1->mtx_witness, w))
1316 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1172 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1317
1318out:
1319#ifdef DDB
1320 if (witness_ddb && go_into_ddb)
1321 Debugger("witness_enter");
1322#endif /* DDB */
1323 w->w_file = file;
1324 w->w_line = line;

--- 26 unchanged lines hidden (view full) ---

1351 m->mtx_description, file, line);
1352 if (mtx_recursed(m)) {
1353 if ((m->mtx_flags & MTX_RECURSE) == 0)
1354 panic("mutex_try_enter: recursion on"
1355 " non-recursive mutex %s @ %s:%d",
1356 m->mtx_description, file, line);
1357 return;
1358 }
1173
1174out:
1175#ifdef DDB
1176 if (witness_ddb && go_into_ddb)
1177 Debugger("witness_enter");
1178#endif /* DDB */
1179 w->w_file = file;
1180 w->w_line = line;

--- 26 unchanged lines hidden (view full) ---

1207 m->mtx_description, file, line);
1208 if (mtx_recursed(m)) {
1209 if ((m->mtx_flags & MTX_RECURSE) == 0)
1210 panic("mutex_try_enter: recursion on"
1211 " non-recursive mutex %s @ %s:%d",
1212 m->mtx_description, file, line);
1213 return;
1214 }
1359 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
1215 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
1360 PCPU_SET(witness_spin_check,
1361 PCPU_GET(witness_spin_check) | w->w_level);
1216 PCPU_SET(witness_spin_check,
1217 PCPU_GET(witness_spin_check) | w->w_level);
1362 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1218 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1363 w->w_file = file;
1364 w->w_line = line;
1365 m->mtx_line = line;
1366 m->mtx_file = file;
1367 return;
1368 }
1369
1370 if ((m->mtx_flags & MTX_SPIN) != 0)

--- 31 unchanged lines hidden (view full) ---

1402 " %s:%d", m->mtx_description, file, line);
1403 if (mtx_recursed(m)) {
1404 if ((m->mtx_flags & MTX_RECURSE) == 0)
1405 panic("mutex_exit: recursion on non-recursive"
1406 " mutex %s @ %s:%d", m->mtx_description,
1407 file, line);
1408 return;
1409 }
1219 w->w_file = file;
1220 w->w_line = line;
1221 m->mtx_line = line;
1222 m->mtx_file = file;
1223 return;
1224 }
1225
1226 if ((m->mtx_flags & MTX_SPIN) != 0)

--- 31 unchanged lines hidden (view full) ---

1258 " %s:%d", m->mtx_description, file, line);
1259 if (mtx_recursed(m)) {
1260 if ((m->mtx_flags & MTX_RECURSE) == 0)
1261 panic("mutex_exit: recursion on non-recursive"
1262 " mutex %s @ %s:%d", m->mtx_description,
1263 file, line);
1264 return;
1265 }
1410 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
1266 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
1411 PCPU_SET(witness_spin_check,
1412 PCPU_GET(witness_spin_check) & ~w->w_level);
1267 PCPU_SET(witness_spin_check,
1268 PCPU_GET(witness_spin_check) & ~w->w_level);
1413 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1269 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1414 return;
1415 }
1416 if ((m->mtx_flags & MTX_SPIN) != 0)
1417 panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
1418 m->mtx_description, file, line);
1419
1420 if (mtx_recursed(m)) {
1421 if ((m->mtx_flags & MTX_RECURSE) == 0)
1422 panic("mutex_exit: recursion on non-recursive"
1423 " mutex %s @ %s:%d", m->mtx_description,
1424 file, line);
1425 return;
1426 }
1427
1428 if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
1270 return;
1271 }
1272 if ((m->mtx_flags & MTX_SPIN) != 0)
1273 panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
1274 m->mtx_description, file, line);
1275
1276 if (mtx_recursed(m)) {
1277 if ((m->mtx_flags & MTX_RECURSE) == 0)
1278 panic("mutex_exit: recursion on non-recursive"
1279 " mutex %s @ %s:%d", m->mtx_description,
1280 file, line);
1281 return;
1282 }
1283
1284 if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
1429 panic("switchable mtx_exit() of %s when not legal @ %s:%d",
1285 panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
1430 m->mtx_description, file, line);
1431 LIST_REMOVE(m, mtx_held);
1432 m->mtx_held.le_prev = NULL;
1433}
1434
1435int
1436witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
1437{

--- 54 unchanged lines hidden (view full) ---

1492 w1->w_file = "order list";
1493 itismychild(w, w1);
1494 w = w1;
1495 }
1496 }
1497 }
1498 if ((flag & MTX_SPIN) && witness_skipspin)
1499 return (NULL);
1286 m->mtx_description, file, line);
1287 LIST_REMOVE(m, mtx_held);
1288 m->mtx_held.le_prev = NULL;
1289}
1290
1291int
1292witness_sleep(int check_only, struct mtx *mtx, const char *file, int line)
1293{

--- 54 unchanged lines hidden (view full) ---

1348 w1->w_file = "order list";
1349 itismychild(w, w1);
1350 w = w1;
1351 }
1352 }
1353 }
1354 if ((flag & MTX_SPIN) && witness_skipspin)
1355 return (NULL);
1500 mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
1356 mtx_lock_spin_flags(&w_mtx, MTX_QUIET);
1501 for (w = w_all; w; w = w->w_next) {
1502 if (strcmp(description, w->w_description) == 0) {
1357 for (w = w_all; w; w = w->w_next) {
1358 if (strcmp(description, w->w_description) == 0) {
1503 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1359 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1504 return (w);
1505 }
1506 }
1507 if ((w = witness_get()) == NULL)
1508 return (NULL);
1509 w->w_next = w_all;
1510 w_all = w;
1511 w->w_description = description;
1360 return (w);
1361 }
1362 }
1363 if ((w = witness_get()) == NULL)
1364 return (NULL);
1365 w->w_next = w_all;
1366 w_all = w;
1367 w->w_description = description;
1512 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1368 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1513 if (flag & MTX_SPIN) {
1514 w->w_spin = 1;
1515
1516 i = 1;
1517 for (order = spin_order_list; *order != NULL; order++) {
1518 if (strcmp(description, *order) == 0)
1519 break;
1520 i <<= 1;

--- 205 unchanged lines hidden (view full) ---

1726
1727static struct witness *
1728witness_get()
1729{
1730 struct witness *w;
1731
1732 if ((w = w_free) == NULL) {
1733 witness_dead = 1;
1369 if (flag & MTX_SPIN) {
1370 w->w_spin = 1;
1371
1372 i = 1;
1373 for (order = spin_order_list; *order != NULL; order++) {
1374 if (strcmp(description, *order) == 0)
1375 break;
1376 i <<= 1;

--- 205 unchanged lines hidden (view full) ---

1582
1583static struct witness *
1584witness_get()
1585{
1586 struct witness *w;
1587
1588 if ((w = w_free) == NULL) {
1589 witness_dead = 1;
1734 mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
1590 mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
1735 printf("witness exhausted\n");
1736 return (NULL);
1737 }
1738 w_free = w->w_next;
1739 bzero(w, sizeof(*w));
1740 return (w);
1741}
1742

--- 61 unchanged lines hidden --- 		1591 printf("witness exhausted\n");
1592 return (NULL);
1593 }
1594 w_free = w->w_next;
1595 bzero(w, sizeof(*w));
1596 return (w);
1597}
1598

--- 61 unchanged lines hidden ---