Deleted Added
sdiff udiff text old ( 227758 ) new ( 228424 )
full compact
kern_mutex.c (227758) kern_mutex.c (228424)
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 20 unchanged lines hidden (view full) ---

29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32/*
33 * Machine independent bits of mutex implementation.
34 */
35
36#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 20 unchanged lines hidden (view full) ---

29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32/*
33 * Machine independent bits of mutex implementation.
34 */
35
36#include <sys/cdefs.h>
37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 227758 2011-11-20 16:33:09Z attilio $");
37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 228424 2011-12-11 21:02:01Z avg $");
38
39#include "opt_adaptive_mutexes.h"
40#include "opt_ddb.h"
41#include "opt_global.h"
42#include "opt_kdtrace.h"
43#include "opt_sched.h"
44
45#include <sys/param.h>

--- 141 unchanged lines hidden (view full) ---

187/*
188 * Function versions of the inlined __mtx_* macros. These are used by
189 * modules and can also be called from assembly language if needed.
190 */
191void
192_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
193{
194
38
39#include "opt_adaptive_mutexes.h"
40#include "opt_ddb.h"
41#include "opt_global.h"
42#include "opt_kdtrace.h"
43#include "opt_sched.h"
44
45#include <sys/param.h>

--- 141 unchanged lines hidden (view full) ---

187/*
188 * Function versions of the inlined __mtx_* macros. These are used by
189 * modules and can also be called from assembly language if needed.
190 */
191void
192_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
193{
194
195 if (SCHEDULER_STOPPED())
196 return;
195 MPASS(curthread != NULL);
196 KASSERT(m->mtx_lock != MTX_DESTROYED,
197 ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
198 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
199 ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
200 file, line));
201 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
202 file, line, NULL);
203
204 __mtx_lock(m, curthread, opts, file, line);
205 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
206 line);
207 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
208 curthread->td_locks++;
209}
210
211void
212_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
213{
197 MPASS(curthread != NULL);
198 KASSERT(m->mtx_lock != MTX_DESTROYED,
199 ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
200 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
201 ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
202 file, line));
203 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
204 file, line, NULL);
205
206 __mtx_lock(m, curthread, opts, file, line);
207 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
208 line);
209 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
210 curthread->td_locks++;
211}
212
213void
214_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
215{
216
217 if (SCHEDULER_STOPPED())
218 return;
214 MPASS(curthread != NULL);
215 KASSERT(m->mtx_lock != MTX_DESTROYED,
216 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
217 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
218 ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
219 file, line));
220 curthread->td_locks--;
221 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);

--- 5 unchanged lines hidden (view full) ---

227 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m);
228 __mtx_unlock(m, curthread, opts, file, line);
229}
230
231void
232_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
233{
234
219 MPASS(curthread != NULL);
220 KASSERT(m->mtx_lock != MTX_DESTROYED,
221 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
222 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
223 ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
224 file, line));
225 curthread->td_locks--;
226 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);

--- 5 unchanged lines hidden (view full) ---

232 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m);
233 __mtx_unlock(m, curthread, opts, file, line);
234}
235
236void
237_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
238{
239
240 if (SCHEDULER_STOPPED())
241 return;
235 MPASS(curthread != NULL);
236 KASSERT(m->mtx_lock != MTX_DESTROYED,
237 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
238 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
239 ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
240 m->lock_object.lo_name, file, line));
241 if (mtx_owned(m))
242 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,

--- 6 unchanged lines hidden (view full) ---

249 line);
250 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
251}
252
253void
254_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
255{
256
242 MPASS(curthread != NULL);
243 KASSERT(m->mtx_lock != MTX_DESTROYED,
244 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
245 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
246 ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
247 m->lock_object.lo_name, file, line));
248 if (mtx_owned(m))
249 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,

--- 6 unchanged lines hidden (view full) ---

256 line);
257 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
258}
259
260void
261_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
262{
263
264 if (SCHEDULER_STOPPED())
265 return;
257 MPASS(curthread != NULL);
258 KASSERT(m->mtx_lock != MTX_DESTROYED,
259 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
260 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
261 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
262 m->lock_object.lo_name, file, line));
263 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
264 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,

--- 12 unchanged lines hidden (view full) ---

277mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line)
278{
279#ifdef LOCK_PROFILING
280 uint64_t waittime = 0;
281 int contested = 0;
282#endif
283 int rval;
284
266 MPASS(curthread != NULL);
267 KASSERT(m->mtx_lock != MTX_DESTROYED,
268 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
269 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
270 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
271 m->lock_object.lo_name, file, line));
272 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
273 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file,

--- 12 unchanged lines hidden (view full) ---

286mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line)
287{
288#ifdef LOCK_PROFILING
289 uint64_t waittime = 0;
290 int contested = 0;
291#endif
292 int rval;
293
294 if (SCHEDULER_STOPPED())
295 return (1);
296
285 MPASS(curthread != NULL);
286 KASSERT(m->mtx_lock != MTX_DESTROYED,
287 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
288 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
289 ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
290 file, line));
291
292 if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) {

--- 40 unchanged lines hidden (view full) ---

333 uint64_t waittime = 0;
334#endif
335#ifdef KDTRACE_HOOKS
336 uint64_t spin_cnt = 0;
337 uint64_t sleep_cnt = 0;
338 int64_t sleep_time = 0;
339#endif
340
297 MPASS(curthread != NULL);
298 KASSERT(m->mtx_lock != MTX_DESTROYED,
299 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
300 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep,
301 ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name,
302 file, line));
303
304 if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) {

--- 40 unchanged lines hidden (view full) ---

345 uint64_t waittime = 0;
346#endif
347#ifdef KDTRACE_HOOKS
348 uint64_t spin_cnt = 0;
349 uint64_t sleep_cnt = 0;
350 int64_t sleep_time = 0;
351#endif
352
353 if (SCHEDULER_STOPPED())
354 return;
355
341 if (mtx_owned(m)) {
342 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
343 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
344 m->lock_object.lo_name, file, line));
345 m->mtx_recurse++;
346 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
347 if (LOCK_LOG_TEST(&m->lock_object, opts))
348 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);

--- 154 unchanged lines hidden (view full) ---

503 int line)
504{
505 int i = 0;
506#ifdef LOCK_PROFILING
507 int contested = 0;
508 uint64_t waittime = 0;
509#endif
510
356 if (mtx_owned(m)) {
357 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
358 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
359 m->lock_object.lo_name, file, line));
360 m->mtx_recurse++;
361 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
362 if (LOCK_LOG_TEST(&m->lock_object, opts))
363 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);

--- 154 unchanged lines hidden (view full) ---

518 int line)
519{
520 int i = 0;
521#ifdef LOCK_PROFILING
522 int contested = 0;
523 uint64_t waittime = 0;
524#endif
525
526 if (SCHEDULER_STOPPED())
527 return;
528
511 if (LOCK_LOG_TEST(&m->lock_object, opts))
512 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
513
514 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
515 while (!_mtx_obtain_lock(m, tid)) {
516
517 /* Give interrupts a chance while we spin. */
518 spinlock_exit();

--- 31 unchanged lines hidden (view full) ---

550 uint64_t waittime = 0;
551#endif
552#ifdef KDTRACE_HOOKS
553 uint64_t spin_cnt = 0;
554#endif
555
556 i = 0;
557 tid = (uintptr_t)curthread;
529 if (LOCK_LOG_TEST(&m->lock_object, opts))
530 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
531
532 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
533 while (!_mtx_obtain_lock(m, tid)) {
534
535 /* Give interrupts a chance while we spin. */
536 spinlock_exit();

--- 31 unchanged lines hidden (view full) ---

568 uint64_t waittime = 0;
569#endif
570#ifdef KDTRACE_HOOKS
571 uint64_t spin_cnt = 0;
572#endif
573
574 i = 0;
575 tid = (uintptr_t)curthread;
576
577 if (SCHEDULER_STOPPED())
578 return;
579
558 for (;;) {
559retry:
560 spinlock_enter();
561 m = td->td_lock;
562 KASSERT(m->mtx_lock != MTX_DESTROYED,
563 ("thread_lock() of destroyed mutex @ %s:%d", file, line));
564 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
565 ("thread_lock() of sleep mutex %s @ %s:%d",

--- 85 unchanged lines hidden (view full) ---

651 * We are only called here if the lock is recursed or contested (i.e. we
652 * need to wake up a blocked thread).
653 */
654void
655_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
656{
657 struct turnstile *ts;
658
580 for (;;) {
581retry:
582 spinlock_enter();
583 m = td->td_lock;
584 KASSERT(m->mtx_lock != MTX_DESTROYED,
585 ("thread_lock() of destroyed mutex @ %s:%d", file, line));
586 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
587 ("thread_lock() of sleep mutex %s @ %s:%d",

--- 85 unchanged lines hidden (view full) ---

673 * We are only called here if the lock is recursed or contested (i.e. we
674 * need to wake up a blocked thread).
675 */
676void
677_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
678{
679 struct turnstile *ts;
680
681 if (SCHEDULER_STOPPED())
682 return;
683
659 if (mtx_recursed(m)) {
660 if (--(m->mtx_recurse) == 0)
661 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
662 if (LOCK_LOG_TEST(&m->lock_object, opts))
663 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
664 return;
665 }
666

--- 247 unchanged lines hidden --- 		684 if (mtx_recursed(m)) {
685 if (--(m->mtx_recurse) == 0)
686 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
687 if (LOCK_LOG_TEST(&m->lock_object, opts))
688 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m);
689 return;
690 }
691

--- 247 unchanged lines hidden ---