Deleted Added
sdiff udiff text old ( 278694 ) new ( 284998 )
full compact
kern_mutex.c (278694) kern_mutex.c (284998)
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 20 unchanged lines hidden (view full) ---

29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32/*
33 * Machine independent bits of mutex implementation.
34 */
35
36#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 20 unchanged lines hidden (view full) ---

29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32/*
33 * Machine independent bits of mutex implementation.
34 */
35
36#include <sys/cdefs.h>
37__FBSDID("$FreeBSD: stable/10/sys/kern/kern_mutex.c 278694 2015-02-13 19:06:22Z sbruno $");
37__FBSDID("$FreeBSD: stable/10/sys/kern/kern_mutex.c 284998 2015-07-01 10:15:49Z avg $");
38
39#include "opt_adaptive_mutexes.h"
40#include "opt_ddb.h"
41#include "opt_global.h"
42#include "opt_hwpmc_hooks.h"
43#include "opt_kdtrace.h"
44#include "opt_sched.h"
45

--- 337 unchanged lines hidden (view full) ---

383#ifdef LOCK_PROFILING
384 int contested = 0;
385 uint64_t waittime = 0;
386#endif
387#ifdef KDTRACE_HOOKS
388 uint64_t spin_cnt = 0;
389 uint64_t sleep_cnt = 0;
390 int64_t sleep_time = 0;
38
39#include "opt_adaptive_mutexes.h"
40#include "opt_ddb.h"
41#include "opt_global.h"
42#include "opt_hwpmc_hooks.h"
43#include "opt_kdtrace.h"
44#include "opt_sched.h"
45

--- 337 unchanged lines hidden (view full) ---

383#ifdef LOCK_PROFILING
384 int contested = 0;
385 uint64_t waittime = 0;
386#endif
387#ifdef KDTRACE_HOOKS
388 uint64_t spin_cnt = 0;
389 uint64_t sleep_cnt = 0;
390 int64_t sleep_time = 0;
391 int64_t all_time = 0;
391#endif
392
393 if (SCHEDULER_STOPPED())
394 return;
395
396 m = mtxlock2mtx(c);
397
398 if (mtx_owned(m)) {

--- 14 unchanged lines hidden (view full) ---

413 PMC_SOFT_CALL( , , lock, failed);
414#endif
415 lock_profile_obtain_lock_failed(&m->lock_object,
416 &contested, &waittime);
417 if (LOCK_LOG_TEST(&m->lock_object, opts))
418 CTR4(KTR_LOCK,
419 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
420 m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
392#endif
393
394 if (SCHEDULER_STOPPED())
395 return;
396
397 m = mtxlock2mtx(c);
398
399 if (mtx_owned(m)) {

--- 14 unchanged lines hidden (view full) ---

414 PMC_SOFT_CALL( , , lock, failed);
415#endif
416 lock_profile_obtain_lock_failed(&m->lock_object,
417 &contested, &waittime);
418 if (LOCK_LOG_TEST(&m->lock_object, opts))
419 CTR4(KTR_LOCK,
420 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
421 m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
422#ifdef KDTRACE_HOOKS
423 all_time -= lockstat_nsecs();
424#endif
421
422 while (!_mtx_obtain_lock(m, tid)) {
423#ifdef KDTRACE_HOOKS
424 spin_cnt++;
425#endif
426#ifdef ADAPTIVE_MUTEXES
427 /*
428 * If the owner is running on another CPU, spin until the

--- 87 unchanged lines hidden (view full) ---

516 sleep_time -= lockstat_nsecs();
517#endif
518 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
519#ifdef KDTRACE_HOOKS
520 sleep_time += lockstat_nsecs();
521 sleep_cnt++;
522#endif
523 }
425
426 while (!_mtx_obtain_lock(m, tid)) {
427#ifdef KDTRACE_HOOKS
428 spin_cnt++;
429#endif
430#ifdef ADAPTIVE_MUTEXES
431 /*
432 * If the owner is running on another CPU, spin until the

--- 87 unchanged lines hidden (view full) ---

520 sleep_time -= lockstat_nsecs();
521#endif
522 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
523#ifdef KDTRACE_HOOKS
524 sleep_time += lockstat_nsecs();
525 sleep_cnt++;
526#endif
527 }
528#ifdef KDTRACE_HOOKS
529 all_time += lockstat_nsecs();
530#endif
524#ifdef KTR
525 if (cont_logged) {
526 CTR4(KTR_CONTENTION,
527 "contention end: %s acquired by %p at %s:%d",
528 m->lock_object.lo_name, (void *)tid, file, line);
529 }
530#endif
531 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested,
532 waittime, file, line);
533#ifdef KDTRACE_HOOKS
534 if (sleep_time)
535 LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time);
536
537 /*
538 * Only record the loops spinning and not sleeping.
539 */
540 if (spin_cnt > sleep_cnt)
531#ifdef KTR
532 if (cont_logged) {
533 CTR4(KTR_CONTENTION,
534 "contention end: %s acquired by %p at %s:%d",
535 m->lock_object.lo_name, (void *)tid, file, line);
536 }
537#endif
538 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested,
539 waittime, file, line);
540#ifdef KDTRACE_HOOKS
541 if (sleep_time)
542 LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time);
543
544 /*
545 * Only record the loops spinning and not sleeping.
546 */
547 if (spin_cnt > sleep_cnt)
541 LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt));
548 LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (all_time - sleep_time));
542#endif
543}
544
545static void
546_mtx_lock_spin_failed(struct mtx *m)
547{
548 struct thread *td;
549

--- 23 unchanged lines hidden (view full) ---

573 const char *file, int line)
574{
575 struct mtx *m;
576 int i = 0;
577#ifdef LOCK_PROFILING
578 int contested = 0;
579 uint64_t waittime = 0;
580#endif
549#endif
550}
551
552static void
553_mtx_lock_spin_failed(struct mtx *m)
554{
555 struct thread *td;
556

--- 23 unchanged lines hidden (view full) ---

580 const char *file, int line)
581{
582 struct mtx *m;
583 int i = 0;
584#ifdef LOCK_PROFILING
585 int contested = 0;
586 uint64_t waittime = 0;
587#endif
588#ifdef KDTRACE_HOOKS
589 int64_t spin_time = 0;
590#endif
581
582 if (SCHEDULER_STOPPED())
583 return;
584
585 m = mtxlock2mtx(c);
586
587 if (LOCK_LOG_TEST(&m->lock_object, opts))
588 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
589 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
590 "spinning", "lockname:\"%s\"", m->lock_object.lo_name);
591
592#ifdef HWPMC_HOOKS
593 PMC_SOFT_CALL( , , lock, failed);
594#endif
595 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
591
592 if (SCHEDULER_STOPPED())
593 return;
594
595 m = mtxlock2mtx(c);
596
597 if (LOCK_LOG_TEST(&m->lock_object, opts))
598 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
599 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
600 "spinning", "lockname:\"%s\"", m->lock_object.lo_name);
601
602#ifdef HWPMC_HOOKS
603 PMC_SOFT_CALL( , , lock, failed);
604#endif
605 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
606#ifdef KDTRACE_HOOKS
607 spin_time -= lockstat_nsecs();
608#endif
596 while (!_mtx_obtain_lock(m, tid)) {
597
598 /* Give interrupts a chance while we spin. */
599 spinlock_exit();
600 while (m->mtx_lock != MTX_UNOWNED) {
601 if (i++ < 10000000) {
602 cpu_spinwait();
603 continue;
604 }
605 if (i < 60000000 || kdb_active || panicstr != NULL)
606 DELAY(1);
607 else
608 _mtx_lock_spin_failed(m);
609 cpu_spinwait();
610 }
611 spinlock_enter();
612 }
609 while (!_mtx_obtain_lock(m, tid)) {
610
611 /* Give interrupts a chance while we spin. */
612 spinlock_exit();
613 while (m->mtx_lock != MTX_UNOWNED) {
614 if (i++ < 10000000) {
615 cpu_spinwait();
616 continue;
617 }
618 if (i < 60000000 || kdb_active || panicstr != NULL)
619 DELAY(1);
620 else
621 _mtx_lock_spin_failed(m);
622 cpu_spinwait();
623 }
624 spinlock_enter();
625 }
626#ifdef KDTRACE_HOOKS
627 spin_time += lockstat_nsecs();
628#endif
613
614 if (LOCK_LOG_TEST(&m->lock_object, opts))
615 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
616 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
617 "running");
618
619 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m,
620 contested, waittime, (file), (line));
629
630 if (LOCK_LOG_TEST(&m->lock_object, opts))
631 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m);
632 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid),
633 "running");
634
635 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m,
636 contested, waittime, (file), (line));
621 LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i);
637 LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, spin_time);
622}
623#endif /* SMP */
624
625void
626thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
627{
628 struct mtx *m;
629 uintptr_t tid;
630 int i;
631#ifdef LOCK_PROFILING
632 int contested = 0;
633 uint64_t waittime = 0;
634#endif
635#ifdef KDTRACE_HOOKS
638}
639#endif /* SMP */
640
641void
642thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
643{
644 struct mtx *m;
645 uintptr_t tid;
646 int i;
647#ifdef LOCK_PROFILING
648 int contested = 0;
649 uint64_t waittime = 0;
650#endif
651#ifdef KDTRACE_HOOKS
636 uint64_t spin_cnt = 0;
652 int64_t spin_time = 0;
637#endif
638
639 i = 0;
640 tid = (uintptr_t)curthread;
641
642 if (SCHEDULER_STOPPED())
643 return;
644
653#endif
654
655 i = 0;
656 tid = (uintptr_t)curthread;
657
658 if (SCHEDULER_STOPPED())
659 return;
660
661#ifdef KDTRACE_HOOKS
662 spin_time -= lockstat_nsecs();
663#endif
645 for (;;) {
646retry:
647 spinlock_enter();
648 m = td->td_lock;
649 KASSERT(m->mtx_lock != MTX_DESTROYED,
650 ("thread_lock() of destroyed mutex @ %s:%d", file, line));
651 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
652 ("thread_lock() of sleep mutex %s @ %s:%d",
653 m->lock_object.lo_name, file, line));
654 if (mtx_owned(m))
655 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
656 ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
657 m->lock_object.lo_name, file, line));
658 WITNESS_CHECKORDER(&m->lock_object,
659 opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
660 while (!_mtx_obtain_lock(m, tid)) {
664 for (;;) {
665retry:
666 spinlock_enter();
667 m = td->td_lock;
668 KASSERT(m->mtx_lock != MTX_DESTROYED,
669 ("thread_lock() of destroyed mutex @ %s:%d", file, line));
670 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin,
671 ("thread_lock() of sleep mutex %s @ %s:%d",
672 m->lock_object.lo_name, file, line));
673 if (mtx_owned(m))
674 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
675 ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n",
676 m->lock_object.lo_name, file, line));
677 WITNESS_CHECKORDER(&m->lock_object,
678 opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
679 while (!_mtx_obtain_lock(m, tid)) {
661#ifdef KDTRACE_HOOKS
662 spin_cnt++;
663#endif
664 if (m->mtx_lock == tid) {
665 m->mtx_recurse++;
666 break;
667 }
668#ifdef HWPMC_HOOKS
669 PMC_SOFT_CALL( , , lock, failed);
670#endif
671 lock_profile_obtain_lock_failed(&m->lock_object,

--- 12 unchanged lines hidden (view full) ---

684 if (m != td->td_lock)
685 goto retry;
686 }
687 spinlock_enter();
688 }
689 if (m == td->td_lock)
690 break;
691 __mtx_unlock_spin(m); /* does spinlock_exit() */
680 if (m->mtx_lock == tid) {
681 m->mtx_recurse++;
682 break;
683 }
684#ifdef HWPMC_HOOKS
685 PMC_SOFT_CALL( , , lock, failed);
686#endif
687 lock_profile_obtain_lock_failed(&m->lock_object,

--- 12 unchanged lines hidden (view full) ---

700 if (m != td->td_lock)
701 goto retry;
702 }
703 spinlock_enter();
704 }
705 if (m == td->td_lock)
706 break;
707 __mtx_unlock_spin(m); /* does spinlock_exit() */
708 }
692#ifdef KDTRACE_HOOKS
709#ifdef KDTRACE_HOOKS
693 spin_cnt++;
710 spin_time += lockstat_nsecs();
694#endif
711#endif
695 }
696 if (m->mtx_recurse == 0)
697 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE,
698 m, contested, waittime, (file), (line));
699 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
700 line);
701 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
712 if (m->mtx_recurse == 0)
713 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE,
714 m, contested, waittime, (file), (line));
715 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
716 line);
717 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
702 LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt);
718 LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_time);
703}
704
705struct mtx *
706thread_lock_block(struct thread *td)
707{
708 struct mtx *lock;
709
710 THREAD_LOCK_ASSERT(td, MA_OWNED);

--- 310 unchanged lines hidden --- 		719}
720
721struct mtx *
722thread_lock_block(struct thread *td)
723{
724 struct mtx *lock;
725
726 THREAD_LOCK_ASSERT(td, MA_OWNED);

--- 310 unchanged lines hidden ---