kern_mutex.c (284998) | kern_mutex.c (285759) |
---|---|
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 20 unchanged lines hidden (view full) --- 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 20 unchanged lines hidden (view full) --- 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> |
37__FBSDID("$FreeBSD: stable/10/sys/kern/kern_mutex.c 284998 2015-07-01 10:15:49Z avg $"); | 37__FBSDID("$FreeBSD: stable/10/sys/kern/kern_mutex.c 285759 2015-07-21 17:16:37Z markj $"); |
38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_hwpmc_hooks.h" 43#include "opt_kdtrace.h" 44#include "opt_sched.h" 45 --- 369 unchanged lines hidden (view full) --- 415#endif 416 lock_profile_obtain_lock_failed(&m->lock_object, 417 &contested, &waittime); 418 if (LOCK_LOG_TEST(&m->lock_object, opts)) 419 CTR4(KTR_LOCK, 420 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 421 m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 422#ifdef KDTRACE_HOOKS | 38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_hwpmc_hooks.h" 43#include "opt_kdtrace.h" 44#include "opt_sched.h" 45 --- 369 unchanged lines hidden (view full) --- 415#endif 416 lock_profile_obtain_lock_failed(&m->lock_object, 417 &contested, &waittime); 418 if (LOCK_LOG_TEST(&m->lock_object, opts)) 419 CTR4(KTR_LOCK, 420 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 421 m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 422#ifdef KDTRACE_HOOKS |
423 all_time -= lockstat_nsecs(); | 423 all_time -= lockstat_nsecs(&m->lock_object); |
424#endif 425 426 while (!_mtx_obtain_lock(m, tid)) { 427#ifdef KDTRACE_HOOKS 428 spin_cnt++; 429#endif 430#ifdef ADAPTIVE_MUTEXES 431 /* --- 80 unchanged lines hidden (view full) --- 512 cont_logged = 1; 513 } 514#endif 515 516 /* 517 * Block on the turnstile. 518 */ 519#ifdef KDTRACE_HOOKS | 424#endif 425 426 while (!_mtx_obtain_lock(m, tid)) { 427#ifdef KDTRACE_HOOKS 428 spin_cnt++; 429#endif 430#ifdef ADAPTIVE_MUTEXES 431 /* --- 80 unchanged lines hidden (view full) --- 512 cont_logged = 1; 513 } 514#endif 515 516 /* 517 * Block on the turnstile. 518 */ 519#ifdef KDTRACE_HOOKS |
520 sleep_time -= lockstat_nsecs(); | 520 sleep_time -= lockstat_nsecs(&m->lock_object); |
521#endif 522 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 523#ifdef KDTRACE_HOOKS | 521#endif 522 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 523#ifdef KDTRACE_HOOKS |
524 sleep_time += lockstat_nsecs(); | 524 sleep_time += lockstat_nsecs(&m->lock_object); |
525 sleep_cnt++; 526#endif 527 } 528#ifdef KDTRACE_HOOKS | 525 sleep_cnt++; 526#endif 527 } 528#ifdef KDTRACE_HOOKS |
529 all_time += lockstat_nsecs(); | 529 all_time += lockstat_nsecs(&m->lock_object); |
530#endif 531#ifdef KTR 532 if (cont_logged) { 533 CTR4(KTR_CONTENTION, 534 "contention end: %s acquired by %p at %s:%d", 535 m->lock_object.lo_name, (void *)tid, file, line); 536 } 537#endif --- 61 unchanged lines hidden (view full) --- 599 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 600 "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 601 602#ifdef HWPMC_HOOKS 603 PMC_SOFT_CALL( , , lock, failed); 604#endif 605 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 606#ifdef KDTRACE_HOOKS | 530#endif 531#ifdef KTR 532 if (cont_logged) { 533 CTR4(KTR_CONTENTION, 534 "contention end: %s acquired by %p at %s:%d", 535 m->lock_object.lo_name, (void *)tid, file, line); 536 } 537#endif --- 61 unchanged lines hidden (view full) --- 599 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 600 "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 601 602#ifdef HWPMC_HOOKS 603 PMC_SOFT_CALL( , , lock, failed); 604#endif 605 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 606#ifdef KDTRACE_HOOKS |
607 spin_time -= lockstat_nsecs(); | 607 spin_time -= lockstat_nsecs(&m->lock_object); |
608#endif 609 while (!_mtx_obtain_lock(m, tid)) { 610 611 /* Give interrupts a chance while we spin. */ 612 spinlock_exit(); 613 while (m->mtx_lock != MTX_UNOWNED) { 614 if (i++ < 10000000) { 615 cpu_spinwait(); 616 continue; 617 } 618 if (i < 60000000 || kdb_active || panicstr != NULL) 619 DELAY(1); 620 else 621 _mtx_lock_spin_failed(m); 622 cpu_spinwait(); 623 } 624 spinlock_enter(); 625 } 626#ifdef KDTRACE_HOOKS | 608#endif 609 while (!_mtx_obtain_lock(m, tid)) { 610 611 /* Give interrupts a chance while we spin. */ 612 spinlock_exit(); 613 while (m->mtx_lock != MTX_UNOWNED) { 614 if (i++ < 10000000) { 615 cpu_spinwait(); 616 continue; 617 } 618 if (i < 60000000 || kdb_active || panicstr != NULL) 619 DELAY(1); 620 else 621 _mtx_lock_spin_failed(m); 622 cpu_spinwait(); 623 } 624 spinlock_enter(); 625 } 626#ifdef KDTRACE_HOOKS |
627 spin_time += lockstat_nsecs(); | 627 spin_time += lockstat_nsecs(&m->lock_object); |
628#endif 629 630 if (LOCK_LOG_TEST(&m->lock_object, opts)) 631 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 632 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 633 "running"); 634 635 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, 636 contested, waittime, (file), (line)); | 628#endif 629 630 if (LOCK_LOG_TEST(&m->lock_object, opts)) 631 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 632 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 633 "running"); 634 635 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, 636 contested, waittime, (file), (line)); |
637 LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, spin_time); | 637#ifdef KDTRACE_HOOKS 638 if (spin_time != 0) 639 LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, spin_time); 640#endif |
638} 639#endif /* SMP */ 640 641void 642thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 643{ 644 struct mtx *m; 645 uintptr_t tid; --- 8 unchanged lines hidden (view full) --- 654 655 i = 0; 656 tid = (uintptr_t)curthread; 657 658 if (SCHEDULER_STOPPED()) 659 return; 660 661#ifdef KDTRACE_HOOKS | 641} 642#endif /* SMP */ 643 644void 645thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 646{ 647 struct mtx *m; 648 uintptr_t tid; --- 8 unchanged lines hidden (view full) --- 657 658 i = 0; 659 tid = (uintptr_t)curthread; 660 661 if (SCHEDULER_STOPPED()) 662 return; 663 664#ifdef KDTRACE_HOOKS |
662 spin_time -= lockstat_nsecs(); | 665 spin_time -= lockstat_nsecs(&td->td_lock->lock_object); |
663#endif 664 for (;;) { 665retry: 666 spinlock_enter(); 667 m = td->td_lock; 668 KASSERT(m->mtx_lock != MTX_DESTROYED, 669 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 670 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, --- 31 unchanged lines hidden (view full) --- 702 } 703 spinlock_enter(); 704 } 705 if (m == td->td_lock) 706 break; 707 __mtx_unlock_spin(m); /* does spinlock_exit() */ 708 } 709#ifdef KDTRACE_HOOKS | 666#endif 667 for (;;) { 668retry: 669 spinlock_enter(); 670 m = td->td_lock; 671 KASSERT(m->mtx_lock != MTX_DESTROYED, 672 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 673 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, --- 31 unchanged lines hidden (view full) --- 705 } 706 spinlock_enter(); 707 } 708 if (m == td->td_lock) 709 break; 710 __mtx_unlock_spin(m); /* does spinlock_exit() */ 711 } 712#ifdef KDTRACE_HOOKS |
710 spin_time += lockstat_nsecs(); | 713 spin_time += lockstat_nsecs(&m->lock_object); |
711#endif 712 if (m->mtx_recurse == 0) 713 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, 714 m, contested, waittime, (file), (line)); 715 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 716 line); 717 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 718 LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_time); --- 318 unchanged lines hidden --- | 714#endif 715 if (m->mtx_recurse == 0) 716 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, 717 m, contested, waittime, (file), (line)); 718 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 719 line); 720 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 721 LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_time); --- 318 unchanged lines hidden --- |