kern_mutex.c (278694) | kern_mutex.c (284998) |
---|---|
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 20 unchanged lines hidden (view full) --- 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 20 unchanged lines hidden (view full) --- 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> |
37__FBSDID("$FreeBSD: stable/10/sys/kern/kern_mutex.c 278694 2015-02-13 19:06:22Z sbruno $"); | 37__FBSDID("$FreeBSD: stable/10/sys/kern/kern_mutex.c 284998 2015-07-01 10:15:49Z avg $"); |
38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_hwpmc_hooks.h" 43#include "opt_kdtrace.h" 44#include "opt_sched.h" 45 --- 337 unchanged lines hidden (view full) --- 383#ifdef LOCK_PROFILING 384 int contested = 0; 385 uint64_t waittime = 0; 386#endif 387#ifdef KDTRACE_HOOKS 388 uint64_t spin_cnt = 0; 389 uint64_t sleep_cnt = 0; 390 int64_t sleep_time = 0; | 38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_hwpmc_hooks.h" 43#include "opt_kdtrace.h" 44#include "opt_sched.h" 45 --- 337 unchanged lines hidden (view full) --- 383#ifdef LOCK_PROFILING 384 int contested = 0; 385 uint64_t waittime = 0; 386#endif 387#ifdef KDTRACE_HOOKS 388 uint64_t spin_cnt = 0; 389 uint64_t sleep_cnt = 0; 390 int64_t sleep_time = 0; |
391 int64_t all_time = 0; |
|
391#endif 392 393 if (SCHEDULER_STOPPED()) 394 return; 395 396 m = mtxlock2mtx(c); 397 398 if (mtx_owned(m)) { --- 14 unchanged lines hidden (view full) --- 413 PMC_SOFT_CALL( , , lock, failed); 414#endif 415 lock_profile_obtain_lock_failed(&m->lock_object, 416 &contested, &waittime); 417 if (LOCK_LOG_TEST(&m->lock_object, opts)) 418 CTR4(KTR_LOCK, 419 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 420 m->lock_object.lo_name, (void *)m->mtx_lock, file, line); | 392#endif 393 394 if (SCHEDULER_STOPPED()) 395 return; 396 397 m = mtxlock2mtx(c); 398 399 if (mtx_owned(m)) { --- 14 unchanged lines hidden (view full) --- 414 PMC_SOFT_CALL( , , lock, failed); 415#endif 416 lock_profile_obtain_lock_failed(&m->lock_object, 417 &contested, &waittime); 418 if (LOCK_LOG_TEST(&m->lock_object, opts)) 419 CTR4(KTR_LOCK, 420 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 421 m->lock_object.lo_name, (void *)m->mtx_lock, file, line); |
422#ifdef KDTRACE_HOOKS 423 all_time -= lockstat_nsecs(); 424#endif |
|
421 422 while (!_mtx_obtain_lock(m, tid)) { 423#ifdef KDTRACE_HOOKS 424 spin_cnt++; 425#endif 426#ifdef ADAPTIVE_MUTEXES 427 /* 428 * If the owner is running on another CPU, spin until the --- 87 unchanged lines hidden (view full) --- 516 sleep_time -= lockstat_nsecs(); 517#endif 518 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 519#ifdef KDTRACE_HOOKS 520 sleep_time += lockstat_nsecs(); 521 sleep_cnt++; 522#endif 523 } | 425 426 while (!_mtx_obtain_lock(m, tid)) { 427#ifdef KDTRACE_HOOKS 428 spin_cnt++; 429#endif 430#ifdef ADAPTIVE_MUTEXES 431 /* 432 * If the owner is running on another CPU, spin until the --- 87 unchanged lines hidden (view full) --- 520 sleep_time -= lockstat_nsecs(); 521#endif 522 turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); 523#ifdef KDTRACE_HOOKS 524 sleep_time += lockstat_nsecs(); 525 sleep_cnt++; 526#endif 527 } |
528#ifdef KDTRACE_HOOKS 529 all_time += lockstat_nsecs(); 530#endif |
|
524#ifdef KTR 525 if (cont_logged) { 526 CTR4(KTR_CONTENTION, 527 "contention end: %s acquired by %p at %s:%d", 528 m->lock_object.lo_name, (void *)tid, file, line); 529 } 530#endif 531 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested, 532 waittime, file, line); 533#ifdef KDTRACE_HOOKS 534 if (sleep_time) 535 LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time); 536 537 /* 538 * Only record the loops spinning and not sleeping. 539 */ 540 if (spin_cnt > sleep_cnt) | 531#ifdef KTR 532 if (cont_logged) { 533 CTR4(KTR_CONTENTION, 534 "contention end: %s acquired by %p at %s:%d", 535 m->lock_object.lo_name, (void *)tid, file, line); 536 } 537#endif 538 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested, 539 waittime, file, line); 540#ifdef KDTRACE_HOOKS 541 if (sleep_time) 542 LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time); 543 544 /* 545 * Only record the loops spinning and not sleeping. 546 */ 547 if (spin_cnt > sleep_cnt) |
541 LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt)); | 548 LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (all_time - sleep_time)); |
542#endif 543} 544 545static void 546_mtx_lock_spin_failed(struct mtx *m) 547{ 548 struct thread *td; 549 --- 23 unchanged lines hidden (view full) --- 573 const char *file, int line) 574{ 575 struct mtx *m; 576 int i = 0; 577#ifdef LOCK_PROFILING 578 int contested = 0; 579 uint64_t waittime = 0; 580#endif | 549#endif 550} 551 552static void 553_mtx_lock_spin_failed(struct mtx *m) 554{ 555 struct thread *td; 556 --- 23 unchanged lines hidden (view full) --- 580 const char *file, int line) 581{ 582 struct mtx *m; 583 int i = 0; 584#ifdef LOCK_PROFILING 585 int contested = 0; 586 uint64_t waittime = 0; 587#endif |
588#ifdef KDTRACE_HOOKS 589 int64_t spin_time = 0; 590#endif |
|
581 582 if (SCHEDULER_STOPPED()) 583 return; 584 585 m = mtxlock2mtx(c); 586 587 if (LOCK_LOG_TEST(&m->lock_object, opts)) 588 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 589 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 590 "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 591 592#ifdef HWPMC_HOOKS 593 PMC_SOFT_CALL( , , lock, failed); 594#endif 595 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); | 591 592 if (SCHEDULER_STOPPED()) 593 return; 594 595 m = mtxlock2mtx(c); 596 597 if (LOCK_LOG_TEST(&m->lock_object, opts)) 598 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 599 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 600 "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 601 602#ifdef HWPMC_HOOKS 603 PMC_SOFT_CALL( , , lock, failed); 604#endif 605 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); |
606#ifdef KDTRACE_HOOKS 607 spin_time -= lockstat_nsecs(); 608#endif |
|
596 while (!_mtx_obtain_lock(m, tid)) { 597 598 /* Give interrupts a chance while we spin. */ 599 spinlock_exit(); 600 while (m->mtx_lock != MTX_UNOWNED) { 601 if (i++ < 10000000) { 602 cpu_spinwait(); 603 continue; 604 } 605 if (i < 60000000 || kdb_active || panicstr != NULL) 606 DELAY(1); 607 else 608 _mtx_lock_spin_failed(m); 609 cpu_spinwait(); 610 } 611 spinlock_enter(); 612 } | 609 while (!_mtx_obtain_lock(m, tid)) { 610 611 /* Give interrupts a chance while we spin. */ 612 spinlock_exit(); 613 while (m->mtx_lock != MTX_UNOWNED) { 614 if (i++ < 10000000) { 615 cpu_spinwait(); 616 continue; 617 } 618 if (i < 60000000 || kdb_active || panicstr != NULL) 619 DELAY(1); 620 else 621 _mtx_lock_spin_failed(m); 622 cpu_spinwait(); 623 } 624 spinlock_enter(); 625 } |
626#ifdef KDTRACE_HOOKS 627 spin_time += lockstat_nsecs(); 628#endif |
|
613 614 if (LOCK_LOG_TEST(&m->lock_object, opts)) 615 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 616 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 617 "running"); 618 619 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, 620 contested, waittime, (file), (line)); | 629 630 if (LOCK_LOG_TEST(&m->lock_object, opts)) 631 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 632 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 633 "running"); 634 635 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, 636 contested, waittime, (file), (line)); |
621 LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i); | 637 LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, spin_time); |
622} 623#endif /* SMP */ 624 625void 626thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 627{ 628 struct mtx *m; 629 uintptr_t tid; 630 int i; 631#ifdef LOCK_PROFILING 632 int contested = 0; 633 uint64_t waittime = 0; 634#endif 635#ifdef KDTRACE_HOOKS | 638} 639#endif /* SMP */ 640 641void 642thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 643{ 644 struct mtx *m; 645 uintptr_t tid; 646 int i; 647#ifdef LOCK_PROFILING 648 int contested = 0; 649 uint64_t waittime = 0; 650#endif 651#ifdef KDTRACE_HOOKS |
636 uint64_t spin_cnt = 0; | 652 int64_t spin_time = 0; |
637#endif 638 639 i = 0; 640 tid = (uintptr_t)curthread; 641 642 if (SCHEDULER_STOPPED()) 643 return; 644 | 653#endif 654 655 i = 0; 656 tid = (uintptr_t)curthread; 657 658 if (SCHEDULER_STOPPED()) 659 return; 660 |
661#ifdef KDTRACE_HOOKS 662 spin_time -= lockstat_nsecs(); 663#endif |
|
645 for (;;) { 646retry: 647 spinlock_enter(); 648 m = td->td_lock; 649 KASSERT(m->mtx_lock != MTX_DESTROYED, 650 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 651 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 652 ("thread_lock() of sleep mutex %s @ %s:%d", 653 m->lock_object.lo_name, file, line)); 654 if (mtx_owned(m)) 655 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 656 ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 657 m->lock_object.lo_name, file, line)); 658 WITNESS_CHECKORDER(&m->lock_object, 659 opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 660 while (!_mtx_obtain_lock(m, tid)) { | 664 for (;;) { 665retry: 666 spinlock_enter(); 667 m = td->td_lock; 668 KASSERT(m->mtx_lock != MTX_DESTROYED, 669 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 670 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 671 ("thread_lock() of sleep mutex %s @ %s:%d", 672 m->lock_object.lo_name, file, line)); 673 if (mtx_owned(m)) 674 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 675 ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 676 m->lock_object.lo_name, file, line)); 677 WITNESS_CHECKORDER(&m->lock_object, 678 opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 679 while (!_mtx_obtain_lock(m, tid)) { |
661#ifdef KDTRACE_HOOKS 662 spin_cnt++; 663#endif | |
664 if (m->mtx_lock == tid) { 665 m->mtx_recurse++; 666 break; 667 } 668#ifdef HWPMC_HOOKS 669 PMC_SOFT_CALL( , , lock, failed); 670#endif 671 lock_profile_obtain_lock_failed(&m->lock_object, --- 12 unchanged lines hidden (view full) --- 684 if (m != td->td_lock) 685 goto retry; 686 } 687 spinlock_enter(); 688 } 689 if (m == td->td_lock) 690 break; 691 __mtx_unlock_spin(m); /* does spinlock_exit() */ | 680 if (m->mtx_lock == tid) { 681 m->mtx_recurse++; 682 break; 683 } 684#ifdef HWPMC_HOOKS 685 PMC_SOFT_CALL( , , lock, failed); 686#endif 687 lock_profile_obtain_lock_failed(&m->lock_object, --- 12 unchanged lines hidden (view full) --- 700 if (m != td->td_lock) 701 goto retry; 702 } 703 spinlock_enter(); 704 } 705 if (m == td->td_lock) 706 break; 707 __mtx_unlock_spin(m); /* does spinlock_exit() */ |
708 } |
|
692#ifdef KDTRACE_HOOKS | 709#ifdef KDTRACE_HOOKS |
693 spin_cnt++; | 710 spin_time += lockstat_nsecs(); |
694#endif | 711#endif |
695 } | |
696 if (m->mtx_recurse == 0) 697 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, 698 m, contested, waittime, (file), (line)); 699 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 700 line); 701 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); | 712 if (m->mtx_recurse == 0) 713 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, 714 m, contested, waittime, (file), (line)); 715 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 716 line); 717 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); |
702 LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt); | 718 LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_time); |
703} 704 705struct mtx * 706thread_lock_block(struct thread *td) 707{ 708 struct mtx *lock; 709 710 THREAD_LOCK_ASSERT(td, MA_OWNED); --- 310 unchanged lines hidden --- | 719} 720 721struct mtx * 722thread_lock_block(struct thread *td) 723{ 724 struct mtx *lock; 725 726 THREAD_LOCK_ASSERT(td, MA_OWNED); --- 310 unchanged lines hidden --- |