kern_sx.c (278694) | kern_sx.c (284998) |
---|---|
1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 28 unchanged lines hidden (view full) --- 37 */ 38 39#include "opt_ddb.h" 40#include "opt_hwpmc_hooks.h" 41#include "opt_kdtrace.h" 42#include "opt_no_adaptive_sx.h" 43 44#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 28 unchanged lines hidden (view full) --- 37 */ 38 39#include "opt_ddb.h" 40#include "opt_hwpmc_hooks.h" 41#include "opt_kdtrace.h" 42#include "opt_no_adaptive_sx.h" 43 44#include <sys/cdefs.h> |
45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 278694 2015-02-13 19:06:22Z sbruno $"); | 45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 284998 2015-07-01 10:15:49Z avg $"); |
46 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/kdb.h> 50#include <sys/ktr.h> 51#include <sys/lock.h> 52#include <sys/mutex.h> 53#include <sys/proc.h> --- 229 unchanged lines hidden (view full) --- 283 x = sx->sx_lock; 284 KASSERT(x != SX_LOCK_DESTROYED, 285 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 286 if (!(x & SX_LOCK_SHARED)) 287 break; 288 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 289 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 290 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); | 46 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/kdb.h> 50#include <sys/ktr.h> 51#include <sys/lock.h> 52#include <sys/mutex.h> 53#include <sys/proc.h> --- 229 unchanged lines hidden (view full) --- 283 x = sx->sx_lock; 284 KASSERT(x != SX_LOCK_DESTROYED, 285 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 286 if (!(x & SX_LOCK_SHARED)) 287 break; 288 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 289 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 290 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); |
291 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, 292 sx, 0, 0, file, line); |
|
291 curthread->td_locks++; 292 return (1); 293 } 294 } 295 296 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 297 return (0); 298} --- 44 unchanged lines hidden (view full) --- 343 rval = 1; 344 } else 345 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 346 (uintptr_t)curthread); 347 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 348 if (rval) { 349 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 350 file, line); | 293 curthread->td_locks++; 294 return (1); 295 } 296 } 297 298 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 299 return (0); 300} --- 44 unchanged lines hidden (view full) --- 345 rval = 1; 346 } else 347 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 348 (uintptr_t)curthread); 349 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 350 if (rval) { 351 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 352 file, line); |
353 if (!sx_recursed(sx)) 354 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, 355 sx, 0, 0, file, line); |
|
351 curthread->td_locks++; 352 } 353 354 return (rval); 355} 356 357void 358_sx_sunlock(struct sx *sx, const char *file, int line) --- 148 unchanged lines hidden (view full) --- 507#endif 508 uintptr_t x; 509#ifdef LOCK_PROFILING 510 uint64_t waittime = 0; 511 int contested = 0; 512#endif 513 int error = 0; 514#ifdef KDTRACE_HOOKS | 356 curthread->td_locks++; 357 } 358 359 return (rval); 360} 361 362void 363_sx_sunlock(struct sx *sx, const char *file, int line) --- 148 unchanged lines hidden (view full) --- 512#endif 513 uintptr_t x; 514#ifdef LOCK_PROFILING 515 uint64_t waittime = 0; 516 int contested = 0; 517#endif 518 int error = 0; 519#ifdef KDTRACE_HOOKS |
520 uintptr_t state; |
|
515 uint64_t spin_cnt = 0; 516 uint64_t sleep_cnt = 0; 517 int64_t sleep_time = 0; | 521 uint64_t spin_cnt = 0; 522 uint64_t sleep_cnt = 0; 523 int64_t sleep_time = 0; |
524 int64_t all_time = 0; |
|
518#endif 519 520 if (SCHEDULER_STOPPED()) 521 return (0); 522 523 /* If we already hold an exclusive lock, then recurse. */ 524 if (sx_xlocked(sx)) { 525 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, --- 5 unchanged lines hidden (view full) --- 531 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 532 return (0); 533 } 534 535 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 536 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 537 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 538 | 525#endif 526 527 if (SCHEDULER_STOPPED()) 528 return (0); 529 530 /* If we already hold an exclusive lock, then recurse. */ 531 if (sx_xlocked(sx)) { 532 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, --- 5 unchanged lines hidden (view full) --- 538 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 539 return (0); 540 } 541 542 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 543 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 544 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 545 |
546#ifdef KDTRACE_HOOKS 547 all_time -= lockstat_nsecs(); 548 state = sx->sx_lock; 549#endif |
|
539 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 540#ifdef KDTRACE_HOOKS 541 spin_cnt++; 542#endif 543#ifdef HWPMC_HOOKS 544 PMC_SOFT_CALL( , , lock, failed); 545#endif 546 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, --- 156 unchanged lines hidden (view full) --- 703 "%s: interruptible sleep by %p suspended by signal", 704 __func__, sx); 705 break; 706 } 707 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 708 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 709 __func__, sx); 710 } | 550 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 551#ifdef KDTRACE_HOOKS 552 spin_cnt++; 553#endif 554#ifdef HWPMC_HOOKS 555 PMC_SOFT_CALL( , , lock, failed); 556#endif 557 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, --- 156 unchanged lines hidden (view full) --- 714 "%s: interruptible sleep by %p suspended by signal", 715 __func__, sx); 716 break; 717 } 718 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 719 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 720 __func__, sx); 721 } |
711 712 GIANT_RESTORE(); 713 if (!error) 714 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx, 715 contested, waittime, file, line); | |
716#ifdef KDTRACE_HOOKS | 722#ifdef KDTRACE_HOOKS |
723 all_time += lockstat_nsecs(); |
|
717 if (sleep_time) | 724 if (sleep_time) |
718 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time); | 725 LOCKSTAT_RECORD4(LS_SX_XLOCK_BLOCK, sx, sleep_time, 726 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 727 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); |
719 if (spin_cnt > sleep_cnt) | 728 if (spin_cnt > sleep_cnt) |
720 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt)); | 729 LOCKSTAT_RECORD4(LS_SX_XLOCK_SPIN, sx, all_time - sleep_time, 730 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 731 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); |
721#endif | 732#endif |
733 if (!error) 734 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx, 735 contested, waittime, file, line); 736 GIANT_RESTORE(); |
|
722 return (error); 723} 724 725/* 726 * This function represents the so-called 'hard case' for sx_xunlock 727 * operation. All 'easy case' failures are redirected to this. Note 728 * that ideally this would be a static function, but it needs to be 729 * accessible from at least sx.h. --- 69 unchanged lines hidden (view full) --- 799#endif 800#ifdef LOCK_PROFILING 801 uint64_t waittime = 0; 802 int contested = 0; 803#endif 804 uintptr_t x; 805 int error = 0; 806#ifdef KDTRACE_HOOKS | 737 return (error); 738} 739 740/* 741 * This function represents the so-called 'hard case' for sx_xunlock 742 * operation. All 'easy case' failures are redirected to this. Note 743 * that ideally this would be a static function, but it needs to be 744 * accessible from at least sx.h. --- 69 unchanged lines hidden (view full) --- 814#endif 815#ifdef LOCK_PROFILING 816 uint64_t waittime = 0; 817 int contested = 0; 818#endif 819 uintptr_t x; 820 int error = 0; 821#ifdef KDTRACE_HOOKS |
822 uintptr_t state; |
|
807 uint64_t spin_cnt = 0; 808 uint64_t sleep_cnt = 0; 809 int64_t sleep_time = 0; | 823 uint64_t spin_cnt = 0; 824 uint64_t sleep_cnt = 0; 825 int64_t sleep_time = 0; |
826 int64_t all_time = 0; |
|
810#endif 811 812 if (SCHEDULER_STOPPED()) 813 return (0); 814 | 827#endif 828 829 if (SCHEDULER_STOPPED()) 830 return (0); 831 |
832#ifdef KDTRACE_HOOKS 833 state = sx->sx_lock; 834 all_time -= lockstat_nsecs(); 835#endif 836 |
|
815 /* 816 * As with rwlocks, we don't make any attempt to try to block 817 * shared locks once there is an exclusive waiter. 818 */ 819 for (;;) { 820#ifdef KDTRACE_HOOKS 821 spin_cnt++; 822#endif --- 133 unchanged lines hidden (view full) --- 956 "%s: interruptible sleep by %p suspended by signal", 957 __func__, sx); 958 break; 959 } 960 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 961 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 962 __func__, sx); 963 } | 837 /* 838 * As with rwlocks, we don't make any attempt to try to block 839 * shared locks once there is an exclusive waiter. 840 */ 841 for (;;) { 842#ifdef KDTRACE_HOOKS 843 spin_cnt++; 844#endif --- 133 unchanged lines hidden (view full) --- 978 "%s: interruptible sleep by %p suspended by signal", 979 __func__, sx); 980 break; 981 } 982 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 983 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 984 __func__, sx); 985 } |
964 if (error == 0) 965 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 966 contested, waittime, file, line); | |
967#ifdef KDTRACE_HOOKS | 986#ifdef KDTRACE_HOOKS |
987 all_time += lockstat_nsecs(); |
|
968 if (sleep_time) | 988 if (sleep_time) |
969 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time); | 989 LOCKSTAT_RECORD4(LS_SX_SLOCK_BLOCK, sx, sleep_time, 990 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 991 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); |
970 if (spin_cnt > sleep_cnt) | 992 if (spin_cnt > sleep_cnt) |
971 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt)); | 993 LOCKSTAT_RECORD4(LS_SX_SLOCK_SPIN, sx, all_time - sleep_time, 994 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 995 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); |
972#endif | 996#endif |
997 if (error == 0) 998 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 999 contested, waittime, file, line); |
|
973 GIANT_RESTORE(); 974 return (error); 975} 976 977/* 978 * This function represents the so-called 'hard case' for sx_sunlock 979 * operation. All 'easy case' failures are redirected to this. Note 980 * that ideally this would be a static function, but it needs to be --- 251 unchanged lines hidden --- | 1000 GIANT_RESTORE(); 1001 return (error); 1002} 1003 1004/* 1005 * This function represents the so-called 'hard case' for sx_sunlock 1006 * operation. All 'easy case' failures are redirected to this. Note 1007 * that ideally this would be a static function, but it needs to be --- 251 unchanged lines hidden --- |