| kern_sx.c (284998) | kern_sx.c (285759) |
|---|---|
| 1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 28 unchanged lines hidden (view full) --- 37 */ 38 39#include "opt_ddb.h" 40#include "opt_hwpmc_hooks.h" 41#include "opt_kdtrace.h" 42#include "opt_no_adaptive_sx.h" 43 44#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 28 unchanged lines hidden (view full) --- 37 */ 38 39#include "opt_ddb.h" 40#include "opt_hwpmc_hooks.h" 41#include "opt_kdtrace.h" 42#include "opt_no_adaptive_sx.h" 43 44#include <sys/cdefs.h> |
| 45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 284998 2015-07-01 10:15:49Z avg $"); | 45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 285759 2015-07-21 17:16:37Z markj $"); |
| 46 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/kdb.h> 50#include <sys/ktr.h> 51#include <sys/lock.h> 52#include <sys/mutex.h> 53#include <sys/proc.h> --- 485 unchanged lines hidden (view full) --- 539 return (0); 540 } 541 542 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 543 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 544 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 545 546#ifdef KDTRACE_HOOKS | 46 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/kdb.h> 50#include <sys/ktr.h> 51#include <sys/lock.h> 52#include <sys/mutex.h> 53#include <sys/proc.h> --- 485 unchanged lines hidden (view full) --- 539 return (0); 540 } 541 542 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 543 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 544 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 545 546#ifdef KDTRACE_HOOKS |
| 547 all_time -= lockstat_nsecs(); | 547 all_time -= lockstat_nsecs(&sx->lock_object); |
| 548 state = sx->sx_lock; 549#endif 550 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 551#ifdef KDTRACE_HOOKS 552 spin_cnt++; 553#endif 554#ifdef HWPMC_HOOKS 555 PMC_SOFT_CALL( , , lock, failed); --- 133 unchanged lines hidden (view full) --- 689 * lock and the exclusive waiters flag is set, we have 690 * to sleep. 691 */ 692 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 693 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 694 __func__, sx); 695 696#ifdef KDTRACE_HOOKS | 548 state = sx->sx_lock; 549#endif 550 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 551#ifdef KDTRACE_HOOKS 552 spin_cnt++; 553#endif 554#ifdef HWPMC_HOOKS 555 PMC_SOFT_CALL( , , lock, failed); --- 133 unchanged lines hidden (view full) --- 689 * lock and the exclusive waiters flag is set, we have 690 * to sleep. 691 */ 692 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 693 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 694 __func__, sx); 695 696#ifdef KDTRACE_HOOKS |
| 697 sleep_time -= lockstat_nsecs(); | 697 sleep_time -= lockstat_nsecs(&sx->lock_object); |
| 698#endif 699 GIANT_SAVE(); 700 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 701 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 702 SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE); 703 if (!(opts & SX_INTERRUPTIBLE)) 704 sleepq_wait(&sx->lock_object, 0); 705 else 706 error = sleepq_wait_sig(&sx->lock_object, 0); 707#ifdef KDTRACE_HOOKS | 698#endif 699 GIANT_SAVE(); 700 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 701 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 702 SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE); 703 if (!(opts & SX_INTERRUPTIBLE)) 704 sleepq_wait(&sx->lock_object, 0); 705 else 706 error = sleepq_wait_sig(&sx->lock_object, 0); 707#ifdef KDTRACE_HOOKS |
| 708 sleep_time += lockstat_nsecs(); | 708 sleep_time += lockstat_nsecs(&sx->lock_object); |
| 709 sleep_cnt++; 710#endif 711 if (error) { 712 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 713 CTR2(KTR_LOCK, 714 "%s: interruptible sleep by %p suspended by signal", 715 __func__, sx); 716 break; 717 } 718 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 719 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 720 __func__, sx); 721 } 722#ifdef KDTRACE_HOOKS | 709 sleep_cnt++; 710#endif 711 if (error) { 712 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 713 CTR2(KTR_LOCK, 714 "%s: interruptible sleep by %p suspended by signal", 715 __func__, sx); 716 break; 717 } 718 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 719 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 720 __func__, sx); 721 } 722#ifdef KDTRACE_HOOKS |
| 723 all_time += lockstat_nsecs(); | 723 all_time += lockstat_nsecs(&sx->lock_object); |
| 724 if (sleep_time) 725 LOCKSTAT_RECORD4(LS_SX_XLOCK_BLOCK, sx, sleep_time, 726 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 727 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 728 if (spin_cnt > sleep_cnt) 729 LOCKSTAT_RECORD4(LS_SX_XLOCK_SPIN, sx, all_time - sleep_time, 730 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 731 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); --- 94 unchanged lines hidden (view full) --- 826 int64_t all_time = 0; 827#endif 828 829 if (SCHEDULER_STOPPED()) 830 return (0); 831 832#ifdef KDTRACE_HOOKS 833 state = sx->sx_lock; | 724 if (sleep_time) 725 LOCKSTAT_RECORD4(LS_SX_XLOCK_BLOCK, sx, sleep_time, 726 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 727 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 728 if (spin_cnt > sleep_cnt) 729 LOCKSTAT_RECORD4(LS_SX_XLOCK_SPIN, sx, all_time - sleep_time, 730 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 731 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); --- 94 unchanged lines hidden (view full) --- 826 int64_t all_time = 0; 827#endif 828 829 if (SCHEDULER_STOPPED()) 830 return (0); 831 832#ifdef KDTRACE_HOOKS 833 state = sx->sx_lock; |
| 834 all_time -= lockstat_nsecs(); | 834 all_time -= lockstat_nsecs(&sx->lock_object); |
| 835#endif 836 837 /* 838 * As with rwlocks, we don't make any attempt to try to block 839 * shared locks once there is an exclusive waiter. 840 */ 841 for (;;) { 842#ifdef KDTRACE_HOOKS --- 110 unchanged lines hidden (view full) --- 953 * Since we have been unable to acquire the shared lock, 954 * we have to sleep. 955 */ 956 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 957 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 958 __func__, sx); 959 960#ifdef KDTRACE_HOOKS | 835#endif 836 837 /* 838 * As with rwlocks, we don't make any attempt to try to block 839 * shared locks once there is an exclusive waiter. 840 */ 841 for (;;) { 842#ifdef KDTRACE_HOOKS --- 110 unchanged lines hidden (view full) --- 953 * Since we have been unable to acquire the shared lock, 954 * we have to sleep. 955 */ 956 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 957 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 958 __func__, sx); 959 960#ifdef KDTRACE_HOOKS |
| 961 sleep_time -= lockstat_nsecs(); | 961 sleep_time -= lockstat_nsecs(&sx->lock_object); |
| 962#endif 963 GIANT_SAVE(); 964 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 965 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 966 SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE); 967 if (!(opts & SX_INTERRUPTIBLE)) 968 sleepq_wait(&sx->lock_object, 0); 969 else 970 error = sleepq_wait_sig(&sx->lock_object, 0); 971#ifdef KDTRACE_HOOKS | 962#endif 963 GIANT_SAVE(); 964 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 965 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 966 SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE); 967 if (!(opts & SX_INTERRUPTIBLE)) 968 sleepq_wait(&sx->lock_object, 0); 969 else 970 error = sleepq_wait_sig(&sx->lock_object, 0); 971#ifdef KDTRACE_HOOKS |
| 972 sleep_time += lockstat_nsecs(); | 972 sleep_time += lockstat_nsecs(&sx->lock_object); |
| 973 sleep_cnt++; 974#endif 975 if (error) { 976 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 977 CTR2(KTR_LOCK, 978 "%s: interruptible sleep by %p suspended by signal", 979 __func__, sx); 980 break; 981 } 982 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 983 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 984 __func__, sx); 985 } 986#ifdef KDTRACE_HOOKS | 973 sleep_cnt++; 974#endif 975 if (error) { 976 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 977 CTR2(KTR_LOCK, 978 "%s: interruptible sleep by %p suspended by signal", 979 __func__, sx); 980 break; 981 } 982 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 983 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 984 __func__, sx); 985 } 986#ifdef KDTRACE_HOOKS |
| 987 all_time += lockstat_nsecs(); | 987 all_time += lockstat_nsecs(&sx->lock_object); |
| 988 if (sleep_time) 989 LOCKSTAT_RECORD4(LS_SX_SLOCK_BLOCK, sx, sleep_time, 990 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 991 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 992 if (spin_cnt > sleep_cnt) 993 LOCKSTAT_RECORD4(LS_SX_SLOCK_SPIN, sx, all_time - sleep_time, 994 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 995 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); --- 263 unchanged lines hidden --- | 988 if (sleep_time) 989 LOCKSTAT_RECORD4(LS_SX_SLOCK_BLOCK, sx, sleep_time, 990 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 991 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 992 if (spin_cnt > sleep_cnt) 993 LOCKSTAT_RECORD4(LS_SX_SLOCK_SPIN, sx, all_time - sleep_time, 994 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 995 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); --- 263 unchanged lines hidden --- |