Deleted Added
sdiff udiff text old ( 278694 ) new ( 284998 )
full compact
kern_sx.c (278694) kern_sx.c (284998)
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 28 unchanged lines hidden (view full) ---

37 */
38
39#include "opt_ddb.h"
40#include "opt_hwpmc_hooks.h"
41#include "opt_kdtrace.h"
42#include "opt_no_adaptive_sx.h"
43
44#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 28 unchanged lines hidden (view full) ---

37 */
38
39#include "opt_ddb.h"
40#include "opt_hwpmc_hooks.h"
41#include "opt_kdtrace.h"
42#include "opt_no_adaptive_sx.h"
43
44#include <sys/cdefs.h>
45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 278694 2015-02-13 19:06:22Z sbruno $");
45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 284998 2015-07-01 10:15:49Z avg $");
46
47#include <sys/param.h>
48#include <sys/systm.h>
49#include <sys/kdb.h>
50#include <sys/ktr.h>
51#include <sys/lock.h>
52#include <sys/mutex.h>
53#include <sys/proc.h>

--- 229 unchanged lines hidden (view full) ---

283 x = sx->sx_lock;
284 KASSERT(x != SX_LOCK_DESTROYED,
285 ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
286 if (!(x & SX_LOCK_SHARED))
287 break;
288 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
289 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
290 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
46
47#include <sys/param.h>
48#include <sys/systm.h>
49#include <sys/kdb.h>
50#include <sys/ktr.h>
51#include <sys/lock.h>
52#include <sys/mutex.h>
53#include <sys/proc.h>

--- 229 unchanged lines hidden (view full) ---

283 x = sx->sx_lock;
284 KASSERT(x != SX_LOCK_DESTROYED,
285 ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
286 if (!(x & SX_LOCK_SHARED))
287 break;
288 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
289 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
290 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
291 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE,
292 sx, 0, 0, file, line);
291 curthread->td_locks++;
292 return (1);
293 }
294 }
295
296 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
297 return (0);
298}

--- 44 unchanged lines hidden (view full) ---

343 rval = 1;
344 } else
345 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
346 (uintptr_t)curthread);
347 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
348 if (rval) {
349 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
350 file, line);
293 curthread->td_locks++;
294 return (1);
295 }
296 }
297
298 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
299 return (0);
300}

--- 44 unchanged lines hidden (view full) ---

345 rval = 1;
346 } else
347 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
348 (uintptr_t)curthread);
349 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
350 if (rval) {
351 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
352 file, line);
353 if (!sx_recursed(sx))
354 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE,
355 sx, 0, 0, file, line);
351 curthread->td_locks++;
352 }
353
354 return (rval);
355}
356
357void
358_sx_sunlock(struct sx *sx, const char *file, int line)

--- 148 unchanged lines hidden (view full) ---

507#endif
508 uintptr_t x;
509#ifdef LOCK_PROFILING
510 uint64_t waittime = 0;
511 int contested = 0;
512#endif
513 int error = 0;
514#ifdef KDTRACE_HOOKS
356 curthread->td_locks++;
357 }
358
359 return (rval);
360}
361
362void
363_sx_sunlock(struct sx *sx, const char *file, int line)

--- 148 unchanged lines hidden (view full) ---

512#endif
513 uintptr_t x;
514#ifdef LOCK_PROFILING
515 uint64_t waittime = 0;
516 int contested = 0;
517#endif
518 int error = 0;
519#ifdef KDTRACE_HOOKS
520 uintptr_t state;
515 uint64_t spin_cnt = 0;
516 uint64_t sleep_cnt = 0;
517 int64_t sleep_time = 0;
521 uint64_t spin_cnt = 0;
522 uint64_t sleep_cnt = 0;
523 int64_t sleep_time = 0;
524 int64_t all_time = 0;
518#endif
519
520 if (SCHEDULER_STOPPED())
521 return (0);
522
523 /* If we already hold an exclusive lock, then recurse. */
524 if (sx_xlocked(sx)) {
525 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,

--- 5 unchanged lines hidden (view full) ---

531 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
532 return (0);
533 }
534
535 if (LOCK_LOG_TEST(&sx->lock_object, 0))
536 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
537 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
538
525#endif
526
527 if (SCHEDULER_STOPPED())
528 return (0);
529
530 /* If we already hold an exclusive lock, then recurse. */
531 if (sx_xlocked(sx)) {
532 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,

--- 5 unchanged lines hidden (view full) ---

538 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
539 return (0);
540 }
541
542 if (LOCK_LOG_TEST(&sx->lock_object, 0))
543 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
544 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
545
546#ifdef KDTRACE_HOOKS
547 all_time -= lockstat_nsecs();
548 state = sx->sx_lock;
549#endif
539 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
540#ifdef KDTRACE_HOOKS
541 spin_cnt++;
542#endif
543#ifdef HWPMC_HOOKS
544 PMC_SOFT_CALL( , , lock, failed);
545#endif
546 lock_profile_obtain_lock_failed(&sx->lock_object, &contested,

--- 156 unchanged lines hidden (view full) ---

703 "%s: interruptible sleep by %p suspended by signal",
704 __func__, sx);
705 break;
706 }
707 if (LOCK_LOG_TEST(&sx->lock_object, 0))
708 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
709 __func__, sx);
710 }
550 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
551#ifdef KDTRACE_HOOKS
552 spin_cnt++;
553#endif
554#ifdef HWPMC_HOOKS
555 PMC_SOFT_CALL( , , lock, failed);
556#endif
557 lock_profile_obtain_lock_failed(&sx->lock_object, &contested,

--- 156 unchanged lines hidden (view full) ---

714 "%s: interruptible sleep by %p suspended by signal",
715 __func__, sx);
716 break;
717 }
718 if (LOCK_LOG_TEST(&sx->lock_object, 0))
719 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
720 __func__, sx);
721 }
711
712 GIANT_RESTORE();
713 if (!error)
714 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx,
715 contested, waittime, file, line);
716#ifdef KDTRACE_HOOKS
722#ifdef KDTRACE_HOOKS
723 all_time += lockstat_nsecs();
717 if (sleep_time)
724 if (sleep_time)
718 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time);
725 LOCKSTAT_RECORD4(LS_SX_XLOCK_BLOCK, sx, sleep_time,
726 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
727 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
719 if (spin_cnt > sleep_cnt)
728 if (spin_cnt > sleep_cnt)
720 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt));
729 LOCKSTAT_RECORD4(LS_SX_XLOCK_SPIN, sx, all_time - sleep_time,
730 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
731 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
721#endif
732#endif
733 if (!error)
734 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx,
735 contested, waittime, file, line);
736 GIANT_RESTORE();
722 return (error);
723}
724
725/*
726 * This function represents the so-called 'hard case' for sx_xunlock
727 * operation. All 'easy case' failures are redirected to this. Note
728 * that ideally this would be a static function, but it needs to be
729 * accessible from at least sx.h.

--- 69 unchanged lines hidden (view full) ---

799#endif
800#ifdef LOCK_PROFILING
801 uint64_t waittime = 0;
802 int contested = 0;
803#endif
804 uintptr_t x;
805 int error = 0;
806#ifdef KDTRACE_HOOKS
737 return (error);
738}
739
740/*
741 * This function represents the so-called 'hard case' for sx_xunlock
742 * operation. All 'easy case' failures are redirected to this. Note
743 * that ideally this would be a static function, but it needs to be
744 * accessible from at least sx.h.

--- 69 unchanged lines hidden (view full) ---

814#endif
815#ifdef LOCK_PROFILING
816 uint64_t waittime = 0;
817 int contested = 0;
818#endif
819 uintptr_t x;
820 int error = 0;
821#ifdef KDTRACE_HOOKS
822 uintptr_t state;
807 uint64_t spin_cnt = 0;
808 uint64_t sleep_cnt = 0;
809 int64_t sleep_time = 0;
823 uint64_t spin_cnt = 0;
824 uint64_t sleep_cnt = 0;
825 int64_t sleep_time = 0;
826 int64_t all_time = 0;
810#endif
811
812 if (SCHEDULER_STOPPED())
813 return (0);
814
827#endif
828
829 if (SCHEDULER_STOPPED())
830 return (0);
831
832#ifdef KDTRACE_HOOKS
833 state = sx->sx_lock;
834 all_time -= lockstat_nsecs();
835#endif
836
815 /*
816 * As with rwlocks, we don't make any attempt to try to block
817 * shared locks once there is an exclusive waiter.
818 */
819 for (;;) {
820#ifdef KDTRACE_HOOKS
821 spin_cnt++;
822#endif

--- 133 unchanged lines hidden (view full) ---

956 "%s: interruptible sleep by %p suspended by signal",
957 __func__, sx);
958 break;
959 }
960 if (LOCK_LOG_TEST(&sx->lock_object, 0))
961 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
962 __func__, sx);
963 }
837 /*
838 * As with rwlocks, we don't make any attempt to try to block
839 * shared locks once there is an exclusive waiter.
840 */
841 for (;;) {
842#ifdef KDTRACE_HOOKS
843 spin_cnt++;
844#endif

--- 133 unchanged lines hidden (view full) ---

978 "%s: interruptible sleep by %p suspended by signal",
979 __func__, sx);
980 break;
981 }
982 if (LOCK_LOG_TEST(&sx->lock_object, 0))
983 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
984 __func__, sx);
985 }
964 if (error == 0)
965 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx,
966 contested, waittime, file, line);
967#ifdef KDTRACE_HOOKS
986#ifdef KDTRACE_HOOKS
987 all_time += lockstat_nsecs();
968 if (sleep_time)
988 if (sleep_time)
969 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time);
989 LOCKSTAT_RECORD4(LS_SX_SLOCK_BLOCK, sx, sleep_time,
990 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
991 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
970 if (spin_cnt > sleep_cnt)
992 if (spin_cnt > sleep_cnt)
971 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt));
993 LOCKSTAT_RECORD4(LS_SX_SLOCK_SPIN, sx, all_time - sleep_time,
994 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
995 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
972#endif
996#endif
997 if (error == 0)
998 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx,
999 contested, waittime, file, line);
973 GIANT_RESTORE();
974 return (error);
975}
976
977/*
978 * This function represents the so-called 'hard case' for sx_sunlock
979 * operation. All 'easy case' failures are redirected to this. Note
980 * that ideally this would be a static function, but it needs to be

--- 251 unchanged lines hidden --- 		1000 GIANT_RESTORE();
1001 return (error);
1002}
1003
1004/*
1005 * This function represents the so-called 'hard case' for sx_sunlock
1006 * operation. All 'easy case' failures are redirected to this. Note
1007 * that ideally this would be a static function, but it needs to be

--- 251 unchanged lines hidden ---