Deleted Added
sdiff udiff text old ( 278694 ) new ( 284998 )
full compact
kern_rwlock.c (278694) kern_rwlock.c (284998)
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 15 unchanged lines hidden (view full) ---

24 * SUCH DAMAGE.
25 */
26
27/*
28 * Machine independent bits of reader/writer lock implementation.
29 */
30
31#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 15 unchanged lines hidden (view full) ---

24 * SUCH DAMAGE.
25 */
26
27/*
28 * Machine independent bits of reader/writer lock implementation.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: stable/10/sys/kern/kern_rwlock.c 278694 2015-02-13 19:06:22Z sbruno $");
32__FBSDID("$FreeBSD: stable/10/sys/kern/kern_rwlock.c 284998 2015-07-01 10:15:49Z avg $");
33
34#include "opt_ddb.h"
35#include "opt_hwpmc_hooks.h"
36#include "opt_kdtrace.h"
37#include "opt_no_adaptive_rwlocks.h"
38
39#include <sys/param.h>
40#include <sys/kdb.h>

--- 253 unchanged lines hidden (view full) ---

294 } else
295 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
296 (uintptr_t)curthread);
297
298 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
299 if (rval) {
300 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
301 file, line);
33
34#include "opt_ddb.h"
35#include "opt_hwpmc_hooks.h"
36#include "opt_kdtrace.h"
37#include "opt_no_adaptive_rwlocks.h"
38
39#include <sys/param.h>
40#include <sys/kdb.h>

--- 253 unchanged lines hidden (view full) ---

294 } else
295 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED,
296 (uintptr_t)curthread);
297
298 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
299 if (rval) {
300 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
301 file, line);
302 if (!rw_recursed(rw))
303 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE,
304 rw, 0, 0, file, line);
302 curthread->td_locks++;
303 }
304 return (rval);
305}
306
307void
308_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
309{

--- 38 unchanged lines hidden (view full) ---

348 int i;
349#endif
350#ifdef LOCK_PROFILING
351 uint64_t waittime = 0;
352 int contested = 0;
353#endif
354 uintptr_t v;
355#ifdef KDTRACE_HOOKS
305 curthread->td_locks++;
306 }
307 return (rval);
308}
309
310void
311_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
312{

--- 38 unchanged lines hidden (view full) ---

351 int i;
352#endif
353#ifdef LOCK_PROFILING
354 uint64_t waittime = 0;
355 int contested = 0;
356#endif
357 uintptr_t v;
358#ifdef KDTRACE_HOOKS
359 uintptr_t state;
356 uint64_t spin_cnt = 0;
357 uint64_t sleep_cnt = 0;
358 int64_t sleep_time = 0;
360 uint64_t spin_cnt = 0;
361 uint64_t sleep_cnt = 0;
362 int64_t sleep_time = 0;
363 int64_t all_time = 0;
359#endif
360
361 if (SCHEDULER_STOPPED())
362 return;
363
364 rw = rwlock2rw(c);
365
366 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
367 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d",
368 curthread, rw->lock_object.lo_name, file, line));
369 KASSERT(rw->rw_lock != RW_DESTROYED,
370 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
371 KASSERT(rw_wowner(rw) != curthread,
372 ("rw_rlock: wlock already held for %s @ %s:%d",
373 rw->lock_object.lo_name, file, line));
374 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
375
364#endif
365
366 if (SCHEDULER_STOPPED())
367 return;
368
369 rw = rwlock2rw(c);
370
371 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
372 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d",
373 curthread, rw->lock_object.lo_name, file, line));
374 KASSERT(rw->rw_lock != RW_DESTROYED,
375 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
376 KASSERT(rw_wowner(rw) != curthread,
377 ("rw_rlock: wlock already held for %s @ %s:%d",
378 rw->lock_object.lo_name, file, line));
379 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
380
381#ifdef KDTRACE_HOOKS
382 all_time -= lockstat_nsecs();
383 state = rw->rw_lock;
384#endif
376 for (;;) {
377#ifdef KDTRACE_HOOKS
378 spin_cnt++;
379#endif
380 /*
381 * Handle the easy case. If no other thread has a write
382 * lock, then try to bump up the count of read locks. Note
383 * that we have to preserve the current state of the

--- 143 unchanged lines hidden (view full) ---

527#ifdef KDTRACE_HOOKS
528 sleep_time += lockstat_nsecs();
529 sleep_cnt++;
530#endif
531 if (LOCK_LOG_TEST(&rw->lock_object, 0))
532 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
533 __func__, rw);
534 }
385 for (;;) {
386#ifdef KDTRACE_HOOKS
387 spin_cnt++;
388#endif
389 /*
390 * Handle the easy case. If no other thread has a write
391 * lock, then try to bump up the count of read locks. Note
392 * that we have to preserve the current state of the

--- 143 unchanged lines hidden (view full) ---

536#ifdef KDTRACE_HOOKS
537 sleep_time += lockstat_nsecs();
538 sleep_cnt++;
539#endif
540 if (LOCK_LOG_TEST(&rw->lock_object, 0))
541 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
542 __func__, rw);
543 }
544#ifdef KDTRACE_HOOKS
545 all_time += lockstat_nsecs();
546 if (sleep_time)
547 LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time,
548 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
549 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
535
550
551 /* Record only the loops spinning and not sleeping. */
552 if (spin_cnt > sleep_cnt)
553 LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, all_time - sleep_time,
554 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
555 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
556#endif
536 /*
537 * TODO: acquire "owner of record" here. Here be turnstile dragons
538 * however. turnstiles don't like owners changing between calls to
539 * turnstile_wait() currently.
540 */
541 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested,
542 waittime, file, line);
543 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
544 WITNESS_LOCK(&rw->lock_object, 0, file, line);
545 curthread->td_locks++;
546 curthread->td_rw_rlocks++;
557 /*
558 * TODO: acquire "owner of record" here. Here be turnstile dragons
559 * however. turnstiles don't like owners changing between calls to
560 * turnstile_wait() currently.
561 */
562 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested,
563 waittime, file, line);
564 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
565 WITNESS_LOCK(&rw->lock_object, 0, file, line);
566 curthread->td_locks++;
567 curthread->td_rw_rlocks++;
547#ifdef KDTRACE_HOOKS
548 if (sleep_time)
549 LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time);
550
551 /*
552 * Record only the loops spinning and not sleeping.
553 */
554 if (spin_cnt > sleep_cnt)
555 LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt));
556#endif
557}
558
559int
560__rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
561{
562 struct rwlock *rw;
563 uintptr_t x;
564

--- 11 unchanged lines hidden (view full) ---

576 KASSERT(rw->rw_lock != RW_DESTROYED,
577 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
578 if (!(x & RW_LOCK_READ))
579 break;
580 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
581 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
582 line);
583 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
568}
569
570int
571__rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
572{
573 struct rwlock *rw;
574 uintptr_t x;
575

--- 11 unchanged lines hidden (view full) ---

587 KASSERT(rw->rw_lock != RW_DESTROYED,
588 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
589 if (!(x & RW_LOCK_READ))
590 break;
591 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
592 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
593 line);
594 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
595 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE,
596 rw, 0, 0, file, line);
584 curthread->td_locks++;
585 curthread->td_rw_rlocks++;
586 return (1);
587 }
588 }
589
590 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
591 return (0);

--- 128 unchanged lines hidden (view full) ---

720 int i;
721#endif
722 uintptr_t v, x;
723#ifdef LOCK_PROFILING
724 uint64_t waittime = 0;
725 int contested = 0;
726#endif
727#ifdef KDTRACE_HOOKS
597 curthread->td_locks++;
598 curthread->td_rw_rlocks++;
599 return (1);
600 }
601 }
602
603 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
604 return (0);

--- 128 unchanged lines hidden (view full) ---

733 int i;
734#endif
735 uintptr_t v, x;
736#ifdef LOCK_PROFILING
737 uint64_t waittime = 0;
738 int contested = 0;
739#endif
740#ifdef KDTRACE_HOOKS
741 uintptr_t state;
728 uint64_t spin_cnt = 0;
729 uint64_t sleep_cnt = 0;
730 int64_t sleep_time = 0;
742 uint64_t spin_cnt = 0;
743 uint64_t sleep_cnt = 0;
744 int64_t sleep_time = 0;
745 int64_t all_time = 0;
731#endif
732
733 if (SCHEDULER_STOPPED())
734 return;
735
736 rw = rwlock2rw(c);
737
738 if (rw_wlocked(rw)) {

--- 5 unchanged lines hidden (view full) ---

744 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
745 return;
746 }
747
748 if (LOCK_LOG_TEST(&rw->lock_object, 0))
749 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
750 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
751
746#endif
747
748 if (SCHEDULER_STOPPED())
749 return;
750
751 rw = rwlock2rw(c);
752
753 if (rw_wlocked(rw)) {

--- 5 unchanged lines hidden (view full) ---

759 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
760 return;
761 }
762
763 if (LOCK_LOG_TEST(&rw->lock_object, 0))
764 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
765 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
766
767#ifdef KDTRACE_HOOKS
768 all_time -= lockstat_nsecs();
769 state = rw->rw_lock;
770#endif
752 while (!_rw_write_lock(rw, tid)) {
753#ifdef KDTRACE_HOOKS
754 spin_cnt++;
755#endif
756#ifdef HWPMC_HOOKS
757 PMC_SOFT_CALL( , , lock, failed);
758#endif
759 lock_profile_obtain_lock_failed(&rw->lock_object,

--- 121 unchanged lines hidden (view full) ---

881#endif
882 if (LOCK_LOG_TEST(&rw->lock_object, 0))
883 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
884 __func__, rw);
885#ifdef ADAPTIVE_RWLOCKS
886 spintries = 0;
887#endif
888 }
771 while (!_rw_write_lock(rw, tid)) {
772#ifdef KDTRACE_HOOKS
773 spin_cnt++;
774#endif
775#ifdef HWPMC_HOOKS
776 PMC_SOFT_CALL( , , lock, failed);
777#endif
778 lock_profile_obtain_lock_failed(&rw->lock_object,

--- 121 unchanged lines hidden (view full) ---

900#endif
901 if (LOCK_LOG_TEST(&rw->lock_object, 0))
902 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
903 __func__, rw);
904#ifdef ADAPTIVE_RWLOCKS
905 spintries = 0;
906#endif
907 }
889 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested,
890 waittime, file, line);
891#ifdef KDTRACE_HOOKS
908#ifdef KDTRACE_HOOKS
909 all_time += lockstat_nsecs();
892 if (sleep_time)
910 if (sleep_time)
893 LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time);
911 LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time,
912 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0,
913 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
894
914
895 /*
896 * Record only the loops spinning and not sleeping.
897 */
915 /* Record only the loops spinning and not sleeping. */
898 if (spin_cnt > sleep_cnt)
916 if (spin_cnt > sleep_cnt)
899 LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt));
917 LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, all_time - sleep_time,
918 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
919 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
900#endif
920#endif
921 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested,
922 waittime, file, line);
901}
902
903/*
904 * This function is called if the first try at releasing a write lock failed.
905 * This means that one of the 2 waiter bits must be set indicating that at
906 * least one thread is waiting on this lock.
907 */
908void

--- 342 unchanged lines hidden --- 		923}
924
925/*
926 * This function is called if the first try at releasing a write lock failed.
927 * This means that one of the 2 waiter bits must be set indicating that at
928 * least one thread is waiting on this lock.
929 */
930void

--- 342 unchanged lines hidden ---