Deleted Added
sdiff udiff text old ( 170295 ) new ( 171052 )
full compact
kern_rwlock.c (170295) kern_rwlock.c (171052)
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 170295 2007-06-04 23:51:44Z jeff $");
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 171052 2007-06-26 21:31:56Z attilio $");
36
37#include "opt_ddb.h"
38#include "opt_no_adaptive_rwlocks.h"
39
40#include <sys/param.h>
41#include <sys/ktr.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>
44#include <sys/proc.h>
45#include <sys/rwlock.h>
46#include <sys/systm.h>
47#include <sys/turnstile.h>
48#include <sys/lock_profile.h>
49#include <machine/cpu.h>
50
36
37#include "opt_ddb.h"
38#include "opt_no_adaptive_rwlocks.h"
39
40#include <sys/param.h>
41#include <sys/ktr.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>
44#include <sys/proc.h>
45#include <sys/rwlock.h>
46#include <sys/systm.h>
47#include <sys/turnstile.h>
48#include <sys/lock_profile.h>
49#include <machine/cpu.h>
50
51CTASSERT((RW_RECURSE & LO_CLASSFLAGS) == RW_RECURSE);
52
51#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
52#define ADAPTIVE_RWLOCKS
53#endif
54
55#ifdef DDB
56#include <ddb/ddb.h>
57
58static void db_show_rwlock(struct lock_object *lock);

--- 15 unchanged lines hidden (view full) ---

74 * Return a pointer to the owning thread if the lock is write-locked or
75 * NULL if the lock is unlocked or read-locked.
76 */
77#define rw_wowner(rw) \
78 ((rw)->rw_lock & RW_LOCK_READ ? NULL : \
79 (struct thread *)RW_OWNER((rw)->rw_lock))
80
81/*
53#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
54#define ADAPTIVE_RWLOCKS
55#endif
56
57#ifdef DDB
58#include <ddb/ddb.h>
59
60static void db_show_rwlock(struct lock_object *lock);

--- 15 unchanged lines hidden (view full) ---

76 * Return a pointer to the owning thread if the lock is write-locked or
77 * NULL if the lock is unlocked or read-locked.
78 */
79#define rw_wowner(rw) \
80 ((rw)->rw_lock & RW_LOCK_READ ? NULL : \
81 (struct thread *)RW_OWNER((rw)->rw_lock))
82
83/*
84 * Returns if a write owner is recursed. Write ownership is not assured
85 * here and should be previously checked.
86 */
87#define rw_recursed(rw) ((rw)->rw_recurse != 0)
88
89/*
90 * Return true if curthread helds the lock.
91 */
92#define rw_wlocked(rw) (rw_wowner((rw)) == curthread)
93
94/*
82 * Return a pointer to the owning thread for this lock who should receive
83 * any priority lent by threads that block on this lock. Currently this
84 * is identical to rw_wowner().
85 */
86#define rw_owner(rw) rw_wowner(rw)
87
88#ifndef INVARIANTS
89#define _rw_assert(rw, what, file, line)

--- 23 unchanged lines hidden (view full) ---

113 return (0);
114 } else {
115 rw_wunlock(rw);
116 return (1);
117 }
118}
119
120void
95 * Return a pointer to the owning thread for this lock who should receive
96 * any priority lent by threads that block on this lock. Currently this
97 * is identical to rw_wowner().
98 */
99#define rw_owner(rw) rw_wowner(rw)
100
101#ifndef INVARIANTS
102#define _rw_assert(rw, what, file, line)

--- 23 unchanged lines hidden (view full) ---

126 return (0);
127 } else {
128 rw_wunlock(rw);
129 return (1);
130 }
131}
132
133void
121rw_init(struct rwlock *rw, const char *name)
134rw_init_flags(struct rwlock *rw, const char *name, int opts)
122{
135{
136 int flags;
123
137
124 rw->rw_lock = RW_UNLOCKED;
138 MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET |
139 RW_RECURSE)) == 0);
125
140
126 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, LO_WITNESS |
127 LO_RECURSABLE | LO_UPGRADABLE);
141 flags = LO_UPGRADABLE | LO_RECURSABLE;
142 if (opts & RW_DUPOK)
143 flags |= LO_DUPOK;
144 if (opts & RW_NOPROFILE)
145 flags |= LO_NOPROFILE;
146 if (!(opts & RW_NOWITNESS))
147 flags |= LO_WITNESS;
148 if (opts & RW_QUIET)
149 flags |= LO_QUIET;
150 flags |= opts & RW_RECURSE;
151
152 rw->rw_lock = RW_UNLOCKED;
153 rw->rw_recurse = 0;
154 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags);
128}
129
130void
131rw_destroy(struct rwlock *rw)
132{
133
134 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked"));
155}
156
157void
158rw_destroy(struct rwlock *rw)
159{
160
161 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked"));
162 KASSERT(rw->rw_recurse == 0, ("rw lock still recursed"));
135 rw->rw_lock = RW_DESTROYED;
136 lock_destroy(&rw->lock_object);
137}
138
139void
140rw_sysinit(void *arg)
141{
142 struct rw_args *args = arg;

--- 16 unchanged lines hidden (view full) ---

159 KASSERT(rw->rw_lock != RW_DESTROYED,
160 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
161 KASSERT(rw_wowner(rw) != curthread,
162 ("%s (%s): wlock already held @ %s:%d", __func__,
163 rw->lock_object.lo_name, file, line));
164 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
165 line);
166 __rw_wlock(rw, curthread, file, line);
163 rw->rw_lock = RW_DESTROYED;
164 lock_destroy(&rw->lock_object);
165}
166
167void
168rw_sysinit(void *arg)
169{
170 struct rw_args *args = arg;

--- 16 unchanged lines hidden (view full) ---

187 KASSERT(rw->rw_lock != RW_DESTROYED,
188 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
189 KASSERT(rw_wowner(rw) != curthread,
190 ("%s (%s): wlock already held @ %s:%d", __func__,
191 rw->lock_object.lo_name, file, line));
192 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
193 line);
194 __rw_wlock(rw, curthread, file, line);
167 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, 0, file, line);
195 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
168 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
169 curthread->td_locks++;
170}
171
172void
173_rw_wunlock(struct rwlock *rw, const char *file, int line)
174{
175
176 MPASS(curthread != NULL);
177 KASSERT(rw->rw_lock != RW_DESTROYED,
178 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
179 _rw_assert(rw, RA_WLOCKED, file, line);
180 curthread->td_locks--;
181 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
196 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
197 curthread->td_locks++;
198}
199
200void
201_rw_wunlock(struct rwlock *rw, const char *file, int line)
202{
203
204 MPASS(curthread != NULL);
205 KASSERT(rw->rw_lock != RW_DESTROYED,
206 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
207 _rw_assert(rw, RA_WLOCKED, file, line);
208 curthread->td_locks--;
209 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
182 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, 0, file, line);
183 lock_profile_release_lock(&rw->lock_object);
210 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
211 line);
212 if (!rw_recursed(rw))
213 lock_profile_release_lock(&rw->lock_object);
184 __rw_wunlock(rw, curthread, file, line);
185}
186
187void
188_rw_rlock(struct rwlock *rw, const char *file, int line)
189{
190 struct turnstile *ts;
191#ifdef ADAPTIVE_RWLOCKS

--- 269 unchanged lines hidden (view full) ---

461_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
462{
463 struct turnstile *ts;
464#ifdef ADAPTIVE_RWLOCKS
465 volatile struct thread *owner;
466#endif
467 uintptr_t v;
468
214 __rw_wunlock(rw, curthread, file, line);
215}
216
217void
218_rw_rlock(struct rwlock *rw, const char *file, int line)
219{
220 struct turnstile *ts;
221#ifdef ADAPTIVE_RWLOCKS

--- 269 unchanged lines hidden (view full) ---

491_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
492{
493 struct turnstile *ts;
494#ifdef ADAPTIVE_RWLOCKS
495 volatile struct thread *owner;
496#endif
497 uintptr_t v;
498
499 if (rw_wlocked(rw)) {
500 KASSERT(rw->lock_object.lo_flags & RW_RECURSE,
501 ("%s: recursing but non-recursive rw %s @ %s:%d\n",
502 __func__, rw->lock_object.lo_name, file, line));
503 rw->rw_recurse++;
504 atomic_set_ptr(&rw->rw_lock, RW_LOCK_RECURSED);
505 if (LOCK_LOG_TEST(&rw->lock_object, 0))
506 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
507 return;
508 }
509
469 if (LOCK_LOG_TEST(&rw->lock_object, 0))
470 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
471 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
472
473 while (!_rw_write_lock(rw, tid)) {
474 ts = turnstile_trywait(&rw->lock_object);
475 v = rw->rw_lock;
476

--- 88 unchanged lines hidden (view full) ---

565 */
566void
567_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
568{
569 struct turnstile *ts;
570 uintptr_t v;
571 int queue;
572
510 if (LOCK_LOG_TEST(&rw->lock_object, 0))
511 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
512 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
513
514 while (!_rw_write_lock(rw, tid)) {
515 ts = turnstile_trywait(&rw->lock_object);
516 v = rw->rw_lock;
517

--- 88 unchanged lines hidden (view full) ---

606 */
607void
608_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
609{
610 struct turnstile *ts;
611 uintptr_t v;
612 int queue;
613
614 if (rw_wlocked(rw) && rw_recursed(rw)) {
615 if ((--rw->rw_recurse) == 0)
616 atomic_clear_ptr(&rw->rw_lock, RW_LOCK_RECURSED);
617 if (LOCK_LOG_TEST(&rw->lock_object, 0))
618 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
619 return;
620 }
621
573 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
574 ("%s: neither of the waiter flags are set", __func__));
575
576 if (LOCK_LOG_TEST(&rw->lock_object, 0))
577 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
578
579 turnstile_chain_lock(&rw->lock_object);
580 ts = turnstile_lookup(&rw->lock_object);

--- 146 unchanged lines hidden (view full) ---

727void
728_rw_downgrade(struct rwlock *rw, const char *file, int line)
729{
730 struct turnstile *ts;
731 uintptr_t tid, v;
732
733 KASSERT(rw->rw_lock != RW_DESTROYED,
734 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
622 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
623 ("%s: neither of the waiter flags are set", __func__));
624
625 if (LOCK_LOG_TEST(&rw->lock_object, 0))
626 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
627
628 turnstile_chain_lock(&rw->lock_object);
629 ts = turnstile_lookup(&rw->lock_object);

--- 146 unchanged lines hidden (view full) ---

776void
777_rw_downgrade(struct rwlock *rw, const char *file, int line)
778{
779 struct turnstile *ts;
780 uintptr_t tid, v;
781
782 KASSERT(rw->rw_lock != RW_DESTROYED,
783 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
735 _rw_assert(rw, RA_WLOCKED, file, line);
784 _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line);
785#ifndef INVARIANTS
786 if (rw_recursed(rw))
787 panic("downgrade of a recursed lock");
788#endif
736
737 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
738
739 /*
740 * Convert from a writer to a single reader. First we handle
741 * the easy case with no waiters. If there are any waiters, we
742 * lock the turnstile, "disown" the lock, and awaken any read
743 * waiters.

--- 60 unchanged lines hidden (view full) ---

804void
805_rw_assert(struct rwlock *rw, int what, const char *file, int line)
806{
807
808 if (panicstr != NULL)
809 return;
810 switch (what) {
811 case RA_LOCKED:
789
790 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
791
792 /*
793 * Convert from a writer to a single reader. First we handle
794 * the easy case with no waiters. If there are any waiters, we
795 * lock the turnstile, "disown" the lock, and awaken any read
796 * waiters.

--- 60 unchanged lines hidden (view full) ---

857void
858_rw_assert(struct rwlock *rw, int what, const char *file, int line)
859{
860
861 if (panicstr != NULL)
862 return;
863 switch (what) {
864 case RA_LOCKED:
812 case RA_LOCKED | LA_NOTRECURSED:
865 case RA_LOCKED | RA_RECURSED:
866 case RA_LOCKED | RA_NOTRECURSED:
813 case RA_RLOCKED:
814#ifdef WITNESS
815 witness_assert(&rw->lock_object, what, file, line);
816#else
817 /*
818 * If some other thread has a write lock or we have one
819 * and are asserting a read lock, fail. Also, if no one
820 * has a lock at all, fail.
821 */
822 if (rw->rw_lock == RW_UNLOCKED ||
823 (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED ||
824 rw_wowner(rw) != curthread)))
825 panic("Lock %s not %slocked @ %s:%d\n",
826 rw->lock_object.lo_name, (what == RA_RLOCKED) ?
827 "read " : "", file, line);
867 case RA_RLOCKED:
868#ifdef WITNESS
869 witness_assert(&rw->lock_object, what, file, line);
870#else
871 /*
872 * If some other thread has a write lock or we have one
873 * and are asserting a read lock, fail. Also, if no one
874 * has a lock at all, fail.
875 */
876 if (rw->rw_lock == RW_UNLOCKED ||
877 (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED ||
878 rw_wowner(rw) != curthread)))
879 panic("Lock %s not %slocked @ %s:%d\n",
880 rw->lock_object.lo_name, (what == RA_RLOCKED) ?
881 "read " : "", file, line);
882
883 if (!(rw->rw_lock & RW_LOCK_READ)) {
884 if (rw_recursed(rw)) {
885 if (what & RA_NOTRECURSED)
886 panic("Lock %s recursed @ %s:%d\n",
887 rw->lock_object.lo_name, file,
888 line);
889 } else if (what & RA_RECURSED)
890 panic("Lock %s not recursed @ %s:%d\n",
891 rw->lock_object.lo_name, file, line);
892 }
828#endif
829 break;
830 case RA_WLOCKED:
893#endif
894 break;
895 case RA_WLOCKED:
896 case RA_WLOCKED | RA_RECURSED:
897 case RA_WLOCKED | RA_NOTRECURSED:
831 if (rw_wowner(rw) != curthread)
832 panic("Lock %s not exclusively locked @ %s:%d\n",
833 rw->lock_object.lo_name, file, line);
898 if (rw_wowner(rw) != curthread)
899 panic("Lock %s not exclusively locked @ %s:%d\n",
900 rw->lock_object.lo_name, file, line);
901 if (rw_recursed(rw)) {
902 if (what & RA_NOTRECURSED)
903 panic("Lock %s recursed @ %s:%d\n",
904 rw->lock_object.lo_name, file, line);
905 } else if (what & RA_RECURSED)
906 panic("Lock %s not recursed @ %s:%d\n",
907 rw->lock_object.lo_name, file, line);
834 break;
835 case RA_UNLOCKED:
836#ifdef WITNESS
837 witness_assert(&rw->lock_object, what, file, line);
838#else
839 /*
840 * If we hold a write lock fail. We can't reliably check
841 * to see if we hold a read lock or not.

--- 27 unchanged lines hidden (view full) ---

869 return;
870 } else if (rw->rw_lock & RW_LOCK_READ)
871 db_printf("RLOCK: %ju locks\n",
872 (uintmax_t)(RW_READERS(rw->rw_lock)));
873 else {
874 td = rw_wowner(rw);
875 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
876 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
908 break;
909 case RA_UNLOCKED:
910#ifdef WITNESS
911 witness_assert(&rw->lock_object, what, file, line);
912#else
913 /*
914 * If we hold a write lock fail. We can't reliably check
915 * to see if we hold a read lock or not.

--- 27 unchanged lines hidden (view full) ---

943 return;
944 } else if (rw->rw_lock & RW_LOCK_READ)
945 db_printf("RLOCK: %ju locks\n",
946 (uintmax_t)(RW_READERS(rw->rw_lock)));
947 else {
948 td = rw_wowner(rw);
949 db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
950 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
951 if (rw_recursed(rw))
952 db_printf(" recursed: %u\n", rw->rw_recurse);
877 }
878 db_printf(" waiters: ");
879 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {
880 case RW_LOCK_READ_WAITERS:
881 db_printf("readers\n");
882 break;
883 case RW_LOCK_WRITE_WAITERS:
884 db_printf("writers\n");
885 break;
886 case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS:
887 db_printf("readers and writers\n");
888 break;
889 default:
890 db_printf("none\n");
891 break;
892 }
893}
894
895#endif
953 }
954 db_printf(" waiters: ");
955 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {
956 case RW_LOCK_READ_WAITERS:
957 db_printf("readers\n");
958 break;
959 case RW_LOCK_WRITE_WAITERS:
960 db_printf("writers\n");
961 break;
962 case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS:
963 db_printf("readers and writers\n");
964 break;
965 default:
966 db_printf("none\n");
967 break;
968 }
969}
970
971#endif