Deleted Added
sdiff udiff text old ( 227588 ) new ( 228424 )
full compact
kern_rwlock.c (227588) kern_rwlock.c (228424)
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 227588 2011-11-16 21:51:17Z pjd $");
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 228424 2011-12-11 21:02:01Z avg $");
36
37#include "opt_ddb.h"
38#include "opt_kdtrace.h"
39#include "opt_no_adaptive_rwlocks.h"
40
41#include <sys/param.h>
42#include <sys/ktr.h>
43#include <sys/kernel.h>

--- 184 unchanged lines hidden (view full) ---

228
229 return (rw_wowner(rw) == curthread);
230}
231
232void
233_rw_wlock(struct rwlock *rw, const char *file, int line)
234{
235
36
37#include "opt_ddb.h"
38#include "opt_kdtrace.h"
39#include "opt_no_adaptive_rwlocks.h"
40
41#include <sys/param.h>
42#include <sys/ktr.h>
43#include <sys/kernel.h>

--- 184 unchanged lines hidden (view full) ---

228
229 return (rw_wowner(rw) == curthread);
230}
231
232void
233_rw_wlock(struct rwlock *rw, const char *file, int line)
234{
235
236 if (SCHEDULER_STOPPED())
237 return;
236 MPASS(curthread != NULL);
237 KASSERT(rw->rw_lock != RW_DESTROYED,
238 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
239 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
240 line, NULL);
241 __rw_wlock(rw, curthread, file, line);
242 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
243 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
244 curthread->td_locks++;
245}
246
247int
248_rw_try_wlock(struct rwlock *rw, const char *file, int line)
249{
250 int rval;
251
238 MPASS(curthread != NULL);
239 KASSERT(rw->rw_lock != RW_DESTROYED,
240 ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
241 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
242 line, NULL);
243 __rw_wlock(rw, curthread, file, line);
244 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
245 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
246 curthread->td_locks++;
247}
248
249int
250_rw_try_wlock(struct rwlock *rw, const char *file, int line)
251{
252 int rval;
253
254 if (SCHEDULER_STOPPED())
255 return (1);
256
252 KASSERT(rw->rw_lock != RW_DESTROYED,
253 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
254
255 if (rw_wlocked(rw) &&
256 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
257 rw->rw_recurse++;
258 rval = 1;
259 } else

--- 8 unchanged lines hidden (view full) ---

268 }
269 return (rval);
270}
271
272void
273_rw_wunlock(struct rwlock *rw, const char *file, int line)
274{
275
257 KASSERT(rw->rw_lock != RW_DESTROYED,
258 ("rw_try_wlock() of destroyed rwlock @ %s:%d", file, line));
259
260 if (rw_wlocked(rw) &&
261 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) {
262 rw->rw_recurse++;
263 rval = 1;
264 } else

--- 8 unchanged lines hidden (view full) ---

273 }
274 return (rval);
275}
276
277void
278_rw_wunlock(struct rwlock *rw, const char *file, int line)
279{
280
281 if (SCHEDULER_STOPPED())
282 return;
276 MPASS(curthread != NULL);
277 KASSERT(rw->rw_lock != RW_DESTROYED,
278 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
279 _rw_assert(rw, RA_WLOCKED, file, line);
280 curthread->td_locks--;
281 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
282 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
283 line);

--- 28 unchanged lines hidden (view full) ---

312#endif
313 uintptr_t v;
314#ifdef KDTRACE_HOOKS
315 uint64_t spin_cnt = 0;
316 uint64_t sleep_cnt = 0;
317 int64_t sleep_time = 0;
318#endif
319
283 MPASS(curthread != NULL);
284 KASSERT(rw->rw_lock != RW_DESTROYED,
285 ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
286 _rw_assert(rw, RA_WLOCKED, file, line);
287 curthread->td_locks--;
288 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
289 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
290 line);

--- 28 unchanged lines hidden (view full) ---

319#endif
320 uintptr_t v;
321#ifdef KDTRACE_HOOKS
322 uint64_t spin_cnt = 0;
323 uint64_t sleep_cnt = 0;
324 int64_t sleep_time = 0;
325#endif
326
327 if (SCHEDULER_STOPPED())
328 return;
329
320 KASSERT(rw->rw_lock != RW_DESTROYED,
321 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
322 KASSERT(rw_wowner(rw) != curthread,
323 ("%s (%s): wlock already held @ %s:%d", __func__,
324 rw->lock_object.lo_name, file, line));
325 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
326
327 for (;;) {

--- 166 unchanged lines hidden (view full) ---

494#endif
495}
496
497int
498_rw_try_rlock(struct rwlock *rw, const char *file, int line)
499{
500 uintptr_t x;
501
330 KASSERT(rw->rw_lock != RW_DESTROYED,
331 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
332 KASSERT(rw_wowner(rw) != curthread,
333 ("%s (%s): wlock already held @ %s:%d", __func__,
334 rw->lock_object.lo_name, file, line));
335 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
336
337 for (;;) {

--- 166 unchanged lines hidden (view full) ---

504#endif
505}
506
507int
508_rw_try_rlock(struct rwlock *rw, const char *file, int line)
509{
510 uintptr_t x;
511
512 if (SCHEDULER_STOPPED())
513 return (1);
514
502 for (;;) {
503 x = rw->rw_lock;
504 KASSERT(rw->rw_lock != RW_DESTROYED,
505 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
506 if (!(x & RW_LOCK_READ))
507 break;
508 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
509 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,

--- 10 unchanged lines hidden (view full) ---

520}
521
522void
523_rw_runlock(struct rwlock *rw, const char *file, int line)
524{
525 struct turnstile *ts;
526 uintptr_t x, v, queue;
527
515 for (;;) {
516 x = rw->rw_lock;
517 KASSERT(rw->rw_lock != RW_DESTROYED,
518 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line));
519 if (!(x & RW_LOCK_READ))
520 break;
521 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) {
522 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,

--- 10 unchanged lines hidden (view full) ---

533}
534
535void
536_rw_runlock(struct rwlock *rw, const char *file, int line)
537{
538 struct turnstile *ts;
539 uintptr_t x, v, queue;
540
541 if (SCHEDULER_STOPPED())
542 return;
543
528 KASSERT(rw->rw_lock != RW_DESTROYED,
529 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
530 _rw_assert(rw, RA_RLOCKED, file, line);
531 curthread->td_locks--;
532 curthread->td_rw_rlocks--;
533 WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
534 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
535

--- 109 unchanged lines hidden (view full) ---

645 int contested = 0;
646#endif
647#ifdef KDTRACE_HOOKS
648 uint64_t spin_cnt = 0;
649 uint64_t sleep_cnt = 0;
650 int64_t sleep_time = 0;
651#endif
652
544 KASSERT(rw->rw_lock != RW_DESTROYED,
545 ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
546 _rw_assert(rw, RA_RLOCKED, file, line);
547 curthread->td_locks--;
548 curthread->td_rw_rlocks--;
549 WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
550 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
551

--- 109 unchanged lines hidden (view full) ---

661 int contested = 0;
662#endif
663#ifdef KDTRACE_HOOKS
664 uint64_t spin_cnt = 0;
665 uint64_t sleep_cnt = 0;
666 int64_t sleep_time = 0;
667#endif
668
669 if (SCHEDULER_STOPPED())
670 return;
671
653 if (rw_wlocked(rw)) {
654 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
655 ("%s: recursing but non-recursive rw %s @ %s:%d\n",
656 __func__, rw->lock_object.lo_name, file, line));
657 rw->rw_recurse++;
658 if (LOCK_LOG_TEST(&rw->lock_object, 0))
659 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
660 return;

--- 148 unchanged lines hidden (view full) ---

809 */
810void
811_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
812{
813 struct turnstile *ts;
814 uintptr_t v;
815 int queue;
816
672 if (rw_wlocked(rw)) {
673 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
674 ("%s: recursing but non-recursive rw %s @ %s:%d\n",
675 __func__, rw->lock_object.lo_name, file, line));
676 rw->rw_recurse++;
677 if (LOCK_LOG_TEST(&rw->lock_object, 0))
678 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
679 return;

--- 148 unchanged lines hidden (view full) ---

828 */
829void
830_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
831{
832 struct turnstile *ts;
833 uintptr_t v;
834 int queue;
835
836 if (SCHEDULER_STOPPED())
837 return;
838
817 if (rw_wlocked(rw) && rw_recursed(rw)) {
818 rw->rw_recurse--;
819 if (LOCK_LOG_TEST(&rw->lock_object, 0))
820 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
821 return;
822 }
823
824 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),

--- 46 unchanged lines hidden (view full) ---

871 */
872int
873_rw_try_upgrade(struct rwlock *rw, const char *file, int line)
874{
875 uintptr_t v, x, tid;
876 struct turnstile *ts;
877 int success;
878
839 if (rw_wlocked(rw) && rw_recursed(rw)) {
840 rw->rw_recurse--;
841 if (LOCK_LOG_TEST(&rw->lock_object, 0))
842 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
843 return;
844 }
845
846 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),

--- 46 unchanged lines hidden (view full) ---

893 */
894int
895_rw_try_upgrade(struct rwlock *rw, const char *file, int line)
896{
897 uintptr_t v, x, tid;
898 struct turnstile *ts;
899 int success;
900
901 if (SCHEDULER_STOPPED())
902 return (1);
903
879 KASSERT(rw->rw_lock != RW_DESTROYED,
880 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
881 _rw_assert(rw, RA_RLOCKED, file, line);
882
883 /*
884 * Attempt to switch from one reader to a writer. If there
885 * are any write waiters, then we will have to lock the
886 * turnstile first to prevent races with another writer

--- 54 unchanged lines hidden (view full) ---

941 */
942void
943_rw_downgrade(struct rwlock *rw, const char *file, int line)
944{
945 struct turnstile *ts;
946 uintptr_t tid, v;
947 int rwait, wwait;
948
904 KASSERT(rw->rw_lock != RW_DESTROYED,
905 ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
906 _rw_assert(rw, RA_RLOCKED, file, line);
907
908 /*
909 * Attempt to switch from one reader to a writer. If there
910 * are any write waiters, then we will have to lock the
911 * turnstile first to prevent races with another writer

--- 54 unchanged lines hidden (view full) ---

966 */
967void
968_rw_downgrade(struct rwlock *rw, const char *file, int line)
969{
970 struct turnstile *ts;
971 uintptr_t tid, v;
972 int rwait, wwait;
973
974 if (SCHEDULER_STOPPED())
975 return;
976
949 KASSERT(rw->rw_lock != RW_DESTROYED,
950 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
951 _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line);
952#ifndef INVARIANTS
953 if (rw_recursed(rw))
954 panic("downgrade of a recursed lock");
955#endif
956

--- 171 unchanged lines hidden --- 		977 KASSERT(rw->rw_lock != RW_DESTROYED,
978 ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
979 _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line);
980#ifndef INVARIANTS
981 if (rw_recursed(rw))
982 panic("downgrade of a recursed lock");
983#endif
984

--- 171 unchanged lines hidden ---