Lines Matching refs:rw
81 .lc_name = "rw",
129 #define rw_wowner(rw) lv_rw_wowner(RW_READ_VALUE(rw))
135 #define rw_recursed(rw) ((rw)->rw_recurse != 0)
140 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread)
147 #define rw_owner(rw) rw_wowner(rw)
163 struct rwlock *rw;
165 rw = (struct rwlock *)lock;
167 rw_rlock(rw);
169 rw_wlock(rw);
175 struct rwlock *rw;
177 rw = (struct rwlock *)lock;
178 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED);
179 if (rw->rw_lock & RW_LOCK_READ) {
180 rw_runlock(rw);
183 rw_wunlock(rw);
192 const struct rwlock *rw = (const struct rwlock *)lock;
193 uintptr_t x = rw->rw_lock;
195 *owner = rw_wowner(rw);
204 struct rwlock *rw;
207 rw = rwlock2rw(c);
211 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock,
213 &rw->rw_lock));
229 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags);
230 rw->rw_lock = RW_UNLOCKED;
231 rw->rw_recurse = 0;
237 struct rwlock *rw;
239 rw = rwlock2rw(c);
241 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw));
242 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw));
243 rw->rw_lock = RW_DESTROYED;
244 lock_destroy(&rw->lock_object);
267 struct rwlock *rw;
270 rw = rwlock2rw(c);
275 curthread, rw->lock_object.lo_name, file, line));
276 KASSERT(rw->rw_lock != RW_DESTROYED,
278 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
282 if (!_rw_write_lock_fetch(rw, &v, tid))
283 _rw_wlock_hard(rw, v, file, line);
285 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
288 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
289 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
294 __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
308 curthread, rw->lock_object.lo_name, file, line));
309 KASSERT(rw->rw_lock != RW_DESTROYED,
316 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid))
320 if (v == tid && (rw->lock_object.lo_flags & LO_RECURSABLE)) {
321 rw->rw_recurse++;
322 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
329 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
331 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
335 rw, 0, 0, file, line, LOCKSTAT_WRITER);
344 struct rwlock *rw;
346 rw = rwlock2rw(c);
347 return (__rw_try_wlock_int(rw LOCK_FILE_LINE_ARG));
353 struct rwlock *rw;
355 rw = rwlock2rw(c);
357 KASSERT(rw->rw_lock != RW_DESTROYED,
360 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
361 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
365 _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
367 __rw_wunlock(rw, curthread, file, line);
393 __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp
408 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp,
410 if (LOCK_LOG_TEST(&rw->lock_object, 0))
413 rw, (void *)*vp,
423 __rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
451 if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
454 all_time -= lockstat_nsecs(&rw->lock_object);
475 lock_profile_obtain_lock_failed(&rw->lock_object,
479 if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
494 if (LOCK_LOG_TEST(&rw->lock_object, 0))
497 __func__, rw, owner);
500 "lockname:\"%s\"", rw->lock_object.lo_name);
503 v = RW_READ_VALUE(rw);
514 rw->lock_object.lo_name);
518 v = RW_READ_VALUE(rw);
538 ts = turnstile_trywait(&rw->lock_object);
544 v = RW_READ_VALUE(rw);
581 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v,
584 if (LOCK_LOG_TEST(&rw->lock_object, 0))
586 __func__, rw);
593 if (LOCK_LOG_TEST(&rw->lock_object, 0))
595 rw);
597 sleep_time -= lockstat_nsecs(&rw->lock_object);
599 MPASS(owner == rw_owner(rw));
602 sleep_time += lockstat_nsecs(&rw->lock_object);
605 if (LOCK_LOG_TEST(&rw->lock_object, 0))
607 __func__, rw);
608 v = RW_READ_VALUE(rw);
615 all_time += lockstat_nsecs(&rw->lock_object);
617 LOCKSTAT_RECORD4(rw__block, rw, sleep_time,
623 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
633 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
638 __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
648 td, rw->lock_object.lo_name, file, line));
649 KASSERT(rw->rw_lock != RW_DESTROYED,
651 KASSERT(rw_wowner(rw) != td,
653 rw->lock_object.lo_name, file, line));
654 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
656 v = RW_READ_VALUE(rw);
658 !__rw_rlock_try(rw, td, &v, true LOCK_FILE_LINE_ARG)))
659 __rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
661 lock_profile_obtain_lock_success(&rw->lock_object, 0, 0,
664 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
665 WITNESS_LOCK(&rw->lock_object, 0, file, line);
672 struct rwlock *rw;
674 rw = rwlock2rw(c);
675 __rw_rlock_int(rw LOCK_FILE_LINE_ARG);
679 __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
688 curthread, rw->lock_object.lo_name, file, line));
690 x = rw->rw_lock;
692 KASSERT(rw->rw_lock != RW_DESTROYED,
696 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &x, x + RW_ONE_READER)) {
697 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
699 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
701 rw, 0, 0, file, line, LOCKSTAT_READER);
708 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
715 struct rwlock *rw;
717 rw = rwlock2rw(c);
718 return (__rw_try_rlock_int(rw LOCK_FILE_LINE_ARG));
722 __rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp)
731 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp,
733 if (LOCK_LOG_TEST(&rw->lock_object, 0))
736 __func__, rw, (void *)*vp,
750 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp,
752 if (LOCK_LOG_TEST(&rw->lock_object, 0))
754 __func__, rw);
766 __rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
775 if (__rw_runlock_try(rw, td, &v))
782 turnstile_chain_lock(&rw->lock_object);
783 v = RW_READ_VALUE(rw);
785 if (__rw_runlock_try(rw, td, &v))
814 if (!atomic_fcmpset_rel_ptr(&rw->rw_lock, &v, setv))
816 if (LOCK_LOG_TEST(&rw->lock_object, 0))
818 __func__, rw);
827 ts = turnstile_lookup(&rw->lock_object);
834 turnstile_chain_unlock(&rw->lock_object);
836 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER);
840 _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
845 KASSERT(rw->rw_lock != RW_DESTROYED,
847 __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
848 WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
849 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
852 v = RW_READ_VALUE(rw);
855 !__rw_runlock_try(rw, td, &v)))
856 __rw_runlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
858 lock_profile_release_lock(&rw->lock_object);
866 struct rwlock *rw;
868 rw = rwlock2rw(c);
869 _rw_runlock_cookie_int(rw LOCK_FILE_LINE_ARG);
881 struct rwlock *rw;
908 rw = rwlock2rw(c);
913 if (_rw_write_lock_fetch(rw, &v, tid))
917 all_time -= lockstat_nsecs(&rw->lock_object);
935 v = RW_READ_VALUE(rw);
938 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
939 ("%s: recursing but non-recursive rw %s @ %s:%d\n",
940 __func__, rw->lock_object.lo_name, file, line));
941 rw->rw_recurse++;
942 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
943 if (LOCK_LOG_TEST(&rw->lock_object, 0))
944 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
948 if (LOCK_LOG_TEST(&rw->lock_object, 0))
950 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
955 lock_profile_obtain_lock_failed(&rw->lock_object,
960 if (_rw_write_lock_fetch(rw, &v, tid))
979 if (LOCK_LOG_TEST(&rw->lock_object, 0))
981 __func__, rw, owner);
984 rw->lock_object.lo_name);
987 v = RW_READ_VALUE(rw);
998 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v,
1006 rw->lock_object.lo_name);
1010 v = RW_READ_VALUE(rw);
1024 ts = turnstile_trywait(&rw->lock_object);
1025 v = RW_READ_VALUE(rw);
1057 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid | x)) {
1072 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v,
1075 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1077 __func__, rw);
1083 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1085 rw);
1087 sleep_time -= lockstat_nsecs(&rw->lock_object);
1089 MPASS(owner == rw_owner(rw));
1092 sleep_time += lockstat_nsecs(&rw->lock_object);
1095 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1097 __func__, rw);
1101 v = RW_READ_VALUE(rw);
1108 all_time += lockstat_nsecs(&rw->lock_object);
1110 LOCKSTAT_RECORD4(rw__block, rw, sleep_time,
1116 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
1121 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
1134 struct rwlock *rw;
1143 rw = rwlock2rw(c);
1145 v = RW_READ_VALUE(rw);
1148 if (--(rw->rw_recurse) == 0)
1149 atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
1150 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1151 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
1155 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_WRITER);
1156 if (v == tid && _rw_write_unlock(rw, tid))
1159 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
1162 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1163 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
1165 turnstile_chain_lock(&rw->lock_object);
1184 v = RW_READ_VALUE(rw);
1190 atomic_store_rel_ptr(&rw->rw_lock, setv);
1193 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1194 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
1197 ts = turnstile_lookup(&rw->lock_object);
1201 turnstile_chain_unlock(&rw->lock_object);
1210 __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
1219 KASSERT(rw->rw_lock != RW_DESTROYED,
1221 __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
1232 v = RW_READ_VALUE(rw);
1237 success = atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid);
1246 ts = turnstile_trywait(&rw->lock_object);
1247 v = RW_READ_VALUE(rw);
1260 success = atomic_fcmpset_ptr(&rw->rw_lock, &v, setv);
1270 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
1273 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
1275 LOCKSTAT_RECORD0(rw__upgrade, rw);
1283 struct rwlock *rw;
1285 rw = rwlock2rw(c);
1286 return (__rw_try_upgrade_int(rw LOCK_FILE_LINE_ARG));
1293 __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
1302 KASSERT(rw->rw_lock != RW_DESTROYED,
1304 __rw_assert(&rw->rw_lock, RA_WLOCKED | RA_NOTRECURSED, file, line);
1306 if (rw_recursed(rw))
1310 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
1318 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
1325 turnstile_chain_lock(&rw->lock_object);
1326 v = rw->rw_lock & RW_LOCK_WAITERS;
1335 ts = turnstile_lookup(&rw->lock_object);
1339 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v);
1349 turnstile_chain_unlock(&rw->lock_object);
1352 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
1353 LOCKSTAT_RECORD0(rw__downgrade, rw);
1359 struct rwlock *rw;
1361 rw = rwlock2rw(c);
1362 __rw_downgrade_int(rw LOCK_FILE_LINE_ARG);
1378 const struct rwlock *rw;
1383 rw = rwlock2rw(c);
1393 witness_assert(&rw->lock_object, what, file, line);
1400 if (rw->rw_lock == RW_UNLOCKED ||
1401 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED ||
1402 rw_wowner(rw) != curthread)))
1404 rw->lock_object.lo_name, (what & RA_RLOCKED) ?
1407 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) {
1408 if (rw_recursed(rw)) {
1411 rw->lock_object.lo_name, file,
1415 rw->lock_object.lo_name, file, line);
1422 if (rw_wowner(rw) != curthread)
1424 rw->lock_object.lo_name, file, line);
1425 if (rw_recursed(rw)) {
1428 rw->lock_object.lo_name, file, line);
1431 rw->lock_object.lo_name, file, line);
1435 witness_assert(&rw->lock_object, what, file, line);
1441 if (rw_wowner(rw) == curthread)
1443 rw->lock_object.lo_name, file, line);
1447 panic("Unknown rw lock assertion: %d @ %s:%d", what, file,
1457 const struct rwlock *rw;
1460 rw = (const struct rwlock *)lock;
1463 if (rw->rw_lock == RW_UNLOCKED)
1465 else if (rw->rw_lock == RW_DESTROYED) {
1468 } else if (rw->rw_lock & RW_LOCK_READ)
1470 (uintmax_t)(RW_READERS(rw->rw_lock)));
1472 td = rw_wowner(rw);
1475 if (rw_recursed(rw))
1476 db_printf(" recursed: %u\n", rw->rw_recurse);
1479 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {