sys/kern/kern_rmlock.c

66 #define	rm_destroyed(rm)						\
67 	(LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
87 	.lc_name = "rm",
101 	.lc_name = "sleepable rm",
130 	struct rmlock *rm;
132 	rm = (struct rmlock *)lock;
134 		rm_wlock(rm);
144 	struct rmlock *rm;
146 	rm = (struct rmlock *)lock;
147 	rm_wunlock(rm);
155 	struct rmlock *rm;
158 	rm = (struct rmlock *)lock;
159 	lc = LOCK_CLASS(&rm->rm_wlock_object);
160 	return (lc->lc_owner(&rm->rm_wlock_object, owner));
193  * has on this CPU for the lock 'rm'.
196 rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
207 		if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
232 	struct rmlock *rm = arg;
240 		if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
243 			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
251 rm_init_flags(struct rmlock *rm, const char *name, int opts)
261 	rm->rm_writecpus = all_cpus;
262 	LIST_INIT(&rm->rm_activeReaders);
266 		sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS);
269 		mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS);
271 	lock_init(&rm->lock_object, lc, name, NULL, liflags);
275 rm_init(struct rmlock *rm, const char *name)
278 	rm_init_flags(rm, name, 0);
282 rm_destroy(struct rmlock *rm)
285 	rm_assert(rm, RA_UNLOCKED);
286 	LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
287 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
288 		sx_destroy(&rm->rm_lock_sx);
290 		mtx_destroy(&rm->rm_lock_mtx);
291 	lock_destroy(&rm->lock_object);
295 rm_wowned(struct rmlock *rm)
298 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
299 		return (sx_xlocked(&rm->rm_lock_sx));
301 		return (mtx_owned(&rm->rm_lock_mtx));
321 _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
329 	if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
349 	if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
354 		if (rm_trackers_present(pc, rm, curthread) != 0) {
356 			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
370 		if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
371 			if (!sx_try_xlock(&rm->rm_lock_sx))
374 			if (!mtx_trylock(&rm->rm_lock_mtx))
378 		if (rm->lock_object.lo_flags & LO_SLEEPABLE)
379 			sx_xlock(&rm->rm_lock_sx);
381 			mtx_lock(&rm->rm_lock_mtx);
386 	CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
391 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
392 		sx_xunlock(&rm->rm_lock_sx);
394 		mtx_unlock(&rm->rm_lock_mtx);
400 _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
410 	tracker->rmp_rmlock = rm;
431 	    CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
435 	return _rm_rlock_hard(rm, tracker, trylock);
454 		struct rmlock *rm;
457 		rm = tracker->rmp_rmlock;
459 		turnstile_chain_lock(&rm->lock_object);
462 		ts = turnstile_lookup(&rm->lock_object);
466 		turnstile_chain_unlock(&rm->lock_object);
472 _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
493 _rm_wlock(struct rmlock *rm)
502 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
503 		sx_xlock(&rm->rm_lock_sx);
505 		mtx_lock(&rm->rm_lock_mtx);
507 	if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
510 		CPU_NAND(&readcpus, &rm->rm_writecpus);
511 		rm->rm_writecpus = all_cpus;
514 		 * Assumes rm->rm_writecpus update is visible on other CPUs
522 		    rm);
525 		rm_cleanIPI(rm);
529 		while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
530 			ts = turnstile_trywait(&rm->lock_object);
542 _rm_wunlock(struct rmlock *rm)
545 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
546 		sx_xunlock(&rm->rm_lock_sx);
548 		mtx_unlock(&rm->rm_lock_mtx);
554 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
562 	    curthread, rm->lock_object.lo_name, file, line));
563 	KASSERT(!rm_destroyed(rm),
565 	_rm_assert(rm, RA_UNLOCKED, file, line);
567 	WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
570 	_rm_wlock(rm);
572 	LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
574 	WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
581 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
587 	KASSERT(!rm_destroyed(rm),
589 	_rm_assert(rm, RA_WLOCKED, file, line);
590 	WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
591 	LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
592 	_rm_wunlock(rm);
597 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
605 	if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
607 		KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
610 		    rm->lock_object.lo_name, file, line));
616 	    curthread, rm->lock_object.lo_name, file, line));
617 	KASSERT(!rm_destroyed(rm),
620 		KASSERT(!rm_wowned(rm),
622 		    rm->lock_object.lo_name, file, line));
623 		WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
627 	if (_rm_rlock(rm, tracker, trylock)) {
629 			LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
632 			LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
634 		WITNESS_LOCK(&rm->lock_object, 0, file, line);
640 		LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
646 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
653 	KASSERT(!rm_destroyed(rm),
655 	_rm_assert(rm, RA_RLOCKED, file, line);
656 	WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
657 	LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
658 	_rm_runlock(rm, tracker);
669 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
672 	_rm_wlock(rm);
676 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
679 	_rm_wunlock(rm);
683 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
687 	return _rm_rlock(rm, tracker, trylock);
691 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
695 	_rm_runlock(rm, tracker);
711 _rm_assert(struct rmlock *rm, int what, const char *file, int line)
728 		if (rm_wowned(rm)) {
731 				    rm->lock_object.lo_name, file, line);
734 				    rm->lock_object.lo_name, file, line);
739 		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
744 			    rm->lock_object.lo_name, (what & RA_RLOCKED) ?
749 				    rm->lock_object.lo_name, file, line);
752 			    rm->lock_object.lo_name, file, line);
755 		if (!rm_wowned(rm))
757 			    rm->lock_object.lo_name, file, line);
760 		if (rm_wowned(rm))
762 			    rm->lock_object.lo_name, file, line);
765 		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
770 			    rm->lock_object.lo_name, file, line);
773 		panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
802 	struct rmlock *rm;
806 	rm = (struct rmlock *)lock;
808 	ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
815 			if (tr->rmp_rmlock == rm)
819 	LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
821 	lc = LOCK_CLASS(&rm->rm_wlock_object);
823 	lc->lc_ddb_show(&rm->rm_wlock_object);