sys/kern/kern_rmlock.c

65 #define	rm_destroyed(rm)						\
66 	(LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
86 	.lc_name = "rm",
100 	.lc_name = "sleepable rm",
123 	struct rmlock *rm;
126 	rm = (struct rmlock *)lock;
128 		rm_wlock(rm);
131 		rm_rlock(rm, tracker);
140 	struct rmlock *rm;
145 	rm = (struct rmlock *)lock;
148 	rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
149 	if (rm_wowned(rm))
150 		rm_wunlock(rm);
163 				if ((tracker->rmp_rmlock == rm) &&
172 		rm_runlock(rm, tracker);
181 	const struct rmlock *rm;
184 	rm = (const struct rmlock *)lock;
185 	lc = LOCK_CLASS(&rm->rm_wlock_object);
186 	return (lc->lc_owner(&rm->rm_wlock_object, owner));
219  * has on this CPU for the lock 'rm'.
222 rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
233 		if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
258 	struct rmlock *rm = arg;
266 		if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
269 			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
277 rm_init_flags(struct rmlock *rm, const char *name, int opts)
289 	rm->rm_writecpus = all_cpus;
290 	LIST_INIT(&rm->rm_activeReaders);
295 		sx_init_flags(&rm->rm_lock_sx, "rmlock_sx",
300 		mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx",
303 	lock_init(&rm->lock_object, lc, name, NULL, liflags);
307 rm_init(struct rmlock *rm, const char *name)
310 	rm_init_flags(rm, name, 0);
314 rm_destroy(struct rmlock *rm)
317 	rm_assert(rm, RA_UNLOCKED);
318 	LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
319 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
320 		sx_destroy(&rm->rm_lock_sx);
322 		mtx_destroy(&rm->rm_lock_mtx);
323 	lock_destroy(&rm->lock_object);
327 rm_wowned(const struct rmlock *rm)
330 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
331 		return (sx_xlocked(&rm->rm_lock_sx));
333 		return (mtx_owned(&rm->rm_lock_mtx));
346 _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
354 	if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
378 	if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
383 		if (rm_trackers_present(pc, rm, curthread) != 0) {
385 			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
399 		if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
400 			if (!sx_try_xlock(&rm->rm_lock_sx))
403 			if (!mtx_trylock(&rm->rm_lock_mtx))
407 		if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
409 			sx_xlock(&rm->rm_lock_sx);
412 			mtx_lock(&rm->rm_lock_mtx);
417 	CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
422 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
423 		sx_xunlock(&rm->rm_lock_sx);
425 		mtx_unlock(&rm->rm_lock_mtx);
431 _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
441 	tracker->rmp_rmlock = rm;
443 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
465 	    CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
469 	return _rm_rlock_hard(rm, tracker, trylock);
488 		struct rmlock *rm;
491 		rm = tracker->rmp_rmlock;
493 		turnstile_chain_lock(&rm->lock_object);
496 		ts = turnstile_lookup(&rm->lock_object);
500 		turnstile_chain_unlock(&rm->lock_object);
506 _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
520 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
530 _rm_wlock(struct rmlock *rm)
539 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
540 		sx_xlock(&rm->rm_lock_sx);
542 		mtx_lock(&rm->rm_lock_mtx);
544 	if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
547 		CPU_NAND(&readcpus, &rm->rm_writecpus);
548 		rm->rm_writecpus = all_cpus;
551 		 * Assumes rm->rm_writecpus update is visible on other CPUs
559 		    rm);
562 		rm_cleanIPI(rm);
566 		while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
567 			ts = turnstile_trywait(&rm->lock_object);
579 _rm_wunlock(struct rmlock *rm)
582 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
583 		sx_xunlock(&rm->rm_lock_sx);
585 		mtx_unlock(&rm->rm_lock_mtx);
591 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
599 	    curthread, rm->lock_object.lo_name, file, line));
600 	KASSERT(!rm_destroyed(rm),
602 	_rm_assert(rm, RA_UNLOCKED, file, line);
604 	WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
607 	_rm_wlock(rm);
609 	LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
610 	WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
615 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
621 	KASSERT(!rm_destroyed(rm),
623 	_rm_assert(rm, RA_WLOCKED, file, line);
624 	WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
625 	LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
626 	_rm_wunlock(rm);
631 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
639 	if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
641 		KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
644 		    rm->lock_object.lo_name, file, line));
650 	    curthread, rm->lock_object.lo_name, file, line));
651 	KASSERT(!rm_destroyed(rm),
654 		KASSERT(!rm_wowned(rm),
656 		    rm->lock_object.lo_name, file, line));
657 		WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
661 	if (_rm_rlock(rm, tracker, trylock)) {
663 			LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
666 			LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
668 		WITNESS_LOCK(&rm->lock_object, 0, file, line);
672 		LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
678 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
685 	KASSERT(!rm_destroyed(rm),
687 	_rm_assert(rm, RA_RLOCKED, file, line);
688 	WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
689 	LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
690 	_rm_runlock(rm, tracker);
701 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
704 	_rm_wlock(rm);
708 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
711 	_rm_wunlock(rm);
715 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
719 	return _rm_rlock(rm, tracker, trylock);
723 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
727 	_rm_runlock(rm, tracker);
743 _rm_assert(const struct rmlock *rm, int what, const char *file, int line)
760 		if (rm_wowned(rm)) {
763 				    rm->lock_object.lo_name, file, line);
766 				    rm->lock_object.lo_name, file, line);
771 		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
776 			    rm->lock_object.lo_name, (what & RA_RLOCKED) ?
781 				    rm->lock_object.lo_name, file, line);
784 			    rm->lock_object.lo_name, file, line);
787 		if (!rm_wowned(rm))
789 			    rm->lock_object.lo_name, file, line);
792 		if (rm_wowned(rm))
794 			    rm->lock_object.lo_name, file, line);
797 		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
802 			    rm->lock_object.lo_name, file, line);
805 		panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
834 	const struct rmlock *rm;
838 	rm = (const struct rmlock *)lock;
840 	ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
847 			if (tr->rmp_rmlock == rm)
851 	LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
853 	lc = LOCK_CLASS(&rm->rm_wlock_object);
855 	lc->lc_ddb_show(&rm->rm_wlock_object);