sys/kern/kern_sx.c

188 	uintptr_t x = sx->sx_lock;
211 	ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock,
212 	    ("%s: sx_lock not aligned for %s: %p", __func__, description,
213 	    &sx->sx_lock));
228 	sx->sx_lock = SX_LOCK_UNLOCKED;
237 	KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
239 	sx->sx_lock = SX_LOCK_DESTROYED;
253 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
279 		x = sx->sx_lock;
284 		if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
306 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
332 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
338 		atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
341 		rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
359 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
375 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
401 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
410 	x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS;
411 	success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x,
434 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
455 	x = sx->sx_lock;
457 	    atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) |
474 	x = sx->sx_lock;
475 	atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
525 		atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
533 		    sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
535 	while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
550 		x = sx->sx_lock;
561 					while (SX_OWNER(sx->sx_lock) == x &&
578 					x = sx->sx_lock;
594 		x = sx->sx_lock;
626 		 * woken up and acquired the lock yet, sx_lock will be
634 			if (atomic_cmpset_acq_ptr(&sx->sx_lock,
651 			if (!atomic_cmpset_ptr(&sx->sx_lock, x,
725 	MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
730 			atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
735 	MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS |
752 	if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 &&
755 		x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS);
764 	atomic_store_rel_ptr(&sx->sx_lock, x);
808 		x = sx->sx_lock;
818 			if (atomic_cmpset_acq_ptr(&sx->sx_lock, x,
850 				while (SX_OWNER(sx->sx_lock) == x &&
867 		x = sx->sx_lock;
900 			if (!atomic_cmpset_ptr(&sx->sx_lock, x,
973 		x = sx->sx_lock;
987 			if (atomic_cmpset_rel_ptr(&sx->sx_lock, x,
1005 			if (atomic_cmpset_rel_ptr(&sx->sx_lock,
1029 		if (!atomic_cmpset_rel_ptr(&sx->sx_lock,
1085 		if (sx->sx_lock == SX_LOCK_UNLOCKED ||
1086 		    (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked ||
1092 		if (!(sx->sx_lock & SX_LOCK_SHARED)) {
1149 	if (sx->sx_lock == SX_LOCK_UNLOCKED)
1151 	else if (sx->sx_lock == SX_LOCK_DESTROYED) {
1154 	} else if (sx->sx_lock & SX_LOCK_SHARED)
1155 		db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
1165 	switch(sx->sx_lock &
1204 	if (sx->sx_lock & SX_LOCK_SHARED)
1206 		    (uintmax_t)SX_SHARERS(sx->sx_lock));