kernel/locking/mutex.c

34 #include <trace/events/lock.h>
46 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
48 	atomic_long_set(&lock->owner, 0);
49 	raw_spin_lock_init(&lock->wait_lock);
50 	INIT_LIST_HEAD(&lock->wait_list);
52 	osq_lock_init(&lock->osq);
55 	debug_mutex_init(lock, name, key);
60  * @owner: contains: 'struct task_struct *' to the current lock owner,
65  * Bit1 indicates unlock needs to hand the lock to the top-waiter
79 static inline struct task_struct *__mutex_owner(struct mutex *lock)
81 	return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
89 bool mutex_is_locked(struct mutex *lock)
91 	return __mutex_owner(lock) != NULL;
101  * Returns: __mutex_owner(lock) on failure or NULL on success.
103 static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
107 	owner = atomic_long_read(&lock->owner);
129 		if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {
142 static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)
144 	return !__mutex_trylock_common(lock, handoff);
150 static inline bool __mutex_trylock(struct mutex *lock)
152 	return !__mutex_trylock_common(lock, false);
166 static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
171 	if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
177 static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
181 	return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);
185 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
187 	atomic_long_or(flag, &lock->owner);
190 static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
192 	atomic_long_andnot(flag, &lock->owner);
195 static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
197 	return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
201  * Add @waiter to a given location in the lock wait_list and set the
205 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
208 	debug_mutex_add_waiter(lock, waiter, current);
211 	if (__mutex_waiter_is_first(lock, waiter))
212 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
216 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
219 	if (likely(list_empty(&lock->wait_list)))
220 		__mutex_clear_flag(lock, MUTEX_FLAGS);
222 	debug_mutex_remove_waiter(lock, waiter, current);
231 static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
233 	unsigned long owner = atomic_long_read(&lock->owner);
246 		if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))
253  * We split the mutex lock/unlock logic into separate fastpath and
258 static void __sched __mutex_lock_slowpath(struct mutex *lock);
262  * @lock: the mutex to be acquired
281 void __sched mutex_lock(struct mutex *lock)
285 	if (!__mutex_trylock_fast(lock))
286 		__mutex_lock_slowpath(lock);
298 static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
300 	return __mutex_trylock_common(lock, false);
304 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
309 	ww = container_of(lock, struct ww_mutex, base);
328 	 * lock from a waiter with an earlier stamp, since the
329 	 * other thread may already own a lock that we also
332 	if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))
339 	if (waiter && !__mutex_waiter_is_first(lock, waiter))
352 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
359 	while (__mutex_owner(lock) == owner) {
362 		 * checking lock->owner still matches owner. And we already
371 		 * Use vcpu_is_preempted to detect lock holder preemption issue.
378 		if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {
392 static inline int mutex_can_spin_on_owner(struct mutex *lock)
407 	owner = __mutex_owner(lock);
412 	 * If lock->owner is not set, the mutex has been released. Return true
422  * We try to spin for acquisition when we find that the lock owner
424  * need to reschedule. The rationale is that if the lock owner is
425  * running, it is likely to release the lock soon.
427  * The mutex spinners are queued up using MCS lock so that only one
429  * going to happen, there is no point in going through the lock/unlock
432  * Returns true when the lock was taken, otherwise false, indicating
436  * queue. The waiter-spinner will spin on the lock directly and concurrently
441 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
449 		 * is not going to take OSQ lock anyway, there is no need
452 		if (!mutex_can_spin_on_owner(lock))
458 		 * MCS (queued) lock first before spinning on the owner field.
460 		if (!osq_lock(&lock->osq))
468 		owner = __mutex_trylock_or_owner(lock);
474 		 * release the lock or go to sleep.
476 		if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))
489 		osq_unlock(&lock->osq);
496 		osq_unlock(&lock->osq);
501 	 * reschedule now, before we try-lock the mutex. This avoids getting
517 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
524 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
528  * @lock: the mutex to be released
542 void __sched mutex_unlock(struct mutex *lock)
545 	if (__mutex_unlock_fast(lock))
548 	__mutex_unlock_slowpath(lock, _RET_IP_);
554  * @lock: the mutex to be released
563 void __sched ww_mutex_unlock(struct ww_mutex *lock)
565 	__ww_mutex_unlock(lock);
566 	mutex_unlock(&lock->base);
574 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
587 	MUTEX_WARN_ON(lock->magic != lock);
589 	ww = container_of(lock, struct ww_mutex, base);
608 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
610 	trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
611 	if (__mutex_trylock(lock) ||
612 	    mutex_optimistic_spin(lock, ww_ctx, NULL)) {
613 		/* got the lock, yay! */
614 		lock_acquired(&lock->dep_map, ip);
617 		trace_contention_end(lock, 0);
622 	raw_spin_lock(&lock->wait_lock);
626 	if (__mutex_trylock(lock)) {
628 			__ww_mutex_check_waiters(lock, ww_ctx);
633 	debug_mutex_lock_common(lock, &waiter);
638 	lock_contended(&lock->dep_map, ip);
642 		__mutex_add_waiter(lock, &waiter, &lock->wait_list);
648 		ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx);
654 	trace_contention_begin(lock, LCB_F_MUTEX);
660 		 * mutex_unlock() handing the lock off to us, do a trylock
664 		if (__mutex_trylock(lock))
669 		 * wait_lock. This ensures the lock cancellation is ordered
678 			ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);
683 		raw_spin_unlock(&lock->wait_lock);
686 		first = __mutex_waiter_is_first(lock, &waiter);
694 		if (__mutex_trylock_or_handoff(lock, first))
698 			trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
699 			if (mutex_optimistic_spin(lock, ww_ctx, &waiter))
701 			trace_contention_begin(lock, LCB_F_MUTEX);
704 		raw_spin_lock(&lock->wait_lock);
706 	raw_spin_lock(&lock->wait_lock);
712 		 * Wound-Wait; we stole the lock (!first_waiter), check the
716 		    !__mutex_waiter_is_first(lock, &waiter))
717 			__ww_mutex_check_waiters(lock, ww_ctx);
720 	__mutex_remove_waiter(lock, &waiter);
725 	/* got the lock - cleanup and rejoice! */
726 	lock_acquired(&lock->dep_map, ip);
727 	trace_contention_end(lock, 0);
732 	raw_spin_unlock(&lock->wait_lock);
738 	__mutex_remove_waiter(lock, &waiter);
740 	trace_contention_end(lock, ret);
741 	raw_spin_unlock(&lock->wait_lock);
743 	mutex_release(&lock->dep_map, ip);
749 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
752 	return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);
756 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
759 	return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);
764  * @ww: mutex to lock
802 mutex_lock_nested(struct mutex *lock, unsigned int subclass)
804 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
810 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
812 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
817 mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
819 	return __mutex_lock(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);
824 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
826 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
831 mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
838 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
845 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
859 		ctx->contending_lock = lock;
861 		ww_mutex_unlock(lock);
871 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
876 	ret =  __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,
879 		return ww_mutex_deadlock_injection(lock, ctx);
886 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
891 	ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,
895 		return ww_mutex_deadlock_injection(lock, ctx);
904  * Release the lock, slowpath:
906 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
912 	mutex_release(&lock->dep_map, ip);
915 	 * Release the lock before (potentially) taking the spinlock such that
921 	owner = atomic_long_read(&lock->owner);
929 		if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {
937 	raw_spin_lock(&lock->wait_lock);
938 	debug_mutex_unlock(lock);
939 	if (!list_empty(&lock->wait_list)) {
942 			list_first_entry(&lock->wait_list,
947 		debug_mutex_wake_waiter(lock, waiter);
952 		__mutex_handoff(lock, next);
954 	raw_spin_unlock(&lock->wait_lock);
965 __mutex_lock_killable_slowpath(struct mutex *lock);
968 __mutex_lock_interruptible_slowpath(struct mutex *lock);
972  * @lock: The mutex to be acquired.
979  * Return: 0 if the lock was successfully acquired or %-EINTR if a
982 int __sched mutex_lock_interruptible(struct mutex *lock)
986 	if (__mutex_trylock_fast(lock))
989 	return __mutex_lock_interruptible_slowpath(lock);
996  * @lock: The mutex to be acquired.
1003  * Return: 0 if the lock was successfully acquired or %-EINTR if a
1006 int __sched mutex_lock_killable(struct mutex *lock)
1010 	if (__mutex_trylock_fast(lock))
1013 	return __mutex_lock_killable_slowpath(lock);
1019  * @lock: The mutex to be acquired.
1027 void __sched mutex_lock_io(struct mutex *lock)
1032 	mutex_lock(lock);
1038 __mutex_lock_slowpath(struct mutex *lock)
1040 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
1044 __mutex_lock_killable_slowpath(struct mutex *lock)
1046 	return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
1050 __mutex_lock_interruptible_slowpath(struct mutex *lock)
1052 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
1056 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1058 	return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,
1063 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
1066 	return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,
1074  * @lock: the mutex to be acquired
1086 int __sched mutex_trylock(struct mutex *lock)
1090 	MUTEX_WARN_ON(lock->magic != lock);
1092 	locked = __mutex_trylock(lock);
1094 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1102 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1106 	if (__mutex_trylock_fast(&lock->base)) {
1108 			ww_mutex_set_context_fastpath(lock, ctx);
1112 	return __ww_mutex_lock_slowpath(lock, ctx);
1117 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1121 	if (__mutex_trylock_fast(&lock->base)) {
1123 			ww_mutex_set_context_fastpath(lock, ctx);
1127 	return __ww_mutex_lock_interruptible_slowpath(lock, ctx);
1140  * @lock: the mutex to return holding if we dec to 0
1142  * return true and hold lock if we dec to 0, return false otherwise
1144 int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
1149 	/* we might hit 0, so take the lock */
1150 	mutex_lock(lock);
1153 		mutex_unlock(lock);
1156 	/* we hit 0, and we hold the lock */