kernel/futex/pi.c

15 	struct futex_pi_state *pi_state;
20 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);
22 	if (!pi_state)
25 	INIT_LIST_HEAD(&pi_state->list);
27 	pi_state->owner = NULL;
28 	refcount_set(&pi_state->refcount, 1);
29 	pi_state->key = FUTEX_KEY_INIT;
31 	current->pi_state_cache = pi_state;
38 	struct futex_pi_state *pi_state = current->pi_state_cache;
40 	WARN_ON(!pi_state);
43 	return pi_state;
46 static void pi_state_update_owner(struct futex_pi_state *pi_state,
49 	struct task_struct *old_owner = pi_state->owner;
51 	lockdep_assert_held(&pi_state->pi_mutex.wait_lock);
55 		WARN_ON(list_empty(&pi_state->list));
56 		list_del_init(&pi_state->list);
62 		WARN_ON(!list_empty(&pi_state->list));
63 		list_add(&pi_state->list, &new_owner->pi_state_list);
64 		pi_state->owner = new_owner;
69 void get_pi_state(struct futex_pi_state *pi_state)
71 	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));
75  * Drops a reference to the pi_state object and frees or caches it
78 void put_pi_state(struct futex_pi_state *pi_state)
80 	if (!pi_state)
83 	if (!refcount_dec_and_test(&pi_state->refcount))
87 	 * If pi_state->owner is NULL, the owner is most probably dying
88 	 * and has cleaned up the pi_state already
90 	if (pi_state->owner) {
93 		raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);
94 		pi_state_update_owner(pi_state, NULL);
95 		rt_mutex_proxy_unlock(&pi_state->pi_mutex);
96 		raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);
100 		kfree(pi_state);
103 		 * pi_state->list is already empty.
104 		 * clear pi_state->owner.
107 		pi_state->owner = NULL;
108 		refcount_set(&pi_state->refcount, 1);
109 		current->pi_state_cache = pi_state;
116  *      Waiter | pi_state | pi->owner | uTID      | uODIED | ?
149  *	the pi_state but cannot access the user space value.
151  * [7]	pi_state->owner can only be NULL when the OWNER_DIED bit is set.
169  *	futex_q -> pi_state, relation
176  *	{uval, pi_state}
182  *	p->pi_state_list -> pi_state->list, relation
183  *	pi_mutex->owner -> pi_state->owner, relation
185  * pi_state->refcount:
187  *	pi_state lifetime
199  * Validate that the existing waiter has a pi_state and sanity check
200  * the pi_state against the user space value. If correct, attach to
204 			      struct futex_pi_state *pi_state,
214 	if (unlikely(!pi_state))
222 	 * our pi_state.
224 	 * The waiter holding a reference on @pi_state also protects against
227 	 * free pi_state before we can take a reference ourselves.
229 	WARN_ON(!refcount_read(&pi_state->refcount));
232 	 * Now that we have a pi_state, we can acquire wait_lock
235 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
238 	 * Since {uval, pi_state} is serialized by wait_lock, and our current
256 		 * pi_state->rt_mutex will fixup owner.
258 		if (!pi_state->owner) {
283 		 * If the owner died bit is not set, then the pi_state
286 		if (!pi_state->owner)
295 	if (pid != task_pid_vnr(pi_state->owner))
299 	get_pi_state(pi_state);
300 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
301 	*ps = pi_state;
317 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
383 	 * This creates pi_state, we have hb->lock held, this means nothing can
386 	struct futex_pi_state *pi_state = alloc_pi_state();
392 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
395 	pi_state->key = *key;
397 	WARN_ON(!list_empty(&pi_state->list));
398 	list_add(&pi_state->list, &p->pi_state_list);
400 	 * Assignment without holding pi_state->pi_mutex.wait_lock is safe
403 	pi_state->owner = p;
405 	*ps = pi_state;
420 	 * the new pi_state to it, but bail out when TID = 0 [1]
496  * @ps:			the pi_state pointer where we store the result of the
547 	 * its pi_state.
551 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
612  * Caller must hold a reference on @pi_state.
615 			 struct futex_pi_state *pi_state,
658 		pi_state_update_owner(pi_state, new_owner);
659 		postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh);
663 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
674 	struct futex_pi_state *pi_state = q->pi_state;
679 	oldowner = pi_state->owner;
684 	 *  - we stole the lock and pi_state->owner needs updating to reflect
695 	 * Note: We write the user space value _before_ changing the pi_state
699 	 * Modifying pi_state _before_ the user space value would leave the
700 	 * pi_state in an inconsistent state when we fault here, because we
714 		if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {
715 			/* We got the lock. pi_state is correct. Tell caller. */
723 		newowner = rt_mutex_owner(&pi_state->pi_mutex);
750 	if (!pi_state->owner)
770 	 * We fixed up user space. Now we need to fix the pi_state
773 	pi_state_update_owner(pi_state, newowner);
781 	 * the rtmutex) the chance to try the fixup of the pi_state. So once we
782 	 * are back from handling the fault we need to check the pi_state after
791 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
810 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
815 	if (pi_state->owner != oldowner)
828 	 * Ensure that the rtmutex owner is also the pi_state owner despite
838 	pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex));
846 	struct futex_pi_state *pi_state = q->pi_state;
851 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
853 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
858  * fixup_pi_owner() - Post lock pi_state and corner case management
860  * @q:		futex_q (contains pi_state and access to the rt_mutex)
864  * the pi_state owner as well as handle race conditions that may allow us to
879 		 * Speculative pi_state->owner read (we don't hold wait_lock);
880 		 * since we own the lock pi_state->owner == current is the
883 		if (q->pi_state->owner != current)
893 	 * Another speculative read; pi_state->owner == current is unstable
896 	if (q->pi_state->owner == current)
903 	if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current))
943 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
979 	WARN_ON(!q.pi_state);
987 		ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
1011 	 * handoff -- such that the latter can observe the futex_q::pi_state
1014 	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
1019 	 * it sees the futex_q::pi_state.
1021 	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);
1022 	raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);
1033 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
1043 	 * state between hb and pi_state, but only in the direction of not
1054 	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
1069 	 * Fixup the pi_state owner and possibly acquire the lock if we
1147 		struct futex_pi_state *pi_state = top_waiter->pi_state;
1151 		if (!pi_state)
1155 		 * If current does not own the pi_state then the futex is
1158 		if (pi_state->owner != current)
1176 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
1187 		rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
1190 			raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1194 		get_pi_state(pi_state);
1197 		/* drops pi_state->pi_mutex.wait_lock */
1198 		ret = wake_futex_pi(uaddr, uval, pi_state, rt_waiter);
1200 		put_pi_state(pi_state);