kernel/futex/waitwake.c

11  * READ this before attempting to hack on futexes!
30  * optimization to work, ordering guarantees must exist so that the waiter
31  * being added to the list is acknowledged when the list is concurrently being
50  * This would cause the waiter on CPU 0 to wait forever because it
51  * missed the transition of the user space value from val to newval
84  * to futex and the waiters read (see futex_hb_waiters_pending()).
100  * Refer to the comment in futex_q_lock().
102  * Similarly, in order to account for waiters being requeued on another
112 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
120 	 * to prevent the following store to lock_ptr from getting ahead of the
131  * must ensure to later call wake_up_q() for the actual
132  * wakeups to occur.
175 	/* Make sure we really have tasks to wakeup */
218 			pr_info_ratelimited("futex_wake_op: %s tries to shift op by %d; fix this program\n",
251  * to this virtual address:
340  * @q:		the futex_q to queue up on
347 	 * The task state is guaranteed to be set before another task can
350 	 * access to the hash list and forcing another memory barrier.
361 	 * has tried to wake us, and we can skip the call to schedule().
367 		 * is no timeout, or if it has yet to expire.
377  * @v:	   The list of futexes to unqueue
380  * Helper to unqueue a list of futexes. This can't fail.
399  * futex_wait_multiple_setup - Prepare to wait and enqueue multiple futexes
400  * @vs:		The futex list to wait on
402  * @woken:	Index of the last woken futex, if any. Used to notify the
403  *		caller that it can return this index to userspace (return parameter)
407  * task is ready to interruptible sleep.
422 	 * Enqueuing multiple futexes is tricky, because we need to enqueue
423 	 * each futex on the list before dealing with the next one to avoid
424 	 * deadlocking on the hash bucket. But, before enqueuing, we need to
432 	 * Private futexes doesn't need to recalculate hash in retry, so skip
473 		 * was woken, we don't return error and return this index to
482 			 * If we need to handle a page fault, we need to do so
504  * @vs:    List of futexes to wait for
506  * @to:    Timeout
512 				 struct hrtimer_sleeper *to)
514 	if (to && !to->task)
526  * futex_wait_multiple - Prepare to wait on and enqueue several futexes
527  * @vs:		The list of futexes to wait on
529  * @to:		Timeout before giving up and returning to userspace
536  *  - >=0 - Hint to the futex that was awoken
540 			struct hrtimer_sleeper *to)
544 	if (to)
545 		hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
557 		futex_sleep_multiple(vs, count, to);
565 		if (to && !to->task)
577  * futex_wait_setup() - Prepare to wait on a futex
582  * @hb:		storage for hash_bucket pointer to be returned to caller
606 	 * if cond(var) is known to be true at the time of blocking, for
648 		 struct hrtimer_sleeper *to, u32 bitset)
661 	 * Prepare to wait on uaddr. On success, it holds hb->lock and q
669 	futex_wait_queue(hb, &q, to);
675 	if (to && !to->task)
690 	struct hrtimer_sleeper timeout, *to;
694 	to = futex_setup_timer(abs_time, &timeout, flags,
697 	ret = __futex_wait(uaddr, flags, val, to, bitset);
699 	/* No timeout, nothing to clean up. */
700 	if (!to)
703 	hrtimer_cancel(&to->timer);
704 	destroy_hrtimer_on_stack(&to->timer);