1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _FUTEX_H
3#define _FUTEX_H
4
5#include <linux/futex.h>
6#include <linux/rtmutex.h>
7#include <linux/sched/wake_q.h>
8#include <linux/compat.h>
9
10#ifdef CONFIG_PREEMPT_RT
11#include <linux/rcuwait.h>
12#endif
13
14#include <asm/futex.h>
15
16/*
17 * Futex flags used to encode options to functions and preserve them across
18 * restarts.
19 */
20#define FLAGS_SIZE_8		0x0000
21#define FLAGS_SIZE_16		0x0001
22#define FLAGS_SIZE_32		0x0002
23#define FLAGS_SIZE_64		0x0003
24
25#define FLAGS_SIZE_MASK		0x0003
26
27#ifdef CONFIG_MMU
28# define FLAGS_SHARED		0x0010
29#else
30/*
31 * NOMMU does not have per process address space. Let the compiler optimize
32 * code away.
33 */
34# define FLAGS_SHARED		0x0000
35#endif
36#define FLAGS_CLOCKRT		0x0020
37#define FLAGS_HAS_TIMEOUT	0x0040
38#define FLAGS_NUMA		0x0080
39#define FLAGS_STRICT		0x0100
40
41/* FUTEX_ to FLAGS_ */
42static inline unsigned int futex_to_flags(unsigned int op)
43{
44	unsigned int flags = FLAGS_SIZE_32;
45
46	if (!(op & FUTEX_PRIVATE_FLAG))
47		flags |= FLAGS_SHARED;
48
49	if (op & FUTEX_CLOCK_REALTIME)
50		flags |= FLAGS_CLOCKRT;
51
52	return flags;
53}
54
55#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE)
56
57/* FUTEX2_ to FLAGS_ */
58static inline unsigned int futex2_to_flags(unsigned int flags2)
59{
60	unsigned int flags = flags2 & FUTEX2_SIZE_MASK;
61
62	if (!(flags2 & FUTEX2_PRIVATE))
63		flags |= FLAGS_SHARED;
64
65	if (flags2 & FUTEX2_NUMA)
66		flags |= FLAGS_NUMA;
67
68	return flags;
69}
70
71static inline unsigned int futex_size(unsigned int flags)
72{
73	return 1 << (flags & FLAGS_SIZE_MASK);
74}
75
76static inline bool futex_flags_valid(unsigned int flags)
77{
78	/* Only 64bit futexes for 64bit code */
79	if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) {
80		if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
81			return false;
82	}
83
84	/* Only 32bit futexes are implemented -- for now */
85	if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
86		return false;
87
88	return true;
89}
90
91static inline bool futex_validate_input(unsigned int flags, u64 val)
92{
93	int bits = 8 * futex_size(flags);
94
95	if (bits < 64 && (val >> bits))
96		return false;
97
98	return true;
99}
100
101#ifdef CONFIG_FAIL_FUTEX
102extern bool should_fail_futex(bool fshared);
103#else
104static inline bool should_fail_futex(bool fshared)
105{
106	return false;
107}
108#endif
109
110/*
111 * Hash buckets are shared by all the futex_keys that hash to the same
112 * location.  Each key may have multiple futex_q structures, one for each task
113 * waiting on a futex.
114 */
115struct futex_hash_bucket {
116	atomic_t waiters;
117	spinlock_t lock;
118	struct plist_head chain;
119} ____cacheline_aligned_in_smp;
120
121/*
122 * Priority Inheritance state:
123 */
124struct futex_pi_state {
125	/*
126	 * list of 'owned' pi_state instances - these have to be
127	 * cleaned up in do_exit() if the task exits prematurely:
128	 */
129	struct list_head list;
130
131	/*
132	 * The PI object:
133	 */
134	struct rt_mutex_base pi_mutex;
135
136	struct task_struct *owner;
137	refcount_t refcount;
138
139	union futex_key key;
140} __randomize_layout;
141
142struct futex_q;
143typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q);
144
145/**
146 * struct futex_q - The hashed futex queue entry, one per waiting task
147 * @list:		priority-sorted list of tasks waiting on this futex
148 * @task:		the task waiting on the futex
149 * @lock_ptr:		the hash bucket lock
150 * @wake:		the wake handler for this queue
151 * @wake_data:		data associated with the wake handler
152 * @key:		the key the futex is hashed on
153 * @pi_state:		optional priority inheritance state
154 * @rt_waiter:		rt_waiter storage for use with requeue_pi
155 * @requeue_pi_key:	the requeue_pi target futex key
156 * @bitset:		bitset for the optional bitmasked wakeup
157 * @requeue_state:	State field for futex_requeue_pi()
158 * @requeue_wait:	RCU wait for futex_requeue_pi() (RT only)
159 *
160 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
161 * we can wake only the relevant ones (hashed queues may be shared).
162 *
163 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
164 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
165 * The order of wakeup is always to make the first condition true, then
166 * the second.
167 *
168 * PI futexes are typically woken before they are removed from the hash list via
169 * the rt_mutex code. See futex_unqueue_pi().
170 */
171struct futex_q {
172	struct plist_node list;
173
174	struct task_struct *task;
175	spinlock_t *lock_ptr;
176	futex_wake_fn *wake;
177	void *wake_data;
178	union futex_key key;
179	struct futex_pi_state *pi_state;
180	struct rt_mutex_waiter *rt_waiter;
181	union futex_key *requeue_pi_key;
182	u32 bitset;
183	atomic_t requeue_state;
184#ifdef CONFIG_PREEMPT_RT
185	struct rcuwait requeue_wait;
186#endif
187} __randomize_layout;
188
189extern const struct futex_q futex_q_init;
190
191enum futex_access {
192	FUTEX_READ,
193	FUTEX_WRITE
194};
195
196extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
197			 enum futex_access rw);
198
199extern struct hrtimer_sleeper *
200futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
201		  int flags, u64 range_ns);
202
203extern struct futex_hash_bucket *futex_hash(union futex_key *key);
204
205/**
206 * futex_match - Check whether two futex keys are equal
207 * @key1:	Pointer to key1
208 * @key2:	Pointer to key2
209 *
210 * Return 1 if two futex_keys are equal, 0 otherwise.
211 */
212static inline int futex_match(union futex_key *key1, union futex_key *key2)
213{
214	return (key1 && key2
215		&& key1->both.word == key2->both.word
216		&& key1->both.ptr == key2->both.ptr
217		&& key1->both.offset == key2->both.offset);
218}
219
220extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
221			    struct futex_q *q, struct futex_hash_bucket **hb);
222extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
223				   struct hrtimer_sleeper *timeout);
224extern bool __futex_wake_mark(struct futex_q *q);
225extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
226
227extern int fault_in_user_writeable(u32 __user *uaddr);
228extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval);
229extern int futex_get_value_locked(u32 *dest, u32 __user *from);
230extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
231
232extern void __futex_unqueue(struct futex_q *q);
233extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb);
234extern int futex_unqueue(struct futex_q *q);
235
236/**
237 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
238 * @q:	The futex_q to enqueue
239 * @hb:	The destination hash bucket
240 *
241 * The hb->lock must be held by the caller, and is released here. A call to
242 * futex_queue() is typically paired with exactly one call to futex_unqueue().  The
243 * exceptions involve the PI related operations, which may use futex_unqueue_pi()
244 * or nothing if the unqueue is done as part of the wake process and the unqueue
245 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
246 * an example).
247 */
248static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb)
249	__releases(&hb->lock)
250{
251	__futex_queue(q, hb);
252	spin_unlock(&hb->lock);
253}
254
255extern void futex_unqueue_pi(struct futex_q *q);
256
257extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
258
259/*
260 * Reflects a new waiter being added to the waitqueue.
261 */
262static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
263{
264#ifdef CONFIG_SMP
265	atomic_inc(&hb->waiters);
266	/*
267	 * Full barrier (A), see the ordering comment above.
268	 */
269	smp_mb__after_atomic();
270#endif
271}
272
273/*
274 * Reflects a waiter being removed from the waitqueue by wakeup
275 * paths.
276 */
277static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
278{
279#ifdef CONFIG_SMP
280	atomic_dec(&hb->waiters);
281#endif
282}
283
284static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
285{
286#ifdef CONFIG_SMP
287	/*
288	 * Full barrier (B), see the ordering comment above.
289	 */
290	smp_mb();
291	return atomic_read(&hb->waiters);
292#else
293	return 1;
294#endif
295}
296
297extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q);
298extern void futex_q_unlock(struct futex_hash_bucket *hb);
299
300
301extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
302				union futex_key *key,
303				struct futex_pi_state **ps,
304				struct task_struct *task,
305				struct task_struct **exiting,
306				int set_waiters);
307
308extern int refill_pi_state_cache(void);
309extern void get_pi_state(struct futex_pi_state *pi_state);
310extern void put_pi_state(struct futex_pi_state *pi_state);
311extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
312
313/*
314 * Express the locking dependencies for lockdep:
315 */
316static inline void
317double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
318{
319	if (hb1 > hb2)
320		swap(hb1, hb2);
321
322	spin_lock(&hb1->lock);
323	if (hb1 != hb2)
324		spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
325}
326
327static inline void
328double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
329{
330	spin_unlock(&hb1->lock);
331	if (hb1 != hb2)
332		spin_unlock(&hb2->lock);
333}
334
335/* syscalls */
336
337extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
338				 val, ktime_t *abs_time, u32 bitset, u32 __user
339				 *uaddr2);
340
341extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
342			 u32 __user *uaddr2, unsigned int flags2,
343			 int nr_wake, int nr_requeue,
344			 u32 *cmpval, int requeue_pi);
345
346extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
347			struct hrtimer_sleeper *to, u32 bitset);
348
349extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
350		      ktime_t *abs_time, u32 bitset);
351
352/**
353 * struct futex_vector - Auxiliary struct for futex_waitv()
354 * @w: Userspace provided data
355 * @q: Kernel side data
356 *
357 * Struct used to build an array with all data need for futex_waitv()
358 */
359struct futex_vector {
360	struct futex_waitv w;
361	struct futex_q q;
362};
363
364extern int futex_parse_waitv(struct futex_vector *futexv,
365			     struct futex_waitv __user *uwaitv,
366			     unsigned int nr_futexes, futex_wake_fn *wake,
367			     void *wake_data);
368
369extern int futex_wait_multiple_setup(struct futex_vector *vs, int count,
370				     int *woken);
371
372extern int futex_unqueue_multiple(struct futex_vector *v, int count);
373
374extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
375			       struct hrtimer_sleeper *to);
376
377extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
378
379extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
380			 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
381
382extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
383
384extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
385
386#endif /* _FUTEX_H */
387