1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef ASM_X86_CMPXCHG_H
3#define ASM_X86_CMPXCHG_H
4
5#include <linux/compiler.h>
6#include <asm/cpufeatures.h>
7#include <asm/alternative.h> /* Provides LOCK_PREFIX */
8
9/*
10 * Non-existent functions to indicate usage errors at link time
11 * (or compile-time if the compiler implements __compiletime_error().
12 */
13extern void __xchg_wrong_size(void)
14	__compiletime_error("Bad argument size for xchg");
15extern void __cmpxchg_wrong_size(void)
16	__compiletime_error("Bad argument size for cmpxchg");
17extern void __xadd_wrong_size(void)
18	__compiletime_error("Bad argument size for xadd");
19extern void __add_wrong_size(void)
20	__compiletime_error("Bad argument size for add");
21
22/*
23 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
24 * -1 because sizeof will never return -1, thereby making those switch
25 * case statements guaranteed dead code which the compiler will
26 * eliminate, and allowing the "missing symbol in the default case" to
27 * indicate a usage error.
28 */
29#define __X86_CASE_B	1
30#define __X86_CASE_W	2
31#define __X86_CASE_L	4
32#ifdef CONFIG_64BIT
33#define __X86_CASE_Q	8
34#else
35#define	__X86_CASE_Q	-1		/* sizeof will never return -1 */
36#endif
37
38/*
39 * An exchange-type operation, which takes a value and a pointer, and
40 * returns the old value.
41 */
42#define __xchg_op(ptr, arg, op, lock)					\
43	({								\
44	        __typeof__ (*(ptr)) __ret = (arg);			\
45		switch (sizeof(*(ptr))) {				\
46		case __X86_CASE_B:					\
47			asm volatile (lock #op "b %b0, %1\n"		\
48				      : "+q" (__ret), "+m" (*(ptr))	\
49				      : : "memory", "cc");		\
50			break;						\
51		case __X86_CASE_W:					\
52			asm volatile (lock #op "w %w0, %1\n"		\
53				      : "+r" (__ret), "+m" (*(ptr))	\
54				      : : "memory", "cc");		\
55			break;						\
56		case __X86_CASE_L:					\
57			asm volatile (lock #op "l %0, %1\n"		\
58				      : "+r" (__ret), "+m" (*(ptr))	\
59				      : : "memory", "cc");		\
60			break;						\
61		case __X86_CASE_Q:					\
62			asm volatile (lock #op "q %q0, %1\n"		\
63				      : "+r" (__ret), "+m" (*(ptr))	\
64				      : : "memory", "cc");		\
65			break;						\
66		default:						\
67			__ ## op ## _wrong_size();			\
68		}							\
69		__ret;							\
70	})
71
72/*
73 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
74 * Since this is generally used to protect other memory information, we
75 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
76 * information around.
77 */
78#define arch_xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")
79
80/*
81 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
82 * store NEW in MEM.  Return the initial value in MEM.  Success is
83 * indicated by comparing RETURN with OLD.
84 */
85#define __raw_cmpxchg(ptr, old, new, size, lock)			\
86({									\
87	__typeof__(*(ptr)) __ret;					\
88	__typeof__(*(ptr)) __old = (old);				\
89	__typeof__(*(ptr)) __new = (new);				\
90	switch (size) {							\
91	case __X86_CASE_B:						\
92	{								\
93		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
94		asm volatile(lock "cmpxchgb %2,%1"			\
95			     : "=a" (__ret), "+m" (*__ptr)		\
96			     : "q" (__new), "0" (__old)			\
97			     : "memory");				\
98		break;							\
99	}								\
100	case __X86_CASE_W:						\
101	{								\
102		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
103		asm volatile(lock "cmpxchgw %2,%1"			\
104			     : "=a" (__ret), "+m" (*__ptr)		\
105			     : "r" (__new), "0" (__old)			\
106			     : "memory");				\
107		break;							\
108	}								\
109	case __X86_CASE_L:						\
110	{								\
111		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
112		asm volatile(lock "cmpxchgl %2,%1"			\
113			     : "=a" (__ret), "+m" (*__ptr)		\
114			     : "r" (__new), "0" (__old)			\
115			     : "memory");				\
116		break;							\
117	}								\
118	case __X86_CASE_Q:						\
119	{								\
120		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
121		asm volatile(lock "cmpxchgq %2,%1"			\
122			     : "=a" (__ret), "+m" (*__ptr)		\
123			     : "r" (__new), "0" (__old)			\
124			     : "memory");				\
125		break;							\
126	}								\
127	default:							\
128		__cmpxchg_wrong_size();					\
129	}								\
130	__ret;								\
131})
132
133#define __cmpxchg(ptr, old, new, size)					\
134	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
135
136#define __sync_cmpxchg(ptr, old, new, size)				\
137	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
138
139#define __cmpxchg_local(ptr, old, new, size)				\
140	__raw_cmpxchg((ptr), (old), (new), (size), "")
141
142#ifdef CONFIG_X86_32
143# include <asm/cmpxchg_32.h>
144#else
145# include <asm/cmpxchg_64.h>
146#endif
147
148#define arch_cmpxchg(ptr, old, new)					\
149	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
150
151#define arch_sync_cmpxchg(ptr, old, new)				\
152	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
153
154#define arch_cmpxchg_local(ptr, old, new)				\
155	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
156
157
158#define __raw_try_cmpxchg(_ptr, _pold, _new, size, lock)		\
159({									\
160	bool success;							\
161	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
162	__typeof__(*(_ptr)) __old = *_old;				\
163	__typeof__(*(_ptr)) __new = (_new);				\
164	switch (size) {							\
165	case __X86_CASE_B:						\
166	{								\
167		volatile u8 *__ptr = (volatile u8 *)(_ptr);		\
168		asm volatile(lock "cmpxchgb %[new], %[ptr]"		\
169			     CC_SET(z)					\
170			     : CC_OUT(z) (success),			\
171			       [ptr] "+m" (*__ptr),			\
172			       [old] "+a" (__old)			\
173			     : [new] "q" (__new)			\
174			     : "memory");				\
175		break;							\
176	}								\
177	case __X86_CASE_W:						\
178	{								\
179		volatile u16 *__ptr = (volatile u16 *)(_ptr);		\
180		asm volatile(lock "cmpxchgw %[new], %[ptr]"		\
181			     CC_SET(z)					\
182			     : CC_OUT(z) (success),			\
183			       [ptr] "+m" (*__ptr),			\
184			       [old] "+a" (__old)			\
185			     : [new] "r" (__new)			\
186			     : "memory");				\
187		break;							\
188	}								\
189	case __X86_CASE_L:						\
190	{								\
191		volatile u32 *__ptr = (volatile u32 *)(_ptr);		\
192		asm volatile(lock "cmpxchgl %[new], %[ptr]"		\
193			     CC_SET(z)					\
194			     : CC_OUT(z) (success),			\
195			       [ptr] "+m" (*__ptr),			\
196			       [old] "+a" (__old)			\
197			     : [new] "r" (__new)			\
198			     : "memory");				\
199		break;							\
200	}								\
201	case __X86_CASE_Q:						\
202	{								\
203		volatile u64 *__ptr = (volatile u64 *)(_ptr);		\
204		asm volatile(lock "cmpxchgq %[new], %[ptr]"		\
205			     CC_SET(z)					\
206			     : CC_OUT(z) (success),			\
207			       [ptr] "+m" (*__ptr),			\
208			       [old] "+a" (__old)			\
209			     : [new] "r" (__new)			\
210			     : "memory");				\
211		break;							\
212	}								\
213	default:							\
214		__cmpxchg_wrong_size();					\
215	}								\
216	if (unlikely(!success))						\
217		*_old = __old;						\
218	likely(success);						\
219})
220
221#define __try_cmpxchg(ptr, pold, new, size)				\
222	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
223
224#define __sync_try_cmpxchg(ptr, pold, new, size)			\
225	__raw_try_cmpxchg((ptr), (pold), (new), (size), "lock; ")
226
227#define __try_cmpxchg_local(ptr, pold, new, size)			\
228	__raw_try_cmpxchg((ptr), (pold), (new), (size), "")
229
230#define arch_try_cmpxchg(ptr, pold, new) 				\
231	__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
232
233#define arch_sync_try_cmpxchg(ptr, pold, new) 				\
234	__sync_try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
235
236#define arch_try_cmpxchg_local(ptr, pold, new)				\
237	__try_cmpxchg_local((ptr), (pold), (new), sizeof(*(ptr)))
238
239/*
240 * xadd() adds "inc" to "*ptr" and atomically returns the previous
241 * value of "*ptr".
242 *
243 * xadd() is locked when multiple CPUs are online
244 */
245#define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)
246#define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)
247
248#endif	/* ASM_X86_CMPXCHG_H */
249