1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License.  See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7 */
8#ifndef __ASM_CMPXCHG_H
9#define __ASM_CMPXCHG_H
10
11#include <linux/bug.h>
12#include <linux/irqflags.h>
13#include <asm/asm.h>
14#include <asm/compiler.h>
15#include <asm/sync.h>
16
17/*
18 * These functions doesn't exist, so if they are called you'll either:
19 *
20 * - Get an error at compile-time due to __compiletime_error, if supported by
21 *   your compiler.
22 *
23 * or:
24 *
25 * - Get an error at link-time due to the call to the missing function.
26 */
27extern unsigned long __cmpxchg_called_with_bad_pointer(void)
28	__compiletime_error("Bad argument size for cmpxchg");
29extern unsigned long __cmpxchg64_unsupported(void)
30	__compiletime_error("cmpxchg64 not available; cpu_has_64bits may be false");
31extern unsigned long __xchg_called_with_bad_pointer(void)
32	__compiletime_error("Bad argument size for xchg");
33
34#define __xchg_asm(ld, st, m, val)					\
35({									\
36	__typeof(*(m)) __ret;						\
37									\
38	if (kernel_uses_llsc) {						\
39		__asm__ __volatile__(					\
40		"	.set	push				\n"	\
41		"	.set	noat				\n"	\
42		"	.set	push				\n"	\
43		"	.set	" MIPS_ISA_ARCH_LEVEL "		\n"	\
44		"	" __SYNC(full, loongson3_war) "		\n"	\
45		"1:	" ld "	%0, %2		# __xchg_asm	\n"	\
46		"	.set	pop				\n"	\
47		"	move	$1, %z3				\n"	\
48		"	.set	" MIPS_ISA_ARCH_LEVEL "		\n"	\
49		"	" st "	$1, %1				\n"	\
50		"\t" __stringify(SC_BEQZ)	"	$1, 1b	\n"	\
51		"	.set	pop				\n"	\
52		: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m)		\
53		: GCC_OFF_SMALL_ASM() (*m), "Jr" (val)			\
54		: __LLSC_CLOBBER);					\
55	} else {							\
56		unsigned long __flags;					\
57									\
58		raw_local_irq_save(__flags);				\
59		__ret = *m;						\
60		*m = val;						\
61		raw_local_irq_restore(__flags);				\
62	}								\
63									\
64	__ret;								\
65})
66
67extern unsigned long __xchg_small(volatile void *ptr, unsigned long val,
68				  unsigned int size);
69
70static __always_inline
71unsigned long __arch_xchg(volatile void *ptr, unsigned long x, int size)
72{
73	switch (size) {
74	case 1:
75	case 2:
76		return __xchg_small(ptr, x, size);
77
78	case 4:
79		return __xchg_asm("ll", "sc", (volatile u32 *)ptr, x);
80
81	case 8:
82		if (!IS_ENABLED(CONFIG_64BIT))
83			return __xchg_called_with_bad_pointer();
84
85		return __xchg_asm("lld", "scd", (volatile u64 *)ptr, x);
86
87	default:
88		return __xchg_called_with_bad_pointer();
89	}
90}
91
92#define arch_xchg(ptr, x)						\
93({									\
94	__typeof__(*(ptr)) __res;					\
95									\
96	/*								\
97	 * In the Loongson3 workaround case __xchg_asm() already	\
98	 * contains a completion barrier prior to the LL, so we don't	\
99	 * need to emit an extra one here.				\
100	 */								\
101	if (__SYNC_loongson3_war == 0)					\
102		smp_mb__before_llsc();					\
103									\
104	__res = (__typeof__(*(ptr)))					\
105		__arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr)));	\
106									\
107	smp_llsc_mb();							\
108									\
109	__res;								\
110})
111
112#define __cmpxchg_asm(ld, st, m, old, new)				\
113({									\
114	__typeof(*(m)) __ret;						\
115									\
116	if (kernel_uses_llsc) {						\
117		__asm__ __volatile__(					\
118		"	.set	push				\n"	\
119		"	.set	noat				\n"	\
120		"	.set	push				\n"	\
121		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"	\
122		"	" __SYNC(full, loongson3_war) "		\n"	\
123		"1:	" ld "	%0, %2		# __cmpxchg_asm \n"	\
124		"	bne	%0, %z3, 2f			\n"	\
125		"	.set	pop				\n"	\
126		"	move	$1, %z4				\n"	\
127		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"	\
128		"	" st "	$1, %1				\n"	\
129		"\t" __stringify(SC_BEQZ)	"	$1, 1b	\n"	\
130		"	.set	pop				\n"	\
131		"2:	" __SYNC(full, loongson3_war) "		\n"	\
132		: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m)		\
133		: GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new)	\
134		: __LLSC_CLOBBER);					\
135	} else {							\
136		unsigned long __flags;					\
137									\
138		raw_local_irq_save(__flags);				\
139		__ret = *m;						\
140		if (__ret == old)					\
141			*m = new;					\
142		raw_local_irq_restore(__flags);				\
143	}								\
144									\
145	__ret;								\
146})
147
148extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old,
149				     unsigned long new, unsigned int size);
150
151static __always_inline
152unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
153			unsigned long new, unsigned int size)
154{
155	switch (size) {
156	case 1:
157	case 2:
158		return __cmpxchg_small(ptr, old, new, size);
159
160	case 4:
161		return __cmpxchg_asm("ll", "sc", (volatile u32 *)ptr,
162				     (u32)old, new);
163
164	case 8:
165		/* lld/scd are only available for MIPS64 */
166		if (!IS_ENABLED(CONFIG_64BIT))
167			return __cmpxchg_called_with_bad_pointer();
168
169		return __cmpxchg_asm("lld", "scd", (volatile u64 *)ptr,
170				     (u64)old, new);
171
172	default:
173		return __cmpxchg_called_with_bad_pointer();
174	}
175}
176
177#define arch_cmpxchg_local(ptr, old, new)				\
178	((__typeof__(*(ptr)))						\
179		__cmpxchg((ptr),					\
180			  (unsigned long)(__typeof__(*(ptr)))(old),	\
181			  (unsigned long)(__typeof__(*(ptr)))(new),	\
182			  sizeof(*(ptr))))
183
184#define arch_cmpxchg(ptr, old, new)					\
185({									\
186	__typeof__(*(ptr)) __res;					\
187									\
188	/*								\
189	 * In the Loongson3 workaround case __cmpxchg_asm() already	\
190	 * contains a completion barrier prior to the LL, so we don't	\
191	 * need to emit an extra one here.				\
192	 */								\
193	if (__SYNC_loongson3_war == 0)					\
194		smp_mb__before_llsc();					\
195									\
196	__res = arch_cmpxchg_local((ptr), (old), (new));		\
197									\
198	/*								\
199	 * In the Loongson3 workaround case __cmpxchg_asm() already	\
200	 * contains a completion barrier after the SC, so we don't	\
201	 * need to emit an extra one here.				\
202	 */								\
203	if (__SYNC_loongson3_war == 0)					\
204		smp_llsc_mb();						\
205									\
206	__res;								\
207})
208
209#ifdef CONFIG_64BIT
210#define arch_cmpxchg64_local(ptr, o, n)					\
211  ({									\
212	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
213	arch_cmpxchg_local((ptr), (o), (n));				\
214  })
215
216#define arch_cmpxchg64(ptr, o, n)					\
217  ({									\
218	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
219	arch_cmpxchg((ptr), (o), (n));					\
220  })
221#else
222
223# include <asm-generic/cmpxchg-local.h>
224# define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
225
226# ifdef CONFIG_SMP
227
228static inline unsigned long __cmpxchg64(volatile void *ptr,
229					unsigned long long old,
230					unsigned long long new)
231{
232	unsigned long long tmp, ret;
233	unsigned long flags;
234
235	/*
236	 * The assembly below has to combine 32 bit values into a 64 bit
237	 * register, and split 64 bit values from one register into two. If we
238	 * were to take an interrupt in the middle of this we'd only save the
239	 * least significant 32 bits of each register & probably clobber the
240	 * most significant 32 bits of the 64 bit values we're using. In order
241	 * to avoid this we must disable interrupts.
242	 */
243	local_irq_save(flags);
244
245	asm volatile(
246	"	.set	push				\n"
247	"	.set	" MIPS_ISA_ARCH_LEVEL "		\n"
248	/* Load 64 bits from ptr */
249	"	" __SYNC(full, loongson3_war) "		\n"
250	"1:	lld	%L0, %3		# __cmpxchg64	\n"
251	"	.set	pop				\n"
252	/*
253	 * Split the 64 bit value we loaded into the 2 registers that hold the
254	 * ret variable.
255	 */
256	"	dsra	%M0, %L0, 32			\n"
257	"	sll	%L0, %L0, 0			\n"
258	/*
259	 * Compare ret against old, breaking out of the loop if they don't
260	 * match.
261	 */
262	"	bne	%M0, %M4, 2f			\n"
263	"	bne	%L0, %L4, 2f			\n"
264	/*
265	 * Combine the 32 bit halves from the 2 registers that hold the new
266	 * variable into a single 64 bit register.
267	 */
268#  if MIPS_ISA_REV >= 2
269	"	move	%L1, %L5			\n"
270	"	dins	%L1, %M5, 32, 32		\n"
271#  else
272	"	dsll	%L1, %L5, 32			\n"
273	"	dsrl	%L1, %L1, 32			\n"
274	"	.set	noat				\n"
275	"	dsll	$at, %M5, 32			\n"
276	"	or	%L1, %L1, $at			\n"
277	"	.set	at				\n"
278#  endif
279	"	.set	push				\n"
280	"	.set	" MIPS_ISA_ARCH_LEVEL "		\n"
281	/* Attempt to store new at ptr */
282	"	scd	%L1, %2				\n"
283	/* If we failed, loop! */
284	"\t" __stringify(SC_BEQZ) "	%L1, 1b		\n"
285	"2:	" __SYNC(full, loongson3_war) "		\n"
286	"	.set	pop				\n"
287	: "=&r"(ret),
288	  "=&r"(tmp),
289	  "=" GCC_OFF_SMALL_ASM() (*(unsigned long long *)ptr)
290	: GCC_OFF_SMALL_ASM() (*(unsigned long long *)ptr),
291	  "r" (old),
292	  "r" (new)
293	: "memory");
294
295	local_irq_restore(flags);
296	return ret;
297}
298
299#  define arch_cmpxchg64(ptr, o, n) ({					\
300	unsigned long long __old = (__typeof__(*(ptr)))(o);		\
301	unsigned long long __new = (__typeof__(*(ptr)))(n);		\
302	__typeof__(*(ptr)) __res;					\
303									\
304	/*								\
305	 * We can only use cmpxchg64 if we know that the CPU supports	\
306	 * 64-bits, ie. lld & scd. Our call to __cmpxchg64_unsupported	\
307	 * will cause a build error unless cpu_has_64bits is a		\
308	 * compile-time constant 1.					\
309	 */								\
310	if (cpu_has_64bits && kernel_uses_llsc) {			\
311		smp_mb__before_llsc();					\
312		__res = __cmpxchg64((ptr), __old, __new);		\
313		smp_llsc_mb();						\
314	} else {							\
315		__res = __cmpxchg64_unsupported();			\
316	}								\
317									\
318	__res;								\
319})
320
321# else /* !CONFIG_SMP */
322#  define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n))
323# endif /* !CONFIG_SMP */
324#endif /* !CONFIG_64BIT */
325
326#endif /* __ASM_CMPXCHG_H */
327