1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef __ASM_CMPXCHG_H
6#define __ASM_CMPXCHG_H
7
8#include <linux/bits.h>
9#include <linux/build_bug.h>
10#include <asm/barrier.h>
11
12#define __xchg_asm(amswap_db, m, val)		\
13({						\
14		__typeof(val) __ret;		\
15						\
16		__asm__ __volatile__ (		\
17		" "amswap_db" %1, %z2, %0 \n"	\
18		: "+ZB" (*m), "=&r" (__ret)	\
19		: "Jr" (val)			\
20		: "memory");			\
21						\
22		__ret;				\
23})
24
25static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,
26					unsigned int size)
27{
28	unsigned int shift;
29	u32 old32, mask, temp;
30	volatile u32 *ptr32;
31
32	/* Mask value to the correct size. */
33	mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
34	val &= mask;
35
36	/*
37	 * Calculate a shift & mask that correspond to the value we wish to
38	 * exchange within the naturally aligned 4 byte integerthat includes
39	 * it.
40	 */
41	shift = (unsigned long)ptr & 0x3;
42	shift *= BITS_PER_BYTE;
43	mask <<= shift;
44
45	/*
46	 * Calculate a pointer to the naturally aligned 4 byte integer that
47	 * includes our byte of interest, and load its value.
48	 */
49	ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
50
51	asm volatile (
52	"1:	ll.w		%0, %3		\n"
53	"	andn		%1, %0, %z4	\n"
54	"	or		%1, %1, %z5	\n"
55	"	sc.w		%1, %2		\n"
56	"	beqz		%1, 1b		\n"
57	: "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
58	: "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift)
59	: "memory");
60
61	return (old32 & mask) >> shift;
62}
63
64static __always_inline unsigned long
65__arch_xchg(volatile void *ptr, unsigned long x, int size)
66{
67	switch (size) {
68	case 1:
69	case 2:
70		return __xchg_small(ptr, x, size);
71
72	case 4:
73		return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
74
75	case 8:
76		return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
77
78	default:
79		BUILD_BUG();
80	}
81
82	return 0;
83}
84
85#define arch_xchg(ptr, x)						\
86({									\
87	__typeof__(*(ptr)) __res;					\
88									\
89	__res = (__typeof__(*(ptr)))					\
90		__arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr)));	\
91									\
92	__res;								\
93})
94
95#define __cmpxchg_asm(ld, st, m, old, new)				\
96({									\
97	__typeof(old) __ret;						\
98									\
99	__asm__ __volatile__(						\
100	"1:	" ld "	%0, %2		# __cmpxchg_asm \n"		\
101	"	bne	%0, %z3, 2f			\n"		\
102	"	move	$t0, %z4			\n"		\
103	"	" st "	$t0, %1				\n"		\
104	"	beqz	$t0, 1b				\n"		\
105	"2:						\n"		\
106	__WEAK_LLSC_MB							\
107	: "=&r" (__ret), "=ZB"(*m)					\
108	: "ZB"(*m), "Jr" (old), "Jr" (new)				\
109	: "t0", "memory");						\
110									\
111	__ret;								\
112})
113
114static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old,
115					   unsigned int new, unsigned int size)
116{
117	unsigned int shift;
118	u32 old32, mask, temp;
119	volatile u32 *ptr32;
120
121	/* Mask inputs to the correct size. */
122	mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
123	old &= mask;
124	new &= mask;
125
126	/*
127	 * Calculate a shift & mask that correspond to the value we wish to
128	 * compare & exchange within the naturally aligned 4 byte integer
129	 * that includes it.
130	 */
131	shift = (unsigned long)ptr & 0x3;
132	shift *= BITS_PER_BYTE;
133	old <<= shift;
134	new <<= shift;
135	mask <<= shift;
136
137	/*
138	 * Calculate a pointer to the naturally aligned 4 byte integer that
139	 * includes our byte of interest, and load its value.
140	 */
141	ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
142
143	asm volatile (
144	"1:	ll.w		%0, %3		\n"
145	"	and		%1, %0, %z4	\n"
146	"	bne		%1, %z5, 2f	\n"
147	"	andn		%1, %0, %z4	\n"
148	"	or		%1, %1, %z6	\n"
149	"	sc.w		%1, %2		\n"
150	"	beqz		%1, 1b		\n"
151	"	b		3f		\n"
152	"2:					\n"
153	__WEAK_LLSC_MB
154	"3:					\n"
155	: "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
156	: "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new)
157	: "memory");
158
159	return (old32 & mask) >> shift;
160}
161
162static __always_inline unsigned long
163__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size)
164{
165	switch (size) {
166	case 1:
167	case 2:
168		return __cmpxchg_small(ptr, old, new, size);
169
170	case 4:
171		return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr,
172				     (u32)old, new);
173
174	case 8:
175		return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr,
176				     (u64)old, new);
177
178	default:
179		BUILD_BUG();
180	}
181
182	return 0;
183}
184
185#define arch_cmpxchg_local(ptr, old, new)				\
186	((__typeof__(*(ptr)))						\
187		__cmpxchg((ptr),					\
188			  (unsigned long)(__typeof__(*(ptr)))(old),	\
189			  (unsigned long)(__typeof__(*(ptr)))(new),	\
190			  sizeof(*(ptr))))
191
192#define arch_cmpxchg(ptr, old, new)					\
193({									\
194	__typeof__(*(ptr)) __res;					\
195									\
196	__res = arch_cmpxchg_local((ptr), (old), (new));		\
197									\
198	__res;								\
199})
200
201#ifdef CONFIG_64BIT
202#define arch_cmpxchg64_local(ptr, o, n)					\
203  ({									\
204	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
205	arch_cmpxchg_local((ptr), (o), (n));				\
206  })
207
208#define arch_cmpxchg64(ptr, o, n)					\
209  ({									\
210	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
211	arch_cmpxchg((ptr), (o), (n));					\
212  })
213#else
214#include <asm-generic/cmpxchg-local.h>
215#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
216#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n))
217#endif
218
219#endif /* __ASM_CMPXCHG_H */
220