1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ALPHA_CMPXCHG_H
3#error Do not include xchg.h directly!
4#else
5/*
6 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
7 * except that local version do not have the expensive memory barrier.
8 * So this file is included twice from asm/cmpxchg.h.
9 */
10
11/*
12 * Atomic exchange.
13 * Since it can be used to implement critical sections
14 * it must clobber "memory" (also for interrupts in UP).
15 */
16
17static inline unsigned long
18____xchg(_u8, volatile char *m, unsigned long val)
19{
20	unsigned long ret, tmp, addr64;
21
22	__asm__ __volatile__(
23	"	andnot	%4,7,%3\n"
24	"	insbl	%1,%4,%1\n"
25	"1:	ldq_l	%2,0(%3)\n"
26	"	extbl	%2,%4,%0\n"
27	"	mskbl	%2,%4,%2\n"
28	"	or	%1,%2,%2\n"
29	"	stq_c	%2,0(%3)\n"
30	"	beq	%2,2f\n"
31	".subsection 2\n"
32	"2:	br	1b\n"
33	".previous"
34	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
35	: "r" ((long)m), "1" (val) : "memory");
36
37	return ret;
38}
39
40static inline unsigned long
41____xchg(_u16, volatile short *m, unsigned long val)
42{
43	unsigned long ret, tmp, addr64;
44
45	__asm__ __volatile__(
46	"	andnot	%4,7,%3\n"
47	"	inswl	%1,%4,%1\n"
48	"1:	ldq_l	%2,0(%3)\n"
49	"	extwl	%2,%4,%0\n"
50	"	mskwl	%2,%4,%2\n"
51	"	or	%1,%2,%2\n"
52	"	stq_c	%2,0(%3)\n"
53	"	beq	%2,2f\n"
54	".subsection 2\n"
55	"2:	br	1b\n"
56	".previous"
57	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
58	: "r" ((long)m), "1" (val) : "memory");
59
60	return ret;
61}
62
63static inline unsigned long
64____xchg(_u32, volatile int *m, unsigned long val)
65{
66	unsigned long dummy;
67
68	__asm__ __volatile__(
69	"1:	ldl_l %0,%4\n"
70	"	bis $31,%3,%1\n"
71	"	stl_c %1,%2\n"
72	"	beq %1,2f\n"
73	".subsection 2\n"
74	"2:	br 1b\n"
75	".previous"
76	: "=&r" (val), "=&r" (dummy), "=m" (*m)
77	: "rI" (val), "m" (*m) : "memory");
78
79	return val;
80}
81
82static inline unsigned long
83____xchg(_u64, volatile long *m, unsigned long val)
84{
85	unsigned long dummy;
86
87	__asm__ __volatile__(
88	"1:	ldq_l %0,%4\n"
89	"	bis $31,%3,%1\n"
90	"	stq_c %1,%2\n"
91	"	beq %1,2f\n"
92	".subsection 2\n"
93	"2:	br 1b\n"
94	".previous"
95	: "=&r" (val), "=&r" (dummy), "=m" (*m)
96	: "rI" (val), "m" (*m) : "memory");
97
98	return val;
99}
100
101/* This function doesn't exist, so you'll get a linker error
102   if something tries to do an invalid xchg().  */
103extern void __xchg_called_with_bad_pointer(void);
104
105static __always_inline unsigned long
106____xchg(, volatile void *ptr, unsigned long x, int size)
107{
108	switch (size) {
109		case 1:
110			return ____xchg(_u8, ptr, x);
111		case 2:
112			return ____xchg(_u16, ptr, x);
113		case 4:
114			return ____xchg(_u32, ptr, x);
115		case 8:
116			return ____xchg(_u64, ptr, x);
117	}
118	__xchg_called_with_bad_pointer();
119	return x;
120}
121
122/*
123 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
124 * store NEW in MEM.  Return the initial value in MEM.  Success is
125 * indicated by comparing RETURN with OLD.
126 */
127
128static inline unsigned long
129____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
130{
131	unsigned long prev, tmp, cmp, addr64;
132
133	__asm__ __volatile__(
134	"	andnot	%5,7,%4\n"
135	"	insbl	%1,%5,%1\n"
136	"1:	ldq_l	%2,0(%4)\n"
137	"	extbl	%2,%5,%0\n"
138	"	cmpeq	%0,%6,%3\n"
139	"	beq	%3,2f\n"
140	"	mskbl	%2,%5,%2\n"
141	"	or	%1,%2,%2\n"
142	"	stq_c	%2,0(%4)\n"
143	"	beq	%2,3f\n"
144	"2:\n"
145	".subsection 2\n"
146	"3:	br	1b\n"
147	".previous"
148	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
149	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
150
151	return prev;
152}
153
154static inline unsigned long
155____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
156{
157	unsigned long prev, tmp, cmp, addr64;
158
159	__asm__ __volatile__(
160	"	andnot	%5,7,%4\n"
161	"	inswl	%1,%5,%1\n"
162	"1:	ldq_l	%2,0(%4)\n"
163	"	extwl	%2,%5,%0\n"
164	"	cmpeq	%0,%6,%3\n"
165	"	beq	%3,2f\n"
166	"	mskwl	%2,%5,%2\n"
167	"	or	%1,%2,%2\n"
168	"	stq_c	%2,0(%4)\n"
169	"	beq	%2,3f\n"
170	"2:\n"
171	".subsection 2\n"
172	"3:	br	1b\n"
173	".previous"
174	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
175	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
176
177	return prev;
178}
179
180static inline unsigned long
181____cmpxchg(_u32, volatile int *m, int old, int new)
182{
183	unsigned long prev, cmp;
184
185	__asm__ __volatile__(
186	"1:	ldl_l %0,%5\n"
187	"	cmpeq %0,%3,%1\n"
188	"	beq %1,2f\n"
189	"	mov %4,%1\n"
190	"	stl_c %1,%2\n"
191	"	beq %1,3f\n"
192	"2:\n"
193	".subsection 2\n"
194	"3:	br 1b\n"
195	".previous"
196	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
197	: "r"((long) old), "r"(new), "m"(*m) : "memory");
198
199	return prev;
200}
201
202static inline unsigned long
203____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
204{
205	unsigned long prev, cmp;
206
207	__asm__ __volatile__(
208	"1:	ldq_l %0,%5\n"
209	"	cmpeq %0,%3,%1\n"
210	"	beq %1,2f\n"
211	"	mov %4,%1\n"
212	"	stq_c %1,%2\n"
213	"	beq %1,3f\n"
214	"2:\n"
215	".subsection 2\n"
216	"3:	br 1b\n"
217	".previous"
218	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
219	: "r"((long) old), "r"(new), "m"(*m) : "memory");
220
221	return prev;
222}
223
224/* This function doesn't exist, so you'll get a linker error
225   if something tries to do an invalid cmpxchg().  */
226extern void __cmpxchg_called_with_bad_pointer(void);
227
228static __always_inline unsigned long
229____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
230	      int size)
231{
232	switch (size) {
233		case 1:
234			return ____cmpxchg(_u8, ptr, old, new);
235		case 2:
236			return ____cmpxchg(_u16, ptr, old, new);
237		case 4:
238			return ____cmpxchg(_u32, ptr, old, new);
239		case 8:
240			return ____cmpxchg(_u64, ptr, old, new);
241	}
242	__cmpxchg_called_with_bad_pointer();
243	return old;
244}
245
246#endif
247