1#ifndef _ATOMIC_H
2#define _ATOMIC_H
3
4#include <stdint.h>
5
6#include "atomic_arch.h"
7
8#ifdef a_ll
9
10#ifndef a_pre_llsc
11#define a_pre_llsc()
12#endif
13
14#ifndef a_post_llsc
15#define a_post_llsc()
16#endif
17
18#ifndef a_cas
19#define a_cas a_cas
20static inline int a_cas(volatile int *p, int t, int s)
21{
22	int old;
23	a_pre_llsc();
24	do old = a_ll(p);
25	while (old==t && !a_sc(p, s));
26	a_post_llsc();
27	return old;
28}
29#endif
30
31#ifndef a_swap
32#define a_swap a_swap
33static inline int a_swap(volatile int *p, int v)
34{
35	int old;
36	a_pre_llsc();
37	do old = a_ll(p);
38	while (!a_sc(p, v));
39	a_post_llsc();
40	return old;
41}
42#endif
43
44#ifndef a_fetch_add
45#define a_fetch_add a_fetch_add
46static inline int a_fetch_add(volatile int *p, int v)
47{
48	int old;
49	a_pre_llsc();
50	do old = a_ll(p);
51	while (!a_sc(p, (unsigned)old + v));
52	a_post_llsc();
53	return old;
54}
55#endif
56
57#ifndef a_fetch_and
58#define a_fetch_and a_fetch_and
59static inline int a_fetch_and(volatile int *p, int v)
60{
61	int old;
62	a_pre_llsc();
63	do old = a_ll(p);
64	while (!a_sc(p, old & v));
65	a_post_llsc();
66	return old;
67}
68#endif
69
70#ifndef a_fetch_or
71#define a_fetch_or a_fetch_or
72static inline int a_fetch_or(volatile int *p, int v)
73{
74	int old;
75	a_pre_llsc();
76	do old = a_ll(p);
77	while (!a_sc(p, old | v));
78	a_post_llsc();
79	return old;
80}
81#endif
82
83#endif
84
85#ifdef a_ll_p
86
87#ifndef a_cas_p
88#define a_cas_p a_cas_p
89static inline void *a_cas_p(volatile void *p, void *t, void *s)
90{
91	void *old;
92	a_pre_llsc();
93	do old = a_ll_p(p);
94	while (old==t && !a_sc_p(p, s));
95	a_post_llsc();
96	return old;
97}
98#endif
99
100#endif
101
102#ifndef a_cas
103#error missing definition of a_cas
104#endif
105
106#ifndef a_swap
107#define a_swap a_swap
108static inline int a_swap(volatile int *p, int v)
109{
110	int old;
111	do old = *p;
112	while (a_cas(p, old, v) != old);
113	return old;
114}
115#endif
116
117#ifndef a_fetch_add
118#define a_fetch_add a_fetch_add
119static inline int a_fetch_add(volatile int *p, int v)
120{
121	int old;
122	do old = *p;
123	while (a_cas(p, old, (unsigned)old+v) != old);
124	return old;
125}
126#endif
127
128#ifndef a_fetch_and
129#define a_fetch_and a_fetch_and
130static inline int a_fetch_and(volatile int *p, int v)
131{
132	int old;
133	do old = *p;
134	while (a_cas(p, old, old&v) != old);
135	return old;
136}
137#endif
138#ifndef a_fetch_or
139#define a_fetch_or a_fetch_or
140static inline int a_fetch_or(volatile int *p, int v)
141{
142	int old;
143	do old = *p;
144	while (a_cas(p, old, old|v) != old);
145	return old;
146}
147#endif
148
149#ifndef a_and
150#define a_and a_and
151static inline void a_and(volatile int *p, int v)
152{
153	a_fetch_and(p, v);
154}
155#endif
156
157#ifndef a_or
158#define a_or a_or
159static inline void a_or(volatile int *p, int v)
160{
161	a_fetch_or(p, v);
162}
163#endif
164
165#ifndef a_inc
166#define a_inc a_inc
167static inline void a_inc(volatile int *p)
168{
169	a_fetch_add(p, 1);
170}
171#endif
172
173#ifndef a_dec
174#define a_dec a_dec
175static inline void a_dec(volatile int *p)
176{
177	a_fetch_add(p, -1);
178}
179#endif
180
181#ifndef a_store
182#define a_store a_store
183static inline void a_store(volatile int *p, int v)
184{
185#ifdef a_barrier
186	a_barrier();
187	*p = v;
188	a_barrier();
189#else
190	a_swap(p, v);
191#endif
192}
193#endif
194
195#ifndef a_barrier
196#define a_barrier a_barrier
197static void a_barrier()
198{
199	volatile int tmp = 0;
200	a_cas(&tmp, 0, 0);
201}
202#endif
203
204#ifndef a_spin
205#define a_spin a_barrier
206#endif
207
208#ifndef a_and_64
209#define a_and_64 a_and_64
210static inline void a_and_64(volatile uint64_t *p, uint64_t v)
211{
212	union { uint64_t v; uint32_t r[2]; } u = { v };
213	if (u.r[0]+1) a_and((int *)p, u.r[0]);
214	if (u.r[1]+1) a_and((int *)p+1, u.r[1]);
215}
216#endif
217
218#ifndef a_or_64
219#define a_or_64 a_or_64
220static inline void a_or_64(volatile uint64_t *p, uint64_t v)
221{
222	union { uint64_t v; uint32_t r[2]; } u = { v };
223	if (u.r[0]) a_or((int *)p, u.r[0]);
224	if (u.r[1]) a_or((int *)p+1, u.r[1]);
225}
226#endif
227
228#ifndef a_cas_p
229typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1];
230#define a_cas_p a_cas_p
231static inline void *a_cas_p(volatile void *p, void *t, void *s)
232{
233	return (void *)a_cas((volatile int *)p, (int)t, (int)s);
234}
235#endif
236
237#ifndef a_or_l
238#define a_or_l a_or_l
239static inline void a_or_l(volatile void *p, long v)
240{
241	if (sizeof(long) == sizeof(int)) a_or(p, v);
242	else a_or_64(p, v);
243}
244#endif
245
246#ifndef a_crash
247#define a_crash a_crash
248static inline void a_crash()
249{
250	*(volatile char *)0=0;
251}
252#endif
253
254#ifndef a_ctz_64
255#define a_ctz_64 a_ctz_64
256static inline int a_ctz_64(uint64_t x)
257{
258	static const char debruijn64[64] = {
259		0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
260		62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
261		63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
262		51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
263	};
264	static const char debruijn32[32] = {
265		0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
266		31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
267	};
268	if (sizeof(long) < 8) {
269		uint32_t y = x;
270		if (!y) {
271			y = x>>32;
272			return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
273		}
274		return debruijn32[(y&-y)*0x076be629 >> 27];
275	}
276	return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
277}
278#endif
279
280#ifndef a_ctz_l
281#define a_ctz_l a_ctz_l
282static inline int a_ctz_l(unsigned long x)
283{
284	static const char debruijn32[32] = {
285		0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
286		31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
287	};
288	if (sizeof(long) == 8) return a_ctz_64(x);
289	return debruijn32[(x&-x)*0x076be629 >> 27];
290}
291#endif
292
293#endif
294