50#ifndef I32_bit
51#define I32_bit (1 << 7) /* IRQ disable */
52#endif
53#ifndef F32_bit
54#define F32_bit (1 << 6) /* FIQ disable */
55#endif
56
57#define __with_interrupts_disabled(expr) \
58 do { \
59 u_int cpsr_save, tmp; \
60 \
61 __asm __volatile( \
62 "mrs %0, cpsr;" \
63 "orr %1, %0, %2;" \
64 "msr cpsr_all, %1;" \
65 : "=r" (cpsr_save), "=r" (tmp) \
66 : "I" (I32_bit | F32_bit) \
67 : "cc" ); \
68 (expr); \
69 __asm __volatile( \
70 "msr cpsr_all, %0" \
71 : /* no output */ \
72 : "r" (cpsr_save) \
73 : "cc" ); \
74 } while(0)
75
76static __inline uint32_t
77__swp(uint32_t val, volatile uint32_t *ptr)
78{
79 __asm __volatile("swp %0, %2, [%3]"
80 : "=&r" (val), "=m" (*ptr)
81 : "r" (val), "r" (ptr), "m" (*ptr)
82 : "memory");
83 return (val);
84}
85
86
87#ifdef _KERNEL
88static __inline void
89atomic_set_32(volatile uint32_t *address, uint32_t setmask)
90{
91 __with_interrupts_disabled(*address |= setmask);
92}
93
94static __inline void
95atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
96{
97 __with_interrupts_disabled(*address &= ~clearmask);
98}
99
100static __inline u_int32_t
101atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
102{
103 int ret;
104
105 __with_interrupts_disabled(
106 {
107 if (*p == cmpval) {
108 *p = newval;
109 ret = 1;
110 } else {
111 ret = 0;
112 }
113 });
114 return (ret);
115}
116
117static __inline void
118atomic_add_32(volatile u_int32_t *p, u_int32_t val)
119{
120 __with_interrupts_disabled(*p += val);
121}
122
123static __inline void
124atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
125{
126 __with_interrupts_disabled(*p -= val);
127}
128
129static __inline uint32_t
130atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
131{
132 uint32_t value;
133
134 __with_interrupts_disabled(
135 {
136 value = *p;
137 *p += v;
138 });
139 return (value);
140}
141
142#else /* !_KERNEL */
143
144static __inline u_int32_t
145atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
146{
147 register int done, ras_start = ARM_RAS_START;
148
149 __asm __volatile("1:\n"
150 "adr %1, 1b\n"
151 "str %1, [%0]\n"
152 "adr %1, 2f\n"
153 "str %1, [%0, #4]\n"
154 "ldr %1, [%2]\n"
155 "cmp %1, %3\n"
156 "streq %4, [%2]\n"
157 "2:\n"
158 "mov %1, #0\n"
159 "str %1, [%0]\n"
160 "mov %1, #0xffffffff\n"
161 "str %1, [%0, #4]\n"
162 "moveq %1, #1\n"
163 "movne %1, #0\n"
164 : "+r" (ras_start), "=r" (done)
165 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "memory");
166 return (done);
167}
168
169static __inline void
170atomic_add_32(volatile u_int32_t *p, u_int32_t val)
171{
172 int start, ras_start = ARM_RAS_START;
173
174 __asm __volatile("1:\n"
175 "adr %1, 1b\n"
176 "str %1, [%0]\n"
177 "adr %1, 2f\n"
178 "str %1, [%0, #4]\n"
179 "ldr %1, [%2]\n"
180 "add %1, %1, %3\n"
181 "str %1, [%2]\n"
182 "2:\n"
183 "mov %1, #0\n"
184 "str %1, [%0]\n"
185 "mov %1, #0xffffffff\n"
186 "str %1, [%0, #4]\n"
187 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
188 : : "memory");
189}
190
191static __inline void
192atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
193{
194 int start, ras_start = ARM_RAS_START;
195
196 __asm __volatile("1:\n"
197 "adr %1, 1b\n"
198 "str %1, [%0]\n"
199 "adr %1, 2f\n"
200 "str %1, [%0, #4]\n"
201 "ldr %1, [%2]\n"
202 "sub %1, %1, %3\n"
203 "str %1, [%2]\n"
204 "2:\n"
205 "mov %1, #0\n"
206 "str %1, [%0]\n"
207 "mov %1, #0xffffffff\n"
208 "str %1, [%0, #4]\n"
209
210 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
211 : : "memory");
212}
213
214static __inline void
215atomic_set_32(volatile uint32_t *address, uint32_t setmask)
216{
217 int start, ras_start = ARM_RAS_START;
218
219 __asm __volatile("1:\n"
220 "adr %1, 1b\n"
221 "str %1, [%0]\n"
222 "adr %1, 2f\n"
223 "str %1, [%0, #4]\n"
224 "ldr %1, [%2]\n"
225 "orr %1, %1, %3\n"
226 "str %1, [%2]\n"
227 "2:\n"
228 "mov %1, #0\n"
229 "str %1, [%0]\n"
230 "mov %1, #0xffffffff\n"
231 "str %1, [%0, #4]\n"
232
233 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask)
234 : : "memory");
235}
236
237static __inline void
238atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
239{
240 int start, ras_start = ARM_RAS_START;
241
242 __asm __volatile("1:\n"
243 "adr %1, 1b\n"
244 "str %1, [%0]\n"
245 "adr %1, 2f\n"
246 "str %1, [%0, #4]\n"
247 "ldr %1, [%2]\n"
248 "bic %1, %1, %3\n"
249 "str %1, [%2]\n"
250 "2:\n"
251 "mov %1, #0\n"
252 "str %1, [%0]\n"
253 "mov %1, #0xffffffff\n"
254 "str %1, [%0, #4]\n"
255 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask)
256 : : "memory");
257
258}
259
260static __inline uint32_t
261atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
262{
263 uint32_t start, ras_start = ARM_RAS_START;
264
265 __asm __volatile("1:\n"
266 "adr %1, 1b\n"
267 "str %1, [%0]\n"
268 "adr %1, 2f\n"
269 "str %1, [%0, #4]\n"
270 "ldr %1, [%2]\n"
271 "add %1, %1, %3\n"
272 "str %0, [%2]\n"
273 "2:\n"
274 "mov %3, #0\n"
275 "str %3, [%0]\n"
276 "mov %3, #0xffffffff\n"
277 "str %3, [%0, #4]\n"
278 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (v)
279 : : "memory");
280 return (start);
281}
282
283
284#endif /* _KERNEL */
285
286static __inline int
287atomic_load_32(volatile uint32_t *v)
288{
289
290 return (*v);
291}
292
293static __inline void
294atomic_store_32(volatile uint32_t *dst, uint32_t src)
295{
296 *dst = src;
297}
298
299static __inline uint32_t
300atomic_readandclear_32(volatile u_int32_t *p)
301{
302
303 return (__swp(0, p));
304}
305
306#undef __with_interrupts_disabled
307
308#endif /* _LOCORE */
309
310#define atomic_add_long(p, v) \
311 atomic_add_32((volatile u_int *)(p), (u_int)(v))
312#define atomic_add_acq_long atomic_add_long
313#define atomic_add_rel_long atomic_add_long
314#define atomic_subtract_long(p, v) \
315 atomic_subtract_32((volatile u_int *)(p), (u_int)(v))
316#define atomic_subtract_acq_long atomic_subtract_long
317#define atomic_subtract_rel_long atomic_subtract_long
318#define atomic_clear_long(p, v) \
319 atomic_clear_32((volatile u_int *)(p), (u_int)(v))
320#define atomic_clear_acq_long atomic_clear_long
321#define atomic_clear_rel_long atomic_clear_long
322#define atomic_set_long(p, v) \
323 atomic_set_32((volatile u_int *)(p), (u_int)(v))
324#define atomic_set_acq_long atomic_set_long
325#define atomic_set_rel_long atomic_set_long
326#define atomic_cmpset_long(dst, old, new) \
327 atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
328#define atomic_cmpset_acq_long atomic_cmpset_long
329#define atomic_cmpset_rel_long atomic_cmpset_long
330#define atomic_fetchadd_long(p, v) \
331 atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v))
332#define atomic_readandclear_long(p) \
333 atomic_readandclear_long((volatile u_int *)(p))
334#define atomic_load_long(p) \
335 atomic_load_32((volatile u_int *)(p))
336#define atomic_load_acq_long atomic_load_long
337#define atomic_store_rel_long(p, v) \
338 atomic_store_rel_32((volatile u_int *)(p), (u_int)(v))
339
340
341#define atomic_clear_ptr atomic_clear_32
342#define atomic_set_ptr atomic_set_32
343#define atomic_cmpset_ptr atomic_cmpset_32
344#define atomic_cmpset_rel_ptr atomic_cmpset_ptr
345#define atomic_cmpset_acq_ptr atomic_cmpset_ptr
346#define atomic_store_ptr atomic_store_32
347#define atomic_store_rel_ptr atomic_store_ptr
348
349#define atomic_add_int atomic_add_32
350#define atomic_add_acq_int atomic_add_int
351#define atomic_add_rel_int atomic_add_int
352#define atomic_subtract_int atomic_subtract_32
353#define atomic_subtract_acq_int atomic_subtract_int
354#define atomic_subtract_rel_int atomic_subtract_int
355#define atomic_clear_int atomic_clear_32
356#define atomic_clear_acq_int atomic_clear_int
357#define atomic_clear_rel_int atomic_clear_int
358#define atomic_set_int atomic_set_32
359#define atomic_set_acq_int atomic_set_int
360#define atomic_set_rel_int atomic_set_int
361#define atomic_cmpset_int atomic_cmpset_32
362#define atomic_cmpset_acq_int atomic_cmpset_int
363#define atomic_cmpset_rel_int atomic_cmpset_int
364#define atomic_fetchadd_int atomic_fetchadd_32
365#define atomic_readandclear_int atomic_readandclear_32
366#define atomic_load_acq_int atomic_load_32
367#define atomic_store_rel_int atomic_store_32
368
369#define atomic_add_acq_32 atomic_add_32
370#define atomic_add_rel_32 atomic_add_32
371#define atomic_subtract_acq_32 atomic_subtract_32
372#define atomic_subtract_rel_32 atomic_subtract_32
373#define atomic_clear_acq_32 atomic_clear_32
374#define atomic_clear_rel_32 atomic_clear_32
375#define atomic_set_acq_32 atomic_set_32
376#define atomic_set_rel_32 atomic_set_32
377#define atomic_cmpset_acq_32 atomic_cmpset_32
378#define atomic_cmpset_rel_32 atomic_cmpset_32
379#define atomic_load_acq_32 atomic_load_32
380#define atomic_store_rel_32 atomic_store_32
381
382#endif /* _MACHINE_ATOMIC_H_ */
| 54#ifndef I32_bit
55#define I32_bit (1 << 7) /* IRQ disable */
56#endif
57#ifndef F32_bit
58#define F32_bit (1 << 6) /* FIQ disable */
59#endif
60
61#define __with_interrupts_disabled(expr) \
62 do { \
63 u_int cpsr_save, tmp; \
64 \
65 __asm __volatile( \
66 "mrs %0, cpsr;" \
67 "orr %1, %0, %2;" \
68 "msr cpsr_all, %1;" \
69 : "=r" (cpsr_save), "=r" (tmp) \
70 : "I" (I32_bit | F32_bit) \
71 : "cc" ); \
72 (expr); \
73 __asm __volatile( \
74 "msr cpsr_all, %0" \
75 : /* no output */ \
76 : "r" (cpsr_save) \
77 : "cc" ); \
78 } while(0)
79
80static __inline uint32_t
81__swp(uint32_t val, volatile uint32_t *ptr)
82{
83 __asm __volatile("swp %0, %2, [%3]"
84 : "=&r" (val), "=m" (*ptr)
85 : "r" (val), "r" (ptr), "m" (*ptr)
86 : "memory");
87 return (val);
88}
89
90
91#ifdef _KERNEL
92static __inline void
93atomic_set_32(volatile uint32_t *address, uint32_t setmask)
94{
95 __with_interrupts_disabled(*address |= setmask);
96}
97
98static __inline void
99atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
100{
101 __with_interrupts_disabled(*address &= ~clearmask);
102}
103
104static __inline u_int32_t
105atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
106{
107 int ret;
108
109 __with_interrupts_disabled(
110 {
111 if (*p == cmpval) {
112 *p = newval;
113 ret = 1;
114 } else {
115 ret = 0;
116 }
117 });
118 return (ret);
119}
120
121static __inline void
122atomic_add_32(volatile u_int32_t *p, u_int32_t val)
123{
124 __with_interrupts_disabled(*p += val);
125}
126
127static __inline void
128atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
129{
130 __with_interrupts_disabled(*p -= val);
131}
132
133static __inline uint32_t
134atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
135{
136 uint32_t value;
137
138 __with_interrupts_disabled(
139 {
140 value = *p;
141 *p += v;
142 });
143 return (value);
144}
145
146#else /* !_KERNEL */
147
148static __inline u_int32_t
149atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
150{
151 register int done, ras_start = ARM_RAS_START;
152
153 __asm __volatile("1:\n"
154 "adr %1, 1b\n"
155 "str %1, [%0]\n"
156 "adr %1, 2f\n"
157 "str %1, [%0, #4]\n"
158 "ldr %1, [%2]\n"
159 "cmp %1, %3\n"
160 "streq %4, [%2]\n"
161 "2:\n"
162 "mov %1, #0\n"
163 "str %1, [%0]\n"
164 "mov %1, #0xffffffff\n"
165 "str %1, [%0, #4]\n"
166 "moveq %1, #1\n"
167 "movne %1, #0\n"
168 : "+r" (ras_start), "=r" (done)
169 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "memory");
170 return (done);
171}
172
173static __inline void
174atomic_add_32(volatile u_int32_t *p, u_int32_t val)
175{
176 int start, ras_start = ARM_RAS_START;
177
178 __asm __volatile("1:\n"
179 "adr %1, 1b\n"
180 "str %1, [%0]\n"
181 "adr %1, 2f\n"
182 "str %1, [%0, #4]\n"
183 "ldr %1, [%2]\n"
184 "add %1, %1, %3\n"
185 "str %1, [%2]\n"
186 "2:\n"
187 "mov %1, #0\n"
188 "str %1, [%0]\n"
189 "mov %1, #0xffffffff\n"
190 "str %1, [%0, #4]\n"
191 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
192 : : "memory");
193}
194
195static __inline void
196atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
197{
198 int start, ras_start = ARM_RAS_START;
199
200 __asm __volatile("1:\n"
201 "adr %1, 1b\n"
202 "str %1, [%0]\n"
203 "adr %1, 2f\n"
204 "str %1, [%0, #4]\n"
205 "ldr %1, [%2]\n"
206 "sub %1, %1, %3\n"
207 "str %1, [%2]\n"
208 "2:\n"
209 "mov %1, #0\n"
210 "str %1, [%0]\n"
211 "mov %1, #0xffffffff\n"
212 "str %1, [%0, #4]\n"
213
214 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
215 : : "memory");
216}
217
218static __inline void
219atomic_set_32(volatile uint32_t *address, uint32_t setmask)
220{
221 int start, ras_start = ARM_RAS_START;
222
223 __asm __volatile("1:\n"
224 "adr %1, 1b\n"
225 "str %1, [%0]\n"
226 "adr %1, 2f\n"
227 "str %1, [%0, #4]\n"
228 "ldr %1, [%2]\n"
229 "orr %1, %1, %3\n"
230 "str %1, [%2]\n"
231 "2:\n"
232 "mov %1, #0\n"
233 "str %1, [%0]\n"
234 "mov %1, #0xffffffff\n"
235 "str %1, [%0, #4]\n"
236
237 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask)
238 : : "memory");
239}
240
241static __inline void
242atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
243{
244 int start, ras_start = ARM_RAS_START;
245
246 __asm __volatile("1:\n"
247 "adr %1, 1b\n"
248 "str %1, [%0]\n"
249 "adr %1, 2f\n"
250 "str %1, [%0, #4]\n"
251 "ldr %1, [%2]\n"
252 "bic %1, %1, %3\n"
253 "str %1, [%2]\n"
254 "2:\n"
255 "mov %1, #0\n"
256 "str %1, [%0]\n"
257 "mov %1, #0xffffffff\n"
258 "str %1, [%0, #4]\n"
259 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask)
260 : : "memory");
261
262}
263
264static __inline uint32_t
265atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
266{
267 uint32_t start, ras_start = ARM_RAS_START;
268
269 __asm __volatile("1:\n"
270 "adr %1, 1b\n"
271 "str %1, [%0]\n"
272 "adr %1, 2f\n"
273 "str %1, [%0, #4]\n"
274 "ldr %1, [%2]\n"
275 "add %1, %1, %3\n"
276 "str %0, [%2]\n"
277 "2:\n"
278 "mov %3, #0\n"
279 "str %3, [%0]\n"
280 "mov %3, #0xffffffff\n"
281 "str %3, [%0, #4]\n"
282 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (v)
283 : : "memory");
284 return (start);
285}
286
287
288#endif /* _KERNEL */
289
290static __inline int
291atomic_load_32(volatile uint32_t *v)
292{
293
294 return (*v);
295}
296
297static __inline void
298atomic_store_32(volatile uint32_t *dst, uint32_t src)
299{
300 *dst = src;
301}
302
303static __inline uint32_t
304atomic_readandclear_32(volatile u_int32_t *p)
305{
306
307 return (__swp(0, p));
308}
309
310#undef __with_interrupts_disabled
311
312#endif /* _LOCORE */
313
314#define atomic_add_long(p, v) \
315 atomic_add_32((volatile u_int *)(p), (u_int)(v))
316#define atomic_add_acq_long atomic_add_long
317#define atomic_add_rel_long atomic_add_long
318#define atomic_subtract_long(p, v) \
319 atomic_subtract_32((volatile u_int *)(p), (u_int)(v))
320#define atomic_subtract_acq_long atomic_subtract_long
321#define atomic_subtract_rel_long atomic_subtract_long
322#define atomic_clear_long(p, v) \
323 atomic_clear_32((volatile u_int *)(p), (u_int)(v))
324#define atomic_clear_acq_long atomic_clear_long
325#define atomic_clear_rel_long atomic_clear_long
326#define atomic_set_long(p, v) \
327 atomic_set_32((volatile u_int *)(p), (u_int)(v))
328#define atomic_set_acq_long atomic_set_long
329#define atomic_set_rel_long atomic_set_long
330#define atomic_cmpset_long(dst, old, new) \
331 atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
332#define atomic_cmpset_acq_long atomic_cmpset_long
333#define atomic_cmpset_rel_long atomic_cmpset_long
334#define atomic_fetchadd_long(p, v) \
335 atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v))
336#define atomic_readandclear_long(p) \
337 atomic_readandclear_long((volatile u_int *)(p))
338#define atomic_load_long(p) \
339 atomic_load_32((volatile u_int *)(p))
340#define atomic_load_acq_long atomic_load_long
341#define atomic_store_rel_long(p, v) \
342 atomic_store_rel_32((volatile u_int *)(p), (u_int)(v))
343
344
345#define atomic_clear_ptr atomic_clear_32
346#define atomic_set_ptr atomic_set_32
347#define atomic_cmpset_ptr atomic_cmpset_32
348#define atomic_cmpset_rel_ptr atomic_cmpset_ptr
349#define atomic_cmpset_acq_ptr atomic_cmpset_ptr
350#define atomic_store_ptr atomic_store_32
351#define atomic_store_rel_ptr atomic_store_ptr
352
353#define atomic_add_int atomic_add_32
354#define atomic_add_acq_int atomic_add_int
355#define atomic_add_rel_int atomic_add_int
356#define atomic_subtract_int atomic_subtract_32
357#define atomic_subtract_acq_int atomic_subtract_int
358#define atomic_subtract_rel_int atomic_subtract_int
359#define atomic_clear_int atomic_clear_32
360#define atomic_clear_acq_int atomic_clear_int
361#define atomic_clear_rel_int atomic_clear_int
362#define atomic_set_int atomic_set_32
363#define atomic_set_acq_int atomic_set_int
364#define atomic_set_rel_int atomic_set_int
365#define atomic_cmpset_int atomic_cmpset_32
366#define atomic_cmpset_acq_int atomic_cmpset_int
367#define atomic_cmpset_rel_int atomic_cmpset_int
368#define atomic_fetchadd_int atomic_fetchadd_32
369#define atomic_readandclear_int atomic_readandclear_32
370#define atomic_load_acq_int atomic_load_32
371#define atomic_store_rel_int atomic_store_32
372
373#define atomic_add_acq_32 atomic_add_32
374#define atomic_add_rel_32 atomic_add_32
375#define atomic_subtract_acq_32 atomic_subtract_32
376#define atomic_subtract_rel_32 atomic_subtract_32
377#define atomic_clear_acq_32 atomic_clear_32
378#define atomic_clear_rel_32 atomic_clear_32
379#define atomic_set_acq_32 atomic_set_32
380#define atomic_set_rel_32 atomic_set_32
381#define atomic_cmpset_acq_32 atomic_cmpset_32
382#define atomic_cmpset_rel_32 atomic_cmpset_32
383#define atomic_load_acq_32 atomic_load_32
384#define atomic_store_rel_32 atomic_store_32
385
386#endif /* _MACHINE_ATOMIC_H_ */
|