atomic-v6.h revision 308327 
1/* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */
2
3/*-
4 * Copyright (C) 2003-2004 Olivier Houchard
5 * Copyright (C) 1994-1997 Mark Brinicombe
6 * Copyright (C) 1994 Brini
7 * All rights reserved.
8 *
9 * This code is derived from software written for Brini by Mark Brinicombe
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 *    notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in the
18 *    documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 *    must display the following acknowledgement:
21 *	This product includes software developed by Brini.
22 * 4. The name of Brini may not be used to endorse or promote products
23 *    derived from this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR
26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
31 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
33 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
34 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * $FreeBSD: stable/11/sys/arm/include/atomic-v6.h 308327 2016-11-05 04:36:12Z mmel $
37 */
38
39#ifndef _MACHINE_ATOMIC_V6_H_
40#define	_MACHINE_ATOMIC_V6_H_
41
42#ifndef _MACHINE_ATOMIC_H_
43#error Do not include this file directly, use <machine/atomic.h>
44#endif
45
46#if __ARM_ARCH >= 7
47#define isb()  __asm __volatile("isb" : : : "memory")
48#define dsb()  __asm __volatile("dsb" : : : "memory")
49#define dmb()  __asm __volatile("dmb" : : : "memory")
50#elif __ARM_ARCH >= 6
51#define isb()  __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
52#define dsb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
53#define dmb()  __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
54#else
55#error Only use this file with ARMv6 and later
56#endif
57
58#define mb()   dmb()
59#define wmb()  dmb()
60#define rmb()  dmb()
61
62#define	ARM_HAVE_ATOMIC64
63
64#define ATOMIC_ACQ_REL_LONG(NAME)					\
65static __inline void							\
66atomic_##NAME##_acq_long(__volatile u_long *p, u_long v)		\
67{									\
68	atomic_##NAME##_long(p, v);					\
69	dmb();								\
70}									\
71									\
72static __inline  void							\
73atomic_##NAME##_rel_long(__volatile u_long *p, u_long v)		\
74{									\
75	dmb();								\
76	atomic_##NAME##_long(p, v);					\
77}
78
79#define	ATOMIC_ACQ_REL(NAME, WIDTH)					\
80static __inline  void							\
81atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
82{									\
83	atomic_##NAME##_##WIDTH(p, v);					\
84	dmb();								\
85}									\
86									\
87static __inline  void							\
88atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
89{									\
90	dmb();								\
91	atomic_##NAME##_##WIDTH(p, v);					\
92}
93
94
95static __inline void
96atomic_add_32(volatile uint32_t *p, uint32_t val)
97{
98	uint32_t tmp = 0, tmp2 = 0;
99
100	__asm __volatile(
101	    "1: ldrex	%0, [%2]	\n"
102	    "   add	%0, %0, %3	\n"
103	    "   strex	%1, %0, [%2]	\n"
104	    "   cmp	%1, #0		\n"
105	    "   it	ne		\n"
106	    "   bne	1b		\n"
107	    : "=&r" (tmp), "+r" (tmp2)
108	    ,"+r" (p), "+r" (val) : : "cc", "memory");
109}
110
111static __inline void
112atomic_add_64(volatile uint64_t *p, uint64_t val)
113{
114	uint64_t tmp;
115	uint32_t exflag;
116
117	__asm __volatile(
118	    "1:							\n"
119	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
120	    "   adds	%Q[tmp], %Q[val]			\n"
121	    "   adc	%R[tmp], %R[tmp], %R[val]		\n"
122	    "   strexd	%[exf], %Q[tmp], %R[tmp], [%[ptr]]	\n"
123	    "   teq	%[exf], #0				\n"
124	    "   it	ne					\n"
125	    "   bne	1b					\n"
126	    : [exf] "=&r" (exflag),
127	      [tmp] "=&r" (tmp)
128	    : [ptr] "r"   (p),
129	      [val] "r"   (val)
130	    : "cc", "memory");
131}
132
133static __inline void
134atomic_add_long(volatile u_long *p, u_long val)
135{
136
137	atomic_add_32((volatile uint32_t *)p, val);
138}
139
140ATOMIC_ACQ_REL(add, 32)
141ATOMIC_ACQ_REL(add, 64)
142ATOMIC_ACQ_REL_LONG(add)
143
144static __inline void
145atomic_clear_32(volatile uint32_t *address, uint32_t setmask)
146{
147	uint32_t tmp = 0, tmp2 = 0;
148
149	__asm __volatile(
150	    "1: ldrex	%0, [%2]	\n"
151	    "   bic	%0, %0, %3	\n"
152	    "   strex	%1, %0, [%2]	\n"
153	    "   cmp	%1, #0		\n"
154	    "   it	ne		\n"
155	    "   bne	1b		\n"
156	    : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask)
157	    : : "cc", "memory");
158}
159
160static __inline void
161atomic_clear_64(volatile uint64_t *p, uint64_t val)
162{
163	uint64_t tmp;
164	uint32_t exflag;
165
166	__asm __volatile(
167	    "1:							\n"
168	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
169	    "   bic	%Q[tmp], %Q[val]			\n"
170	    "   bic	%R[tmp], %R[val]			\n"
171	    "   strexd	%[exf], %Q[tmp], %R[tmp], [%[ptr]]	\n"
172	    "   teq	%[exf], #0				\n"
173	    "   it	ne					\n"
174	    "   bne	1b					\n"
175	    : [exf] "=&r" (exflag),
176	      [tmp] "=&r" (tmp)
177	    : [ptr] "r"   (p),
178	      [val] "r"   (val)
179	    : "cc", "memory");
180}
181
182static __inline void
183atomic_clear_long(volatile u_long *address, u_long setmask)
184{
185
186	atomic_clear_32((volatile uint32_t *)address, setmask);
187}
188
189ATOMIC_ACQ_REL(clear, 32)
190ATOMIC_ACQ_REL(clear, 64)
191ATOMIC_ACQ_REL_LONG(clear)
192
193static __inline uint32_t
194atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
195{
196	uint32_t ret;
197
198	__asm __volatile(
199	    "1: ldrex	%0, [%1]	\n"
200	    "   cmp	%0, %2		\n"
201	    "   itt	ne		\n"
202	    "   movne	%0, #0		\n"
203	    "   bne	2f		\n"
204	    "   strex	%0, %3, [%1]	\n"
205	    "   cmp	%0, #0		\n"
206	    "   ite	eq		\n"
207	    "   moveq	%0, #1		\n"
208	    "   bne	1b		\n"
209	    "2:"
210	    : "=&r" (ret), "+r" (p), "+r" (cmpval), "+r" (newval)
211	    : : "cc", "memory");
212	return (ret);
213}
214
215static __inline int
216atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
217{
218	uint64_t tmp;
219	uint32_t ret;
220
221	__asm __volatile(
222	    "1:							\n"
223	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
224	    "   teq	%Q[tmp], %Q[cmpval]			\n"
225	    "   itee	eq					\n"
226	    "   teqeq	%R[tmp], %R[cmpval]			\n"
227	    "   movne	%[ret], #0				\n"
228	    "   bne	2f					\n"
229	    "   strexd	%[ret], %Q[newval], %R[newval], [%[ptr]]\n"
230	    "   teq	%[ret], #0				\n"
231	    "   it	ne					\n"
232	    "   bne	1b					\n"
233	    "   mov	%[ret], #1				\n"
234	    "2:							\n"
235	    : [ret]    "=&r" (ret),
236	      [tmp]    "=&r" (tmp)
237	    : [ptr]    "r"   (p),
238	      [cmpval] "r"   (cmpval),
239	      [newval] "r"   (newval)
240	    : "cc", "memory");
241	return (ret);
242}
243
244static __inline u_long
245atomic_cmpset_long(volatile u_long *p, u_long cmpval, u_long newval)
246{
247
248	return (atomic_cmpset_32((volatile uint32_t *)p, cmpval, newval));
249}
250
251static __inline uint32_t
252atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
253{
254	uint32_t ret;
255
256	ret = atomic_cmpset_32(p, cmpval, newval);
257	dmb();
258	return (ret);
259}
260
261static __inline uint64_t
262atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
263{
264	uint64_t ret;
265
266	ret = atomic_cmpset_64(p, cmpval, newval);
267	dmb();
268	return (ret);
269}
270
271static __inline u_long
272atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
273{
274	u_long ret;
275
276	ret = atomic_cmpset_long(p, cmpval, newval);
277	dmb();
278	return (ret);
279}
280
281static __inline uint32_t
282atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
283{
284
285	dmb();
286	return (atomic_cmpset_32(p, cmpval, newval));
287}
288
289static __inline uint64_t
290atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
291{
292
293	dmb();
294	return (atomic_cmpset_64(p, cmpval, newval));
295}
296
297static __inline u_long
298atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
299{
300
301	dmb();
302	return (atomic_cmpset_long(p, cmpval, newval));
303}
304
305static __inline uint32_t
306atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
307{
308	uint32_t tmp = 0, tmp2 = 0, ret = 0;
309
310	__asm __volatile(
311	    "1: ldrex	%0, [%3]	\n"
312	    "   add	%1, %0, %4	\n"
313	    "   strex	%2, %1, [%3]	\n"
314	    "   cmp	%2, #0		\n"
315	    "   it	ne		\n"
316	    "   bne	1b		\n"
317	    : "+r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val)
318	    : : "cc", "memory");
319	return (ret);
320}
321
322static __inline uint64_t
323atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
324{
325	uint64_t ret, tmp;
326	uint32_t exflag;
327
328	__asm __volatile(
329	    "1:							\n"
330	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
331	    "   adds	%Q[tmp], %Q[ret], %Q[val]		\n"
332	    "   adc	%R[tmp], %R[ret], %R[val]		\n"
333	    "   strexd	%[exf], %Q[tmp], %R[tmp], [%[ptr]]	\n"
334	    "   teq	%[exf], #0				\n"
335	    "   it	ne					\n"
336	    "   bne	1b					\n"
337	    : [ret] "=&r" (ret),
338	      [exf] "=&r" (exflag),
339	      [tmp] "=&r" (tmp)
340	    : [ptr] "r"   (p),
341	      [val] "r"   (val)
342	    : "cc", "memory");
343	return (ret);
344}
345
346static __inline u_long
347atomic_fetchadd_long(volatile u_long *p, u_long val)
348{
349
350	return (atomic_fetchadd_32((volatile uint32_t *)p, val));
351}
352
353static __inline uint32_t
354atomic_load_acq_32(volatile uint32_t *p)
355{
356	uint32_t v;
357
358	v = *p;
359	dmb();
360	return (v);
361}
362
363static __inline uint64_t
364atomic_load_64(volatile uint64_t *p)
365{
366	uint64_t ret;
367
368	/*
369	 * The only way to atomically load 64 bits is with LDREXD which puts the
370	 * exclusive monitor into the exclusive state, so reset it to open state
371	 * with CLREX because we don't actually need to store anything.
372	 */
373	__asm __volatile(
374	    "ldrexd	%Q[ret], %R[ret], [%[ptr]]	\n"
375	    "clrex					\n"
376	    : [ret] "=&r" (ret)
377	    : [ptr] "r"   (p)
378	    : "cc", "memory");
379	return (ret);
380}
381
382static __inline uint64_t
383atomic_load_acq_64(volatile uint64_t *p)
384{
385	uint64_t ret;
386
387	ret = atomic_load_64(p);
388	dmb();
389	return (ret);
390}
391
392static __inline u_long
393atomic_load_acq_long(volatile u_long *p)
394{
395	u_long v;
396
397	v = *p;
398	dmb();
399	return (v);
400}
401
402static __inline uint32_t
403atomic_readandclear_32(volatile uint32_t *p)
404{
405	uint32_t ret, tmp = 0, tmp2 = 0;
406
407	__asm __volatile(
408	    "1: ldrex	%0, [%3]	\n"
409	    "   mov	%1, #0		\n"
410	    "   strex	%2, %1, [%3]	\n"
411	    "   cmp	%2, #0		\n"
412	    "   it	ne		\n"
413	    "   bne	1b		\n"
414	    : "=r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p)
415	    : : "cc", "memory");
416	return (ret);
417}
418
419static __inline uint64_t
420atomic_readandclear_64(volatile uint64_t *p)
421{
422	uint64_t ret, tmp;
423	uint32_t exflag;
424
425	__asm __volatile(
426	    "1:							\n"
427	    "   ldrexd	%Q[ret], %R[ret], [%[ptr]]		\n"
428	    "   mov	%Q[tmp], #0				\n"
429	    "   mov	%R[tmp], #0				\n"
430	    "   strexd	%[exf], %Q[tmp], %R[tmp], [%[ptr]]	\n"
431	    "   teq	%[exf], #0				\n"
432	    "   it	ne					\n"
433	    "   bne	1b					\n"
434	    : [ret] "=&r" (ret),
435	      [exf] "=&r" (exflag),
436	      [tmp] "=&r" (tmp)
437	    : [ptr] "r"   (p)
438	    : "cc", "memory");
439	return (ret);
440}
441
442static __inline u_long
443atomic_readandclear_long(volatile u_long *p)
444{
445
446	return (atomic_readandclear_32((volatile uint32_t *)p));
447}
448
449static __inline void
450atomic_set_32(volatile uint32_t *address, uint32_t setmask)
451{
452	uint32_t tmp = 0, tmp2 = 0;
453
454	__asm __volatile(
455	    "1: ldrex	%0, [%2]	\n"
456	    "   orr	%0, %0, %3	\n"
457	    "   strex	%1, %0, [%2]	\n"
458	    "   cmp	%1, #0		\n"
459	    "   it	ne		\n"
460	    "   bne	1b		\n"
461	    : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask)
462	    : : "cc", "memory");
463}
464
465static __inline void
466atomic_set_64(volatile uint64_t *p, uint64_t val)
467{
468	uint64_t tmp;
469	uint32_t exflag;
470
471	__asm __volatile(
472	    "1:							\n"
473	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
474	    "   orr	%Q[tmp], %Q[val]			\n"
475	    "   orr	%R[tmp], %R[val]			\n"
476	    "   strexd	%[exf], %Q[tmp], %R[tmp], [%[ptr]]	\n"
477	    "   teq	%[exf], #0				\n"
478	    "   it	ne					\n"
479	    "   bne	1b					\n"
480	    : [exf] "=&r" (exflag),
481	      [tmp] "=&r" (tmp)
482	    : [ptr] "r"   (p),
483	      [val] "r"   (val)
484	    : "cc", "memory");
485}
486
487static __inline void
488atomic_set_long(volatile u_long *address, u_long setmask)
489{
490
491	atomic_set_32((volatile uint32_t *)address, setmask);
492}
493
494ATOMIC_ACQ_REL(set, 32)
495ATOMIC_ACQ_REL(set, 64)
496ATOMIC_ACQ_REL_LONG(set)
497
498static __inline void
499atomic_subtract_32(volatile uint32_t *p, uint32_t val)
500{
501	uint32_t tmp = 0, tmp2 = 0;
502
503	__asm __volatile(
504	    "1: ldrex	%0, [%2]	\n"
505	    "   sub	%0, %0, %3	\n"
506	    "   strex	%1, %0, [%2]	\n"
507	    "   cmp	%1, #0		\n"
508	    "   it	ne		\n"
509	    "   bne	1b		\n"
510	    : "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val)
511	    : : "cc", "memory");
512}
513
514static __inline void
515atomic_subtract_64(volatile uint64_t *p, uint64_t val)
516{
517	uint64_t tmp;
518	uint32_t exflag;
519
520	__asm __volatile(
521	    "1:							\n"
522	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
523	    "   subs	%Q[tmp], %Q[val]			\n"
524	    "   sbc	%R[tmp], %R[tmp], %R[val]		\n"
525	    "   strexd	%[exf], %Q[tmp], %R[tmp], [%[ptr]]	\n"
526	    "   teq	%[exf], #0				\n"
527	    "   it	ne					\n"
528	    "   bne	1b					\n"
529	    : [exf] "=&r" (exflag),
530	      [tmp] "=&r" (tmp)
531	    : [ptr] "r"   (p),
532	      [val] "r"   (val)
533	    : "cc", "memory");
534}
535
536static __inline void
537atomic_subtract_long(volatile u_long *p, u_long val)
538{
539
540	atomic_subtract_32((volatile uint32_t *)p, val);
541}
542
543ATOMIC_ACQ_REL(subtract, 32)
544ATOMIC_ACQ_REL(subtract, 64)
545ATOMIC_ACQ_REL_LONG(subtract)
546
547static __inline void
548atomic_store_64(volatile uint64_t *p, uint64_t val)
549{
550	uint64_t tmp;
551	uint32_t exflag;
552
553	/*
554	 * The only way to atomically store 64 bits is with STREXD, which will
555	 * succeed only if paired up with a preceeding LDREXD using the same
556	 * address, so we read and discard the existing value before storing.
557	 */
558	__asm __volatile(
559	    "1:							\n"
560	    "   ldrexd	%Q[tmp], %R[tmp], [%[ptr]]		\n"
561	    "   strexd	%[exf], %Q[val], %R[val], [%[ptr]]	\n"
562	    "   teq	%[exf], #0				\n"
563	    "   it	ne					\n"
564	    "   bne	1b					\n"
565	    : [tmp] "=&r" (tmp),
566	      [exf] "=&r" (exflag)
567	    : [ptr] "r"   (p),
568	      [val] "r"   (val)
569	    : "cc", "memory");
570}
571
572static __inline void
573atomic_store_rel_32(volatile uint32_t *p, uint32_t v)
574{
575
576	dmb();
577	*p = v;
578}
579
580static __inline void
581atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
582{
583
584	dmb();
585	atomic_store_64(p, val);
586}
587
588static __inline void
589atomic_store_rel_long(volatile u_long *p, u_long v)
590{
591
592	dmb();
593	*p = v;
594}
595
596static __inline int
597atomic_testandset_32(volatile uint32_t *p, u_int v)
598{
599	uint32_t tmp, tmp2, res, mask;
600
601	mask = 1u << (v & 0x1f);
602	tmp = tmp2 = 0;
603	__asm __volatile(
604	"1:	ldrex	%0, [%4]	\n"
605	"	orr	%1, %0, %3	\n"
606	"	strex	%2, %1, [%4]	\n"
607	"	cmp	%2, #0		\n"
608	"	it	ne		\n"
609	"	bne	1b		\n"
610	: "=&r" (res), "=&r" (tmp), "=&r" (tmp2)
611	: "r" (mask), "r" (p)
612	: "cc", "memory");
613	return ((res & mask) != 0);
614}
615
616static __inline int
617atomic_testandset_int(volatile u_int *p, u_int v)
618{
619
620	return (atomic_testandset_32((volatile uint32_t *)p, v));
621}
622
623static __inline int
624atomic_testandset_long(volatile u_long *p, u_int v)
625{
626
627	return (atomic_testandset_32((volatile uint32_t *)p, v));
628}
629
630static __inline int
631atomic_testandset_64(volatile uint64_t *p, u_int v)
632{
633	volatile uint32_t *p32;
634
635	p32 = (volatile uint32_t *)p;
636	/* Assume little-endian */
637	if (v >= 32) {
638		v &= 0x1f;
639		p32++;
640	}
641	return (atomic_testandset_32(p32, v));
642}
643
644static __inline uint32_t
645atomic_swap_32(volatile uint32_t *p, uint32_t v)
646{
647	uint32_t ret, exflag;
648
649	__asm __volatile(
650	    "1: ldrex	%[ret], [%[ptr]]		\n"
651	    "   strex	%[exf], %[val], [%[ptr]]	\n"
652	    "   teq	%[exf], #0			\n"
653	    "   it	ne				\n"
654	    "   bne	1b				\n"
655	    : [ret] "=&r"  (ret),
656	      [exf] "=&r" (exflag)
657	    : [val] "r"  (v),
658	      [ptr] "r"  (p)
659	    : "cc", "memory");
660	return (ret);
661}
662
663static __inline uint64_t
664atomic_swap_64(volatile uint64_t *p, uint64_t v)
665{
666	uint64_t ret;
667	uint32_t exflag;
668
669	__asm __volatile(
670	    "1: ldrexd	%Q[ret], %R[ret], [%[ptr]]		\n"
671	    "   strexd	%[exf], %Q[val], %R[val], [%[ptr]]	\n"
672	    "   teq	%[exf], #0				\n"
673	    "   it	ne					\n"
674	    "   bne	1b					\n"
675	    : [ret] "=&r" (ret),
676	      [exf] "=&r" (exflag)
677	    : [val] "r"   (v),
678	      [ptr] "r"   (p)
679	    : "cc", "memory");
680	return (ret);
681}
682
683#undef ATOMIC_ACQ_REL
684#undef ATOMIC_ACQ_REL_LONG
685
686static __inline void
687atomic_thread_fence_acq(void)
688{
689
690	dmb();
691}
692
693static __inline void
694atomic_thread_fence_rel(void)
695{
696
697	dmb();
698}
699
700static __inline void
701atomic_thread_fence_acq_rel(void)
702{
703
704	dmb();
705}
706
707static __inline void
708atomic_thread_fence_seq_cst(void)
709{
710
711	dmb();
712}
713
714#endif /* _MACHINE_ATOMIC_V6_H_ */
715