atomic.h revision 185162 
1/*-
2 * Copyright (c) 1998 Doug Rabson.
3 * Copyright (c) 2001 Jake Burkholder.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 *	from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
28 * $FreeBSD: head/sys/sparc64/include/atomic.h 185162 2008-11-22 05:55:56Z kmacy $
29 */
30
31#ifndef	_MACHINE_ATOMIC_H_
32#define	_MACHINE_ATOMIC_H_
33
34#include <machine/cpufunc.h>
35
36/* Userland needs different ASI's. */
37#ifdef _KERNEL
38#define	__ASI_ATOMIC	ASI_N
39#else
40#define	__ASI_ATOMIC	ASI_P
41#endif
42
43#define mb()	__asm__ __volatile__ ("membar #MemIssue": : :"memory")
44#define wmb()	mb()
45#define rmb()	mb()
46
47/*
48 * Various simple arithmetic on memory which is atomic in the presence
49 * of interrupts and multiple processors.  See atomic(9) for details.
50 * Note that efficient hardware support exists only for the 32 and 64
51 * bit variants; the 8 and 16 bit versions are not provided and should
52 * not be used in MI code.
53 *
54 * This implementation takes advantage of the fact that the sparc64
55 * cas instruction is both a load and a store.  The loop is often coded
56 * as follows:
57 *
58 *	do {
59 *		expect = *p;
60 *		new = expect + 1;
61 *	} while (cas(p, expect, new) != expect);
62 *
63 * which performs an unnnecessary load on each iteration that the cas
64 * operation fails.  Modified as follows:
65 *
66 *	expect = *p;
67 *	for (;;) {
68 *		new = expect + 1;
69 *		result = cas(p, expect, new);
70 *		if (result == expect)
71 *			break;
72 *		expect = result;
73 *	}
74 *
75 * the return value of cas is used to avoid the extra reload.
76 *
77 * The memory barriers provided by the acq and rel variants are intended
78 * to be sufficient for use of relaxed memory ordering.  Due to the
79 * suggested assembly syntax of the membar operands containing a #
80 * character, they cannot be used in macros.  The cmask and mmask bits
81 * are hard coded in machine/cpufunc.h and used here through macros.
82 * Hopefully sun will choose not to change the bit numbers.
83 */
84
85#define	itype(sz)	uint ## sz ## _t
86
87#define	atomic_cas_32(p, e, s)	casa(p, e, s, __ASI_ATOMIC)
88#define	atomic_cas_64(p, e, s)	casxa(p, e, s, __ASI_ATOMIC)
89
90#define	atomic_cas(p, e, s, sz)						\
91	atomic_cas_ ## sz(p, e, s)
92
93#define	atomic_cas_acq(p, e, s, sz) ({					\
94	itype(sz) v;							\
95	v = atomic_cas(p, e, s, sz);					\
96	membar(LoadLoad | LoadStore);					\
97	v;								\
98})
99
100#define	atomic_cas_rel(p, e, s, sz) ({					\
101	itype(sz) v;							\
102	membar(LoadStore | StoreStore);					\
103	v = atomic_cas(p, e, s, sz);					\
104	v;								\
105})
106
107#define	atomic_op(p, op, v, sz) ({					\
108	itype(sz) e, r, s;						\
109	for (e = *(volatile itype(sz) *)p;; e = r) {			\
110		s = e op v;						\
111		r = atomic_cas_ ## sz(p, e, s);				\
112		if (r == e)						\
113			break;						\
114	}								\
115	e;								\
116})
117
118#define	atomic_op_acq(p, op, v, sz) ({					\
119	itype(sz) t;							\
120	t = atomic_op(p, op, v, sz);					\
121	membar(LoadLoad | LoadStore);					\
122	t;								\
123})
124
125#define	atomic_op_rel(p, op, v, sz) ({					\
126	itype(sz) t;							\
127	membar(LoadStore | StoreStore);					\
128	t = atomic_op(p, op, v, sz);					\
129	t;								\
130})
131
132#define	atomic_load(p, sz)						\
133	atomic_cas(p, 0, 0, sz)
134
135#define	atomic_load_acq(p, sz) ({					\
136	itype(sz) v;							\
137	v = atomic_load(p, sz);						\
138	membar(LoadLoad | LoadStore);					\
139	v;								\
140})
141
142#define	atomic_load_clear(p, sz) ({					\
143	itype(sz) e, r;							\
144	for (e = *(volatile itype(sz) *)p;; e = r) {			\
145		r = atomic_cas(p, e, 0, sz);				\
146		if (r == e)						\
147			break;						\
148	}								\
149	e;								\
150})
151
152#define	atomic_store(p, v, sz) do {					\
153	itype(sz) e, r;							\
154	for (e = *(volatile itype(sz) *)p;; e = r) {			\
155		r = atomic_cas(p, e, v, sz);				\
156		if (r == e)						\
157			break;						\
158	}								\
159} while (0)
160
161#define	atomic_store_rel(p, v, sz) do {					\
162	membar(LoadStore | StoreStore);					\
163	atomic_store(p, v, sz);						\
164} while (0)
165
166#define	ATOMIC_GEN(name, ptype, vtype, atype, sz)			\
167									\
168static __inline vtype							\
169atomic_add_ ## name(volatile ptype p, atype v)				\
170{									\
171	return ((vtype)atomic_op(p, +, v, sz));				\
172}									\
173static __inline vtype							\
174atomic_add_acq_ ## name(volatile ptype p, atype v)			\
175{									\
176	return ((vtype)atomic_op_acq(p, +, v, sz));			\
177}									\
178static __inline vtype							\
179atomic_add_rel_ ## name(volatile ptype p, atype v)			\
180{									\
181	return ((vtype)atomic_op_rel(p, +, v, sz));			\
182}									\
183									\
184static __inline vtype							\
185atomic_clear_ ## name(volatile ptype p, atype v)			\
186{									\
187	return ((vtype)atomic_op(p, &, ~v, sz));			\
188}									\
189static __inline vtype							\
190atomic_clear_acq_ ## name(volatile ptype p, atype v)			\
191{									\
192	return ((vtype)atomic_op_acq(p, &, ~v, sz));			\
193}									\
194static __inline vtype							\
195atomic_clear_rel_ ## name(volatile ptype p, atype v)			\
196{									\
197	return ((vtype)atomic_op_rel(p, &, ~v, sz));			\
198}									\
199									\
200static __inline int							\
201atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)		\
202{									\
203	return (((vtype)atomic_cas(p, e, s, sz)) == e);			\
204}									\
205static __inline int							\
206atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)		\
207{									\
208	return (((vtype)atomic_cas_acq(p, e, s, sz)) == e);		\
209}									\
210static __inline int							\
211atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)		\
212{									\
213	return (((vtype)atomic_cas_rel(p, e, s, sz)) == e);		\
214}									\
215									\
216static __inline vtype							\
217atomic_load_ ## name(volatile ptype p)					\
218{									\
219	return ((vtype)atomic_cas(p, 0, 0, sz));			\
220}									\
221static __inline vtype							\
222atomic_load_acq_ ## name(volatile ptype p)				\
223{									\
224	return ((vtype)atomic_cas_acq(p, 0, 0, sz));			\
225}									\
226									\
227static __inline vtype							\
228atomic_readandclear_ ## name(volatile ptype p)				\
229{									\
230	return ((vtype)atomic_load_clear(p, sz));			\
231}									\
232									\
233static __inline vtype							\
234atomic_set_ ## name(volatile ptype p, atype v)				\
235{									\
236	return ((vtype)atomic_op(p, |, v, sz));				\
237}									\
238static __inline vtype							\
239atomic_set_acq_ ## name(volatile ptype p, atype v)			\
240{									\
241	return ((vtype)atomic_op_acq(p, |, v, sz));			\
242}									\
243static __inline vtype							\
244atomic_set_rel_ ## name(volatile ptype p, atype v)			\
245{									\
246	return ((vtype)atomic_op_rel(p, |, v, sz));			\
247}									\
248									\
249static __inline vtype							\
250atomic_subtract_ ## name(volatile ptype p, atype v)			\
251{									\
252	return ((vtype)atomic_op(p, -, v, sz));				\
253}									\
254static __inline vtype							\
255atomic_subtract_acq_ ## name(volatile ptype p, atype v)			\
256{									\
257	return ((vtype)atomic_op_acq(p, -, v, sz));			\
258}									\
259static __inline vtype							\
260atomic_subtract_rel_ ## name(volatile ptype p, atype v)			\
261{									\
262	return ((vtype)atomic_op_rel(p, -, v, sz));			\
263}									\
264									\
265static __inline void							\
266atomic_store_ ## name(volatile ptype p, vtype v)			\
267{									\
268	atomic_store(p, v, sz);						\
269}									\
270static __inline void							\
271atomic_store_rel_ ## name(volatile ptype p, vtype v)			\
272{									\
273	atomic_store_rel(p, v, sz);					\
274}
275
276ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
277ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
278
279ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
280ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
281
282ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
283
284#define	atomic_fetchadd_int	atomic_add_int
285#define	atomic_fetchadd_32	atomic_add_32
286#define	atomic_fetchadd_long	atomic_add_long
287
288#undef ATOMIC_GEN
289#undef atomic_cas
290#undef atomic_cas_acq
291#undef atomic_cas_rel
292#undef atomic_op
293#undef atomic_op_acq
294#undef atomic_op_rel
295#undef atomic_load_acq
296#undef atomic_store_rel
297#undef atomic_load_clear
298
299#endif /* !_MACHINE_ATOMIC_H_ */
300