atomic.h revision 129568 
1/*-
2 * Copyright (c) 1998 Doug Rabson.
3 * Copyright (c) 2001 Jake Burkholder.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 *	from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
28 * $FreeBSD: head/sys/sparc64/include/atomic.h 129568 2004-05-22 00:47:26Z marius $
29 */
30
31#ifndef	_MACHINE_ATOMIC_H_
32#define	_MACHINE_ATOMIC_H_
33
34#include <machine/cpufunc.h>
35
36/* Userland needs different ASI's. */
37#ifdef _KERNEL
38#define	__ASI_ATOMIC	ASI_N
39#else
40#define	__ASI_ATOMIC	ASI_P
41#endif
42
43/*
44 * Various simple arithmetic on memory which is atomic in the presence
45 * of interrupts and multiple processors.  See atomic(9) for details.
46 * Note that efficient hardware support exists only for the 32 and 64
47 * bit variants; the 8 and 16 bit versions are not provided and should
48 * not be used in MI code.
49 *
50 * This implementation takes advantage of the fact that the sparc64
51 * cas instruction is both a load and a store.  The loop is often coded
52 * as follows:
53 *
54 *	do {
55 *		expect = *p;
56 *		new = expect + 1;
57 *	} while (cas(p, expect, new) != expect);
58 *
59 * which performs an unnnecessary load on each iteration that the cas
60 * operation fails.  Modified as follows:
61 *
62 *	expect = *p;
63 *	for (;;) {
64 *		new = expect + 1;
65 *		result = cas(p, expect, new);
66 *		if (result == expect)
67 *			break;
68 *		expect = result;
69 *	}
70 *
71 * the return value of cas is used to avoid the extra reload.
72 *
73 * The memory barriers provided by the acq and rel variants are intended
74 * to be sufficient for use of relaxed memory ordering.  Due to the
75 * suggested assembly syntax of the membar operands containing a #
76 * character, they cannot be used in macros.  The cmask and mmask bits
77 * are hard coded in machine/cpufunc.h and used here through macros.
78 * Hopefully sun will choose not to change the bit numbers.
79 */
80
81#define	itype(sz)	uint ## sz ## _t
82
83#define	atomic_cas_32(p, e, s)	casa(p, e, s, __ASI_ATOMIC)
84#define	atomic_cas_64(p, e, s)	casxa(p, e, s, __ASI_ATOMIC)
85
86#define	atomic_cas(p, e, s, sz)						\
87	atomic_cas_ ## sz(p, e, s)
88
89#define	atomic_cas_acq(p, e, s, sz) ({					\
90	itype(sz) v;							\
91	v = atomic_cas(p, e, s, sz);					\
92	membar(LoadLoad | LoadStore);					\
93	v;								\
94})
95
96#define	atomic_cas_rel(p, e, s, sz) ({					\
97	itype(sz) v;							\
98	membar(LoadStore | StoreStore);					\
99	v = atomic_cas(p, e, s, sz);					\
100	v;								\
101})
102
103#define	atomic_op(p, op, v, sz) ({					\
104	itype(sz) e, r, s;						\
105	for (e = *(volatile itype(sz) *)p;; e = r) {			\
106		s = e op v;						\
107		r = atomic_cas_ ## sz(p, e, s);				\
108		if (r == e)						\
109			break;						\
110	}								\
111	e;								\
112})
113
114#define	atomic_op_acq(p, op, v, sz) ({					\
115	itype(sz) t;							\
116	t = atomic_op(p, op, v, sz);					\
117	membar(LoadLoad | LoadStore);					\
118	t;								\
119})
120
121#define	atomic_op_rel(p, op, v, sz) ({					\
122	itype(sz) t;							\
123	membar(LoadStore | StoreStore);					\
124	t = atomic_op(p, op, v, sz);					\
125	t;								\
126})
127
128#define	atomic_load(p, sz)						\
129	atomic_cas(p, 0, 0, sz)
130
131#define	atomic_load_acq(p, sz) ({					\
132	itype(sz) v;							\
133	v = atomic_load(p, sz);						\
134	membar(LoadLoad | LoadStore);					\
135	v;								\
136})
137
138#define	atomic_load_clear(p, sz) ({					\
139	itype(sz) e, r;							\
140	for (e = *(volatile itype(sz) *)p;; e = r) {			\
141		r = atomic_cas(p, e, 0, sz);				\
142		if (r == e)						\
143			break;						\
144	}								\
145	e;								\
146})
147
148#define	atomic_store(p, v, sz) do {					\
149	itype(sz) e, r;							\
150	for (e = *(volatile itype(sz) *)p;; e = r) {			\
151		r = atomic_cas(p, e, v, sz);				\
152		if (r == e)						\
153			break;						\
154	}								\
155} while (0)
156
157#define	atomic_store_rel(p, v, sz) do {					\
158	membar(LoadStore | StoreStore);					\
159	atomic_store(p, v, sz);						\
160} while (0)
161
162#define	ATOMIC_GEN(name, ptype, vtype, atype, sz)			\
163									\
164static __inline vtype							\
165atomic_add_ ## name(volatile ptype p, atype v)				\
166{									\
167	return ((vtype)atomic_op(p, +, v, sz));				\
168}									\
169static __inline vtype							\
170atomic_add_acq_ ## name(volatile ptype p, atype v)			\
171{									\
172	return ((vtype)atomic_op_acq(p, +, v, sz));			\
173}									\
174static __inline vtype							\
175atomic_add_rel_ ## name(volatile ptype p, atype v)			\
176{									\
177	return ((vtype)atomic_op_rel(p, +, v, sz));			\
178}									\
179									\
180static __inline vtype							\
181atomic_clear_ ## name(volatile ptype p, atype v)			\
182{									\
183	return ((vtype)atomic_op(p, &, ~v, sz));			\
184}									\
185static __inline vtype							\
186atomic_clear_acq_ ## name(volatile ptype p, atype v)			\
187{									\
188	return ((vtype)atomic_op_acq(p, &, ~v, sz));			\
189}									\
190static __inline vtype							\
191atomic_clear_rel_ ## name(volatile ptype p, atype v)			\
192{									\
193	return ((vtype)atomic_op_rel(p, &, ~v, sz));			\
194}									\
195									\
196static __inline int							\
197atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)		\
198{									\
199	return (((vtype)atomic_cas(p, e, s, sz)) == e);			\
200}									\
201static __inline int							\
202atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)		\
203{									\
204	return (((vtype)atomic_cas_acq(p, e, s, sz)) == e);		\
205}									\
206static __inline int							\
207atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)		\
208{									\
209	return (((vtype)atomic_cas_rel(p, e, s, sz)) == e);		\
210}									\
211									\
212static __inline vtype							\
213atomic_load_ ## name(volatile ptype p)					\
214{									\
215	return ((vtype)atomic_cas(p, 0, 0, sz));			\
216}									\
217static __inline vtype							\
218atomic_load_acq_ ## name(volatile ptype p)				\
219{									\
220	return ((vtype)atomic_cas_acq(p, 0, 0, sz));			\
221}									\
222									\
223static __inline vtype							\
224atomic_readandclear_ ## name(volatile ptype p)				\
225{									\
226	return ((vtype)atomic_load_clear(p, sz));			\
227}									\
228									\
229static __inline vtype							\
230atomic_set_ ## name(volatile ptype p, atype v)				\
231{									\
232	return ((vtype)atomic_op(p, |, v, sz));				\
233}									\
234static __inline vtype							\
235atomic_set_acq_ ## name(volatile ptype p, atype v)			\
236{									\
237	return ((vtype)atomic_op_acq(p, |, v, sz));			\
238}									\
239static __inline vtype							\
240atomic_set_rel_ ## name(volatile ptype p, atype v)			\
241{									\
242	return ((vtype)atomic_op_rel(p, |, v, sz));			\
243}									\
244									\
245static __inline vtype							\
246atomic_subtract_ ## name(volatile ptype p, atype v)			\
247{									\
248	return ((vtype)atomic_op(p, -, v, sz));				\
249}									\
250static __inline vtype							\
251atomic_subtract_acq_ ## name(volatile ptype p, atype v)			\
252{									\
253	return ((vtype)atomic_op_acq(p, -, v, sz));			\
254}									\
255static __inline vtype							\
256atomic_subtract_rel_ ## name(volatile ptype p, atype v)			\
257{									\
258	return ((vtype)atomic_op_rel(p, -, v, sz));			\
259}									\
260									\
261static __inline void							\
262atomic_store_ ## name(volatile ptype p, vtype v)			\
263{									\
264	atomic_store(p, v, sz);						\
265}									\
266static __inline void							\
267atomic_store_rel_ ## name(volatile ptype p, vtype v)			\
268{									\
269	atomic_store_rel(p, v, sz);					\
270}
271
272ATOMIC_GEN(int, int *, int, int, 32);
273ATOMIC_GEN(32, int *, int, int, 32);
274
275ATOMIC_GEN(long, long *, long, long, 64);
276ATOMIC_GEN(64, long *, long, long, 64);
277
278ATOMIC_GEN(ptr, void *, void *, uintptr_t, 64);
279
280#undef ATOMIC_GEN
281#undef atomic_cas
282#undef atomic_cas_acq
283#undef atomic_cas_rel
284#undef atomic_op
285#undef atomic_op_acq
286#undef atomic_op_rel
287#undef atomic_load_acq
288#undef atomic_store_rel
289#undef atomic_load_clear
290
291#endif /* !_MACHINE_ATOMIC_H_ */
292