1/*	$OpenBSD: atomic.h,v 1.20 2022/08/29 02:01:18 jsg Exp $	*/
2/* $NetBSD: atomic.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */
3
4/*-
5 * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by RedBack Networks Inc.
10 *
11 * Author: Bill Sommerfeld
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 *    notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 *    notice, this list of conditions and the following disclaimer in the
20 *    documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#ifndef _MACHINE_ATOMIC_H_
36#define _MACHINE_ATOMIC_H_
37
38/*
39 * Perform atomic operations on memory. Should be atomic with respect
40 * to interrupts and multiple processors.
41 *
42 * void atomic_setbits_int(volatile u_int *a, u_int mask) { *a |= mask; }
43 * void atomic_clearbits_int(volatile u_int *a, u_int mas) { *a &= ~mask; }
44 */
45#if !defined(_LOCORE)
46
47#if defined(MULTIPROCESSOR) || !defined(_KERNEL)
48#define _LOCK "lock"
49#else
50#define _LOCK
51#endif
52
53static inline unsigned int
54_atomic_cas_uint(volatile unsigned int *p, unsigned int e, unsigned int n)
55{
56	__asm volatile(_LOCK " cmpxchgl %2, %1"
57	    : "=a" (n), "=m" (*p)
58	    : "r" (n), "a" (e), "m" (*p));
59
60	return (n);
61}
62#define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n))
63
64static inline unsigned long
65_atomic_cas_ulong(volatile unsigned long *p, unsigned long e, unsigned long n)
66{
67	__asm volatile(_LOCK " cmpxchgl %2, %1"
68	    : "=a" (n), "=m" (*p)
69	    : "r" (n), "a" (e), "m" (*p));
70
71	return (n);
72}
73#define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n))
74
75static inline void *
76_atomic_cas_ptr(volatile void *p, void *e, void *n)
77{
78	__asm volatile(_LOCK " cmpxchgl %2, %1"
79	    : "=a" (n), "=m" (*(unsigned long *)p)
80	    : "r" (n), "a" (e), "m" (*(unsigned long *)p));
81
82	return (n);
83}
84#define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n))
85
86static inline unsigned int
87_atomic_swap_uint(volatile unsigned int *p, unsigned int n)
88{
89	__asm volatile("xchgl %0, %1"
90	    : "=a" (n), "=m" (*p)
91	    : "0" (n), "m" (*p));
92
93	return (n);
94}
95#define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n))
96#define atomic_swap_32(_p, _n) _atomic_swap_uint((_p), (_n))
97
98static inline unsigned long
99_atomic_swap_ulong(volatile unsigned long *p, unsigned long n)
100{
101	__asm volatile("xchgl %0, %1"
102	    : "=a" (n), "=m" (*p)
103	    : "0" (n), "m" (*p));
104
105	return (n);
106}
107#define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n))
108
109static inline void *
110_atomic_swap_ptr(volatile void *p, void *n)
111{
112	__asm volatile("xchgl %0, %1"
113	    : "=a" (n), "=m" (*(unsigned long *)p)
114	    : "0" (n), "m" (*(unsigned long *)p));
115
116	return (n);
117}
118#define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n))
119
120static inline void
121_atomic_inc_int(volatile unsigned int *p)
122{
123	__asm volatile(_LOCK " incl %0"
124	    : "+m" (*p));
125}
126#define atomic_inc_int(_p) _atomic_inc_int(_p)
127
128static inline void
129_atomic_inc_long(volatile unsigned long *p)
130{
131	__asm volatile(_LOCK " incl %0"
132	    : "+m" (*p));
133}
134#define atomic_inc_long(_p) _atomic_inc_long(_p)
135
136static inline void
137_atomic_dec_int(volatile unsigned int *p)
138{
139	__asm volatile(_LOCK " decl %0"
140	    : "+m" (*p));
141}
142#define atomic_dec_int(_p) _atomic_dec_int(_p)
143
144static inline void
145_atomic_dec_long(volatile unsigned long *p)
146{
147	__asm volatile(_LOCK " decl %0"
148	    : "+m" (*p));
149}
150#define atomic_dec_long(_p) _atomic_dec_long(_p)
151
152static inline void
153_atomic_add_int(volatile unsigned int *p, unsigned int v)
154{
155	__asm volatile(_LOCK " addl %1,%0"
156	    : "+m" (*p)
157	    : "a" (v));
158}
159#define atomic_add_int(_p, _v) _atomic_add_int(_p, _v)
160
161static inline void
162_atomic_add_long(volatile unsigned long *p, unsigned long v)
163{
164	__asm volatile(_LOCK " addl %1,%0"
165	    : "+m" (*p)
166	    : "a" (v));
167}
168#define atomic_add_long(_p, _v) _atomic_add_long(_p, _v)
169
170static inline void
171_atomic_sub_int(volatile unsigned int *p, unsigned int v)
172{
173	__asm volatile(_LOCK " subl %1,%0"
174	    : "+m" (*p)
175	    : "a" (v));
176}
177#define atomic_sub_int(_p, _v) _atomic_sub_int(_p, _v)
178
179static inline void
180_atomic_sub_long(volatile unsigned long *p, unsigned long v)
181{
182	__asm volatile(_LOCK " subl %1,%0"
183	    : "+m" (*p)
184	    : "a" (v));
185}
186#define atomic_sub_long(_p, _v) _atomic_sub_long(_p, _v)
187
188
189static inline unsigned long
190_atomic_add_int_nv(volatile unsigned int *p, unsigned int v)
191{
192	unsigned int rv = v;
193
194	__asm volatile(_LOCK " xaddl %0,%1"
195	    : "+a" (rv), "+m" (*p));
196
197	return (rv + v);
198}
199#define atomic_add_int_nv(_p, _v) _atomic_add_int_nv(_p, _v)
200
201static inline unsigned long
202_atomic_add_long_nv(volatile unsigned long *p, unsigned long v)
203{
204	unsigned long rv = v;
205
206	__asm volatile(_LOCK " xaddl %0,%1"
207	    : "+a" (rv), "+m" (*p));
208
209	return (rv + v);
210}
211#define atomic_add_long_nv(_p, _v) _atomic_add_long_nv(_p, _v)
212
213static inline unsigned long
214_atomic_sub_int_nv(volatile unsigned int *p, unsigned int v)
215{
216	unsigned int rv = 0 - v;
217
218	__asm volatile(_LOCK " xaddl %0,%1"
219	    : "+a" (rv), "+m" (*p));
220
221	return (rv - v);
222}
223#define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv(_p, _v)
224
225static inline unsigned long
226_atomic_sub_long_nv(volatile unsigned long *p, unsigned long v)
227{
228	unsigned long rv = 0 - v;
229
230	__asm volatile(_LOCK " xaddl %0,%1"
231	    : "+a" (rv), "+m" (*p));
232
233	return (rv - v);
234}
235#define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv(_p, _v)
236
237/*
238 * The IA-32 architecture is rather strongly ordered.  When accessing
239 * normal write-back cacheable memory, only reads may be reordered with
240 * older writes to different locations.  There are a few instructions
241 * (clfush, non-temporal move instructions) that obey weaker ordering
242 * rules, but those instructions will only be used in (inline)
243 * assembly code where we can add the necessary fence instructions
244 * ourselves.
245 */
246
247#define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0)
248
249#if defined(MULTIPROCESSOR) || !defined(_KERNEL)
250#define membar_enter()		__membar("lock; addl $0,0(%%esp)")
251#define membar_exit()		__membar("")
252#define membar_producer()	__membar("")
253#define membar_consumer()	__membar("")
254#define membar_sync()		__membar("lock; addl $0,0(%%esp)")
255#else
256#define membar_enter()		__membar("")
257#define membar_exit()		__membar("")
258#define membar_producer()	__membar("")
259#define membar_consumer()	__membar("")
260#define membar_sync()		__membar("")
261#endif
262
263#define membar_enter_after_atomic()	__membar("")
264#define membar_exit_before_atomic()	__membar("")
265
266#ifdef _KERNEL
267
268/* virtio needs MP membars even on SP kernels */
269#define virtio_membar_producer()	__membar("")
270#define virtio_membar_consumer()	__membar("")
271#define virtio_membar_sync()		__membar("lock; addl $0,0(%%esp)")
272
273static __inline u_int64_t
274i386_atomic_testset_uq(volatile u_int64_t *ptr, u_int64_t val)
275{
276	__asm__ volatile ("\n1:\t" _LOCK " cmpxchg8b (%1); jnz 1b" : "+A" (val) :
277	    "r" (ptr), "b" ((u_int32_t)val), "c" ((u_int32_t)(val >> 32)));
278	return val;
279}
280
281static __inline u_int32_t
282i386_atomic_testset_ul(volatile u_int32_t *ptr, unsigned long val)
283{
284	__asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
285	return val;
286}
287
288static __inline int
289i386_atomic_testset_i(volatile int *ptr, unsigned long val)
290{
291	__asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
292	return val;
293}
294
295static __inline void
296i386_atomic_setbits_l(volatile u_int32_t *ptr, unsigned long bits)
297{
298	__asm volatile(_LOCK " orl %1,%0" :  "=m" (*ptr) : "ir" (bits));
299}
300
301static __inline void
302i386_atomic_clearbits_l(volatile u_int32_t *ptr, unsigned long bits)
303{
304	bits = ~bits;
305	__asm volatile(_LOCK " andl %1,%0" :  "=m" (*ptr) : "ir" (bits));
306}
307
308#define atomic_setbits_int i386_atomic_setbits_l
309#define atomic_clearbits_int i386_atomic_clearbits_l
310
311#endif /* _KERNEL */
312
313#undef _LOCK
314
315#endif /* !defined(_LOCORE) */
316#endif /* _MACHINE_ATOMIC_H_ */
317