1/* $OpenBSD: atomic.h,v 1.20 2022/08/29 02:01:18 jsg Exp $ */ 2/* $NetBSD: atomic.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */ 3 4/*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by RedBack Networks Inc. 10 * 11 * Author: Bill Sommerfeld 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35#ifndef _MACHINE_ATOMIC_H_ 36#define _MACHINE_ATOMIC_H_ 37 38/* 39 * Perform atomic operations on memory. Should be atomic with respect 40 * to interrupts and multiple processors. 41 * 42 * void atomic_setbits_int(volatile u_int *a, u_int mask) { *a |= mask; } 43 * void atomic_clearbits_int(volatile u_int *a, u_int mas) { *a &= ~mask; } 44 */ 45#if !defined(_LOCORE) 46 47#if defined(MULTIPROCESSOR) || !defined(_KERNEL) 48#define _LOCK "lock" 49#else 50#define _LOCK 51#endif 52 53static inline unsigned int 54_atomic_cas_uint(volatile unsigned int *p, unsigned int e, unsigned int n) 55{ 56 __asm volatile(_LOCK " cmpxchgl %2, %1" 57 : "=a" (n), "=m" (*p) 58 : "r" (n), "a" (e), "m" (*p)); 59 60 return (n); 61} 62#define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n)) 63 64static inline unsigned long 65_atomic_cas_ulong(volatile unsigned long *p, unsigned long e, unsigned long n) 66{ 67 __asm volatile(_LOCK " cmpxchgl %2, %1" 68 : "=a" (n), "=m" (*p) 69 : "r" (n), "a" (e), "m" (*p)); 70 71 return (n); 72} 73#define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n)) 74 75static inline void * 76_atomic_cas_ptr(volatile void *p, void *e, void *n) 77{ 78 __asm volatile(_LOCK " cmpxchgl %2, %1" 79 : "=a" (n), "=m" (*(unsigned long *)p) 80 : "r" (n), "a" (e), "m" (*(unsigned long *)p)); 81 82 return (n); 83} 84#define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n)) 85 86static inline unsigned int 87_atomic_swap_uint(volatile unsigned int *p, unsigned int n) 88{ 89 __asm volatile("xchgl %0, %1" 90 : "=a" (n), "=m" (*p) 91 : "0" (n), "m" (*p)); 92 93 return (n); 94} 95#define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n)) 96#define atomic_swap_32(_p, _n) _atomic_swap_uint((_p), (_n)) 97 98static inline unsigned long 99_atomic_swap_ulong(volatile unsigned long *p, unsigned long n) 100{ 101 __asm volatile("xchgl %0, %1" 102 : "=a" (n), "=m" (*p) 103 : "0" (n), "m" (*p)); 104 105 return (n); 106} 107#define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n)) 108 109static inline void * 110_atomic_swap_ptr(volatile void *p, void *n) 111{ 112 __asm volatile("xchgl %0, %1" 113 : "=a" (n), "=m" (*(unsigned long *)p) 114 : "0" (n), "m" (*(unsigned long *)p)); 115 116 return (n); 117} 118#define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n)) 119 120static inline void 121_atomic_inc_int(volatile unsigned int *p) 122{ 123 __asm volatile(_LOCK " incl %0" 124 : "+m" (*p)); 125} 126#define atomic_inc_int(_p) _atomic_inc_int(_p) 127 128static inline void 129_atomic_inc_long(volatile unsigned long *p) 130{ 131 __asm volatile(_LOCK " incl %0" 132 : "+m" (*p)); 133} 134#define atomic_inc_long(_p) _atomic_inc_long(_p) 135 136static inline void 137_atomic_dec_int(volatile unsigned int *p) 138{ 139 __asm volatile(_LOCK " decl %0" 140 : "+m" (*p)); 141} 142#define atomic_dec_int(_p) _atomic_dec_int(_p) 143 144static inline void 145_atomic_dec_long(volatile unsigned long *p) 146{ 147 __asm volatile(_LOCK " decl %0" 148 : "+m" (*p)); 149} 150#define atomic_dec_long(_p) _atomic_dec_long(_p) 151 152static inline void 153_atomic_add_int(volatile unsigned int *p, unsigned int v) 154{ 155 __asm volatile(_LOCK " addl %1,%0" 156 : "+m" (*p) 157 : "a" (v)); 158} 159#define atomic_add_int(_p, _v) _atomic_add_int(_p, _v) 160 161static inline void 162_atomic_add_long(volatile unsigned long *p, unsigned long v) 163{ 164 __asm volatile(_LOCK " addl %1,%0" 165 : "+m" (*p) 166 : "a" (v)); 167} 168#define atomic_add_long(_p, _v) _atomic_add_long(_p, _v) 169 170static inline void 171_atomic_sub_int(volatile unsigned int *p, unsigned int v) 172{ 173 __asm volatile(_LOCK " subl %1,%0" 174 : "+m" (*p) 175 : "a" (v)); 176} 177#define atomic_sub_int(_p, _v) _atomic_sub_int(_p, _v) 178 179static inline void 180_atomic_sub_long(volatile unsigned long *p, unsigned long v) 181{ 182 __asm volatile(_LOCK " subl %1,%0" 183 : "+m" (*p) 184 : "a" (v)); 185} 186#define atomic_sub_long(_p, _v) _atomic_sub_long(_p, _v) 187 188 189static inline unsigned long 190_atomic_add_int_nv(volatile unsigned int *p, unsigned int v) 191{ 192 unsigned int rv = v; 193 194 __asm volatile(_LOCK " xaddl %0,%1" 195 : "+a" (rv), "+m" (*p)); 196 197 return (rv + v); 198} 199#define atomic_add_int_nv(_p, _v) _atomic_add_int_nv(_p, _v) 200 201static inline unsigned long 202_atomic_add_long_nv(volatile unsigned long *p, unsigned long v) 203{ 204 unsigned long rv = v; 205 206 __asm volatile(_LOCK " xaddl %0,%1" 207 : "+a" (rv), "+m" (*p)); 208 209 return (rv + v); 210} 211#define atomic_add_long_nv(_p, _v) _atomic_add_long_nv(_p, _v) 212 213static inline unsigned long 214_atomic_sub_int_nv(volatile unsigned int *p, unsigned int v) 215{ 216 unsigned int rv = 0 - v; 217 218 __asm volatile(_LOCK " xaddl %0,%1" 219 : "+a" (rv), "+m" (*p)); 220 221 return (rv - v); 222} 223#define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv(_p, _v) 224 225static inline unsigned long 226_atomic_sub_long_nv(volatile unsigned long *p, unsigned long v) 227{ 228 unsigned long rv = 0 - v; 229 230 __asm volatile(_LOCK " xaddl %0,%1" 231 : "+a" (rv), "+m" (*p)); 232 233 return (rv - v); 234} 235#define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv(_p, _v) 236 237/* 238 * The IA-32 architecture is rather strongly ordered. When accessing 239 * normal write-back cacheable memory, only reads may be reordered with 240 * older writes to different locations. There are a few instructions 241 * (clfush, non-temporal move instructions) that obey weaker ordering 242 * rules, but those instructions will only be used in (inline) 243 * assembly code where we can add the necessary fence instructions 244 * ourselves. 245 */ 246 247#define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0) 248 249#if defined(MULTIPROCESSOR) || !defined(_KERNEL) 250#define membar_enter() __membar("lock; addl $0,0(%%esp)") 251#define membar_exit() __membar("") 252#define membar_producer() __membar("") 253#define membar_consumer() __membar("") 254#define membar_sync() __membar("lock; addl $0,0(%%esp)") 255#else 256#define membar_enter() __membar("") 257#define membar_exit() __membar("") 258#define membar_producer() __membar("") 259#define membar_consumer() __membar("") 260#define membar_sync() __membar("") 261#endif 262 263#define membar_enter_after_atomic() __membar("") 264#define membar_exit_before_atomic() __membar("") 265 266#ifdef _KERNEL 267 268/* virtio needs MP membars even on SP kernels */ 269#define virtio_membar_producer() __membar("") 270#define virtio_membar_consumer() __membar("") 271#define virtio_membar_sync() __membar("lock; addl $0,0(%%esp)") 272 273static __inline u_int64_t 274i386_atomic_testset_uq(volatile u_int64_t *ptr, u_int64_t val) 275{ 276 __asm__ volatile ("\n1:\t" _LOCK " cmpxchg8b (%1); jnz 1b" : "+A" (val) : 277 "r" (ptr), "b" ((u_int32_t)val), "c" ((u_int32_t)(val >> 32))); 278 return val; 279} 280 281static __inline u_int32_t 282i386_atomic_testset_ul(volatile u_int32_t *ptr, unsigned long val) 283{ 284 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr)); 285 return val; 286} 287 288static __inline int 289i386_atomic_testset_i(volatile int *ptr, unsigned long val) 290{ 291 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr)); 292 return val; 293} 294 295static __inline void 296i386_atomic_setbits_l(volatile u_int32_t *ptr, unsigned long bits) 297{ 298 __asm volatile(_LOCK " orl %1,%0" : "=m" (*ptr) : "ir" (bits)); 299} 300 301static __inline void 302i386_atomic_clearbits_l(volatile u_int32_t *ptr, unsigned long bits) 303{ 304 bits = ~bits; 305 __asm volatile(_LOCK " andl %1,%0" : "=m" (*ptr) : "ir" (bits)); 306} 307 308#define atomic_setbits_int i386_atomic_setbits_l 309#define atomic_clearbits_int i386_atomic_clearbits_l 310 311#endif /* _KERNEL */ 312 313#undef _LOCK 314 315#endif /* !defined(_LOCORE) */ 316#endif /* _MACHINE_ATOMIC_H_ */ 317