atomic.h revision 254619
138517Sdfr/*- 238517Sdfr * Copyright (c) 1998 Doug Rabson 338517Sdfr * All rights reserved. 438517Sdfr * 538517Sdfr * Redistribution and use in source and binary forms, with or without 638517Sdfr * modification, are permitted provided that the following conditions 738517Sdfr * are met: 838517Sdfr * 1. Redistributions of source code must retain the above copyright 938517Sdfr * notice, this list of conditions and the following disclaimer. 1038517Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1138517Sdfr * notice, this list of conditions and the following disclaimer in the 1238517Sdfr * documentation and/or other materials provided with the distribution. 1338517Sdfr * 1438517Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1538517Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1638517Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1738517Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1838517Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1938517Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2038517Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2138517Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2238517Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2338517Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2438517Sdfr * SUCH DAMAGE. 2538517Sdfr * 2650477Speter * $FreeBSD: head/sys/i386/include/atomic.h 254619 2013-08-21 22:27:42Z jkim $ 2738517Sdfr */ 2838517Sdfr#ifndef _MACHINE_ATOMIC_H_ 29147855Sjhb#define _MACHINE_ATOMIC_H_ 3038517Sdfr 31143063Sjoerg#ifndef _SYS_CDEFS_H_ 32143063Sjoerg#error this file needs sys/cdefs.h as a prerequisite 33143063Sjoerg#endif 34143063Sjoerg 35254619Sjkim#ifdef _KERNEL 36254619Sjkim#include <machine/md_var.h> 37254619Sjkim#include <machine/specialreg.h> 38254619Sjkim#endif 39254619Sjkim 40236456Skib#define mb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") 41236456Skib#define wmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") 42236456Skib#define rmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") 43185162Skmacy 4438517Sdfr/* 45165635Sbde * Various simple operations on memory, each of which is atomic in the 46165635Sbde * presence of interrupts and multiple processors. 4738517Sdfr * 48165633Sbde * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) 49165633Sbde * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) 50165633Sbde * atomic_add_char(P, V) (*(u_char *)(P) += (V)) 51165633Sbde * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) 5248797Salc * 53165633Sbde * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) 54165633Sbde * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) 55165633Sbde * atomic_add_short(P, V) (*(u_short *)(P) += (V)) 56165633Sbde * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) 5748797Salc * 58165633Sbde * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) 59165633Sbde * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) 60165633Sbde * atomic_add_int(P, V) (*(u_int *)(P) += (V)) 61165633Sbde * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) 62254617Sjkim * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) 63165635Sbde * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) 6448797Salc * 65165633Sbde * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) 66165633Sbde * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) 67165633Sbde * atomic_add_long(P, V) (*(u_long *)(P) += (V)) 68165633Sbde * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) 69254617Sjkim * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) 70165635Sbde * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) 7138517Sdfr */ 7238517Sdfr 7348797Salc/* 7449999Salc * The above functions are expanded inline in the statically-linked 7549999Salc * kernel. Lock prefixes are generated if an SMP kernel is being 7649999Salc * built. 7749999Salc * 7849999Salc * Kernel modules call real functions which are built into the kernel. 7949999Salc * This allows kernel modules to be portable between UP and SMP systems. 8048797Salc */ 81147855Sjhb#if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM) 82147855Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 83197803Sattiliovoid atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ 84197803Sattiliovoid atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 8549999Salc 86208332Sphkint atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); 87165633Sbdeu_int atomic_fetchadd_int(volatile u_int *p, u_int v); 88254617Sjkimint atomic_testandset_int(volatile u_int *p, u_int v); 8965514Sphk 90236456Skib#define ATOMIC_LOAD(TYPE, LOP) \ 91236456Skibu_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) 92236456Skib#define ATOMIC_STORE(TYPE) \ 93100251Smarkmvoid atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 9471085Sjhb 95254619Sjkimuint64_t atomic_load_acq_64(volatile uint64_t *); 96254619Sjkimvoid atomic_store_rel_64(volatile uint64_t *, uint64_t); 97254619Sjkim 98147855Sjhb#else /* !KLD_MODULE && __GNUCLIKE_ASM */ 9972358Smarkm 10084679Sjhb/* 101165635Sbde * For userland, always use lock prefixes so that the binaries will run 102165635Sbde * on both SMP and !SMP systems. 10384679Sjhb */ 10484679Sjhb#if defined(SMP) || !defined(_KERNEL) 105165630Sbde#define MPLOCKED "lock ; " 10690515Sbde#else 107147855Sjhb#define MPLOCKED 10890515Sbde#endif 10938517Sdfr 11048797Salc/* 111197803Sattilio * The assembly is volatilized to avoid code chunk removal by the compiler. 112197803Sattilio * GCC aggressively reorders operations and memory clobbering is necessary 113197803Sattilio * in order to avoid that for memory barriers. 11448797Salc */ 115147855Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 11648797Salcstatic __inline void \ 11749043Salcatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 11848797Salc{ \ 119165630Sbde __asm __volatile(MPLOCKED OP \ 120254612Sjkim : "+m" (*p) \ 121254612Sjkim : CONS (V) \ 122216524Skib : "cc"); \ 123122827Sbde} \ 124197803Sattilio \ 125197803Sattiliostatic __inline void \ 126197803Sattilioatomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 127197803Sattilio{ \ 128197803Sattilio __asm __volatile(MPLOCKED OP \ 129254612Sjkim : "+m" (*p) \ 130254612Sjkim : CONS (V) \ 131216524Skib : "memory", "cc"); \ 132197803Sattilio} \ 133122827Sbdestruct __hack 134100327Smarkm 13565514Sphk/* 13665514Sphk * Atomic compare and set, used by the mutex functions 13765514Sphk * 138208332Sphk * if (*dst == expect) *dst = src (all 32 bit words) 13965514Sphk * 14065514Sphk * Returns 0 on failure, non-zero on success 14165514Sphk */ 14265514Sphk 143165635Sbde#ifdef CPU_DISABLE_CMPXCHG 144100327Smarkm 145197910Sattiliostatic __inline int 146208332Sphkatomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) 147197910Sattilio{ 148197910Sattilio u_char res; 14965514Sphk 150197910Sattilio __asm __volatile( 151197910Sattilio " pushfl ; " 152197910Sattilio " cli ; " 153254612Sjkim " cmpl %3,%1 ; " 154197910Sattilio " jne 1f ; " 155197910Sattilio " movl %2,%1 ; " 156197910Sattilio "1: " 157197910Sattilio " sete %0 ; " 158197910Sattilio " popfl ; " 159197910Sattilio "# atomic_cmpset_int" 160197910Sattilio : "=q" (res), /* 0 */ 161254612Sjkim "+m" (*dst) /* 1 */ 162197910Sattilio : "r" (src), /* 2 */ 163254612Sjkim "r" (expect) /* 3 */ 164197910Sattilio : "memory"); 165197910Sattilio return (res); 166197910Sattilio} 167197910Sattilio 168165635Sbde#else /* !CPU_DISABLE_CMPXCHG */ 169100327Smarkm 170197910Sattiliostatic __inline int 171208332Sphkatomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) 172197910Sattilio{ 173197910Sattilio u_char res; 17465514Sphk 175197910Sattilio __asm __volatile( 176197910Sattilio " " MPLOCKED " " 177254614Sjkim " cmpxchgl %3,%1 ; " 178197910Sattilio " sete %0 ; " 179197910Sattilio "# atomic_cmpset_int" 180254614Sjkim : "=q" (res), /* 0 */ 181254614Sjkim "+m" (*dst), /* 1 */ 182254614Sjkim "+a" (expect) /* 2 */ 183254614Sjkim : "r" (src) /* 3 */ 184216524Skib : "memory", "cc"); 185197910Sattilio return (res); 186197910Sattilio} 187197910Sattilio 188165635Sbde#endif /* CPU_DISABLE_CMPXCHG */ 189100327Smarkm 190150627Sjhb/* 191150627Sjhb * Atomically add the value of v to the integer pointed to by p and return 192150627Sjhb * the previous value of *p. 193150627Sjhb */ 194150627Sjhbstatic __inline u_int 195150627Sjhbatomic_fetchadd_int(volatile u_int *p, u_int v) 196150627Sjhb{ 197150627Sjhb 198165633Sbde __asm __volatile( 199165630Sbde " " MPLOCKED " " 200254610Sjkim " xaddl %0,%1 ; " 201150627Sjhb "# atomic_fetchadd_int" 202254610Sjkim : "+r" (v), /* 0 */ 203254612Sjkim "+m" (*p) /* 1 */ 204254612Sjkim : : "cc"); 205150627Sjhb return (v); 206150627Sjhb} 207150627Sjhb 208254617Sjkimstatic __inline int 209254617Sjkimatomic_testandset_int(volatile u_int *p, u_int v) 210254617Sjkim{ 211254617Sjkim u_char res; 212254617Sjkim 213254617Sjkim __asm __volatile( 214254617Sjkim " " MPLOCKED " " 215254617Sjkim " btsl %2,%1 ; " 216254617Sjkim " setc %0 ; " 217254617Sjkim "# atomic_testandset_int" 218254617Sjkim : "=q" (res), /* 0 */ 219254617Sjkim "+m" (*p) /* 1 */ 220254617Sjkim : "Ir" (v & 0x1f) /* 2 */ 221254617Sjkim : "cc"); 222254617Sjkim return (res); 223254617Sjkim} 224254617Sjkim 225236456Skib/* 226236456Skib * We assume that a = b will do atomic loads and stores. Due to the 227236456Skib * IA32 memory model, a simple store guarantees release semantics. 228236456Skib * 229236456Skib * However, loads may pass stores, so for atomic_load_acq we have to 230236456Skib * ensure a Store/Load barrier to do the load in SMP kernels. We use 231236456Skib * "lock cmpxchg" as recommended by the AMD Software Optimization 232236456Skib * Guide, and not mfence. For UP kernels, however, the cache of the 233236456Skib * single processor is always consistent, so we only need to take care 234236456Skib * of the compiler. 235236456Skib */ 236236456Skib#define ATOMIC_STORE(TYPE) \ 237236456Skibstatic __inline void \ 238236456Skibatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 239236456Skib{ \ 240241374Sattilio __compiler_membar(); \ 241236456Skib *p = v; \ 242236456Skib} \ 243236456Skibstruct __hack 244236456Skib 245137623Sjhb#if defined(_KERNEL) && !defined(SMP) 246100327Smarkm 247236456Skib#define ATOMIC_LOAD(TYPE, LOP) \ 24867351Sjhbstatic __inline u_##TYPE \ 24967351Sjhbatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 25067351Sjhb{ \ 251197803Sattilio u_##TYPE tmp; \ 252197803Sattilio \ 253197803Sattilio tmp = *p; \ 254241374Sattilio __compiler_membar(); \ 255197803Sattilio return (tmp); \ 25667351Sjhb} \ 257122827Sbdestruct __hack 258100327Smarkm 259165635Sbde#else /* !(_KERNEL && !SMP) */ 26067351Sjhb 261236456Skib#define ATOMIC_LOAD(TYPE, LOP) \ 26271023Sjhbstatic __inline u_##TYPE \ 26371023Sjhbatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 26471023Sjhb{ \ 26571023Sjhb u_##TYPE res; \ 26671023Sjhb \ 267165630Sbde __asm __volatile(MPLOCKED LOP \ 268165635Sbde : "=a" (res), /* 0 */ \ 269254612Sjkim "+m" (*p) /* 1 */ \ 270254612Sjkim : : "memory", "cc"); \ 27171023Sjhb return (res); \ 27271023Sjhb} \ 273122827Sbdestruct __hack 274100327Smarkm 275165635Sbde#endif /* _KERNEL && !SMP */ 276100327Smarkm 277254619Sjkim#ifdef _KERNEL 278254619Sjkim 279254619Sjkim#ifdef WANT_FUNCTIONS 280254619Sjkimuint64_t atomic_load_acq_64_i386(volatile uint64_t *); 281254619Sjkimuint64_t atomic_load_acq_64_i586(volatile uint64_t *); 282254619Sjkimvoid atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); 283254619Sjkimvoid atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); 284254619Sjkim#endif 285254619Sjkim 286254619Sjkim/* I486 does not support SMP or CMPXCHG8B. */ 287254619Sjkimstatic __inline uint64_t 288254619Sjkimatomic_load_acq_64_i386(volatile uint64_t *p) 289254619Sjkim{ 290254619Sjkim volatile uint32_t *q; 291254619Sjkim uint64_t res; 292254619Sjkim 293254619Sjkim q = (volatile uint32_t *)p; 294254619Sjkim __asm __volatile( 295254619Sjkim " pushfl ; " 296254619Sjkim " cli ; " 297254619Sjkim " movl %1,%%eax ; " 298254619Sjkim " movl %2,%%edx ; " 299254619Sjkim " popfl" 300254619Sjkim : "=&A" (res) /* 0 */ 301254619Sjkim : "m" (*q), /* 1 */ 302254619Sjkim "m" (*(q + 1)) /* 2 */ 303254619Sjkim : "memory"); 304254619Sjkim return (res); 305254619Sjkim} 306254619Sjkim 307254619Sjkimstatic __inline void 308254619Sjkimatomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) 309254619Sjkim{ 310254619Sjkim volatile uint32_t *q; 311254619Sjkim 312254619Sjkim q = (volatile uint32_t *)p; 313254619Sjkim __asm __volatile( 314254619Sjkim " pushfl ; " 315254619Sjkim " cli ; " 316254619Sjkim " movl %%eax,%0 ; " 317254619Sjkim " movl %%edx,%1 ; " 318254619Sjkim " popfl" 319254619Sjkim : "=m" (*q), /* 0 */ 320254619Sjkim "=m" (*(q + 1)) /* 1 */ 321254619Sjkim : "A" (v) /* 2 */ 322254619Sjkim : "memory"); 323254619Sjkim} 324254619Sjkim 325254619Sjkimstatic __inline uint64_t 326254619Sjkimatomic_load_acq_64_i586(volatile uint64_t *p) 327254619Sjkim{ 328254619Sjkim uint64_t res; 329254619Sjkim 330254619Sjkim __asm __volatile( 331254619Sjkim " movl %%ebx,%%eax ; " 332254619Sjkim " movl %%ecx,%%edx ; " 333254619Sjkim " " MPLOCKED " " 334254619Sjkim " cmpxchg8b %1" 335254619Sjkim : "=&A" (res), /* 0 */ 336254619Sjkim "+m" (*p) /* 1 */ 337254619Sjkim : : "memory", "cc"); 338254619Sjkim return (res); 339254619Sjkim} 340254619Sjkim 341254619Sjkimstatic __inline void 342254619Sjkimatomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) 343254619Sjkim{ 344254619Sjkim 345254619Sjkim __asm __volatile( 346254619Sjkim " movl %%eax,%%ebx ; " 347254619Sjkim " movl %%edx,%%ecx ; " 348254619Sjkim "1: " 349254619Sjkim " " MPLOCKED " " 350254619Sjkim " cmpxchg8b %0 ; " 351254619Sjkim " jne 1b" 352254619Sjkim : "+m" (*p), /* 0 */ 353254619Sjkim "+A" (v) /* 1 */ 354254619Sjkim : : "ebx", "ecx", "memory", "cc"); 355254619Sjkim} 356254619Sjkim 357254619Sjkimstatic __inline uint64_t 358254619Sjkimatomic_load_acq_64(volatile uint64_t *p) 359254619Sjkim{ 360254619Sjkim 361254619Sjkim if ((cpu_feature & CPUID_CX8) == 0) 362254619Sjkim return (atomic_load_acq_64_i386(p)); 363254619Sjkim else 364254619Sjkim return (atomic_load_acq_64_i586(p)); 365254619Sjkim} 366254619Sjkim 367254619Sjkimstatic __inline void 368254619Sjkimatomic_store_rel_64(volatile uint64_t *p, uint64_t v) 369254619Sjkim{ 370254619Sjkim 371254619Sjkim if ((cpu_feature & CPUID_CX8) == 0) 372254619Sjkim atomic_store_rel_64_i386(p, v); 373254619Sjkim else 374254619Sjkim atomic_store_rel_64_i586(p, v); 375254619Sjkim} 376254619Sjkim 377254619Sjkim#endif /* _KERNEL */ 378254619Sjkim 379147855Sjhb#endif /* KLD_MODULE || !__GNUCLIKE_ASM */ 380100251Smarkm 381100251SmarkmATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); 382100251SmarkmATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); 383100251SmarkmATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); 384100251SmarkmATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); 38571085Sjhb 386100251SmarkmATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); 387100251SmarkmATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); 388100251SmarkmATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); 389100251SmarkmATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); 39071085Sjhb 391100251SmarkmATOMIC_ASM(set, int, "orl %1,%0", "ir", v); 392100251SmarkmATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); 393100251SmarkmATOMIC_ASM(add, int, "addl %1,%0", "ir", v); 394100251SmarkmATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); 39571085Sjhb 396100251SmarkmATOMIC_ASM(set, long, "orl %1,%0", "ir", v); 397100251SmarkmATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); 398100251SmarkmATOMIC_ASM(add, long, "addl %1,%0", "ir", v); 399100251SmarkmATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); 40071085Sjhb 401236456SkibATOMIC_LOAD(char, "cmpxchgb %b0,%1"); 402236456SkibATOMIC_LOAD(short, "cmpxchgw %w0,%1"); 403236456SkibATOMIC_LOAD(int, "cmpxchgl %0,%1"); 404236456SkibATOMIC_LOAD(long, "cmpxchgl %0,%1"); 40571023Sjhb 406236456SkibATOMIC_STORE(char); 407236456SkibATOMIC_STORE(short); 408236456SkibATOMIC_STORE(int); 409236456SkibATOMIC_STORE(long); 410236456Skib 41171085Sjhb#undef ATOMIC_ASM 412236456Skib#undef ATOMIC_LOAD 413236456Skib#undef ATOMIC_STORE 41467351Sjhb 415165635Sbde#ifndef WANT_FUNCTIONS 416147855Sjhb 417147855Sjhbstatic __inline int 418208332Sphkatomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) 419147855Sjhb{ 420147855Sjhb 421208332Sphk return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, 422147855Sjhb (u_int)src)); 423147855Sjhb} 424147855Sjhb 425177276Spjdstatic __inline u_long 426177276Spjdatomic_fetchadd_long(volatile u_long *p, u_long v) 427177276Spjd{ 428177276Spjd 429177276Spjd return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); 430177276Spjd} 431177276Spjd 432254617Sjkimstatic __inline int 433254617Sjkimatomic_testandset_long(volatile u_long *p, u_int v) 434254617Sjkim{ 435254617Sjkim 436254617Sjkim return (atomic_testandset_int((volatile u_int *)p, v)); 437254617Sjkim} 438254617Sjkim 439254617Sjkim/* Read the current value and store a new value in the destination. */ 440147855Sjhb#ifdef __GNUCLIKE_ASM 441147855Sjhb 442147855Sjhbstatic __inline u_int 443254617Sjkimatomic_swap_int(volatile u_int *p, u_int v) 444147855Sjhb{ 445147855Sjhb 446165633Sbde __asm __volatile( 447147855Sjhb " xchgl %1,%0 ; " 448254617Sjkim "# atomic_swap_int" 449254617Sjkim : "+r" (v), /* 0 */ 450254612Sjkim "+m" (*p)); /* 1 */ 451254617Sjkim return (v); 452147855Sjhb} 453147855Sjhb 454147855Sjhbstatic __inline u_long 455254617Sjkimatomic_swap_long(volatile u_long *p, u_long v) 456147855Sjhb{ 457147855Sjhb 458254617Sjkim return (atomic_swap_int((volatile u_int *)p, (u_int)v)); 459147855Sjhb} 460147855Sjhb 461147855Sjhb#else /* !__GNUCLIKE_ASM */ 462147855Sjhb 463254617Sjkimu_int atomic_swap_int(volatile u_int *p, u_int v); 464254617Sjkimu_long atomic_swap_long(volatile u_long *p, u_long v); 465147855Sjhb 466147855Sjhb#endif /* __GNUCLIKE_ASM */ 467147855Sjhb 468197803Sattilio#define atomic_set_acq_char atomic_set_barr_char 469197803Sattilio#define atomic_set_rel_char atomic_set_barr_char 470197803Sattilio#define atomic_clear_acq_char atomic_clear_barr_char 471197803Sattilio#define atomic_clear_rel_char atomic_clear_barr_char 472197803Sattilio#define atomic_add_acq_char atomic_add_barr_char 473197803Sattilio#define atomic_add_rel_char atomic_add_barr_char 474197803Sattilio#define atomic_subtract_acq_char atomic_subtract_barr_char 475197803Sattilio#define atomic_subtract_rel_char atomic_subtract_barr_char 47671085Sjhb 477197803Sattilio#define atomic_set_acq_short atomic_set_barr_short 478197803Sattilio#define atomic_set_rel_short atomic_set_barr_short 479197803Sattilio#define atomic_clear_acq_short atomic_clear_barr_short 480197803Sattilio#define atomic_clear_rel_short atomic_clear_barr_short 481197803Sattilio#define atomic_add_acq_short atomic_add_barr_short 482197803Sattilio#define atomic_add_rel_short atomic_add_barr_short 483197803Sattilio#define atomic_subtract_acq_short atomic_subtract_barr_short 484197803Sattilio#define atomic_subtract_rel_short atomic_subtract_barr_short 48571085Sjhb 486197803Sattilio#define atomic_set_acq_int atomic_set_barr_int 487197803Sattilio#define atomic_set_rel_int atomic_set_barr_int 488197803Sattilio#define atomic_clear_acq_int atomic_clear_barr_int 489197803Sattilio#define atomic_clear_rel_int atomic_clear_barr_int 490197803Sattilio#define atomic_add_acq_int atomic_add_barr_int 491197803Sattilio#define atomic_add_rel_int atomic_add_barr_int 492197803Sattilio#define atomic_subtract_acq_int atomic_subtract_barr_int 493197803Sattilio#define atomic_subtract_rel_int atomic_subtract_barr_int 494197910Sattilio#define atomic_cmpset_acq_int atomic_cmpset_int 495197910Sattilio#define atomic_cmpset_rel_int atomic_cmpset_int 49671085Sjhb 497197803Sattilio#define atomic_set_acq_long atomic_set_barr_long 498197803Sattilio#define atomic_set_rel_long atomic_set_barr_long 499197803Sattilio#define atomic_clear_acq_long atomic_clear_barr_long 500197803Sattilio#define atomic_clear_rel_long atomic_clear_barr_long 501197803Sattilio#define atomic_add_acq_long atomic_add_barr_long 502197803Sattilio#define atomic_add_rel_long atomic_add_barr_long 503197803Sattilio#define atomic_subtract_acq_long atomic_subtract_barr_long 504197803Sattilio#define atomic_subtract_rel_long atomic_subtract_barr_long 505197910Sattilio#define atomic_cmpset_acq_long atomic_cmpset_long 506197910Sattilio#define atomic_cmpset_rel_long atomic_cmpset_long 50771085Sjhb 508254617Sjkim#define atomic_readandclear_int(p) atomic_swap_int(p, 0) 509254617Sjkim#define atomic_readandclear_long(p) atomic_swap_long(p, 0) 510254617Sjkim 511147855Sjhb/* Operations on 8-bit bytes. */ 51271085Sjhb#define atomic_set_8 atomic_set_char 51371085Sjhb#define atomic_set_acq_8 atomic_set_acq_char 51471085Sjhb#define atomic_set_rel_8 atomic_set_rel_char 51571085Sjhb#define atomic_clear_8 atomic_clear_char 51671085Sjhb#define atomic_clear_acq_8 atomic_clear_acq_char 51771085Sjhb#define atomic_clear_rel_8 atomic_clear_rel_char 51871085Sjhb#define atomic_add_8 atomic_add_char 51971085Sjhb#define atomic_add_acq_8 atomic_add_acq_char 52071085Sjhb#define atomic_add_rel_8 atomic_add_rel_char 52171085Sjhb#define atomic_subtract_8 atomic_subtract_char 52271085Sjhb#define atomic_subtract_acq_8 atomic_subtract_acq_char 52371085Sjhb#define atomic_subtract_rel_8 atomic_subtract_rel_char 52471085Sjhb#define atomic_load_acq_8 atomic_load_acq_char 52571085Sjhb#define atomic_store_rel_8 atomic_store_rel_char 52671085Sjhb 527147855Sjhb/* Operations on 16-bit words. */ 52871085Sjhb#define atomic_set_16 atomic_set_short 52971085Sjhb#define atomic_set_acq_16 atomic_set_acq_short 53071085Sjhb#define atomic_set_rel_16 atomic_set_rel_short 53171085Sjhb#define atomic_clear_16 atomic_clear_short 53271085Sjhb#define atomic_clear_acq_16 atomic_clear_acq_short 53371085Sjhb#define atomic_clear_rel_16 atomic_clear_rel_short 53471085Sjhb#define atomic_add_16 atomic_add_short 53571085Sjhb#define atomic_add_acq_16 atomic_add_acq_short 53671085Sjhb#define atomic_add_rel_16 atomic_add_rel_short 53771085Sjhb#define atomic_subtract_16 atomic_subtract_short 53871085Sjhb#define atomic_subtract_acq_16 atomic_subtract_acq_short 53971085Sjhb#define atomic_subtract_rel_16 atomic_subtract_rel_short 54071085Sjhb#define atomic_load_acq_16 atomic_load_acq_short 54171085Sjhb#define atomic_store_rel_16 atomic_store_rel_short 54271085Sjhb 543147855Sjhb/* Operations on 32-bit double words. */ 54471085Sjhb#define atomic_set_32 atomic_set_int 54571085Sjhb#define atomic_set_acq_32 atomic_set_acq_int 54671085Sjhb#define atomic_set_rel_32 atomic_set_rel_int 54771085Sjhb#define atomic_clear_32 atomic_clear_int 54871085Sjhb#define atomic_clear_acq_32 atomic_clear_acq_int 54971085Sjhb#define atomic_clear_rel_32 atomic_clear_rel_int 55071085Sjhb#define atomic_add_32 atomic_add_int 55171085Sjhb#define atomic_add_acq_32 atomic_add_acq_int 55271085Sjhb#define atomic_add_rel_32 atomic_add_rel_int 55371085Sjhb#define atomic_subtract_32 atomic_subtract_int 55471085Sjhb#define atomic_subtract_acq_32 atomic_subtract_acq_int 55571085Sjhb#define atomic_subtract_rel_32 atomic_subtract_rel_int 55671085Sjhb#define atomic_load_acq_32 atomic_load_acq_int 55771085Sjhb#define atomic_store_rel_32 atomic_store_rel_int 55871085Sjhb#define atomic_cmpset_32 atomic_cmpset_int 55971085Sjhb#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 56071085Sjhb#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 561254617Sjkim#define atomic_swap_32 atomic_swap_int 56271085Sjhb#define atomic_readandclear_32 atomic_readandclear_int 563150627Sjhb#define atomic_fetchadd_32 atomic_fetchadd_int 564254617Sjkim#define atomic_testandset_32 atomic_testandset_int 56571085Sjhb 566147855Sjhb/* Operations on pointers. */ 567157212Sdes#define atomic_set_ptr(p, v) \ 568157212Sdes atomic_set_int((volatile u_int *)(p), (u_int)(v)) 569157212Sdes#define atomic_set_acq_ptr(p, v) \ 570157212Sdes atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) 571157212Sdes#define atomic_set_rel_ptr(p, v) \ 572157212Sdes atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) 573157212Sdes#define atomic_clear_ptr(p, v) \ 574157212Sdes atomic_clear_int((volatile u_int *)(p), (u_int)(v)) 575157212Sdes#define atomic_clear_acq_ptr(p, v) \ 576157212Sdes atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) 577157212Sdes#define atomic_clear_rel_ptr(p, v) \ 578157212Sdes atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) 579157212Sdes#define atomic_add_ptr(p, v) \ 580157212Sdes atomic_add_int((volatile u_int *)(p), (u_int)(v)) 581157212Sdes#define atomic_add_acq_ptr(p, v) \ 582157212Sdes atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) 583157212Sdes#define atomic_add_rel_ptr(p, v) \ 584157212Sdes atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) 585157212Sdes#define atomic_subtract_ptr(p, v) \ 586157212Sdes atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) 587157212Sdes#define atomic_subtract_acq_ptr(p, v) \ 588157212Sdes atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) 589157212Sdes#define atomic_subtract_rel_ptr(p, v) \ 590157212Sdes atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) 591157212Sdes#define atomic_load_acq_ptr(p) \ 592157212Sdes atomic_load_acq_int((volatile u_int *)(p)) 593157212Sdes#define atomic_store_rel_ptr(p, v) \ 594157212Sdes atomic_store_rel_int((volatile u_int *)(p), (v)) 595157212Sdes#define atomic_cmpset_ptr(dst, old, new) \ 596157212Sdes atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 597157212Sdes#define atomic_cmpset_acq_ptr(dst, old, new) \ 598165633Sbde atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ 599165633Sbde (u_int)(new)) 600157212Sdes#define atomic_cmpset_rel_ptr(dst, old, new) \ 601165633Sbde atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ 602165633Sbde (u_int)(new)) 603254617Sjkim#define atomic_swap_ptr(p, v) \ 604254617Sjkim atomic_swap_int((volatile u_int *)(p), (u_int)(v)) 605157212Sdes#define atomic_readandclear_ptr(p) \ 606157212Sdes atomic_readandclear_int((volatile u_int *)(p)) 60765514Sphk 608165635Sbde#endif /* !WANT_FUNCTIONS */ 609165633Sbde 610165633Sbde#endif /* !_MACHINE_ATOMIC_H_ */ 611