138517Sdfr/*- 238517Sdfr * Copyright (c) 1998 Doug Rabson 338517Sdfr * All rights reserved. 438517Sdfr * 538517Sdfr * Redistribution and use in source and binary forms, with or without 638517Sdfr * modification, are permitted provided that the following conditions 738517Sdfr * are met: 838517Sdfr * 1. Redistributions of source code must retain the above copyright 938517Sdfr * notice, this list of conditions and the following disclaimer. 1038517Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1138517Sdfr * notice, this list of conditions and the following disclaimer in the 1238517Sdfr * documentation and/or other materials provided with the distribution. 1338517Sdfr * 1438517Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1538517Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1638517Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1738517Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1838517Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1938517Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2038517Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2138517Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2238517Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2338517Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2438517Sdfr * SUCH DAMAGE. 2538517Sdfr * 2650477Speter * $FreeBSD: releng/10.3/sys/amd64/include/atomic.h 254618 2013-08-21 22:05:58Z jkim $ 2738517Sdfr */ 2838517Sdfr#ifndef _MACHINE_ATOMIC_H_ 29147855Sjhb#define _MACHINE_ATOMIC_H_ 3038517Sdfr 31143063Sjoerg#ifndef _SYS_CDEFS_H_ 32143063Sjoerg#error this file needs sys/cdefs.h as a prerequisite 33143063Sjoerg#endif 34143063Sjoerg 35197824Sattilio#define mb() __asm __volatile("mfence;" : : : "memory") 36197824Sattilio#define wmb() __asm __volatile("sfence;" : : : "memory") 37197824Sattilio#define rmb() __asm __volatile("lfence;" : : : "memory") 38185162Skmacy 3938517Sdfr/* 40165635Sbde * Various simple operations on memory, each of which is atomic in the 41165635Sbde * presence of interrupts and multiple processors. 4238517Sdfr * 43165633Sbde * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) 44165633Sbde * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) 45165633Sbde * atomic_add_char(P, V) (*(u_char *)(P) += (V)) 46165633Sbde * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) 4748797Salc * 48165633Sbde * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) 49165633Sbde * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) 50165633Sbde * atomic_add_short(P, V) (*(u_short *)(P) += (V)) 51165633Sbde * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) 5248797Salc * 53165633Sbde * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) 54165633Sbde * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) 55165633Sbde * atomic_add_int(P, V) (*(u_int *)(P) += (V)) 56165633Sbde * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) 57254617Sjkim * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) 58165635Sbde * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) 5948797Salc * 60165633Sbde * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) 61165633Sbde * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) 62165633Sbde * atomic_add_long(P, V) (*(u_long *)(P) += (V)) 63165633Sbde * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) 64254617Sjkim * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) 65165635Sbde * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) 6638517Sdfr */ 6738517Sdfr 6848797Salc/* 6949999Salc * The above functions are expanded inline in the statically-linked 7049999Salc * kernel. Lock prefixes are generated if an SMP kernel is being 7149999Salc * built. 7249999Salc * 7349999Salc * Kernel modules call real functions which are built into the kernel. 7449999Salc * This allows kernel modules to be portable between UP and SMP systems. 7548797Salc */ 76165578Sbde#if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM) 77147855Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 78197803Sattiliovoid atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ 79197803Sattiliovoid atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 8049999Salc 81208332Sphkint atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); 82208332Sphkint atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); 83165633Sbdeu_int atomic_fetchadd_int(volatile u_int *p, u_int v); 84177276Spjdu_long atomic_fetchadd_long(volatile u_long *p, u_long v); 85254617Sjkimint atomic_testandset_int(volatile u_int *p, u_int v); 86254617Sjkimint atomic_testandset_long(volatile u_long *p, u_int v); 8765514Sphk 88236456Skib#define ATOMIC_LOAD(TYPE, LOP) \ 89236456Skibu_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) 90236456Skib#define ATOMIC_STORE(TYPE) \ 91100251Smarkmvoid atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 9271085Sjhb 93165578Sbde#else /* !KLD_MODULE && __GNUCLIKE_ASM */ 9472358Smarkm 9584679Sjhb/* 96165635Sbde * For userland, always use lock prefixes so that the binaries will run 97165635Sbde * on both SMP and !SMP systems. 9884679Sjhb */ 99122849Speter#if defined(SMP) || !defined(_KERNEL) 100165630Sbde#define MPLOCKED "lock ; " 10190515Sbde#else 102147855Sjhb#define MPLOCKED 10390515Sbde#endif 10438517Sdfr 10548797Salc/* 106197803Sattilio * The assembly is volatilized to avoid code chunk removal by the compiler. 107197803Sattilio * GCC aggressively reorders operations and memory clobbering is necessary 108197803Sattilio * in order to avoid that for memory barriers. 10948797Salc */ 110147855Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 11148797Salcstatic __inline void \ 11249043Salcatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 11348797Salc{ \ 114165630Sbde __asm __volatile(MPLOCKED OP \ 115254612Sjkim : "+m" (*p) \ 116254612Sjkim : CONS (V) \ 117216524Skib : "cc"); \ 118122940Speter} \ 119197803Sattilio \ 120197803Sattiliostatic __inline void \ 121197803Sattilioatomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 122197803Sattilio{ \ 123197803Sattilio __asm __volatile(MPLOCKED OP \ 124254612Sjkim : "+m" (*p) \ 125254612Sjkim : CONS (V) \ 126216524Skib : "memory", "cc"); \ 127197803Sattilio} \ 128122940Speterstruct __hack 129100327Smarkm 13065514Sphk/* 13165514Sphk * Atomic compare and set, used by the mutex functions 13265514Sphk * 133208332Sphk * if (*dst == expect) *dst = src (all 32 bit words) 13465514Sphk * 13565514Sphk * Returns 0 on failure, non-zero on success 13665514Sphk */ 13765514Sphk 138197910Sattiliostatic __inline int 139208332Sphkatomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) 140197910Sattilio{ 141197910Sattilio u_char res; 14265514Sphk 143197910Sattilio __asm __volatile( 144197910Sattilio " " MPLOCKED " " 145254614Sjkim " cmpxchgl %3,%1 ; " 146197910Sattilio " sete %0 ; " 147197910Sattilio "# atomic_cmpset_int" 148254614Sjkim : "=q" (res), /* 0 */ 149254614Sjkim "+m" (*dst), /* 1 */ 150254614Sjkim "+a" (expect) /* 2 */ 151254614Sjkim : "r" (src) /* 3 */ 152216524Skib : "memory", "cc"); 153197910Sattilio return (res); 154197910Sattilio} 155197910Sattilio 156197910Sattiliostatic __inline int 157208332Sphkatomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) 158197910Sattilio{ 159197910Sattilio u_char res; 160197910Sattilio 161197910Sattilio __asm __volatile( 162197910Sattilio " " MPLOCKED " " 163254614Sjkim " cmpxchgq %3,%1 ; " 164197910Sattilio " sete %0 ; " 165197910Sattilio "# atomic_cmpset_long" 166254614Sjkim : "=q" (res), /* 0 */ 167254614Sjkim "+m" (*dst), /* 1 */ 168254614Sjkim "+a" (expect) /* 2 */ 169254614Sjkim : "r" (src) /* 3 */ 170216524Skib : "memory", "cc"); 171197910Sattilio return (res); 172197910Sattilio} 173197910Sattilio 174150627Sjhb/* 175150627Sjhb * Atomically add the value of v to the integer pointed to by p and return 176150627Sjhb * the previous value of *p. 177150627Sjhb */ 178150627Sjhbstatic __inline u_int 179150627Sjhbatomic_fetchadd_int(volatile u_int *p, u_int v) 180150627Sjhb{ 181150627Sjhb 182165633Sbde __asm __volatile( 183165630Sbde " " MPLOCKED " " 184254610Sjkim " xaddl %0,%1 ; " 185150627Sjhb "# atomic_fetchadd_int" 186254610Sjkim : "+r" (v), /* 0 */ 187254612Sjkim "+m" (*p) /* 1 */ 188254612Sjkim : : "cc"); 189150627Sjhb return (v); 190150627Sjhb} 191150627Sjhb 192177276Spjd/* 193177276Spjd * Atomically add the value of v to the long integer pointed to by p and return 194177276Spjd * the previous value of *p. 195177276Spjd */ 196177276Spjdstatic __inline u_long 197177276Spjdatomic_fetchadd_long(volatile u_long *p, u_long v) 198177276Spjd{ 199177276Spjd 200177276Spjd __asm __volatile( 201177276Spjd " " MPLOCKED " " 202254610Sjkim " xaddq %0,%1 ; " 203177276Spjd "# atomic_fetchadd_long" 204254610Sjkim : "+r" (v), /* 0 */ 205254612Sjkim "+m" (*p) /* 1 */ 206254612Sjkim : : "cc"); 207177276Spjd return (v); 208177276Spjd} 209177276Spjd 210254617Sjkimstatic __inline int 211254617Sjkimatomic_testandset_int(volatile u_int *p, u_int v) 212254617Sjkim{ 213254617Sjkim u_char res; 214254617Sjkim 215254617Sjkim __asm __volatile( 216254617Sjkim " " MPLOCKED " " 217254617Sjkim " btsl %2,%1 ; " 218254617Sjkim " setc %0 ; " 219254617Sjkim "# atomic_testandset_int" 220254617Sjkim : "=q" (res), /* 0 */ 221254617Sjkim "+m" (*p) /* 1 */ 222254617Sjkim : "Ir" (v & 0x1f) /* 2 */ 223254617Sjkim : "cc"); 224254617Sjkim return (res); 225254617Sjkim} 226254617Sjkim 227254617Sjkimstatic __inline int 228254617Sjkimatomic_testandset_long(volatile u_long *p, u_int v) 229254617Sjkim{ 230254617Sjkim u_char res; 231254617Sjkim 232254617Sjkim __asm __volatile( 233254617Sjkim " " MPLOCKED " " 234254617Sjkim " btsq %2,%1 ; " 235254617Sjkim " setc %0 ; " 236254617Sjkim "# atomic_testandset_long" 237254617Sjkim : "=q" (res), /* 0 */ 238254617Sjkim "+m" (*p) /* 1 */ 239254617Sjkim : "Jr" ((u_long)(v & 0x3f)) /* 2 */ 240254617Sjkim : "cc"); 241254617Sjkim return (res); 242254617Sjkim} 243254617Sjkim 244236456Skib/* 245236456Skib * We assume that a = b will do atomic loads and stores. Due to the 246236456Skib * IA32 memory model, a simple store guarantees release semantics. 247236456Skib * 248236456Skib * However, loads may pass stores, so for atomic_load_acq we have to 249236456Skib * ensure a Store/Load barrier to do the load in SMP kernels. We use 250236456Skib * "lock cmpxchg" as recommended by the AMD Software Optimization 251236456Skib * Guide, and not mfence. For UP kernels, however, the cache of the 252236456Skib * single processor is always consistent, so we only need to take care 253236456Skib * of the compiler. 254236456Skib */ 255236456Skib#define ATOMIC_STORE(TYPE) \ 256236456Skibstatic __inline void \ 257236456Skibatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 258236456Skib{ \ 259241374Sattilio __compiler_membar(); \ 260236456Skib *p = v; \ 261236456Skib} \ 262236456Skibstruct __hack 263236456Skib 264148267Speter#if defined(_KERNEL) && !defined(SMP) 265148267Speter 266236456Skib#define ATOMIC_LOAD(TYPE, LOP) \ 26767351Sjhbstatic __inline u_##TYPE \ 26867351Sjhbatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 26967351Sjhb{ \ 270197803Sattilio u_##TYPE tmp; \ 271197803Sattilio \ 272197803Sattilio tmp = *p; \ 273241374Sattilio __compiler_membar(); \ 274197803Sattilio return (tmp); \ 275148267Speter} \ 276148267Speterstruct __hack 277148267Speter 278165635Sbde#else /* !(_KERNEL && !SMP) */ 279148267Speter 280236456Skib#define ATOMIC_LOAD(TYPE, LOP) \ 281148267Speterstatic __inline u_##TYPE \ 282148267Speteratomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 283148267Speter{ \ 28471023Sjhb u_##TYPE res; \ 28571023Sjhb \ 286165630Sbde __asm __volatile(MPLOCKED LOP \ 287165635Sbde : "=a" (res), /* 0 */ \ 288254612Sjkim "+m" (*p) /* 1 */ \ 289254612Sjkim : : "memory", "cc"); \ 29071023Sjhb return (res); \ 29171023Sjhb} \ 292122940Speterstruct __hack 293100327Smarkm 294165635Sbde#endif /* _KERNEL && !SMP */ 295148267Speter 296165578Sbde#endif /* KLD_MODULE || !__GNUCLIKE_ASM */ 297100251Smarkm 298100251SmarkmATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); 299100251SmarkmATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); 300100251SmarkmATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); 301100251SmarkmATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); 30271085Sjhb 303100251SmarkmATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); 304100251SmarkmATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); 305100251SmarkmATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); 306100251SmarkmATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); 30771085Sjhb 308100251SmarkmATOMIC_ASM(set, int, "orl %1,%0", "ir", v); 309100251SmarkmATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); 310100251SmarkmATOMIC_ASM(add, int, "addl %1,%0", "ir", v); 311100251SmarkmATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); 31271085Sjhb 313114349SpeterATOMIC_ASM(set, long, "orq %1,%0", "ir", v); 314114349SpeterATOMIC_ASM(clear, long, "andq %1,%0", "ir", ~v); 315114349SpeterATOMIC_ASM(add, long, "addq %1,%0", "ir", v); 316114349SpeterATOMIC_ASM(subtract, long, "subq %1,%0", "ir", v); 31771085Sjhb 318236456SkibATOMIC_LOAD(char, "cmpxchgb %b0,%1"); 319236456SkibATOMIC_LOAD(short, "cmpxchgw %w0,%1"); 320236456SkibATOMIC_LOAD(int, "cmpxchgl %0,%1"); 321236456SkibATOMIC_LOAD(long, "cmpxchgq %0,%1"); 32271023Sjhb 323236456SkibATOMIC_STORE(char); 324236456SkibATOMIC_STORE(short); 325236456SkibATOMIC_STORE(int); 326236456SkibATOMIC_STORE(long); 327236456Skib 32871085Sjhb#undef ATOMIC_ASM 329236456Skib#undef ATOMIC_LOAD 330236456Skib#undef ATOMIC_STORE 33167351Sjhb 332165635Sbde#ifndef WANT_FUNCTIONS 333147855Sjhb 334254617Sjkim/* Read the current value and store a new value in the destination. */ 335165578Sbde#ifdef __GNUCLIKE_ASM 336147855Sjhb 337147855Sjhbstatic __inline u_int 338254617Sjkimatomic_swap_int(volatile u_int *p, u_int v) 339147855Sjhb{ 340147855Sjhb 341165633Sbde __asm __volatile( 342147855Sjhb " xchgl %1,%0 ; " 343254617Sjkim "# atomic_swap_int" 344254617Sjkim : "+r" (v), /* 0 */ 345254612Sjkim "+m" (*p)); /* 1 */ 346254617Sjkim return (v); 347147855Sjhb} 348147855Sjhb 349147855Sjhbstatic __inline u_long 350254617Sjkimatomic_swap_long(volatile u_long *p, u_long v) 351147855Sjhb{ 352147855Sjhb 353165633Sbde __asm __volatile( 354147855Sjhb " xchgq %1,%0 ; " 355254617Sjkim "# atomic_swap_long" 356254617Sjkim : "+r" (v), /* 0 */ 357254612Sjkim "+m" (*p)); /* 1 */ 358254617Sjkim return (v); 359147855Sjhb} 360147855Sjhb 361165578Sbde#else /* !__GNUCLIKE_ASM */ 362147855Sjhb 363254617Sjkimu_int atomic_swap_int(volatile u_int *p, u_int v); 364254617Sjkimu_long atomic_swap_long(volatile u_long *p, u_long v); 365147855Sjhb 366165578Sbde#endif /* __GNUCLIKE_ASM */ 367147855Sjhb 368197803Sattilio#define atomic_set_acq_char atomic_set_barr_char 369197803Sattilio#define atomic_set_rel_char atomic_set_barr_char 370197803Sattilio#define atomic_clear_acq_char atomic_clear_barr_char 371197803Sattilio#define atomic_clear_rel_char atomic_clear_barr_char 372197803Sattilio#define atomic_add_acq_char atomic_add_barr_char 373197803Sattilio#define atomic_add_rel_char atomic_add_barr_char 374197803Sattilio#define atomic_subtract_acq_char atomic_subtract_barr_char 375197803Sattilio#define atomic_subtract_rel_char atomic_subtract_barr_char 37671085Sjhb 377197803Sattilio#define atomic_set_acq_short atomic_set_barr_short 378197803Sattilio#define atomic_set_rel_short atomic_set_barr_short 379197803Sattilio#define atomic_clear_acq_short atomic_clear_barr_short 380197803Sattilio#define atomic_clear_rel_short atomic_clear_barr_short 381197803Sattilio#define atomic_add_acq_short atomic_add_barr_short 382197803Sattilio#define atomic_add_rel_short atomic_add_barr_short 383197803Sattilio#define atomic_subtract_acq_short atomic_subtract_barr_short 384197803Sattilio#define atomic_subtract_rel_short atomic_subtract_barr_short 38571085Sjhb 386197803Sattilio#define atomic_set_acq_int atomic_set_barr_int 387197803Sattilio#define atomic_set_rel_int atomic_set_barr_int 388197803Sattilio#define atomic_clear_acq_int atomic_clear_barr_int 389197803Sattilio#define atomic_clear_rel_int atomic_clear_barr_int 390197803Sattilio#define atomic_add_acq_int atomic_add_barr_int 391197803Sattilio#define atomic_add_rel_int atomic_add_barr_int 392197803Sattilio#define atomic_subtract_acq_int atomic_subtract_barr_int 393197803Sattilio#define atomic_subtract_rel_int atomic_subtract_barr_int 394197910Sattilio#define atomic_cmpset_acq_int atomic_cmpset_int 395197910Sattilio#define atomic_cmpset_rel_int atomic_cmpset_int 39671085Sjhb 397197803Sattilio#define atomic_set_acq_long atomic_set_barr_long 398197803Sattilio#define atomic_set_rel_long atomic_set_barr_long 399197803Sattilio#define atomic_clear_acq_long atomic_clear_barr_long 400197803Sattilio#define atomic_clear_rel_long atomic_clear_barr_long 401197803Sattilio#define atomic_add_acq_long atomic_add_barr_long 402197803Sattilio#define atomic_add_rel_long atomic_add_barr_long 403197803Sattilio#define atomic_subtract_acq_long atomic_subtract_barr_long 404197803Sattilio#define atomic_subtract_rel_long atomic_subtract_barr_long 405197910Sattilio#define atomic_cmpset_acq_long atomic_cmpset_long 406197910Sattilio#define atomic_cmpset_rel_long atomic_cmpset_long 40771085Sjhb 408254617Sjkim#define atomic_readandclear_int(p) atomic_swap_int(p, 0) 409254617Sjkim#define atomic_readandclear_long(p) atomic_swap_long(p, 0) 410254617Sjkim 411147855Sjhb/* Operations on 8-bit bytes. */ 41271085Sjhb#define atomic_set_8 atomic_set_char 41371085Sjhb#define atomic_set_acq_8 atomic_set_acq_char 41471085Sjhb#define atomic_set_rel_8 atomic_set_rel_char 41571085Sjhb#define atomic_clear_8 atomic_clear_char 41671085Sjhb#define atomic_clear_acq_8 atomic_clear_acq_char 41771085Sjhb#define atomic_clear_rel_8 atomic_clear_rel_char 41871085Sjhb#define atomic_add_8 atomic_add_char 41971085Sjhb#define atomic_add_acq_8 atomic_add_acq_char 42071085Sjhb#define atomic_add_rel_8 atomic_add_rel_char 42171085Sjhb#define atomic_subtract_8 atomic_subtract_char 42271085Sjhb#define atomic_subtract_acq_8 atomic_subtract_acq_char 42371085Sjhb#define atomic_subtract_rel_8 atomic_subtract_rel_char 42471085Sjhb#define atomic_load_acq_8 atomic_load_acq_char 42571085Sjhb#define atomic_store_rel_8 atomic_store_rel_char 42671085Sjhb 427147855Sjhb/* Operations on 16-bit words. */ 42871085Sjhb#define atomic_set_16 atomic_set_short 42971085Sjhb#define atomic_set_acq_16 atomic_set_acq_short 43071085Sjhb#define atomic_set_rel_16 atomic_set_rel_short 43171085Sjhb#define atomic_clear_16 atomic_clear_short 43271085Sjhb#define atomic_clear_acq_16 atomic_clear_acq_short 43371085Sjhb#define atomic_clear_rel_16 atomic_clear_rel_short 43471085Sjhb#define atomic_add_16 atomic_add_short 43571085Sjhb#define atomic_add_acq_16 atomic_add_acq_short 43671085Sjhb#define atomic_add_rel_16 atomic_add_rel_short 43771085Sjhb#define atomic_subtract_16 atomic_subtract_short 43871085Sjhb#define atomic_subtract_acq_16 atomic_subtract_acq_short 43971085Sjhb#define atomic_subtract_rel_16 atomic_subtract_rel_short 44071085Sjhb#define atomic_load_acq_16 atomic_load_acq_short 44171085Sjhb#define atomic_store_rel_16 atomic_store_rel_short 44271085Sjhb 443147855Sjhb/* Operations on 32-bit double words. */ 44471085Sjhb#define atomic_set_32 atomic_set_int 44571085Sjhb#define atomic_set_acq_32 atomic_set_acq_int 44671085Sjhb#define atomic_set_rel_32 atomic_set_rel_int 44771085Sjhb#define atomic_clear_32 atomic_clear_int 44871085Sjhb#define atomic_clear_acq_32 atomic_clear_acq_int 44971085Sjhb#define atomic_clear_rel_32 atomic_clear_rel_int 45071085Sjhb#define atomic_add_32 atomic_add_int 45171085Sjhb#define atomic_add_acq_32 atomic_add_acq_int 45271085Sjhb#define atomic_add_rel_32 atomic_add_rel_int 45371085Sjhb#define atomic_subtract_32 atomic_subtract_int 45471085Sjhb#define atomic_subtract_acq_32 atomic_subtract_acq_int 45571085Sjhb#define atomic_subtract_rel_32 atomic_subtract_rel_int 45671085Sjhb#define atomic_load_acq_32 atomic_load_acq_int 45771085Sjhb#define atomic_store_rel_32 atomic_store_rel_int 45871085Sjhb#define atomic_cmpset_32 atomic_cmpset_int 45971085Sjhb#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 46071085Sjhb#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 461254617Sjkim#define atomic_swap_32 atomic_swap_int 46271085Sjhb#define atomic_readandclear_32 atomic_readandclear_int 463150627Sjhb#define atomic_fetchadd_32 atomic_fetchadd_int 464254617Sjkim#define atomic_testandset_32 atomic_testandset_int 46571085Sjhb 466149233Sjhb/* Operations on 64-bit quad words. */ 467149233Sjhb#define atomic_set_64 atomic_set_long 468149233Sjhb#define atomic_set_acq_64 atomic_set_acq_long 469149233Sjhb#define atomic_set_rel_64 atomic_set_rel_long 470149233Sjhb#define atomic_clear_64 atomic_clear_long 471149233Sjhb#define atomic_clear_acq_64 atomic_clear_acq_long 472149233Sjhb#define atomic_clear_rel_64 atomic_clear_rel_long 473149233Sjhb#define atomic_add_64 atomic_add_long 474149233Sjhb#define atomic_add_acq_64 atomic_add_acq_long 475149233Sjhb#define atomic_add_rel_64 atomic_add_rel_long 476149233Sjhb#define atomic_subtract_64 atomic_subtract_long 477149233Sjhb#define atomic_subtract_acq_64 atomic_subtract_acq_long 478149233Sjhb#define atomic_subtract_rel_64 atomic_subtract_rel_long 479149233Sjhb#define atomic_load_acq_64 atomic_load_acq_long 480149233Sjhb#define atomic_store_rel_64 atomic_store_rel_long 481149233Sjhb#define atomic_cmpset_64 atomic_cmpset_long 482149233Sjhb#define atomic_cmpset_acq_64 atomic_cmpset_acq_long 483149233Sjhb#define atomic_cmpset_rel_64 atomic_cmpset_rel_long 484254617Sjkim#define atomic_swap_64 atomic_swap_long 485149233Sjhb#define atomic_readandclear_64 atomic_readandclear_long 486254617Sjkim#define atomic_testandset_64 atomic_testandset_long 487149233Sjhb 488147855Sjhb/* Operations on pointers. */ 489148067Sjhb#define atomic_set_ptr atomic_set_long 490148067Sjhb#define atomic_set_acq_ptr atomic_set_acq_long 491148067Sjhb#define atomic_set_rel_ptr atomic_set_rel_long 492148067Sjhb#define atomic_clear_ptr atomic_clear_long 493148067Sjhb#define atomic_clear_acq_ptr atomic_clear_acq_long 494148067Sjhb#define atomic_clear_rel_ptr atomic_clear_rel_long 495148067Sjhb#define atomic_add_ptr atomic_add_long 496148067Sjhb#define atomic_add_acq_ptr atomic_add_acq_long 497148067Sjhb#define atomic_add_rel_ptr atomic_add_rel_long 498148067Sjhb#define atomic_subtract_ptr atomic_subtract_long 499148067Sjhb#define atomic_subtract_acq_ptr atomic_subtract_acq_long 500148067Sjhb#define atomic_subtract_rel_ptr atomic_subtract_rel_long 501148067Sjhb#define atomic_load_acq_ptr atomic_load_acq_long 502148067Sjhb#define atomic_store_rel_ptr atomic_store_rel_long 503148067Sjhb#define atomic_cmpset_ptr atomic_cmpset_long 504148067Sjhb#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 505148067Sjhb#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 506254617Sjkim#define atomic_swap_ptr atomic_swap_long 507148067Sjhb#define atomic_readandclear_ptr atomic_readandclear_long 50865514Sphk 509165635Sbde#endif /* !WANT_FUNCTIONS */ 510165633Sbde 511165633Sbde#endif /* !_MACHINE_ATOMIC_H_ */ 512