atomic.h revision 329383
138517Sdfr/*- 238517Sdfr * Copyright (c) 1998 Doug Rabson 338517Sdfr * All rights reserved. 438517Sdfr * 538517Sdfr * Redistribution and use in source and binary forms, with or without 638517Sdfr * modification, are permitted provided that the following conditions 738517Sdfr * are met: 838517Sdfr * 1. Redistributions of source code must retain the above copyright 938517Sdfr * notice, this list of conditions and the following disclaimer. 1038517Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1138517Sdfr * notice, this list of conditions and the following disclaimer in the 1238517Sdfr * documentation and/or other materials provided with the distribution. 1338517Sdfr * 1438517Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1538517Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1638517Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1738517Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1838517Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1938517Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2038517Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2138517Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2238517Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2338517Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2438517Sdfr * SUCH DAMAGE. 2538517Sdfr * 2650477Speter * $FreeBSD: stable/11/sys/i386/include/atomic.h 329383 2018-02-16 16:41:19Z markj $ 2738517Sdfr */ 2838517Sdfr#ifndef _MACHINE_ATOMIC_H_ 29147855Sjhb#define _MACHINE_ATOMIC_H_ 3038517Sdfr 31143063Sjoerg#ifndef _SYS_CDEFS_H_ 32143063Sjoerg#error this file needs sys/cdefs.h as a prerequisite 33143063Sjoerg#endif 34143063Sjoerg 35327195Skib#include <sys/atomic_common.h> 36327195Skib 37254619Sjkim#ifdef _KERNEL 38254619Sjkim#include <machine/md_var.h> 39254619Sjkim#include <machine/specialreg.h> 40254619Sjkim#endif 41254619Sjkim 42286051Skib#ifndef __OFFSETOF_MONITORBUF 43286051Skib/* 44286051Skib * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). 45286051Skib * 46286051Skib * The open-coded number is used instead of the symbolic expression to 47286051Skib * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. 48286051Skib * An assertion in i386/vm_machdep.c ensures that the value is correct. 49286051Skib */ 50286051Skib#define __OFFSETOF_MONITORBUF 0x180 51185162Skmacy 52286051Skibstatic __inline void 53286051Skib__mbk(void) 54286051Skib{ 55286051Skib 56286051Skib __asm __volatile("lock; addl $0,%%fs:%0" 57286051Skib : "+m" (*(u_int *)__OFFSETOF_MONITORBUF) : : "memory", "cc"); 58286051Skib} 59286051Skib 60286051Skibstatic __inline void 61286051Skib__mbu(void) 62286051Skib{ 63286051Skib 64286051Skib __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); 65286051Skib} 66286051Skib#endif 67286051Skib 6838517Sdfr/* 69165635Sbde * Various simple operations on memory, each of which is atomic in the 70165635Sbde * presence of interrupts and multiple processors. 7138517Sdfr * 72165633Sbde * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) 73165633Sbde * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) 74165633Sbde * atomic_add_char(P, V) (*(u_char *)(P) += (V)) 75165633Sbde * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) 7648797Salc * 77165633Sbde * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) 78165633Sbde * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) 79165633Sbde * atomic_add_short(P, V) (*(u_short *)(P) += (V)) 80165633Sbde * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) 8148797Salc * 82165633Sbde * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) 83165633Sbde * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) 84165633Sbde * atomic_add_int(P, V) (*(u_int *)(P) += (V)) 85165633Sbde * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) 86254617Sjkim * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) 87165635Sbde * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) 8848797Salc * 89165633Sbde * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) 90165633Sbde * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) 91165633Sbde * atomic_add_long(P, V) (*(u_long *)(P) += (V)) 92165633Sbde * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) 93254617Sjkim * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) 94165635Sbde * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) 9538517Sdfr */ 9638517Sdfr 9748797Salc/* 9849999Salc * The above functions are expanded inline in the statically-linked 9949999Salc * kernel. Lock prefixes are generated if an SMP kernel is being 10049999Salc * built. 10149999Salc * 10249999Salc * Kernel modules call real functions which are built into the kernel. 10349999Salc * This allows kernel modules to be portable between UP and SMP systems. 10448797Salc */ 105147855Sjhb#if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM) 106147855Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 107197803Sattiliovoid atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ 108197803Sattiliovoid atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 10949999Salc 110329383Smarkjint atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); 111329383Smarkjint atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); 112208332Sphkint atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); 113329383Smarkjint atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); 114329383Smarkjint atomic_fcmpset_short(volatile u_short *dst, u_short *expect, 115329383Smarkj u_short src); 116315371Smjgint atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); 117165633Sbdeu_int atomic_fetchadd_int(volatile u_int *p, u_int v); 118254617Sjkimint atomic_testandset_int(volatile u_int *p, u_int v); 119299912Ssepheint atomic_testandclear_int(volatile u_int *p, u_int v); 120285283Skibvoid atomic_thread_fence_acq(void); 121285283Skibvoid atomic_thread_fence_acq_rel(void); 122285283Skibvoid atomic_thread_fence_rel(void); 123285283Skibvoid atomic_thread_fence_seq_cst(void); 12465514Sphk 125284901Skib#define ATOMIC_LOAD(TYPE) \ 126236456Skibu_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) 127236456Skib#define ATOMIC_STORE(TYPE) \ 128100251Smarkmvoid atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 12971085Sjhb 130254620Sjkimint atomic_cmpset_64(volatile uint64_t *, uint64_t, uint64_t); 131254619Sjkimuint64_t atomic_load_acq_64(volatile uint64_t *); 132254619Sjkimvoid atomic_store_rel_64(volatile uint64_t *, uint64_t); 133254620Sjkimuint64_t atomic_swap_64(volatile uint64_t *, uint64_t); 134326514Shselaskyuint64_t atomic_fetchadd_64(volatile uint64_t *, uint64_t); 135254619Sjkim 136147855Sjhb#else /* !KLD_MODULE && __GNUCLIKE_ASM */ 13772358Smarkm 13884679Sjhb/* 139165635Sbde * For userland, always use lock prefixes so that the binaries will run 140165635Sbde * on both SMP and !SMP systems. 14184679Sjhb */ 14284679Sjhb#if defined(SMP) || !defined(_KERNEL) 143165630Sbde#define MPLOCKED "lock ; " 14490515Sbde#else 145147855Sjhb#define MPLOCKED 14690515Sbde#endif 14738517Sdfr 14848797Salc/* 149197803Sattilio * The assembly is volatilized to avoid code chunk removal by the compiler. 150197803Sattilio * GCC aggressively reorders operations and memory clobbering is necessary 151197803Sattilio * in order to avoid that for memory barriers. 15248797Salc */ 153147855Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 15448797Salcstatic __inline void \ 15549043Salcatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 15648797Salc{ \ 157165630Sbde __asm __volatile(MPLOCKED OP \ 158254612Sjkim : "+m" (*p) \ 159254612Sjkim : CONS (V) \ 160216524Skib : "cc"); \ 161122827Sbde} \ 162197803Sattilio \ 163197803Sattiliostatic __inline void \ 164197803Sattilioatomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 165197803Sattilio{ \ 166197803Sattilio __asm __volatile(MPLOCKED OP \ 167254612Sjkim : "+m" (*p) \ 168254612Sjkim : CONS (V) \ 169216524Skib : "memory", "cc"); \ 170197803Sattilio} \ 171122827Sbdestruct __hack 172100327Smarkm 17365514Sphk/* 174329383Smarkj * Atomic compare and set, used by the mutex functions. 17565514Sphk * 176329383Smarkj * cmpset: 177329383Smarkj * if (*dst == expect) 178329383Smarkj * *dst = src 17965514Sphk * 180329383Smarkj * fcmpset: 181329383Smarkj * if (*dst == *expect) 182329383Smarkj * *dst = src 183329383Smarkj * else 184329383Smarkj * *expect = *dst 185329383Smarkj * 186329383Smarkj * Returns 0 on failure, non-zero on success. 18765514Sphk */ 188329383Smarkj#define ATOMIC_CMPSET(TYPE, CONS) \ 189329383Smarkjstatic __inline int \ 190329383Smarkjatomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ 191329383Smarkj{ \ 192329383Smarkj u_char res; \ 193329383Smarkj \ 194329383Smarkj __asm __volatile( \ 195329383Smarkj " " MPLOCKED " " \ 196329383Smarkj " cmpxchg %3,%1 ; " \ 197329383Smarkj " sete %0 ; " \ 198329383Smarkj "# atomic_cmpset_" #TYPE " " \ 199329383Smarkj : "=q" (res), /* 0 */ \ 200329383Smarkj "+m" (*dst), /* 1 */ \ 201329383Smarkj "+a" (expect) /* 2 */ \ 202329383Smarkj : CONS (src) /* 3 */ \ 203329383Smarkj : "memory", "cc"); \ 204329383Smarkj return (res); \ 205329383Smarkj} \ 206329383Smarkj \ 207329383Smarkjstatic __inline int \ 208329383Smarkjatomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ 209329383Smarkj{ \ 210329383Smarkj u_char res; \ 211329383Smarkj \ 212329383Smarkj __asm __volatile( \ 213329383Smarkj " " MPLOCKED " " \ 214329383Smarkj " cmpxchg %3,%1 ; " \ 215329383Smarkj " sete %0 ; " \ 216329383Smarkj "# atomic_fcmpset_" #TYPE " " \ 217329383Smarkj : "=q" (res), /* 0 */ \ 218329383Smarkj "+m" (*dst), /* 1 */ \ 219329383Smarkj "+a" (*expect) /* 2 */ \ 220329383Smarkj : CONS (src) /* 3 */ \ 221329383Smarkj : "memory", "cc"); \ 222329383Smarkj return (res); \ 223197910Sattilio} 224197910Sattilio 225329383SmarkjATOMIC_CMPSET(char, "q"); 226329383SmarkjATOMIC_CMPSET(short, "r"); 227329383SmarkjATOMIC_CMPSET(int, "r"); 228315371Smjg 229150627Sjhb/* 230150627Sjhb * Atomically add the value of v to the integer pointed to by p and return 231150627Sjhb * the previous value of *p. 232150627Sjhb */ 233150627Sjhbstatic __inline u_int 234150627Sjhbatomic_fetchadd_int(volatile u_int *p, u_int v) 235150627Sjhb{ 236150627Sjhb 237165633Sbde __asm __volatile( 238165630Sbde " " MPLOCKED " " 239254610Sjkim " xaddl %0,%1 ; " 240150627Sjhb "# atomic_fetchadd_int" 241254610Sjkim : "+r" (v), /* 0 */ 242254612Sjkim "+m" (*p) /* 1 */ 243254612Sjkim : : "cc"); 244150627Sjhb return (v); 245150627Sjhb} 246150627Sjhb 247254617Sjkimstatic __inline int 248254617Sjkimatomic_testandset_int(volatile u_int *p, u_int v) 249254617Sjkim{ 250254617Sjkim u_char res; 251254617Sjkim 252254617Sjkim __asm __volatile( 253254617Sjkim " " MPLOCKED " " 254254617Sjkim " btsl %2,%1 ; " 255254617Sjkim " setc %0 ; " 256254617Sjkim "# atomic_testandset_int" 257254617Sjkim : "=q" (res), /* 0 */ 258254617Sjkim "+m" (*p) /* 1 */ 259254617Sjkim : "Ir" (v & 0x1f) /* 2 */ 260254617Sjkim : "cc"); 261254617Sjkim return (res); 262254617Sjkim} 263254617Sjkim 264299912Ssephestatic __inline int 265299912Ssepheatomic_testandclear_int(volatile u_int *p, u_int v) 266299912Ssephe{ 267299912Ssephe u_char res; 268299912Ssephe 269299912Ssephe __asm __volatile( 270299912Ssephe " " MPLOCKED " " 271299912Ssephe " btrl %2,%1 ; " 272299912Ssephe " setc %0 ; " 273299912Ssephe "# atomic_testandclear_int" 274299912Ssephe : "=q" (res), /* 0 */ 275299912Ssephe "+m" (*p) /* 1 */ 276299912Ssephe : "Ir" (v & 0x1f) /* 2 */ 277299912Ssephe : "cc"); 278299912Ssephe return (res); 279299912Ssephe} 280299912Ssephe 281236456Skib/* 282236456Skib * We assume that a = b will do atomic loads and stores. Due to the 283236456Skib * IA32 memory model, a simple store guarantees release semantics. 284236456Skib * 285284901Skib * However, a load may pass a store if they are performed on distinct 286286050Skib * addresses, so we need Store/Load barrier for sequentially 287286050Skib * consistent fences in SMP kernels. We use "lock addl $0,mem" for a 288286050Skib * Store/Load barrier, as recommended by the AMD Software Optimization 289286050Skib * Guide, and not mfence. In the kernel, we use a private per-cpu 290286078Skib * cache line for "mem", to avoid introducing false data 291286078Skib * dependencies. In user space, we use the word at the top of the 292286078Skib * stack. 293284901Skib * 294284901Skib * For UP kernels, however, the memory of the single processor is 295284901Skib * always consistent, so we only need to stop the compiler from 296284901Skib * reordering accesses in a way that violates the semantics of acquire 297284901Skib * and release. 298236456Skib */ 299286051Skib 300284901Skib#if defined(_KERNEL) 301284901Skib#if defined(SMP) 302286051Skib#define __storeload_barrier() __mbk() 303284901Skib#else /* _KERNEL && UP */ 304286051Skib#define __storeload_barrier() __compiler_membar() 305284901Skib#endif /* SMP */ 306284901Skib#else /* !_KERNEL */ 307286051Skib#define __storeload_barrier() __mbu() 308284901Skib#endif /* _KERNEL*/ 309284901Skib 310284901Skib#define ATOMIC_LOAD(TYPE) \ 311284901Skibstatic __inline u_##TYPE \ 312284901Skibatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 313284901Skib{ \ 314284901Skib u_##TYPE res; \ 315284901Skib \ 316284901Skib res = *p; \ 317284901Skib __compiler_membar(); \ 318284901Skib return (res); \ 319284901Skib} \ 320122827Sbdestruct __hack 321100327Smarkm 322284901Skib#define ATOMIC_STORE(TYPE) \ 323284901Skibstatic __inline void \ 324284901Skibatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 325284901Skib{ \ 326284901Skib \ 327284901Skib __compiler_membar(); \ 328284901Skib *p = v; \ 329284901Skib} \ 330284901Skibstruct __hack 331100327Smarkm 332285283Skibstatic __inline void 333285283Skibatomic_thread_fence_acq(void) 334285283Skib{ 335285283Skib 336285283Skib __compiler_membar(); 337285283Skib} 338285283Skib 339285283Skibstatic __inline void 340285283Skibatomic_thread_fence_rel(void) 341285283Skib{ 342285283Skib 343285283Skib __compiler_membar(); 344285283Skib} 345285283Skib 346285283Skibstatic __inline void 347285283Skibatomic_thread_fence_acq_rel(void) 348285283Skib{ 349285283Skib 350285283Skib __compiler_membar(); 351285283Skib} 352285283Skib 353285283Skibstatic __inline void 354285283Skibatomic_thread_fence_seq_cst(void) 355285283Skib{ 356285283Skib 357285283Skib __storeload_barrier(); 358285283Skib} 359285283Skib 360254619Sjkim#ifdef _KERNEL 361254619Sjkim 362254619Sjkim#ifdef WANT_FUNCTIONS 363254620Sjkimint atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); 364254620Sjkimint atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); 365254619Sjkimuint64_t atomic_load_acq_64_i386(volatile uint64_t *); 366254619Sjkimuint64_t atomic_load_acq_64_i586(volatile uint64_t *); 367254619Sjkimvoid atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); 368254619Sjkimvoid atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); 369254620Sjkimuint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); 370254620Sjkimuint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); 371254619Sjkim#endif 372254619Sjkim 373254619Sjkim/* I486 does not support SMP or CMPXCHG8B. */ 374254620Sjkimstatic __inline int 375254620Sjkimatomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) 376254620Sjkim{ 377254620Sjkim volatile uint32_t *p; 378254620Sjkim u_char res; 379254620Sjkim 380254620Sjkim p = (volatile uint32_t *)dst; 381254620Sjkim __asm __volatile( 382254620Sjkim " pushfl ; " 383254620Sjkim " cli ; " 384254620Sjkim " xorl %1,%%eax ; " 385254620Sjkim " xorl %2,%%edx ; " 386254620Sjkim " orl %%edx,%%eax ; " 387254620Sjkim " jne 1f ; " 388254620Sjkim " movl %4,%1 ; " 389254620Sjkim " movl %5,%2 ; " 390254620Sjkim "1: " 391254620Sjkim " sete %3 ; " 392254620Sjkim " popfl" 393254620Sjkim : "+A" (expect), /* 0 */ 394254620Sjkim "+m" (*p), /* 1 */ 395254620Sjkim "+m" (*(p + 1)), /* 2 */ 396254620Sjkim "=q" (res) /* 3 */ 397254620Sjkim : "r" ((uint32_t)src), /* 4 */ 398254620Sjkim "r" ((uint32_t)(src >> 32)) /* 5 */ 399254620Sjkim : "memory", "cc"); 400254620Sjkim return (res); 401254620Sjkim} 402254620Sjkim 403254619Sjkimstatic __inline uint64_t 404254619Sjkimatomic_load_acq_64_i386(volatile uint64_t *p) 405254619Sjkim{ 406254619Sjkim volatile uint32_t *q; 407254619Sjkim uint64_t res; 408254619Sjkim 409254619Sjkim q = (volatile uint32_t *)p; 410254619Sjkim __asm __volatile( 411254619Sjkim " pushfl ; " 412254619Sjkim " cli ; " 413254619Sjkim " movl %1,%%eax ; " 414254619Sjkim " movl %2,%%edx ; " 415254619Sjkim " popfl" 416254619Sjkim : "=&A" (res) /* 0 */ 417254619Sjkim : "m" (*q), /* 1 */ 418254619Sjkim "m" (*(q + 1)) /* 2 */ 419254619Sjkim : "memory"); 420254619Sjkim return (res); 421254619Sjkim} 422254619Sjkim 423254619Sjkimstatic __inline void 424254619Sjkimatomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) 425254619Sjkim{ 426254619Sjkim volatile uint32_t *q; 427254619Sjkim 428254619Sjkim q = (volatile uint32_t *)p; 429254619Sjkim __asm __volatile( 430254619Sjkim " pushfl ; " 431254619Sjkim " cli ; " 432254619Sjkim " movl %%eax,%0 ; " 433254619Sjkim " movl %%edx,%1 ; " 434254619Sjkim " popfl" 435254619Sjkim : "=m" (*q), /* 0 */ 436254619Sjkim "=m" (*(q + 1)) /* 1 */ 437254619Sjkim : "A" (v) /* 2 */ 438254619Sjkim : "memory"); 439254619Sjkim} 440254619Sjkim 441254619Sjkimstatic __inline uint64_t 442254620Sjkimatomic_swap_64_i386(volatile uint64_t *p, uint64_t v) 443254620Sjkim{ 444254620Sjkim volatile uint32_t *q; 445254620Sjkim uint64_t res; 446254620Sjkim 447254620Sjkim q = (volatile uint32_t *)p; 448254620Sjkim __asm __volatile( 449254620Sjkim " pushfl ; " 450254620Sjkim " cli ; " 451254620Sjkim " movl %1,%%eax ; " 452254620Sjkim " movl %2,%%edx ; " 453254620Sjkim " movl %4,%2 ; " 454254620Sjkim " movl %3,%1 ; " 455254620Sjkim " popfl" 456254620Sjkim : "=&A" (res), /* 0 */ 457254620Sjkim "+m" (*q), /* 1 */ 458254620Sjkim "+m" (*(q + 1)) /* 2 */ 459254620Sjkim : "r" ((uint32_t)v), /* 3 */ 460254620Sjkim "r" ((uint32_t)(v >> 32))); /* 4 */ 461254620Sjkim return (res); 462254620Sjkim} 463254620Sjkim 464254620Sjkimstatic __inline int 465254620Sjkimatomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) 466254620Sjkim{ 467254620Sjkim u_char res; 468254620Sjkim 469254620Sjkim __asm __volatile( 470254620Sjkim " " MPLOCKED " " 471254620Sjkim " cmpxchg8b %1 ; " 472254620Sjkim " sete %0" 473254620Sjkim : "=q" (res), /* 0 */ 474254620Sjkim "+m" (*dst), /* 1 */ 475254620Sjkim "+A" (expect) /* 2 */ 476254620Sjkim : "b" ((uint32_t)src), /* 3 */ 477254620Sjkim "c" ((uint32_t)(src >> 32)) /* 4 */ 478254620Sjkim : "memory", "cc"); 479254620Sjkim return (res); 480254620Sjkim} 481254620Sjkim 482254620Sjkimstatic __inline uint64_t 483254619Sjkimatomic_load_acq_64_i586(volatile uint64_t *p) 484254619Sjkim{ 485254619Sjkim uint64_t res; 486254619Sjkim 487254619Sjkim __asm __volatile( 488254619Sjkim " movl %%ebx,%%eax ; " 489254619Sjkim " movl %%ecx,%%edx ; " 490254619Sjkim " " MPLOCKED " " 491254619Sjkim " cmpxchg8b %1" 492254619Sjkim : "=&A" (res), /* 0 */ 493254619Sjkim "+m" (*p) /* 1 */ 494254619Sjkim : : "memory", "cc"); 495254619Sjkim return (res); 496254619Sjkim} 497254619Sjkim 498254619Sjkimstatic __inline void 499254619Sjkimatomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) 500254619Sjkim{ 501254619Sjkim 502254619Sjkim __asm __volatile( 503254619Sjkim " movl %%eax,%%ebx ; " 504254619Sjkim " movl %%edx,%%ecx ; " 505254619Sjkim "1: " 506254619Sjkim " " MPLOCKED " " 507254619Sjkim " cmpxchg8b %0 ; " 508254619Sjkim " jne 1b" 509254619Sjkim : "+m" (*p), /* 0 */ 510254619Sjkim "+A" (v) /* 1 */ 511254619Sjkim : : "ebx", "ecx", "memory", "cc"); 512254619Sjkim} 513254619Sjkim 514254619Sjkimstatic __inline uint64_t 515254620Sjkimatomic_swap_64_i586(volatile uint64_t *p, uint64_t v) 516254620Sjkim{ 517254620Sjkim 518254620Sjkim __asm __volatile( 519254620Sjkim " movl %%eax,%%ebx ; " 520254620Sjkim " movl %%edx,%%ecx ; " 521254620Sjkim "1: " 522254620Sjkim " " MPLOCKED " " 523254620Sjkim " cmpxchg8b %0 ; " 524254620Sjkim " jne 1b" 525254620Sjkim : "+m" (*p), /* 0 */ 526254620Sjkim "+A" (v) /* 1 */ 527254620Sjkim : : "ebx", "ecx", "memory", "cc"); 528254620Sjkim return (v); 529254620Sjkim} 530254620Sjkim 531254620Sjkimstatic __inline int 532254620Sjkimatomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) 533254620Sjkim{ 534254620Sjkim 535254620Sjkim if ((cpu_feature & CPUID_CX8) == 0) 536254620Sjkim return (atomic_cmpset_64_i386(dst, expect, src)); 537254620Sjkim else 538254620Sjkim return (atomic_cmpset_64_i586(dst, expect, src)); 539254620Sjkim} 540254620Sjkim 541254620Sjkimstatic __inline uint64_t 542254619Sjkimatomic_load_acq_64(volatile uint64_t *p) 543254619Sjkim{ 544254619Sjkim 545254619Sjkim if ((cpu_feature & CPUID_CX8) == 0) 546254619Sjkim return (atomic_load_acq_64_i386(p)); 547254619Sjkim else 548254619Sjkim return (atomic_load_acq_64_i586(p)); 549254619Sjkim} 550254619Sjkim 551254619Sjkimstatic __inline void 552254619Sjkimatomic_store_rel_64(volatile uint64_t *p, uint64_t v) 553254619Sjkim{ 554254619Sjkim 555254619Sjkim if ((cpu_feature & CPUID_CX8) == 0) 556254619Sjkim atomic_store_rel_64_i386(p, v); 557254619Sjkim else 558254619Sjkim atomic_store_rel_64_i586(p, v); 559254619Sjkim} 560254619Sjkim 561254620Sjkimstatic __inline uint64_t 562254620Sjkimatomic_swap_64(volatile uint64_t *p, uint64_t v) 563254620Sjkim{ 564254620Sjkim 565254620Sjkim if ((cpu_feature & CPUID_CX8) == 0) 566254620Sjkim return (atomic_swap_64_i386(p, v)); 567254620Sjkim else 568254620Sjkim return (atomic_swap_64_i586(p, v)); 569254620Sjkim} 570254620Sjkim 571326514Shselaskystatic __inline uint64_t 572326514Shselaskyatomic_fetchadd_64(volatile uint64_t *p, uint64_t v) 573326514Shselasky{ 574326514Shselasky 575326514Shselasky for (;;) { 576326514Shselasky uint64_t t = *p; 577326514Shselasky if (atomic_cmpset_64(p, t, t + v)) 578326514Shselasky return (t); 579326514Shselasky } 580326514Shselasky} 581326514Shselasky 582254619Sjkim#endif /* _KERNEL */ 583254619Sjkim 584147855Sjhb#endif /* KLD_MODULE || !__GNUCLIKE_ASM */ 585100251Smarkm 586100251SmarkmATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); 587100251SmarkmATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); 588100251SmarkmATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); 589100251SmarkmATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); 59071085Sjhb 591100251SmarkmATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); 592100251SmarkmATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); 593100251SmarkmATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); 594100251SmarkmATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); 59571085Sjhb 596100251SmarkmATOMIC_ASM(set, int, "orl %1,%0", "ir", v); 597100251SmarkmATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); 598100251SmarkmATOMIC_ASM(add, int, "addl %1,%0", "ir", v); 599100251SmarkmATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); 60071085Sjhb 601100251SmarkmATOMIC_ASM(set, long, "orl %1,%0", "ir", v); 602100251SmarkmATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); 603100251SmarkmATOMIC_ASM(add, long, "addl %1,%0", "ir", v); 604100251SmarkmATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); 60571085Sjhb 606284901Skib#define ATOMIC_LOADSTORE(TYPE) \ 607284901Skib ATOMIC_LOAD(TYPE); \ 608284901Skib ATOMIC_STORE(TYPE) 60971023Sjhb 610284901SkibATOMIC_LOADSTORE(char); 611284901SkibATOMIC_LOADSTORE(short); 612284901SkibATOMIC_LOADSTORE(int); 613284901SkibATOMIC_LOADSTORE(long); 614236456Skib 61571085Sjhb#undef ATOMIC_ASM 616236456Skib#undef ATOMIC_LOAD 617236456Skib#undef ATOMIC_STORE 618284901Skib#undef ATOMIC_LOADSTORE 61967351Sjhb 620165635Sbde#ifndef WANT_FUNCTIONS 621147855Sjhb 622147855Sjhbstatic __inline int 623208332Sphkatomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) 624147855Sjhb{ 625147855Sjhb 626208332Sphk return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, 627147855Sjhb (u_int)src)); 628147855Sjhb} 629147855Sjhb 630177276Spjdstatic __inline u_long 631177276Spjdatomic_fetchadd_long(volatile u_long *p, u_long v) 632177276Spjd{ 633177276Spjd 634177276Spjd return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); 635177276Spjd} 636177276Spjd 637254617Sjkimstatic __inline int 638254617Sjkimatomic_testandset_long(volatile u_long *p, u_int v) 639254617Sjkim{ 640254617Sjkim 641254617Sjkim return (atomic_testandset_int((volatile u_int *)p, v)); 642254617Sjkim} 643254617Sjkim 644299912Ssephestatic __inline int 645299912Ssepheatomic_testandclear_long(volatile u_long *p, u_int v) 646299912Ssephe{ 647299912Ssephe 648299912Ssephe return (atomic_testandclear_int((volatile u_int *)p, v)); 649299912Ssephe} 650299912Ssephe 651254617Sjkim/* Read the current value and store a new value in the destination. */ 652147855Sjhb#ifdef __GNUCLIKE_ASM 653147855Sjhb 654147855Sjhbstatic __inline u_int 655254617Sjkimatomic_swap_int(volatile u_int *p, u_int v) 656147855Sjhb{ 657147855Sjhb 658165633Sbde __asm __volatile( 659147855Sjhb " xchgl %1,%0 ; " 660254617Sjkim "# atomic_swap_int" 661254617Sjkim : "+r" (v), /* 0 */ 662254612Sjkim "+m" (*p)); /* 1 */ 663254617Sjkim return (v); 664147855Sjhb} 665147855Sjhb 666147855Sjhbstatic __inline u_long 667254617Sjkimatomic_swap_long(volatile u_long *p, u_long v) 668147855Sjhb{ 669147855Sjhb 670254617Sjkim return (atomic_swap_int((volatile u_int *)p, (u_int)v)); 671147855Sjhb} 672147855Sjhb 673147855Sjhb#else /* !__GNUCLIKE_ASM */ 674147855Sjhb 675254617Sjkimu_int atomic_swap_int(volatile u_int *p, u_int v); 676254617Sjkimu_long atomic_swap_long(volatile u_long *p, u_long v); 677147855Sjhb 678147855Sjhb#endif /* __GNUCLIKE_ASM */ 679147855Sjhb 680197803Sattilio#define atomic_set_acq_char atomic_set_barr_char 681197803Sattilio#define atomic_set_rel_char atomic_set_barr_char 682197803Sattilio#define atomic_clear_acq_char atomic_clear_barr_char 683197803Sattilio#define atomic_clear_rel_char atomic_clear_barr_char 684197803Sattilio#define atomic_add_acq_char atomic_add_barr_char 685197803Sattilio#define atomic_add_rel_char atomic_add_barr_char 686197803Sattilio#define atomic_subtract_acq_char atomic_subtract_barr_char 687197803Sattilio#define atomic_subtract_rel_char atomic_subtract_barr_char 688329383Smarkj#define atomic_cmpset_acq_char atomic_cmpset_char 689329383Smarkj#define atomic_cmpset_rel_char atomic_cmpset_char 690329383Smarkj#define atomic_fcmpset_acq_char atomic_fcmpset_char 691329383Smarkj#define atomic_fcmpset_rel_char atomic_fcmpset_char 69271085Sjhb 693197803Sattilio#define atomic_set_acq_short atomic_set_barr_short 694197803Sattilio#define atomic_set_rel_short atomic_set_barr_short 695197803Sattilio#define atomic_clear_acq_short atomic_clear_barr_short 696197803Sattilio#define atomic_clear_rel_short atomic_clear_barr_short 697197803Sattilio#define atomic_add_acq_short atomic_add_barr_short 698197803Sattilio#define atomic_add_rel_short atomic_add_barr_short 699197803Sattilio#define atomic_subtract_acq_short atomic_subtract_barr_short 700197803Sattilio#define atomic_subtract_rel_short atomic_subtract_barr_short 701329383Smarkj#define atomic_cmpset_acq_short atomic_cmpset_short 702329383Smarkj#define atomic_cmpset_rel_short atomic_cmpset_short 703329383Smarkj#define atomic_fcmpset_acq_short atomic_fcmpset_short 704329383Smarkj#define atomic_fcmpset_rel_short atomic_fcmpset_short 70571085Sjhb 706197803Sattilio#define atomic_set_acq_int atomic_set_barr_int 707197803Sattilio#define atomic_set_rel_int atomic_set_barr_int 708197803Sattilio#define atomic_clear_acq_int atomic_clear_barr_int 709197803Sattilio#define atomic_clear_rel_int atomic_clear_barr_int 710197803Sattilio#define atomic_add_acq_int atomic_add_barr_int 711197803Sattilio#define atomic_add_rel_int atomic_add_barr_int 712197803Sattilio#define atomic_subtract_acq_int atomic_subtract_barr_int 713197803Sattilio#define atomic_subtract_rel_int atomic_subtract_barr_int 714197910Sattilio#define atomic_cmpset_acq_int atomic_cmpset_int 715197910Sattilio#define atomic_cmpset_rel_int atomic_cmpset_int 716315371Smjg#define atomic_fcmpset_acq_int atomic_fcmpset_int 717315371Smjg#define atomic_fcmpset_rel_int atomic_fcmpset_int 71871085Sjhb 719197803Sattilio#define atomic_set_acq_long atomic_set_barr_long 720197803Sattilio#define atomic_set_rel_long atomic_set_barr_long 721197803Sattilio#define atomic_clear_acq_long atomic_clear_barr_long 722197803Sattilio#define atomic_clear_rel_long atomic_clear_barr_long 723197803Sattilio#define atomic_add_acq_long atomic_add_barr_long 724197803Sattilio#define atomic_add_rel_long atomic_add_barr_long 725197803Sattilio#define atomic_subtract_acq_long atomic_subtract_barr_long 726197803Sattilio#define atomic_subtract_rel_long atomic_subtract_barr_long 727197910Sattilio#define atomic_cmpset_acq_long atomic_cmpset_long 728197910Sattilio#define atomic_cmpset_rel_long atomic_cmpset_long 729315371Smjg#define atomic_fcmpset_acq_long atomic_fcmpset_long 730315371Smjg#define atomic_fcmpset_rel_long atomic_fcmpset_long 73171085Sjhb 732254617Sjkim#define atomic_readandclear_int(p) atomic_swap_int(p, 0) 733254617Sjkim#define atomic_readandclear_long(p) atomic_swap_long(p, 0) 734254617Sjkim 735147855Sjhb/* Operations on 8-bit bytes. */ 73671085Sjhb#define atomic_set_8 atomic_set_char 73771085Sjhb#define atomic_set_acq_8 atomic_set_acq_char 73871085Sjhb#define atomic_set_rel_8 atomic_set_rel_char 73971085Sjhb#define atomic_clear_8 atomic_clear_char 74071085Sjhb#define atomic_clear_acq_8 atomic_clear_acq_char 74171085Sjhb#define atomic_clear_rel_8 atomic_clear_rel_char 74271085Sjhb#define atomic_add_8 atomic_add_char 74371085Sjhb#define atomic_add_acq_8 atomic_add_acq_char 74471085Sjhb#define atomic_add_rel_8 atomic_add_rel_char 74571085Sjhb#define atomic_subtract_8 atomic_subtract_char 74671085Sjhb#define atomic_subtract_acq_8 atomic_subtract_acq_char 74771085Sjhb#define atomic_subtract_rel_8 atomic_subtract_rel_char 74871085Sjhb#define atomic_load_acq_8 atomic_load_acq_char 74971085Sjhb#define atomic_store_rel_8 atomic_store_rel_char 750329383Smarkj#define atomic_cmpset_8 atomic_cmpset_char 751329383Smarkj#define atomic_cmpset_acq_8 atomic_cmpset_acq_char 752329383Smarkj#define atomic_cmpset_rel_8 atomic_cmpset_rel_char 753329383Smarkj#define atomic_fcmpset_8 atomic_fcmpset_char 754329383Smarkj#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 755329383Smarkj#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 75671085Sjhb 757147855Sjhb/* Operations on 16-bit words. */ 75871085Sjhb#define atomic_set_16 atomic_set_short 75971085Sjhb#define atomic_set_acq_16 atomic_set_acq_short 76071085Sjhb#define atomic_set_rel_16 atomic_set_rel_short 76171085Sjhb#define atomic_clear_16 atomic_clear_short 76271085Sjhb#define atomic_clear_acq_16 atomic_clear_acq_short 76371085Sjhb#define atomic_clear_rel_16 atomic_clear_rel_short 76471085Sjhb#define atomic_add_16 atomic_add_short 76571085Sjhb#define atomic_add_acq_16 atomic_add_acq_short 76671085Sjhb#define atomic_add_rel_16 atomic_add_rel_short 76771085Sjhb#define atomic_subtract_16 atomic_subtract_short 76871085Sjhb#define atomic_subtract_acq_16 atomic_subtract_acq_short 76971085Sjhb#define atomic_subtract_rel_16 atomic_subtract_rel_short 77071085Sjhb#define atomic_load_acq_16 atomic_load_acq_short 77171085Sjhb#define atomic_store_rel_16 atomic_store_rel_short 772329383Smarkj#define atomic_cmpset_16 atomic_cmpset_short 773329383Smarkj#define atomic_cmpset_acq_16 atomic_cmpset_acq_short 774329383Smarkj#define atomic_cmpset_rel_16 atomic_cmpset_rel_short 775329383Smarkj#define atomic_fcmpset_16 atomic_fcmpset_short 776329383Smarkj#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 777329383Smarkj#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 77871085Sjhb 779147855Sjhb/* Operations on 32-bit double words. */ 78071085Sjhb#define atomic_set_32 atomic_set_int 78171085Sjhb#define atomic_set_acq_32 atomic_set_acq_int 78271085Sjhb#define atomic_set_rel_32 atomic_set_rel_int 78371085Sjhb#define atomic_clear_32 atomic_clear_int 78471085Sjhb#define atomic_clear_acq_32 atomic_clear_acq_int 78571085Sjhb#define atomic_clear_rel_32 atomic_clear_rel_int 78671085Sjhb#define atomic_add_32 atomic_add_int 78771085Sjhb#define atomic_add_acq_32 atomic_add_acq_int 78871085Sjhb#define atomic_add_rel_32 atomic_add_rel_int 78971085Sjhb#define atomic_subtract_32 atomic_subtract_int 79071085Sjhb#define atomic_subtract_acq_32 atomic_subtract_acq_int 79171085Sjhb#define atomic_subtract_rel_32 atomic_subtract_rel_int 79271085Sjhb#define atomic_load_acq_32 atomic_load_acq_int 79371085Sjhb#define atomic_store_rel_32 atomic_store_rel_int 79471085Sjhb#define atomic_cmpset_32 atomic_cmpset_int 79571085Sjhb#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 79671085Sjhb#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 797315371Smjg#define atomic_fcmpset_32 atomic_fcmpset_int 798315371Smjg#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 799315371Smjg#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 800254617Sjkim#define atomic_swap_32 atomic_swap_int 80171085Sjhb#define atomic_readandclear_32 atomic_readandclear_int 802150627Sjhb#define atomic_fetchadd_32 atomic_fetchadd_int 803254617Sjkim#define atomic_testandset_32 atomic_testandset_int 804299912Ssephe#define atomic_testandclear_32 atomic_testandclear_int 80571085Sjhb 806147855Sjhb/* Operations on pointers. */ 807157212Sdes#define atomic_set_ptr(p, v) \ 808157212Sdes atomic_set_int((volatile u_int *)(p), (u_int)(v)) 809157212Sdes#define atomic_set_acq_ptr(p, v) \ 810157212Sdes atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) 811157212Sdes#define atomic_set_rel_ptr(p, v) \ 812157212Sdes atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) 813157212Sdes#define atomic_clear_ptr(p, v) \ 814157212Sdes atomic_clear_int((volatile u_int *)(p), (u_int)(v)) 815157212Sdes#define atomic_clear_acq_ptr(p, v) \ 816157212Sdes atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) 817157212Sdes#define atomic_clear_rel_ptr(p, v) \ 818157212Sdes atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) 819157212Sdes#define atomic_add_ptr(p, v) \ 820157212Sdes atomic_add_int((volatile u_int *)(p), (u_int)(v)) 821157212Sdes#define atomic_add_acq_ptr(p, v) \ 822157212Sdes atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) 823157212Sdes#define atomic_add_rel_ptr(p, v) \ 824157212Sdes atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) 825157212Sdes#define atomic_subtract_ptr(p, v) \ 826157212Sdes atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) 827157212Sdes#define atomic_subtract_acq_ptr(p, v) \ 828157212Sdes atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) 829157212Sdes#define atomic_subtract_rel_ptr(p, v) \ 830157212Sdes atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) 831157212Sdes#define atomic_load_acq_ptr(p) \ 832157212Sdes atomic_load_acq_int((volatile u_int *)(p)) 833157212Sdes#define atomic_store_rel_ptr(p, v) \ 834157212Sdes atomic_store_rel_int((volatile u_int *)(p), (v)) 835157212Sdes#define atomic_cmpset_ptr(dst, old, new) \ 836157212Sdes atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 837157212Sdes#define atomic_cmpset_acq_ptr(dst, old, new) \ 838165633Sbde atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ 839165633Sbde (u_int)(new)) 840157212Sdes#define atomic_cmpset_rel_ptr(dst, old, new) \ 841165633Sbde atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ 842165633Sbde (u_int)(new)) 843315371Smjg#define atomic_fcmpset_ptr(dst, old, new) \ 844315371Smjg atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new)) 845315371Smjg#define atomic_fcmpset_acq_ptr(dst, old, new) \ 846315371Smjg atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \ 847315371Smjg (u_int)(new)) 848315371Smjg#define atomic_fcmpset_rel_ptr(dst, old, new) \ 849315371Smjg atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \ 850315371Smjg (u_int)(new)) 851254617Sjkim#define atomic_swap_ptr(p, v) \ 852254617Sjkim atomic_swap_int((volatile u_int *)(p), (u_int)(v)) 853157212Sdes#define atomic_readandclear_ptr(p) \ 854157212Sdes atomic_readandclear_int((volatile u_int *)(p)) 85565514Sphk 856165635Sbde#endif /* !WANT_FUNCTIONS */ 857165633Sbde 858286051Skib#if defined(_KERNEL) 859286051Skib#define mb() __mbk() 860286051Skib#define wmb() __mbk() 861286051Skib#define rmb() __mbk() 862286051Skib#else 863286051Skib#define mb() __mbu() 864286051Skib#define wmb() __mbu() 865286051Skib#define rmb() __mbu() 866286051Skib#endif 867286051Skib 868165633Sbde#endif /* !_MACHINE_ATOMIC_H_ */ 869