atomic.h revision 286050
1169689Skan/*- 2169689Skan * Copyright (c) 1998 Doug Rabson 3169689Skan * All rights reserved. 4169689Skan * 5169689Skan * Redistribution and use in source and binary forms, with or without 6169689Skan * modification, are permitted provided that the following conditions 7169689Skan * are met: 8169689Skan * 1. Redistributions of source code must retain the above copyright 9169689Skan * notice, this list of conditions and the following disclaimer. 10169689Skan * 2. Redistributions in binary form must reproduce the above copyright 11169689Skan * notice, this list of conditions and the following disclaimer in the 12169689Skan * documentation and/or other materials provided with the distribution. 13169689Skan * 14169689Skan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15169689Skan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16169689Skan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17169689Skan * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18169689Skan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19169689Skan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20169689Skan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21169689Skan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22169689Skan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23169689Skan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24169689Skan * SUCH DAMAGE. 25169689Skan * 26169689Skan * $FreeBSD: head/sys/i386/include/atomic.h 286050 2015-07-29 23:59:17Z kib $ 27169689Skan */ 28169689Skan#ifndef _MACHINE_ATOMIC_H_ 29169689Skan#define _MACHINE_ATOMIC_H_ 30169689Skan 31169689Skan#ifndef _SYS_CDEFS_H_ 32169689Skan#error this file needs sys/cdefs.h as a prerequisite 33169689Skan#endif 34169689Skan 35169689Skan#ifdef _KERNEL 36169689Skan#include <machine/md_var.h> 37169689Skan#include <machine/specialreg.h> 38169689Skan#endif 39169689Skan 40169689Skan#define mb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") 41169689Skan#define wmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") 42169689Skan#define rmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") 43169689Skan 44169689Skan/* 45169689Skan * Various simple operations on memory, each of which is atomic in the 46169689Skan * presence of interrupts and multiple processors. 47169689Skan * 48169689Skan * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) 49169689Skan * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) 50169689Skan * atomic_add_char(P, V) (*(u_char *)(P) += (V)) 51169689Skan * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) 52169689Skan * 53169689Skan * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) 54169689Skan * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) 55169689Skan * atomic_add_short(P, V) (*(u_short *)(P) += (V)) 56169689Skan * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) 57169689Skan * 58169689Skan * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) 59169689Skan * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) 60169689Skan * atomic_add_int(P, V) (*(u_int *)(P) += (V)) 61169689Skan * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) 62169689Skan * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) 63169689Skan * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) 64169689Skan * 65169689Skan * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) 66169689Skan * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) 67169689Skan * atomic_add_long(P, V) (*(u_long *)(P) += (V)) 68169689Skan * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) 69169689Skan * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) 70169689Skan * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) 71169689Skan */ 72169689Skan 73169689Skan/* 74169689Skan * The above functions are expanded inline in the statically-linked 75169689Skan * kernel. Lock prefixes are generated if an SMP kernel is being 76169689Skan * built. 77169689Skan * 78169689Skan * Kernel modules call real functions which are built into the kernel. 79169689Skan * This allows kernel modules to be portable between UP and SMP systems. 80169689Skan */ 81169689Skan#if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM) 82169689Skan#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 83169689Skanvoid atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ 84169689Skanvoid atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 85169689Skan 86169689Skanint atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); 87169689Skanu_int atomic_fetchadd_int(volatile u_int *p, u_int v); 88169689Skanint atomic_testandset_int(volatile u_int *p, u_int v); 89169689Skanvoid atomic_thread_fence_acq(void); 90169689Skanvoid atomic_thread_fence_acq_rel(void); 91169689Skanvoid atomic_thread_fence_rel(void); 92169689Skanvoid atomic_thread_fence_seq_cst(void); 93169689Skan 94169689Skan#define ATOMIC_LOAD(TYPE) \ 95169689Skanu_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) 96169689Skan#define ATOMIC_STORE(TYPE) \ 97169689Skanvoid atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) 98169689Skan 99169689Skanint atomic_cmpset_64(volatile uint64_t *, uint64_t, uint64_t); 100169689Skanuint64_t atomic_load_acq_64(volatile uint64_t *); 101169689Skanvoid atomic_store_rel_64(volatile uint64_t *, uint64_t); 102169689Skanuint64_t atomic_swap_64(volatile uint64_t *, uint64_t); 103169689Skan 104169689Skan#else /* !KLD_MODULE && __GNUCLIKE_ASM */ 105169689Skan 106169689Skan/* 107169689Skan * For userland, always use lock prefixes so that the binaries will run 108169689Skan * on both SMP and !SMP systems. 109169689Skan */ 110169689Skan#if defined(SMP) || !defined(_KERNEL) 111169689Skan#define MPLOCKED "lock ; " 112169689Skan#else 113169689Skan#define MPLOCKED 114169689Skan#endif 115169689Skan 116169689Skan/* 117169689Skan * The assembly is volatilized to avoid code chunk removal by the compiler. 118169689Skan * GCC aggressively reorders operations and memory clobbering is necessary 119169689Skan * in order to avoid that for memory barriers. 120169689Skan */ 121169689Skan#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 122169689Skanstatic __inline void \ 123169689Skanatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 124169689Skan{ \ 125169689Skan __asm __volatile(MPLOCKED OP \ 126169689Skan : "+m" (*p) \ 127169689Skan : CONS (V) \ 128169689Skan : "cc"); \ 129169689Skan} \ 130169689Skan \ 131169689Skanstatic __inline void \ 132169689Skanatomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 133169689Skan{ \ 134169689Skan __asm __volatile(MPLOCKED OP \ 135169689Skan : "+m" (*p) \ 136169689Skan : CONS (V) \ 137169689Skan : "memory", "cc"); \ 138169689Skan} \ 139169689Skanstruct __hack 140169689Skan 141169689Skan/* 142169689Skan * Atomic compare and set, used by the mutex functions 143169689Skan * 144169689Skan * if (*dst == expect) *dst = src (all 32 bit words) 145169689Skan * 146169689Skan * Returns 0 on failure, non-zero on success 147169689Skan */ 148169689Skan 149169689Skan#ifdef CPU_DISABLE_CMPXCHG 150169689Skan 151169689Skanstatic __inline int 152169689Skanatomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) 153169689Skan{ 154169689Skan u_char res; 155169689Skan 156169689Skan __asm __volatile( 157169689Skan " pushfl ; " 158169689Skan " cli ; " 159169689Skan " cmpl %3,%1 ; " 160169689Skan " jne 1f ; " 161169689Skan " movl %2,%1 ; " 162169689Skan "1: " 163169689Skan " sete %0 ; " 164169689Skan " popfl ; " 165169689Skan "# atomic_cmpset_int" 166169689Skan : "=q" (res), /* 0 */ 167169689Skan "+m" (*dst) /* 1 */ 168169689Skan : "r" (src), /* 2 */ 169169689Skan "r" (expect) /* 3 */ 170169689Skan : "memory"); 171169689Skan return (res); 172169689Skan} 173169689Skan 174169689Skan#else /* !CPU_DISABLE_CMPXCHG */ 175169689Skan 176static __inline int 177atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) 178{ 179 u_char res; 180 181 __asm __volatile( 182 " " MPLOCKED " " 183 " cmpxchgl %3,%1 ; " 184 " sete %0 ; " 185 "# atomic_cmpset_int" 186 : "=q" (res), /* 0 */ 187 "+m" (*dst), /* 1 */ 188 "+a" (expect) /* 2 */ 189 : "r" (src) /* 3 */ 190 : "memory", "cc"); 191 return (res); 192} 193 194#endif /* CPU_DISABLE_CMPXCHG */ 195 196/* 197 * Atomically add the value of v to the integer pointed to by p and return 198 * the previous value of *p. 199 */ 200static __inline u_int 201atomic_fetchadd_int(volatile u_int *p, u_int v) 202{ 203 204 __asm __volatile( 205 " " MPLOCKED " " 206 " xaddl %0,%1 ; " 207 "# atomic_fetchadd_int" 208 : "+r" (v), /* 0 */ 209 "+m" (*p) /* 1 */ 210 : : "cc"); 211 return (v); 212} 213 214static __inline int 215atomic_testandset_int(volatile u_int *p, u_int v) 216{ 217 u_char res; 218 219 __asm __volatile( 220 " " MPLOCKED " " 221 " btsl %2,%1 ; " 222 " setc %0 ; " 223 "# atomic_testandset_int" 224 : "=q" (res), /* 0 */ 225 "+m" (*p) /* 1 */ 226 : "Ir" (v & 0x1f) /* 2 */ 227 : "cc"); 228 return (res); 229} 230 231/* 232 * We assume that a = b will do atomic loads and stores. Due to the 233 * IA32 memory model, a simple store guarantees release semantics. 234 * 235 * However, a load may pass a store if they are performed on distinct 236 * addresses, so we need Store/Load barrier for sequentially 237 * consistent fences in SMP kernels. We use "lock addl $0,mem" for a 238 * Store/Load barrier, as recommended by the AMD Software Optimization 239 * Guide, and not mfence. In the kernel, we use a private per-cpu 240 * cache line as the target for the locked addition, to avoid 241 * introducing false data dependencies. In userspace, a word at the 242 * top of the stack is utilized. 243 * 244 * For UP kernels, however, the memory of the single processor is 245 * always consistent, so we only need to stop the compiler from 246 * reordering accesses in a way that violates the semantics of acquire 247 * and release. 248 */ 249#if defined(_KERNEL) 250 251/* 252 * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). 253 * 254 * The open-coded number is used instead of the symbolic expression to 255 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. 256 * An assertion in i386/vm_machdep.c ensures that the value is correct. 257 */ 258#define OFFSETOF_MONITORBUF 0x180 259 260#if defined(SMP) 261static __inline void 262__storeload_barrier(void) 263{ 264 265 __asm __volatile("lock; addl $0,%%fs:%0" 266 : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); 267} 268#else /* _KERNEL && UP */ 269static __inline void 270__storeload_barrier(void) 271{ 272 273 __compiler_membar(); 274} 275#endif /* SMP */ 276#else /* !_KERNEL */ 277static __inline void 278__storeload_barrier(void) 279{ 280 281 __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); 282} 283#endif /* _KERNEL*/ 284 285#define ATOMIC_LOAD(TYPE) \ 286static __inline u_##TYPE \ 287atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 288{ \ 289 u_##TYPE res; \ 290 \ 291 res = *p; \ 292 __compiler_membar(); \ 293 return (res); \ 294} \ 295struct __hack 296 297#define ATOMIC_STORE(TYPE) \ 298static __inline void \ 299atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 300{ \ 301 \ 302 __compiler_membar(); \ 303 *p = v; \ 304} \ 305struct __hack 306 307static __inline void 308atomic_thread_fence_acq(void) 309{ 310 311 __compiler_membar(); 312} 313 314static __inline void 315atomic_thread_fence_rel(void) 316{ 317 318 __compiler_membar(); 319} 320 321static __inline void 322atomic_thread_fence_acq_rel(void) 323{ 324 325 __compiler_membar(); 326} 327 328static __inline void 329atomic_thread_fence_seq_cst(void) 330{ 331 332 __storeload_barrier(); 333} 334 335#ifdef _KERNEL 336 337#ifdef WANT_FUNCTIONS 338int atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); 339int atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); 340uint64_t atomic_load_acq_64_i386(volatile uint64_t *); 341uint64_t atomic_load_acq_64_i586(volatile uint64_t *); 342void atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); 343void atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); 344uint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); 345uint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); 346#endif 347 348/* I486 does not support SMP or CMPXCHG8B. */ 349static __inline int 350atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) 351{ 352 volatile uint32_t *p; 353 u_char res; 354 355 p = (volatile uint32_t *)dst; 356 __asm __volatile( 357 " pushfl ; " 358 " cli ; " 359 " xorl %1,%%eax ; " 360 " xorl %2,%%edx ; " 361 " orl %%edx,%%eax ; " 362 " jne 1f ; " 363 " movl %4,%1 ; " 364 " movl %5,%2 ; " 365 "1: " 366 " sete %3 ; " 367 " popfl" 368 : "+A" (expect), /* 0 */ 369 "+m" (*p), /* 1 */ 370 "+m" (*(p + 1)), /* 2 */ 371 "=q" (res) /* 3 */ 372 : "r" ((uint32_t)src), /* 4 */ 373 "r" ((uint32_t)(src >> 32)) /* 5 */ 374 : "memory", "cc"); 375 return (res); 376} 377 378static __inline uint64_t 379atomic_load_acq_64_i386(volatile uint64_t *p) 380{ 381 volatile uint32_t *q; 382 uint64_t res; 383 384 q = (volatile uint32_t *)p; 385 __asm __volatile( 386 " pushfl ; " 387 " cli ; " 388 " movl %1,%%eax ; " 389 " movl %2,%%edx ; " 390 " popfl" 391 : "=&A" (res) /* 0 */ 392 : "m" (*q), /* 1 */ 393 "m" (*(q + 1)) /* 2 */ 394 : "memory"); 395 return (res); 396} 397 398static __inline void 399atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) 400{ 401 volatile uint32_t *q; 402 403 q = (volatile uint32_t *)p; 404 __asm __volatile( 405 " pushfl ; " 406 " cli ; " 407 " movl %%eax,%0 ; " 408 " movl %%edx,%1 ; " 409 " popfl" 410 : "=m" (*q), /* 0 */ 411 "=m" (*(q + 1)) /* 1 */ 412 : "A" (v) /* 2 */ 413 : "memory"); 414} 415 416static __inline uint64_t 417atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) 418{ 419 volatile uint32_t *q; 420 uint64_t res; 421 422 q = (volatile uint32_t *)p; 423 __asm __volatile( 424 " pushfl ; " 425 " cli ; " 426 " movl %1,%%eax ; " 427 " movl %2,%%edx ; " 428 " movl %4,%2 ; " 429 " movl %3,%1 ; " 430 " popfl" 431 : "=&A" (res), /* 0 */ 432 "+m" (*q), /* 1 */ 433 "+m" (*(q + 1)) /* 2 */ 434 : "r" ((uint32_t)v), /* 3 */ 435 "r" ((uint32_t)(v >> 32))); /* 4 */ 436 return (res); 437} 438 439static __inline int 440atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) 441{ 442 u_char res; 443 444 __asm __volatile( 445 " " MPLOCKED " " 446 " cmpxchg8b %1 ; " 447 " sete %0" 448 : "=q" (res), /* 0 */ 449 "+m" (*dst), /* 1 */ 450 "+A" (expect) /* 2 */ 451 : "b" ((uint32_t)src), /* 3 */ 452 "c" ((uint32_t)(src >> 32)) /* 4 */ 453 : "memory", "cc"); 454 return (res); 455} 456 457static __inline uint64_t 458atomic_load_acq_64_i586(volatile uint64_t *p) 459{ 460 uint64_t res; 461 462 __asm __volatile( 463 " movl %%ebx,%%eax ; " 464 " movl %%ecx,%%edx ; " 465 " " MPLOCKED " " 466 " cmpxchg8b %1" 467 : "=&A" (res), /* 0 */ 468 "+m" (*p) /* 1 */ 469 : : "memory", "cc"); 470 return (res); 471} 472 473static __inline void 474atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) 475{ 476 477 __asm __volatile( 478 " movl %%eax,%%ebx ; " 479 " movl %%edx,%%ecx ; " 480 "1: " 481 " " MPLOCKED " " 482 " cmpxchg8b %0 ; " 483 " jne 1b" 484 : "+m" (*p), /* 0 */ 485 "+A" (v) /* 1 */ 486 : : "ebx", "ecx", "memory", "cc"); 487} 488 489static __inline uint64_t 490atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) 491{ 492 493 __asm __volatile( 494 " movl %%eax,%%ebx ; " 495 " movl %%edx,%%ecx ; " 496 "1: " 497 " " MPLOCKED " " 498 " cmpxchg8b %0 ; " 499 " jne 1b" 500 : "+m" (*p), /* 0 */ 501 "+A" (v) /* 1 */ 502 : : "ebx", "ecx", "memory", "cc"); 503 return (v); 504} 505 506static __inline int 507atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) 508{ 509 510 if ((cpu_feature & CPUID_CX8) == 0) 511 return (atomic_cmpset_64_i386(dst, expect, src)); 512 else 513 return (atomic_cmpset_64_i586(dst, expect, src)); 514} 515 516static __inline uint64_t 517atomic_load_acq_64(volatile uint64_t *p) 518{ 519 520 if ((cpu_feature & CPUID_CX8) == 0) 521 return (atomic_load_acq_64_i386(p)); 522 else 523 return (atomic_load_acq_64_i586(p)); 524} 525 526static __inline void 527atomic_store_rel_64(volatile uint64_t *p, uint64_t v) 528{ 529 530 if ((cpu_feature & CPUID_CX8) == 0) 531 atomic_store_rel_64_i386(p, v); 532 else 533 atomic_store_rel_64_i586(p, v); 534} 535 536static __inline uint64_t 537atomic_swap_64(volatile uint64_t *p, uint64_t v) 538{ 539 540 if ((cpu_feature & CPUID_CX8) == 0) 541 return (atomic_swap_64_i386(p, v)); 542 else 543 return (atomic_swap_64_i586(p, v)); 544} 545 546#endif /* _KERNEL */ 547 548#endif /* KLD_MODULE || !__GNUCLIKE_ASM */ 549 550ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); 551ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); 552ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); 553ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); 554 555ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); 556ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); 557ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); 558ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); 559 560ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); 561ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); 562ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); 563ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); 564 565ATOMIC_ASM(set, long, "orl %1,%0", "ir", v); 566ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); 567ATOMIC_ASM(add, long, "addl %1,%0", "ir", v); 568ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); 569 570#define ATOMIC_LOADSTORE(TYPE) \ 571 ATOMIC_LOAD(TYPE); \ 572 ATOMIC_STORE(TYPE) 573 574ATOMIC_LOADSTORE(char); 575ATOMIC_LOADSTORE(short); 576ATOMIC_LOADSTORE(int); 577ATOMIC_LOADSTORE(long); 578 579#undef ATOMIC_ASM 580#undef ATOMIC_LOAD 581#undef ATOMIC_STORE 582#undef ATOMIC_LOADSTORE 583 584#ifndef WANT_FUNCTIONS 585 586static __inline int 587atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) 588{ 589 590 return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, 591 (u_int)src)); 592} 593 594static __inline u_long 595atomic_fetchadd_long(volatile u_long *p, u_long v) 596{ 597 598 return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); 599} 600 601static __inline int 602atomic_testandset_long(volatile u_long *p, u_int v) 603{ 604 605 return (atomic_testandset_int((volatile u_int *)p, v)); 606} 607 608/* Read the current value and store a new value in the destination. */ 609#ifdef __GNUCLIKE_ASM 610 611static __inline u_int 612atomic_swap_int(volatile u_int *p, u_int v) 613{ 614 615 __asm __volatile( 616 " xchgl %1,%0 ; " 617 "# atomic_swap_int" 618 : "+r" (v), /* 0 */ 619 "+m" (*p)); /* 1 */ 620 return (v); 621} 622 623static __inline u_long 624atomic_swap_long(volatile u_long *p, u_long v) 625{ 626 627 return (atomic_swap_int((volatile u_int *)p, (u_int)v)); 628} 629 630#else /* !__GNUCLIKE_ASM */ 631 632u_int atomic_swap_int(volatile u_int *p, u_int v); 633u_long atomic_swap_long(volatile u_long *p, u_long v); 634 635#endif /* __GNUCLIKE_ASM */ 636 637#define atomic_set_acq_char atomic_set_barr_char 638#define atomic_set_rel_char atomic_set_barr_char 639#define atomic_clear_acq_char atomic_clear_barr_char 640#define atomic_clear_rel_char atomic_clear_barr_char 641#define atomic_add_acq_char atomic_add_barr_char 642#define atomic_add_rel_char atomic_add_barr_char 643#define atomic_subtract_acq_char atomic_subtract_barr_char 644#define atomic_subtract_rel_char atomic_subtract_barr_char 645 646#define atomic_set_acq_short atomic_set_barr_short 647#define atomic_set_rel_short atomic_set_barr_short 648#define atomic_clear_acq_short atomic_clear_barr_short 649#define atomic_clear_rel_short atomic_clear_barr_short 650#define atomic_add_acq_short atomic_add_barr_short 651#define atomic_add_rel_short atomic_add_barr_short 652#define atomic_subtract_acq_short atomic_subtract_barr_short 653#define atomic_subtract_rel_short atomic_subtract_barr_short 654 655#define atomic_set_acq_int atomic_set_barr_int 656#define atomic_set_rel_int atomic_set_barr_int 657#define atomic_clear_acq_int atomic_clear_barr_int 658#define atomic_clear_rel_int atomic_clear_barr_int 659#define atomic_add_acq_int atomic_add_barr_int 660#define atomic_add_rel_int atomic_add_barr_int 661#define atomic_subtract_acq_int atomic_subtract_barr_int 662#define atomic_subtract_rel_int atomic_subtract_barr_int 663#define atomic_cmpset_acq_int atomic_cmpset_int 664#define atomic_cmpset_rel_int atomic_cmpset_int 665 666#define atomic_set_acq_long atomic_set_barr_long 667#define atomic_set_rel_long atomic_set_barr_long 668#define atomic_clear_acq_long atomic_clear_barr_long 669#define atomic_clear_rel_long atomic_clear_barr_long 670#define atomic_add_acq_long atomic_add_barr_long 671#define atomic_add_rel_long atomic_add_barr_long 672#define atomic_subtract_acq_long atomic_subtract_barr_long 673#define atomic_subtract_rel_long atomic_subtract_barr_long 674#define atomic_cmpset_acq_long atomic_cmpset_long 675#define atomic_cmpset_rel_long atomic_cmpset_long 676 677#define atomic_readandclear_int(p) atomic_swap_int(p, 0) 678#define atomic_readandclear_long(p) atomic_swap_long(p, 0) 679 680/* Operations on 8-bit bytes. */ 681#define atomic_set_8 atomic_set_char 682#define atomic_set_acq_8 atomic_set_acq_char 683#define atomic_set_rel_8 atomic_set_rel_char 684#define atomic_clear_8 atomic_clear_char 685#define atomic_clear_acq_8 atomic_clear_acq_char 686#define atomic_clear_rel_8 atomic_clear_rel_char 687#define atomic_add_8 atomic_add_char 688#define atomic_add_acq_8 atomic_add_acq_char 689#define atomic_add_rel_8 atomic_add_rel_char 690#define atomic_subtract_8 atomic_subtract_char 691#define atomic_subtract_acq_8 atomic_subtract_acq_char 692#define atomic_subtract_rel_8 atomic_subtract_rel_char 693#define atomic_load_acq_8 atomic_load_acq_char 694#define atomic_store_rel_8 atomic_store_rel_char 695 696/* Operations on 16-bit words. */ 697#define atomic_set_16 atomic_set_short 698#define atomic_set_acq_16 atomic_set_acq_short 699#define atomic_set_rel_16 atomic_set_rel_short 700#define atomic_clear_16 atomic_clear_short 701#define atomic_clear_acq_16 atomic_clear_acq_short 702#define atomic_clear_rel_16 atomic_clear_rel_short 703#define atomic_add_16 atomic_add_short 704#define atomic_add_acq_16 atomic_add_acq_short 705#define atomic_add_rel_16 atomic_add_rel_short 706#define atomic_subtract_16 atomic_subtract_short 707#define atomic_subtract_acq_16 atomic_subtract_acq_short 708#define atomic_subtract_rel_16 atomic_subtract_rel_short 709#define atomic_load_acq_16 atomic_load_acq_short 710#define atomic_store_rel_16 atomic_store_rel_short 711 712/* Operations on 32-bit double words. */ 713#define atomic_set_32 atomic_set_int 714#define atomic_set_acq_32 atomic_set_acq_int 715#define atomic_set_rel_32 atomic_set_rel_int 716#define atomic_clear_32 atomic_clear_int 717#define atomic_clear_acq_32 atomic_clear_acq_int 718#define atomic_clear_rel_32 atomic_clear_rel_int 719#define atomic_add_32 atomic_add_int 720#define atomic_add_acq_32 atomic_add_acq_int 721#define atomic_add_rel_32 atomic_add_rel_int 722#define atomic_subtract_32 atomic_subtract_int 723#define atomic_subtract_acq_32 atomic_subtract_acq_int 724#define atomic_subtract_rel_32 atomic_subtract_rel_int 725#define atomic_load_acq_32 atomic_load_acq_int 726#define atomic_store_rel_32 atomic_store_rel_int 727#define atomic_cmpset_32 atomic_cmpset_int 728#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 729#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 730#define atomic_swap_32 atomic_swap_int 731#define atomic_readandclear_32 atomic_readandclear_int 732#define atomic_fetchadd_32 atomic_fetchadd_int 733#define atomic_testandset_32 atomic_testandset_int 734 735/* Operations on pointers. */ 736#define atomic_set_ptr(p, v) \ 737 atomic_set_int((volatile u_int *)(p), (u_int)(v)) 738#define atomic_set_acq_ptr(p, v) \ 739 atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) 740#define atomic_set_rel_ptr(p, v) \ 741 atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) 742#define atomic_clear_ptr(p, v) \ 743 atomic_clear_int((volatile u_int *)(p), (u_int)(v)) 744#define atomic_clear_acq_ptr(p, v) \ 745 atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) 746#define atomic_clear_rel_ptr(p, v) \ 747 atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) 748#define atomic_add_ptr(p, v) \ 749 atomic_add_int((volatile u_int *)(p), (u_int)(v)) 750#define atomic_add_acq_ptr(p, v) \ 751 atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) 752#define atomic_add_rel_ptr(p, v) \ 753 atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) 754#define atomic_subtract_ptr(p, v) \ 755 atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) 756#define atomic_subtract_acq_ptr(p, v) \ 757 atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) 758#define atomic_subtract_rel_ptr(p, v) \ 759 atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) 760#define atomic_load_acq_ptr(p) \ 761 atomic_load_acq_int((volatile u_int *)(p)) 762#define atomic_store_rel_ptr(p, v) \ 763 atomic_store_rel_int((volatile u_int *)(p), (v)) 764#define atomic_cmpset_ptr(dst, old, new) \ 765 atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 766#define atomic_cmpset_acq_ptr(dst, old, new) \ 767 atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ 768 (u_int)(new)) 769#define atomic_cmpset_rel_ptr(dst, old, new) \ 770 atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ 771 (u_int)(new)) 772#define atomic_swap_ptr(p, v) \ 773 atomic_swap_int((volatile u_int *)(p), (u_int)(v)) 774#define atomic_readandclear_ptr(p) \ 775 atomic_readandclear_int((volatile u_int *)(p)) 776 777#endif /* !WANT_FUNCTIONS */ 778 779#endif /* !_MACHINE_ATOMIC_H_ */ 780