atomic.h revision 209975
1303275Sdelphij/*- 2303275Sdelphij * Copyright (c) 2008 Marcel Moolenaar 3303275Sdelphij * Copyright (c) 2001 Benno Rice 4303275Sdelphij * Copyright (c) 2001 David E. O'Brien 5303275Sdelphij * Copyright (c) 1998 Doug Rabson 6303275Sdelphij * All rights reserved. 7303275Sdelphij * 8303275Sdelphij * Redistribution and use in source and binary forms, with or without 9303275Sdelphij * modification, are permitted provided that the following conditions 10303275Sdelphij * are met: 11303275Sdelphij * 1. Redistributions of source code must retain the above copyright 12303275Sdelphij * notice, this list of conditions and the following disclaimer. 13303275Sdelphij * 2. Redistributions in binary form must reproduce the above copyright 14303275Sdelphij * notice, this list of conditions and the following disclaimer in the 15303275Sdelphij * documentation and/or other materials provided with the distribution. 16303275Sdelphij * 17303275Sdelphij * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18303275Sdelphij * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19303275Sdelphij * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20303275Sdelphij * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21303275Sdelphij * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22303275Sdelphij * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23303275Sdelphij * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24303275Sdelphij * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25303275Sdelphij * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26303275Sdelphij * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27303275Sdelphij * SUCH DAMAGE. 28303275Sdelphij * 29303275Sdelphij * $FreeBSD: head/sys/powerpc/include/atomic.h 209975 2010-07-13 05:32:19Z nwhitehorn $ 30303275Sdelphij */ 31303275Sdelphij 32303275Sdelphij#ifndef _MACHINE_ATOMIC_H_ 33303275Sdelphij#define _MACHINE_ATOMIC_H_ 34303275Sdelphij 35303275Sdelphij#ifndef _SYS_CDEFS_H_ 36303275Sdelphij#error this file needs sys/cdefs.h as a prerequisite 37303275Sdelphij#endif 38303275Sdelphij 39303275Sdelphij#define __ATOMIC_BARRIER \ 40303275Sdelphij __asm __volatile("sync" : : : "memory") 41303275Sdelphij 42303275Sdelphij#define mb() __ATOMIC_BARRIER 43303275Sdelphij#define wmb() mb() 44303275Sdelphij#define rmb() mb() 45303275Sdelphij 46303275Sdelphij/* 47303275Sdelphij * atomic_add(p, v) 48303275Sdelphij * { *p += v; } 49303275Sdelphij */ 50303275Sdelphij 51303275Sdelphij#define __ATOMIC_ADD_8(p, v, t) \ 52303275Sdelphij 8-bit atomic_add not implemented 53303275Sdelphij 54303275Sdelphij#define __ATOMIC_ADD_16(p, v, t) \ 55303275Sdelphij 16-bit atomic_add not implemented 56303275Sdelphij 57303275Sdelphij#define __ATOMIC_ADD_32(p, v, t) \ 58303275Sdelphij __asm __volatile( \ 59303275Sdelphij "1: lwarx %0, 0, %2\n" \ 60303275Sdelphij " add %0, %3, %0\n" \ 61303275Sdelphij " stwcx. %0, 0, %2\n" \ 62303275Sdelphij " bne- 1b\n" \ 63303275Sdelphij : "=&r" (t), "=m" (*p) \ 64303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 65303275Sdelphij : "cc", "memory") \ 66303275Sdelphij /* __ATOMIC_ADD_32 */ 67303275Sdelphij 68303275Sdelphij#ifdef __powerpc64__ 69303275Sdelphij#define __ATOMIC_ADD_64(p, v, t) \ 70303275Sdelphij __asm __volatile( \ 71303275Sdelphij "1: ldarx %0, 0, %2\n" \ 72303275Sdelphij " add %0, %3, %0\n" \ 73303275Sdelphij " stdcx. %0, 0, %2\n" \ 74303275Sdelphij " bne- 1b\n" \ 75303275Sdelphij : "=&r" (t), "=m" (*p) \ 76303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 77303275Sdelphij : "cc", "memory") \ 78303275Sdelphij /* __ATOMIC_ADD_64 */ 79303275Sdelphij#else 80303275Sdelphij#define __ATOMIC_ADD_64(p, v, t) \ 81303275Sdelphij 64-bit atomic_add not implemented 82303275Sdelphij#endif 83303275Sdelphij 84303275Sdelphij#define _ATOMIC_ADD(width, suffix, type) \ 85303275Sdelphij static __inline void \ 86303275Sdelphij atomic_add_##suffix(volatile type *p, type v) { \ 87303275Sdelphij type t; \ 88303275Sdelphij __ATOMIC_ADD_##width(p, v, t); \ 89303275Sdelphij } \ 90303275Sdelphij \ 91303275Sdelphij static __inline void \ 92303275Sdelphij atomic_add_acq_##suffix(volatile type *p, type v) { \ 93303275Sdelphij type t; \ 94303275Sdelphij __ATOMIC_ADD_##width(p, v, t); \ 95303275Sdelphij __ATOMIC_BARRIER; \ 96303275Sdelphij } \ 97303275Sdelphij \ 98303275Sdelphij static __inline void \ 99303275Sdelphij atomic_add_rel_##suffix(volatile type *p, type v) { \ 100303275Sdelphij type t; \ 101303275Sdelphij __ATOMIC_BARRIER; \ 102303275Sdelphij __ATOMIC_ADD_##width(p, v, t); \ 103303275Sdelphij } \ 104303275Sdelphij /* _ATOMIC_ADD */ 105303275Sdelphij 106303275Sdelphij#if 0 107303275Sdelphij_ATOMIC_ADD(8, 8, uint8_t) 108303275Sdelphij_ATOMIC_ADD(8, char, u_char) 109303275Sdelphij_ATOMIC_ADD(16, 16, uint16_t) 110303275Sdelphij_ATOMIC_ADD(16, short, u_short) 111303275Sdelphij#endif 112303275Sdelphij_ATOMIC_ADD(32, 32, uint32_t) 113303275Sdelphij_ATOMIC_ADD(32, int, u_int) 114303275Sdelphij#ifdef __powerpc64__ 115303275Sdelphij_ATOMIC_ADD(64, 64, uint64_t) 116303275Sdelphij_ATOMIC_ADD(64, long, u_long) 117303275Sdelphij_ATOMIC_ADD(64, ptr, uintptr_t) 118303275Sdelphij#else 119303275Sdelphij_ATOMIC_ADD(32, long, u_long) 120303275Sdelphij_ATOMIC_ADD(32, ptr, uintptr_t) 121303275Sdelphij#endif 122303275Sdelphij 123303275Sdelphij#undef _ATOMIC_ADD 124303275Sdelphij#undef __ATOMIC_ADD_64 125303275Sdelphij#undef __ATOMIC_ADD_32 126303275Sdelphij#undef __ATOMIC_ADD_16 127303275Sdelphij#undef __ATOMIC_ADD_8 128303275Sdelphij 129303275Sdelphij/* 130303275Sdelphij * atomic_clear(p, v) 131303275Sdelphij * { *p &= ~v; } 132303275Sdelphij */ 133303275Sdelphij 134303275Sdelphij#define __ATOMIC_CLEAR_8(p, v, t) \ 135303275Sdelphij 8-bit atomic_clear not implemented 136303275Sdelphij 137303275Sdelphij#define __ATOMIC_CLEAR_16(p, v, t) \ 138303275Sdelphij 16-bit atomic_clear not implemented 139303275Sdelphij 140303275Sdelphij#define __ATOMIC_CLEAR_32(p, v, t) \ 141303275Sdelphij __asm __volatile( \ 142303275Sdelphij "1: lwarx %0, 0, %2\n" \ 143303275Sdelphij " andc %0, %0, %3\n" \ 144303275Sdelphij " stwcx. %0, 0, %2\n" \ 145303275Sdelphij " bne- 1b\n" \ 146303275Sdelphij : "=&r" (t), "=m" (*p) \ 147303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 148303275Sdelphij : "cc", "memory") \ 149303275Sdelphij /* __ATOMIC_CLEAR_32 */ 150303275Sdelphij 151303275Sdelphij#ifdef __powerpc64__ 152303275Sdelphij#define __ATOMIC_CLEAR_64(p, v, t) \ 153303275Sdelphij __asm __volatile( \ 154303275Sdelphij "1: ldarx %0, 0, %2\n" \ 155303275Sdelphij " andc %0, %0, %3\n" \ 156303275Sdelphij " stdcx. %0, 0, %2\n" \ 157303275Sdelphij " bne- 1b\n" \ 158303275Sdelphij : "=&r" (t), "=m" (*p) \ 159303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 160303275Sdelphij : "cc", "memory") \ 161303275Sdelphij /* __ATOMIC_CLEAR_64 */ 162303275Sdelphij#else 163303275Sdelphij#define __ATOMIC_CLEAR_64(p, v, t) \ 164303275Sdelphij 64-bit atomic_clear not implemented 165303275Sdelphij#endif 166303275Sdelphij 167303275Sdelphij#define _ATOMIC_CLEAR(width, suffix, type) \ 168303275Sdelphij static __inline void \ 169303275Sdelphij atomic_clear_##suffix(volatile type *p, type v) { \ 170303275Sdelphij type t; \ 171303275Sdelphij __ATOMIC_CLEAR_##width(p, v, t); \ 172303275Sdelphij } \ 173303275Sdelphij \ 174303275Sdelphij static __inline void \ 175303275Sdelphij atomic_clear_acq_##suffix(volatile type *p, type v) { \ 176303275Sdelphij type t; \ 177303275Sdelphij __ATOMIC_CLEAR_##width(p, v, t); \ 178303275Sdelphij __ATOMIC_BARRIER; \ 179303275Sdelphij } \ 180303275Sdelphij \ 181303275Sdelphij static __inline void \ 182303275Sdelphij atomic_clear_rel_##suffix(volatile type *p, type v) { \ 183303275Sdelphij type t; \ 184303275Sdelphij __ATOMIC_BARRIER; \ 185303275Sdelphij __ATOMIC_CLEAR_##width(p, v, t); \ 186303275Sdelphij } \ 187303275Sdelphij /* _ATOMIC_CLEAR */ 188303275Sdelphij 189303275Sdelphij#if 0 190303275Sdelphij_ATOMIC_CLEAR(8, 8, uint8_t) 191303275Sdelphij_ATOMIC_CLEAR(8, char, u_char) 192303275Sdelphij_ATOMIC_CLEAR(16, 16, uint16_t) 193303275Sdelphij_ATOMIC_CLEAR(16, short, u_short) 194303275Sdelphij#endif 195303275Sdelphij_ATOMIC_CLEAR(32, 32, uint32_t) 196303275Sdelphij_ATOMIC_CLEAR(32, int, u_int) 197303275Sdelphij#ifdef __powerpc64__ 198303275Sdelphij_ATOMIC_CLEAR(64, 64, uint64_t) 199303275Sdelphij_ATOMIC_CLEAR(64, long, u_long) 200303275Sdelphij_ATOMIC_CLEAR(64, ptr, uintptr_t) 201303275Sdelphij#else 202303275Sdelphij_ATOMIC_CLEAR(32, long, u_long) 203303275Sdelphij_ATOMIC_CLEAR(32, ptr, uintptr_t) 204303275Sdelphij#endif 205303275Sdelphij 206303275Sdelphij#undef _ATOMIC_CLEAR 207303275Sdelphij#undef __ATOMIC_CLEAR_64 208303275Sdelphij#undef __ATOMIC_CLEAR_32 209303275Sdelphij#undef __ATOMIC_CLEAR_16 210303275Sdelphij#undef __ATOMIC_CLEAR_8 211303275Sdelphij 212303275Sdelphij/* 213303275Sdelphij * atomic_cmpset(p, o, n) 214303275Sdelphij */ 215303275Sdelphij/* TODO -- see below */ 216303275Sdelphij 217303275Sdelphij/* 218303275Sdelphij * atomic_load_acq(p) 219303275Sdelphij */ 220303275Sdelphij/* TODO -- see below */ 221303275Sdelphij 222303275Sdelphij/* 223303275Sdelphij * atomic_readandclear(p) 224303275Sdelphij */ 225303275Sdelphij/* TODO -- see below */ 226303275Sdelphij 227303275Sdelphij/* 228303275Sdelphij * atomic_set(p, v) 229303275Sdelphij * { *p |= v; } 230303275Sdelphij */ 231303275Sdelphij 232303275Sdelphij#define __ATOMIC_SET_8(p, v, t) \ 233303275Sdelphij 8-bit atomic_set not implemented 234303275Sdelphij 235303275Sdelphij#define __ATOMIC_SET_16(p, v, t) \ 236303275Sdelphij 16-bit atomic_set not implemented 237303275Sdelphij 238303275Sdelphij#define __ATOMIC_SET_32(p, v, t) \ 239303275Sdelphij __asm __volatile( \ 240303275Sdelphij "1: lwarx %0, 0, %2\n" \ 241303275Sdelphij " or %0, %3, %0\n" \ 242303275Sdelphij " stwcx. %0, 0, %2\n" \ 243303275Sdelphij " bne- 1b\n" \ 244303275Sdelphij : "=&r" (t), "=m" (*p) \ 245303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 246303275Sdelphij : "cc", "memory") \ 247303275Sdelphij /* __ATOMIC_SET_32 */ 248303275Sdelphij 249303275Sdelphij#ifdef __powerpc64__ 250303275Sdelphij#define __ATOMIC_SET_64(p, v, t) \ 251303275Sdelphij __asm __volatile( \ 252303275Sdelphij "1: ldarx %0, 0, %2\n" \ 253303275Sdelphij " or %0, %3, %0\n" \ 254303275Sdelphij " stdcx. %0, 0, %2\n" \ 255303275Sdelphij " bne- 1b\n" \ 256303275Sdelphij : "=&r" (t), "=m" (*p) \ 257303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 258303275Sdelphij : "cc", "memory") \ 259303275Sdelphij /* __ATOMIC_SET_64 */ 260303275Sdelphij#else 261303275Sdelphij#define __ATOMIC_SET_64(p, v, t) \ 262303275Sdelphij 64-bit atomic_set not implemented 263303275Sdelphij#endif 264303275Sdelphij 265303275Sdelphij#define _ATOMIC_SET(width, suffix, type) \ 266303275Sdelphij static __inline void \ 267303275Sdelphij atomic_set_##suffix(volatile type *p, type v) { \ 268303275Sdelphij type t; \ 269303275Sdelphij __ATOMIC_SET_##width(p, v, t); \ 270303275Sdelphij } \ 271303275Sdelphij \ 272303275Sdelphij static __inline void \ 273303275Sdelphij atomic_set_acq_##suffix(volatile type *p, type v) { \ 274303275Sdelphij type t; \ 275303275Sdelphij __ATOMIC_SET_##width(p, v, t); \ 276303275Sdelphij __ATOMIC_BARRIER; \ 277303275Sdelphij } \ 278303275Sdelphij \ 279303275Sdelphij static __inline void \ 280303275Sdelphij atomic_set_rel_##suffix(volatile type *p, type v) { \ 281303275Sdelphij type t; \ 282303275Sdelphij __ATOMIC_BARRIER; \ 283303275Sdelphij __ATOMIC_SET_##width(p, v, t); \ 284303275Sdelphij } \ 285303275Sdelphij /* _ATOMIC_SET */ 286303275Sdelphij 287303275Sdelphij#if 0 288303275Sdelphij_ATOMIC_SET(8, 8, uint8_t) 289303275Sdelphij_ATOMIC_SET(8, char, u_char) 290303275Sdelphij_ATOMIC_SET(16, 16, uint16_t) 291303275Sdelphij_ATOMIC_SET(16, short, u_short) 292303275Sdelphij#endif 293303275Sdelphij_ATOMIC_SET(32, 32, uint32_t) 294303275Sdelphij_ATOMIC_SET(32, int, u_int) 295303275Sdelphij#ifdef __powerpc64__ 296303275Sdelphij_ATOMIC_SET(64, 64, uint64_t) 297303275Sdelphij_ATOMIC_SET(64, long, u_long) 298303275Sdelphij_ATOMIC_SET(64, ptr, uintptr_t) 299303275Sdelphij#else 300303275Sdelphij_ATOMIC_SET(32, long, u_long) 301303275Sdelphij_ATOMIC_SET(32, ptr, uintptr_t) 302303275Sdelphij#endif 303303275Sdelphij 304303275Sdelphij#undef _ATOMIC_SET 305303275Sdelphij#undef __ATOMIC_SET_64 306303275Sdelphij#undef __ATOMIC_SET_32 307303275Sdelphij#undef __ATOMIC_SET_16 308303275Sdelphij#undef __ATOMIC_SET_8 309303275Sdelphij 310303275Sdelphij/* 311303275Sdelphij * atomic_subtract(p, v) 312303275Sdelphij * { *p -= v; } 313303275Sdelphij */ 314303275Sdelphij 315303275Sdelphij#define __ATOMIC_SUBTRACT_8(p, v, t) \ 316303275Sdelphij 8-bit atomic_subtract not implemented 317303275Sdelphij 318303275Sdelphij#define __ATOMIC_SUBTRACT_16(p, v, t) \ 319303275Sdelphij 16-bit atomic_subtract not implemented 320303275Sdelphij 321303275Sdelphij#define __ATOMIC_SUBTRACT_32(p, v, t) \ 322303275Sdelphij __asm __volatile( \ 323303275Sdelphij "1: lwarx %0, 0, %2\n" \ 324303275Sdelphij " subf %0, %3, %0\n" \ 325303275Sdelphij " stwcx. %0, 0, %2\n" \ 326303275Sdelphij " bne- 1b\n" \ 327303275Sdelphij : "=&r" (t), "=m" (*p) \ 328303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 329303275Sdelphij : "cc", "memory") \ 330303275Sdelphij /* __ATOMIC_SUBTRACT_32 */ 331303275Sdelphij 332303275Sdelphij#ifdef __powerpc64__ 333303275Sdelphij#define __ATOMIC_SUBTRACT_64(p, v, t) \ 334303275Sdelphij __asm __volatile( \ 335303275Sdelphij "1: ldarx %0, 0, %2\n" \ 336303275Sdelphij " subf %0, %3, %0\n" \ 337303275Sdelphij " stdcx. %0, 0, %2\n" \ 338303275Sdelphij " bne- 1b\n" \ 339303275Sdelphij : "=&r" (t), "=m" (*p) \ 340303275Sdelphij : "r" (p), "r" (v), "m" (*p) \ 341303275Sdelphij : "cc", "memory") \ 342303275Sdelphij /* __ATOMIC_SUBTRACT_64 */ 343303275Sdelphij#else 344303275Sdelphij#define __ATOMIC_SUBTRACT_64(p, v, t) \ 345303275Sdelphij 64-bit atomic_subtract not implemented 346303275Sdelphij#endif 347303275Sdelphij 348303275Sdelphij#define _ATOMIC_SUBTRACT(width, suffix, type) \ 349303275Sdelphij static __inline void \ 350303275Sdelphij atomic_subtract_##suffix(volatile type *p, type v) { \ 351303275Sdelphij type t; \ 352303275Sdelphij __ATOMIC_SUBTRACT_##width(p, v, t); \ 353303275Sdelphij } \ 354303275Sdelphij \ 355303275Sdelphij static __inline void \ 356303275Sdelphij atomic_subtract_acq_##suffix(volatile type *p, type v) { \ 357303275Sdelphij type t; \ 358303275Sdelphij __ATOMIC_SUBTRACT_##width(p, v, t); \ 359303275Sdelphij __ATOMIC_BARRIER; \ 360303275Sdelphij } \ 361303275Sdelphij \ 362303275Sdelphij static __inline void \ 363303275Sdelphij atomic_subtract_rel_##suffix(volatile type *p, type v) { \ 364303275Sdelphij type t; \ 365303275Sdelphij __ATOMIC_BARRIER; \ 366303275Sdelphij __ATOMIC_SUBTRACT_##width(p, v, t); \ 367303275Sdelphij } \ 368303275Sdelphij /* _ATOMIC_SUBTRACT */ 369303275Sdelphij 370303275Sdelphij#if 0 371303275Sdelphij_ATOMIC_SUBTRACT(8, 8, uint8_t) 372303275Sdelphij_ATOMIC_SUBTRACT(8, char, u_char) 373303275Sdelphij_ATOMIC_SUBTRACT(16, 16, uint16_t) 374303275Sdelphij_ATOMIC_SUBTRACT(16, short, u_short) 375303275Sdelphij#endif 376303275Sdelphij_ATOMIC_SUBTRACT(32, 32, uint32_t) 377303275Sdelphij_ATOMIC_SUBTRACT(32, int, u_int) 378303275Sdelphij#ifdef __powerpc64__ 379303275Sdelphij_ATOMIC_SUBTRACT(64, 64, uint64_t) 380303275Sdelphij_ATOMIC_SUBTRACT(64, long, u_long) 381303275Sdelphij_ATOMIC_SUBTRACT(64, ptr, uintptr_t) 382303275Sdelphij#else 383303275Sdelphij_ATOMIC_SUBTRACT(32, long, u_long) 384303275Sdelphij_ATOMIC_SUBTRACT(32, ptr, uintptr_t) 385303275Sdelphij#endif 386303275Sdelphij 387303275Sdelphij#undef _ATOMIC_SUBTRACT 388303275Sdelphij#undef __ATOMIC_SUBTRACT_64 389303275Sdelphij#undef __ATOMIC_SUBTRACT_32 390303275Sdelphij#undef __ATOMIC_SUBTRACT_16 391303275Sdelphij#undef __ATOMIC_SUBTRACT_8 392303275Sdelphij 393303275Sdelphij/* 394303275Sdelphij * atomic_store_rel(p, v) 395303275Sdelphij */ 396303275Sdelphij/* TODO -- see below */ 397303275Sdelphij 398303275Sdelphij/* 399 * Old/original implementations that still need revisiting. 400 */ 401 402static __inline uint32_t 403atomic_readandclear_32(volatile uint32_t *addr) 404{ 405 uint32_t result,temp; 406 407#ifdef __GNUCLIKE_ASM 408 __asm __volatile ( 409 "\tsync\n" /* drain writes */ 410 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 411 "li %1, 0\n\t" /* load new value */ 412 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 413 "bne- 1b\n\t" /* spin if failed */ 414 : "=&r"(result), "=&r"(temp), "=m" (*addr) 415 : "r" (addr), "m" (*addr) 416 : "cc", "memory"); 417#endif 418 419 return (result); 420} 421 422#ifdef __powerpc64__ 423static __inline uint64_t 424atomic_readandclear_64(volatile uint64_t *addr) 425{ 426 uint64_t result,temp; 427 428#ifdef __GNUCLIKE_ASM 429 __asm __volatile ( 430 "\tsync\n" /* drain writes */ 431 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 432 "li %1, 0\n\t" /* load new value */ 433 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 434 "bne- 1b\n\t" /* spin if failed */ 435 : "=&r"(result), "=&r"(temp), "=m" (*addr) 436 : "r" (addr), "m" (*addr) 437 : "cc", "memory"); 438#endif 439 440 return (result); 441} 442#endif 443 444#define atomic_readandclear_int atomic_readandclear_32 445 446#ifdef __powerpc64__ 447#define atomic_readandclear_long atomic_readandclear_64 448#define atomic_readandclear_ptr atomic_readandclear_64 449#else 450#define atomic_readandclear_long atomic_readandclear_32 451#define atomic_readandclear_ptr atomic_readandclear_32 452#endif 453 454/* 455 * We assume that a = b will do atomic loads and stores. 456 */ 457#define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ 458static __inline u_##TYPE \ 459atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ 460{ \ 461 u_##TYPE v; \ 462 \ 463 v = *p; \ 464 __ATOMIC_BARRIER; \ 465 return (v); \ 466} \ 467 \ 468static __inline void \ 469atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ 470{ \ 471 __ATOMIC_BARRIER; \ 472 *p = v; \ 473} \ 474 \ 475static __inline u_##TYPE \ 476atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 477{ \ 478 u_##TYPE v; \ 479 \ 480 v = *p; \ 481 __ATOMIC_BARRIER; \ 482 return (v); \ 483} \ 484 \ 485static __inline void \ 486atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 487{ \ 488 __ATOMIC_BARRIER; \ 489 *p = v; \ 490} 491 492ATOMIC_STORE_LOAD(char, 8) 493ATOMIC_STORE_LOAD(short, 16) 494ATOMIC_STORE_LOAD(int, 32) 495#ifdef __powerpc64__ 496ATOMIC_STORE_LOAD(long, 64) 497#endif 498 499#ifdef __powerpc64__ 500#define atomic_load_acq_long atomic_load_acq_64 501#define atomic_store_rel_long atomic_store_rel_64 502#define atomic_load_acq_ptr atomic_load_acq_64 503#define atomic_store_rel_ptr atomic_store_rel_64 504#else 505#define atomic_load_acq_long atomic_load_acq_32 506#define atomic_store_rel_long atomic_store_rel_32 507#define atomic_load_acq_ptr atomic_load_acq_32 508#define atomic_store_rel_ptr atomic_store_rel_32 509#endif 510 511#undef ATOMIC_STORE_LOAD 512 513/* 514 * Atomically compare the value stored at *p with cmpval and if the 515 * two values are equal, update the value of *p with newval. Returns 516 * zero if the compare failed, nonzero otherwise. 517 */ 518static __inline int 519atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 520{ 521 int ret; 522 523#ifdef __GNUCLIKE_ASM 524 __asm __volatile ( 525 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 526 "cmplw %3, %0\n\t" /* compare */ 527 "bne 2f\n\t" /* exit if not equal */ 528 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 529 "bne- 1b\n\t" /* spin if failed */ 530 "li %0, 1\n\t" /* success - retval = 1 */ 531 "b 3f\n\t" /* we've succeeded */ 532 "2:\n\t" 533 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 534 "li %0, 0\n\t" /* failure - retval = 0 */ 535 "3:\n\t" 536 : "=&r" (ret), "=m" (*p) 537 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 538 : "cc", "memory"); 539#endif 540 541 return (ret); 542} 543 544static __inline int 545atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 546{ 547 int ret; 548 549#ifdef __GNUCLIKE_ASM 550 __asm __volatile ( 551 #ifdef __powerpc64__ 552 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 553 "cmpld %3, %0\n\t" /* compare */ 554 "bne 2f\n\t" /* exit if not equal */ 555 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 556 #else 557 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 558 "cmplw %3, %0\n\t" /* compare */ 559 "bne 2f\n\t" /* exit if not equal */ 560 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 561 #endif 562 "bne- 1b\n\t" /* spin if failed */ 563 "li %0, 1\n\t" /* success - retval = 1 */ 564 "b 3f\n\t" /* we've succeeded */ 565 "2:\n\t" 566 #ifdef __powerpc64__ 567 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 568 #else 569 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 570 #endif 571 "li %0, 0\n\t" /* failure - retval = 0 */ 572 "3:\n\t" 573 : "=&r" (ret), "=m" (*p) 574 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 575 : "cc", "memory"); 576#endif 577 578 return (ret); 579} 580 581#define atomic_cmpset_int atomic_cmpset_32 582 583#ifdef __powerpc64__ 584#define atomic_cmpset_ptr(dst, old, new) \ 585 atomic_cmpset_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) 586#else 587#define atomic_cmpset_ptr(dst, old, new) \ 588 atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 589#endif 590 591static __inline int 592atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 593{ 594 int retval; 595 596 retval = atomic_cmpset_32(p, cmpval, newval); 597 __ATOMIC_BARRIER; 598 return (retval); 599} 600 601static __inline int 602atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 603{ 604 __ATOMIC_BARRIER; 605 return (atomic_cmpset_32(p, cmpval, newval)); 606} 607 608static __inline int 609atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 610{ 611 u_long retval; 612 613 retval = atomic_cmpset_long(p, cmpval, newval); 614 __ATOMIC_BARRIER; 615 return (retval); 616} 617 618static __inline int 619atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 620{ 621 __ATOMIC_BARRIER; 622 return (atomic_cmpset_long(p, cmpval, newval)); 623} 624 625#define atomic_cmpset_acq_int atomic_cmpset_acq_32 626#define atomic_cmpset_rel_int atomic_cmpset_rel_32 627 628#ifdef __powerpc64__ 629#define atomic_cmpset_acq_ptr(dst, old, new) \ 630 atomic_cmpset_acq_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) 631#define atomic_cmpset_rel_ptr(dst, old, new) \ 632 atomic_cmpset_rel_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) 633#else 634#define atomic_cmpset_acq_ptr(dst, old, new) \ 635 atomic_cmpset_acq_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 636#define atomic_cmpset_rel_ptr(dst, old, new) \ 637 atomic_cmpset_rel_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) 638#endif 639 640static __inline uint32_t 641atomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 642{ 643 uint32_t value; 644 645 do { 646 value = *p; 647 } while (!atomic_cmpset_32(p, value, value + v)); 648 return (value); 649} 650 651#define atomic_fetchadd_int atomic_fetchadd_32 652 653#ifdef __powerpc64__ 654static __inline uint64_t 655atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) 656{ 657 uint64_t value; 658 659 do { 660 value = *p; 661 } while (!atomic_cmpset_long(p, value, value + v)); 662 return (value); 663} 664 665#define atomic_fetchadd_long atomic_fetchadd_64 666#else 667#define atomic_fetchadd_long(p, v) \ 668 (u_long)atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) 669#endif 670 671#endif /* ! _MACHINE_ATOMIC_H_ */ 672