170572Sobrien/*- 2178027Smarcel * Copyright (c) 2008 Marcel Moolenaar 370741Sbenno * Copyright (c) 2001 Benno Rice 470572Sobrien * Copyright (c) 2001 David E. O'Brien 570572Sobrien * Copyright (c) 1998 Doug Rabson 670572Sobrien * All rights reserved. 770572Sobrien * 870572Sobrien * Redistribution and use in source and binary forms, with or without 970572Sobrien * modification, are permitted provided that the following conditions 1070572Sobrien * are met: 1170572Sobrien * 1. Redistributions of source code must retain the above copyright 1270572Sobrien * notice, this list of conditions and the following disclaimer. 1370572Sobrien * 2. Redistributions in binary form must reproduce the above copyright 1470572Sobrien * notice, this list of conditions and the following disclaimer in the 1570572Sobrien * documentation and/or other materials provided with the distribution. 1670572Sobrien * 1770572Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1870572Sobrien * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1970572Sobrien * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2070572Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2170572Sobrien * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2270572Sobrien * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2370572Sobrien * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2470572Sobrien * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2570572Sobrien * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2670572Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2770572Sobrien * SUCH DAMAGE. 2870572Sobrien * 2970572Sobrien * $FreeBSD$ 3070572Sobrien */ 3170572Sobrien 3270572Sobrien#ifndef _MACHINE_ATOMIC_H_ 3370572Sobrien#define _MACHINE_ATOMIC_H_ 3470572Sobrien 35143063Sjoerg#ifndef _SYS_CDEFS_H_ 36143063Sjoerg#error this file needs sys/cdefs.h as a prerequisite 37143063Sjoerg#endif 38143063Sjoerg 39235931Smarcel/* 40235931Smarcel * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 41235931Smarcel * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 42235931Smarcel * of this file. See also Appendix B.2 of Book II of the architecture manual. 43235931Smarcel * 44235931Smarcel * Note that not all Book-E processors accept the light-weight sync variant. 45235931Smarcel * In particular, early models of E500 cores are known to wedge. Bank on all 46235931Smarcel * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 47235931Smarcel * to use the heavier-weight sync. 48235931Smarcel */ 49235931Smarcel 50235931Smarcel#ifdef __powerpc64__ 51234584Snwhitehorn#define mb() __asm __volatile("lwsync" : : : "memory") 52234590Snwhitehorn#define rmb() __asm __volatile("lwsync" : : : "memory") 53234584Snwhitehorn#define wmb() __asm __volatile("lwsync" : : : "memory") 54235931Smarcel#define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 55235943Snwhitehorn#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 56234590Snwhitehorn#else 57235942Smarcel#define mb() __asm __volatile("sync" : : : "memory") 58235942Smarcel#define rmb() __asm __volatile("sync" : : : "memory") 59235946Sbz#define wmb() __asm __volatile("sync" : : : "memory") 60235942Smarcel#define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 61235931Smarcel#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 62234590Snwhitehorn#endif 63234584Snwhitehorn 64234584Snwhitehorn/* 65178027Smarcel * atomic_add(p, v) 66178027Smarcel * { *p += v; } 6770572Sobrien */ 6870572Sobrien 69222198Sattilio#define __atomic_add_int(p, v, t) \ 70178027Smarcel __asm __volatile( \ 71178027Smarcel "1: lwarx %0, 0, %2\n" \ 72178027Smarcel " add %0, %3, %0\n" \ 73178027Smarcel " stwcx. %0, 0, %2\n" \ 74178027Smarcel " bne- 1b\n" \ 75178027Smarcel : "=&r" (t), "=m" (*p) \ 76178027Smarcel : "r" (p), "r" (v), "m" (*p) \ 77178027Smarcel : "cc", "memory") \ 78222198Sattilio /* __atomic_add_int */ 7970572Sobrien 80209975Snwhitehorn#ifdef __powerpc64__ 81222198Sattilio#define __atomic_add_long(p, v, t) \ 82209975Snwhitehorn __asm __volatile( \ 83209975Snwhitehorn "1: ldarx %0, 0, %2\n" \ 84209975Snwhitehorn " add %0, %3, %0\n" \ 85209975Snwhitehorn " stdcx. %0, 0, %2\n" \ 86209975Snwhitehorn " bne- 1b\n" \ 87209975Snwhitehorn : "=&r" (t), "=m" (*p) \ 88209975Snwhitehorn : "r" (p), "r" (v), "m" (*p) \ 89209975Snwhitehorn : "cc", "memory") \ 90222198Sattilio /* __atomic_add_long */ 91209975Snwhitehorn#else 92222198Sattilio#define __atomic_add_long(p, v, t) \ 93222198Sattilio __asm __volatile( \ 94222198Sattilio "1: lwarx %0, 0, %2\n" \ 95222198Sattilio " add %0, %3, %0\n" \ 96222198Sattilio " stwcx. %0, 0, %2\n" \ 97222198Sattilio " bne- 1b\n" \ 98222198Sattilio : "=&r" (t), "=m" (*p) \ 99222198Sattilio : "r" (p), "r" (v), "m" (*p) \ 100222198Sattilio : "cc", "memory") \ 101222198Sattilio /* __atomic_add_long */ 102209975Snwhitehorn#endif 10370572Sobrien 104222198Sattilio#define _ATOMIC_ADD(type) \ 105178027Smarcel static __inline void \ 106222198Sattilio atomic_add_##type(volatile u_##type *p, u_##type v) { \ 107222198Sattilio u_##type t; \ 108222198Sattilio __atomic_add_##type(p, v, t); \ 109178027Smarcel } \ 110178027Smarcel \ 111178027Smarcel static __inline void \ 112222198Sattilio atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 113222198Sattilio u_##type t; \ 114222198Sattilio __atomic_add_##type(p, v, t); \ 115234590Snwhitehorn __ATOMIC_ACQ(); \ 116178027Smarcel } \ 117178027Smarcel \ 118178027Smarcel static __inline void \ 119222198Sattilio atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 120222198Sattilio u_##type t; \ 121234590Snwhitehorn __ATOMIC_REL(); \ 122222198Sattilio __atomic_add_##type(p, v, t); \ 123178027Smarcel } \ 124178027Smarcel /* _ATOMIC_ADD */ 12570572Sobrien 126222198Sattilio_ATOMIC_ADD(int) 127222198Sattilio_ATOMIC_ADD(long) 128222198Sattilio 129222198Sattilio#define atomic_add_32 atomic_add_int 130222198Sattilio#define atomic_add_acq_32 atomic_add_acq_int 131222198Sattilio#define atomic_add_rel_32 atomic_add_rel_int 132222198Sattilio 133209975Snwhitehorn#ifdef __powerpc64__ 134222198Sattilio#define atomic_add_64 atomic_add_long 135222198Sattilio#define atomic_add_acq_64 atomic_add_acq_long 136222198Sattilio#define atomic_add_rel_64 atomic_add_rel_long 137222198Sattilio 138222198Sattilio#define atomic_add_ptr atomic_add_long 139222198Sattilio#define atomic_add_acq_ptr atomic_add_acq_long 140222198Sattilio#define atomic_add_rel_ptr atomic_add_rel_long 141209975Snwhitehorn#else 142222198Sattilio#define atomic_add_ptr atomic_add_int 143222198Sattilio#define atomic_add_acq_ptr atomic_add_acq_int 144222198Sattilio#define atomic_add_rel_ptr atomic_add_rel_int 145178027Smarcel#endif 146178027Smarcel#undef _ATOMIC_ADD 147222198Sattilio#undef __atomic_add_long 148222198Sattilio#undef __atomic_add_int 14970572Sobrien 150178027Smarcel/* 151178027Smarcel * atomic_clear(p, v) 152178027Smarcel * { *p &= ~v; } 153178027Smarcel */ 15470572Sobrien 155222198Sattilio#define __atomic_clear_int(p, v, t) \ 156178027Smarcel __asm __volatile( \ 157178027Smarcel "1: lwarx %0, 0, %2\n" \ 158178027Smarcel " andc %0, %0, %3\n" \ 159178027Smarcel " stwcx. %0, 0, %2\n" \ 160178027Smarcel " bne- 1b\n" \ 161178027Smarcel : "=&r" (t), "=m" (*p) \ 162178027Smarcel : "r" (p), "r" (v), "m" (*p) \ 163178027Smarcel : "cc", "memory") \ 164222198Sattilio /* __atomic_clear_int */ 16570572Sobrien 166209975Snwhitehorn#ifdef __powerpc64__ 167222198Sattilio#define __atomic_clear_long(p, v, t) \ 168209975Snwhitehorn __asm __volatile( \ 169209975Snwhitehorn "1: ldarx %0, 0, %2\n" \ 170209975Snwhitehorn " andc %0, %0, %3\n" \ 171209975Snwhitehorn " stdcx. %0, 0, %2\n" \ 172209975Snwhitehorn " bne- 1b\n" \ 173209975Snwhitehorn : "=&r" (t), "=m" (*p) \ 174209975Snwhitehorn : "r" (p), "r" (v), "m" (*p) \ 175209975Snwhitehorn : "cc", "memory") \ 176222198Sattilio /* __atomic_clear_long */ 177209975Snwhitehorn#else 178222198Sattilio#define __atomic_clear_long(p, v, t) \ 179222198Sattilio __asm __volatile( \ 180222198Sattilio "1: lwarx %0, 0, %2\n" \ 181222198Sattilio " andc %0, %0, %3\n" \ 182222198Sattilio " stwcx. %0, 0, %2\n" \ 183222198Sattilio " bne- 1b\n" \ 184222198Sattilio : "=&r" (t), "=m" (*p) \ 185222198Sattilio : "r" (p), "r" (v), "m" (*p) \ 186222198Sattilio : "cc", "memory") \ 187222198Sattilio /* __atomic_clear_long */ 188209975Snwhitehorn#endif 18970572Sobrien 190222198Sattilio#define _ATOMIC_CLEAR(type) \ 191178027Smarcel static __inline void \ 192222198Sattilio atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 193222198Sattilio u_##type t; \ 194222198Sattilio __atomic_clear_##type(p, v, t); \ 195178027Smarcel } \ 196178027Smarcel \ 197178027Smarcel static __inline void \ 198222198Sattilio atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 199222198Sattilio u_##type t; \ 200222198Sattilio __atomic_clear_##type(p, v, t); \ 201234590Snwhitehorn __ATOMIC_ACQ(); \ 202178027Smarcel } \ 203178027Smarcel \ 204178027Smarcel static __inline void \ 205222198Sattilio atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 206222198Sattilio u_##type t; \ 207234590Snwhitehorn __ATOMIC_REL(); \ 208222198Sattilio __atomic_clear_##type(p, v, t); \ 209178027Smarcel } \ 210178027Smarcel /* _ATOMIC_CLEAR */ 21170572Sobrien 212222198Sattilio 213222198Sattilio_ATOMIC_CLEAR(int) 214222198Sattilio_ATOMIC_CLEAR(long) 215222198Sattilio 216222198Sattilio#define atomic_clear_32 atomic_clear_int 217222198Sattilio#define atomic_clear_acq_32 atomic_clear_acq_int 218222198Sattilio#define atomic_clear_rel_32 atomic_clear_rel_int 219222198Sattilio 220209975Snwhitehorn#ifdef __powerpc64__ 221222198Sattilio#define atomic_clear_64 atomic_clear_long 222222198Sattilio#define atomic_clear_acq_64 atomic_clear_acq_long 223222198Sattilio#define atomic_clear_rel_64 atomic_clear_rel_long 224222198Sattilio 225222198Sattilio#define atomic_clear_ptr atomic_clear_long 226222198Sattilio#define atomic_clear_acq_ptr atomic_clear_acq_long 227222198Sattilio#define atomic_clear_rel_ptr atomic_clear_rel_long 228209975Snwhitehorn#else 229222198Sattilio#define atomic_clear_ptr atomic_clear_int 230222198Sattilio#define atomic_clear_acq_ptr atomic_clear_acq_int 231222198Sattilio#define atomic_clear_rel_ptr atomic_clear_rel_int 232178027Smarcel#endif 233178027Smarcel#undef _ATOMIC_CLEAR 234222198Sattilio#undef __atomic_clear_long 235222198Sattilio#undef __atomic_clear_int 236178027Smarcel 23770741Sbenno/* 238178027Smarcel * atomic_cmpset(p, o, n) 23970741Sbenno */ 240178027Smarcel/* TODO -- see below */ 24170741Sbenno 242178027Smarcel/* 243178027Smarcel * atomic_load_acq(p) 244178027Smarcel */ 245178027Smarcel/* TODO -- see below */ 24670572Sobrien 247178027Smarcel/* 248178027Smarcel * atomic_readandclear(p) 249178027Smarcel */ 250178027Smarcel/* TODO -- see below */ 25170572Sobrien 252178027Smarcel/* 253178027Smarcel * atomic_set(p, v) 254178027Smarcel * { *p |= v; } 255178027Smarcel */ 25670572Sobrien 257222198Sattilio#define __atomic_set_int(p, v, t) \ 258178027Smarcel __asm __volatile( \ 259178027Smarcel "1: lwarx %0, 0, %2\n" \ 260178027Smarcel " or %0, %3, %0\n" \ 261178027Smarcel " stwcx. %0, 0, %2\n" \ 262178027Smarcel " bne- 1b\n" \ 263178027Smarcel : "=&r" (t), "=m" (*p) \ 264178027Smarcel : "r" (p), "r" (v), "m" (*p) \ 265178027Smarcel : "cc", "memory") \ 266222198Sattilio /* __atomic_set_int */ 26770572Sobrien 268209975Snwhitehorn#ifdef __powerpc64__ 269222198Sattilio#define __atomic_set_long(p, v, t) \ 270209975Snwhitehorn __asm __volatile( \ 271209975Snwhitehorn "1: ldarx %0, 0, %2\n" \ 272209975Snwhitehorn " or %0, %3, %0\n" \ 273209975Snwhitehorn " stdcx. %0, 0, %2\n" \ 274209975Snwhitehorn " bne- 1b\n" \ 275209975Snwhitehorn : "=&r" (t), "=m" (*p) \ 276209975Snwhitehorn : "r" (p), "r" (v), "m" (*p) \ 277209975Snwhitehorn : "cc", "memory") \ 278222198Sattilio /* __atomic_set_long */ 279209975Snwhitehorn#else 280222198Sattilio#define __atomic_set_long(p, v, t) \ 281222198Sattilio __asm __volatile( \ 282222198Sattilio "1: lwarx %0, 0, %2\n" \ 283222198Sattilio " or %0, %3, %0\n" \ 284222198Sattilio " stwcx. %0, 0, %2\n" \ 285222198Sattilio " bne- 1b\n" \ 286222198Sattilio : "=&r" (t), "=m" (*p) \ 287222198Sattilio : "r" (p), "r" (v), "m" (*p) \ 288222198Sattilio : "cc", "memory") \ 289222198Sattilio /* __atomic_set_long */ 290209975Snwhitehorn#endif 29170572Sobrien 292222198Sattilio#define _ATOMIC_SET(type) \ 293178027Smarcel static __inline void \ 294222198Sattilio atomic_set_##type(volatile u_##type *p, u_##type v) { \ 295222198Sattilio u_##type t; \ 296222198Sattilio __atomic_set_##type(p, v, t); \ 297178027Smarcel } \ 298178027Smarcel \ 299178027Smarcel static __inline void \ 300222198Sattilio atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 301222198Sattilio u_##type t; \ 302222198Sattilio __atomic_set_##type(p, v, t); \ 303234590Snwhitehorn __ATOMIC_ACQ(); \ 304178027Smarcel } \ 305178027Smarcel \ 306178027Smarcel static __inline void \ 307222198Sattilio atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 308222198Sattilio u_##type t; \ 309234590Snwhitehorn __ATOMIC_REL(); \ 310222198Sattilio __atomic_set_##type(p, v, t); \ 311178027Smarcel } \ 312178027Smarcel /* _ATOMIC_SET */ 31370572Sobrien 314222198Sattilio_ATOMIC_SET(int) 315222198Sattilio_ATOMIC_SET(long) 316222198Sattilio 317222198Sattilio#define atomic_set_32 atomic_set_int 318222198Sattilio#define atomic_set_acq_32 atomic_set_acq_int 319222198Sattilio#define atomic_set_rel_32 atomic_set_rel_int 320222198Sattilio 321209975Snwhitehorn#ifdef __powerpc64__ 322222198Sattilio#define atomic_set_64 atomic_set_long 323222198Sattilio#define atomic_set_acq_64 atomic_set_acq_long 324222198Sattilio#define atomic_set_rel_64 atomic_set_rel_long 325222198Sattilio 326222198Sattilio#define atomic_set_ptr atomic_set_long 327222198Sattilio#define atomic_set_acq_ptr atomic_set_acq_long 328222198Sattilio#define atomic_set_rel_ptr atomic_set_rel_long 329209975Snwhitehorn#else 330222198Sattilio#define atomic_set_ptr atomic_set_int 331222198Sattilio#define atomic_set_acq_ptr atomic_set_acq_int 332222198Sattilio#define atomic_set_rel_ptr atomic_set_rel_int 333178027Smarcel#endif 334178027Smarcel#undef _ATOMIC_SET 335222198Sattilio#undef __atomic_set_long 336222198Sattilio#undef __atomic_set_int 33770572Sobrien 338178027Smarcel/* 339178027Smarcel * atomic_subtract(p, v) 340178027Smarcel * { *p -= v; } 341178027Smarcel */ 34270572Sobrien 343222198Sattilio#define __atomic_subtract_int(p, v, t) \ 344178027Smarcel __asm __volatile( \ 345178027Smarcel "1: lwarx %0, 0, %2\n" \ 346178027Smarcel " subf %0, %3, %0\n" \ 347178027Smarcel " stwcx. %0, 0, %2\n" \ 348178027Smarcel " bne- 1b\n" \ 349178027Smarcel : "=&r" (t), "=m" (*p) \ 350178027Smarcel : "r" (p), "r" (v), "m" (*p) \ 351178027Smarcel : "cc", "memory") \ 352222198Sattilio /* __atomic_subtract_int */ 35370572Sobrien 354209975Snwhitehorn#ifdef __powerpc64__ 355222198Sattilio#define __atomic_subtract_long(p, v, t) \ 356209975Snwhitehorn __asm __volatile( \ 357209975Snwhitehorn "1: ldarx %0, 0, %2\n" \ 358209975Snwhitehorn " subf %0, %3, %0\n" \ 359209975Snwhitehorn " stdcx. %0, 0, %2\n" \ 360209975Snwhitehorn " bne- 1b\n" \ 361209975Snwhitehorn : "=&r" (t), "=m" (*p) \ 362209975Snwhitehorn : "r" (p), "r" (v), "m" (*p) \ 363209975Snwhitehorn : "cc", "memory") \ 364222198Sattilio /* __atomic_subtract_long */ 365209975Snwhitehorn#else 366222198Sattilio#define __atomic_subtract_long(p, v, t) \ 367222198Sattilio __asm __volatile( \ 368222198Sattilio "1: lwarx %0, 0, %2\n" \ 369222198Sattilio " subf %0, %3, %0\n" \ 370222198Sattilio " stwcx. %0, 0, %2\n" \ 371222198Sattilio " bne- 1b\n" \ 372222198Sattilio : "=&r" (t), "=m" (*p) \ 373222198Sattilio : "r" (p), "r" (v), "m" (*p) \ 374222198Sattilio : "cc", "memory") \ 375222198Sattilio /* __atomic_subtract_long */ 376209975Snwhitehorn#endif 37770741Sbenno 378222198Sattilio#define _ATOMIC_SUBTRACT(type) \ 379222198Sattilio static __inline void \ 380222198Sattilio atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 381222198Sattilio u_##type t; \ 382222198Sattilio __atomic_subtract_##type(p, v, t); \ 383222198Sattilio } \ 384222198Sattilio \ 385222198Sattilio static __inline void \ 386222198Sattilio atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 387222198Sattilio u_##type t; \ 388222198Sattilio __atomic_subtract_##type(p, v, t); \ 389234590Snwhitehorn __ATOMIC_ACQ(); \ 390222198Sattilio } \ 391222198Sattilio \ 392222198Sattilio static __inline void \ 393222198Sattilio atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 394222198Sattilio u_##type t; \ 395234590Snwhitehorn __ATOMIC_REL(); \ 396222198Sattilio __atomic_subtract_##type(p, v, t); \ 397222198Sattilio } \ 398178027Smarcel /* _ATOMIC_SUBTRACT */ 39970572Sobrien 400222198Sattilio_ATOMIC_SUBTRACT(int) 401222198Sattilio_ATOMIC_SUBTRACT(long) 402222198Sattilio 403222198Sattilio#define atomic_subtract_32 atomic_subtract_int 404222198Sattilio#define atomic_subtract_acq_32 atomic_subtract_acq_int 405222198Sattilio#define atomic_subtract_rel_32 atomic_subtract_rel_int 406222198Sattilio 407209975Snwhitehorn#ifdef __powerpc64__ 408222198Sattilio#define atomic_subtract_64 atomic_subtract_long 409222198Sattilio#define atomic_subtract_acq_64 atomic_subract_acq_long 410222198Sattilio#define atomic_subtract_rel_64 atomic_subtract_rel_long 411222198Sattilio 412222198Sattilio#define atomic_subtract_ptr atomic_subtract_long 413222198Sattilio#define atomic_subtract_acq_ptr atomic_subtract_acq_long 414222198Sattilio#define atomic_subtract_rel_ptr atomic_subtract_rel_long 415209975Snwhitehorn#else 416222198Sattilio#define atomic_subtract_ptr atomic_subtract_int 417222198Sattilio#define atomic_subtract_acq_ptr atomic_subtract_acq_int 418222198Sattilio#define atomic_subtract_rel_ptr atomic_subtract_rel_int 419178027Smarcel#endif 420178027Smarcel#undef _ATOMIC_SUBTRACT 421222198Sattilio#undef __atomic_subtract_long 422222198Sattilio#undef __atomic_subtract_int 42370741Sbenno 424178027Smarcel/* 425178027Smarcel * atomic_store_rel(p, v) 426178027Smarcel */ 427178027Smarcel/* TODO -- see below */ 42870572Sobrien 429178027Smarcel/* 430178027Smarcel * Old/original implementations that still need revisiting. 431178027Smarcel */ 43270741Sbenno 433222198Sattiliostatic __inline u_int 434222198Sattilioatomic_readandclear_int(volatile u_int *addr) 435178027Smarcel{ 436222198Sattilio u_int result,temp; 43770741Sbenno 438178027Smarcel#ifdef __GNUCLIKE_ASM 439178027Smarcel __asm __volatile ( 440178027Smarcel "\tsync\n" /* drain writes */ 441178027Smarcel "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 442178027Smarcel "li %1, 0\n\t" /* load new value */ 443178027Smarcel "stwcx. %1, 0, %3\n\t" /* attempt to store */ 444178027Smarcel "bne- 1b\n\t" /* spin if failed */ 445178027Smarcel : "=&r"(result), "=&r"(temp), "=m" (*addr) 446178027Smarcel : "r" (addr), "m" (*addr) 447178027Smarcel : "cc", "memory"); 448178027Smarcel#endif 44970741Sbenno 450178027Smarcel return (result); 451178027Smarcel} 45270741Sbenno 453209975Snwhitehorn#ifdef __powerpc64__ 454222198Sattiliostatic __inline u_long 455222198Sattilioatomic_readandclear_long(volatile u_long *addr) 456209975Snwhitehorn{ 457222198Sattilio u_long result,temp; 458209975Snwhitehorn 459209975Snwhitehorn#ifdef __GNUCLIKE_ASM 460209975Snwhitehorn __asm __volatile ( 461209975Snwhitehorn "\tsync\n" /* drain writes */ 462209975Snwhitehorn "1:\tldarx %0, 0, %3\n\t" /* load old value */ 463209975Snwhitehorn "li %1, 0\n\t" /* load new value */ 464209975Snwhitehorn "stdcx. %1, 0, %3\n\t" /* attempt to store */ 465209975Snwhitehorn "bne- 1b\n\t" /* spin if failed */ 466209975Snwhitehorn : "=&r"(result), "=&r"(temp), "=m" (*addr) 467209975Snwhitehorn : "r" (addr), "m" (*addr) 468209975Snwhitehorn : "cc", "memory"); 469209975Snwhitehorn#endif 470209975Snwhitehorn 471209975Snwhitehorn return (result); 472209975Snwhitehorn} 473209975Snwhitehorn#endif 474209975Snwhitehorn 475222198Sattilio#define atomic_readandclear_32 atomic_readandclear_int 476209975Snwhitehorn 477209975Snwhitehorn#ifdef __powerpc64__ 478222198Sattilio#define atomic_readandclear_64 atomic_readandclear_long 479222198Sattilio 480222198Sattilio#define atomic_readandclear_ptr atomic_readandclear_long 481209975Snwhitehorn#else 482222198Sattiliostatic __inline u_long 483222198Sattilioatomic_readandclear_long(volatile u_long *addr) 484222198Sattilio{ 485222198Sattilio 486222198Sattilio return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 487222198Sattilio} 488222198Sattilio 489222198Sattilio#define atomic_readandclear_ptr atomic_readandclear_int 490209975Snwhitehorn#endif 491148067Sjhb 49270572Sobrien/* 49370741Sbenno * We assume that a = b will do atomic loads and stores. 49470741Sbenno */ 495222198Sattilio#define ATOMIC_STORE_LOAD(TYPE) \ 49670741Sbennostatic __inline u_##TYPE \ 49770741Sbennoatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 49870741Sbenno{ \ 499109479Sgrehan u_##TYPE v; \ 500109479Sgrehan \ 501109479Sgrehan v = *p; \ 502234590Snwhitehorn mb(); \ 503109479Sgrehan return (v); \ 50470741Sbenno} \ 50570741Sbenno \ 50670741Sbennostatic __inline void \ 50770741Sbennoatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 50870741Sbenno{ \ 509234590Snwhitehorn mb(); \ 51070741Sbenno *p = v; \ 51170741Sbenno} 51270741Sbenno 513222198SattilioATOMIC_STORE_LOAD(int) 514222198Sattilio 515222198Sattilio#define atomic_load_acq_32 atomic_load_acq_int 516222198Sattilio#define atomic_store_rel_32 atomic_store_rel_int 517222198Sattilio 518209975Snwhitehorn#ifdef __powerpc64__ 519222198SattilioATOMIC_STORE_LOAD(long) 52070741Sbenno 521222198Sattilio#define atomic_load_acq_64 atomic_load_acq_long 522222198Sattilio#define atomic_store_rel_64 atomic_store_rel_long 523222198Sattilio 524222198Sattilio#define atomic_load_acq_ptr atomic_load_acq_long 525222198Sattilio#define atomic_store_rel_ptr atomic_store_rel_long 526209975Snwhitehorn#else 527222198Sattiliostatic __inline u_long 528222198Sattilioatomic_load_acq_long(volatile u_long *addr) 529222198Sattilio{ 530222198Sattilio 531222198Sattilio return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 532222198Sattilio} 533222198Sattilio 534222198Sattiliostatic __inline void 535222198Sattilioatomic_store_rel_long(volatile u_long *addr, u_long val) 536222198Sattilio{ 537222198Sattilio 538222198Sattilio atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 539222198Sattilio} 540222198Sattilio 541222198Sattilio#define atomic_load_acq_ptr atomic_load_acq_int 542222198Sattilio#define atomic_store_rel_ptr atomic_store_rel_int 543209975Snwhitehorn#endif 54470741Sbenno#undef ATOMIC_STORE_LOAD 54570741Sbenno 54670741Sbenno/* 54770572Sobrien * Atomically compare the value stored at *p with cmpval and if the 54870572Sobrien * two values are equal, update the value of *p with newval. Returns 54970572Sobrien * zero if the compare failed, nonzero otherwise. 55070572Sobrien */ 551209975Snwhitehornstatic __inline int 552222198Sattilioatomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 55370572Sobrien{ 554209975Snwhitehorn int ret; 55570572Sobrien 556143063Sjoerg#ifdef __GNUCLIKE_ASM 55770572Sobrien __asm __volatile ( 558109479Sgrehan "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 559109479Sgrehan "cmplw %3, %0\n\t" /* compare */ 56078342Sbenno "bne 2f\n\t" /* exit if not equal */ 561109479Sgrehan "stwcx. %4, 0, %2\n\t" /* attempt to store */ 56278342Sbenno "bne- 1b\n\t" /* spin if failed */ 563109479Sgrehan "li %0, 1\n\t" /* success - retval = 1 */ 56491207Sbenno "b 3f\n\t" /* we've succeeded */ 565109479Sgrehan "2:\n\t" 566109479Sgrehan "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 567109479Sgrehan "li %0, 0\n\t" /* failure - retval = 0 */ 568109479Sgrehan "3:\n\t" 569150182Sjhb : "=&r" (ret), "=m" (*p) 570150182Sjhb : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 571109479Sgrehan : "cc", "memory"); 572109479Sgrehan#endif 57370572Sobrien 574109479Sgrehan return (ret); 57570572Sobrien} 576209975Snwhitehornstatic __inline int 577173928Sjbatomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 578173928Sjb{ 579209975Snwhitehorn int ret; 580173928Sjb 581173928Sjb#ifdef __GNUCLIKE_ASM 582173928Sjb __asm __volatile ( 583209975Snwhitehorn #ifdef __powerpc64__ 584209975Snwhitehorn "1:\tldarx %0, 0, %2\n\t" /* load old value */ 585209975Snwhitehorn "cmpld %3, %0\n\t" /* compare */ 586209975Snwhitehorn "bne 2f\n\t" /* exit if not equal */ 587222198Sattilio "stdcx. %4, 0, %2\n\t" /* attempt to store */ 588209975Snwhitehorn #else 589173928Sjb "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 590173928Sjb "cmplw %3, %0\n\t" /* compare */ 591173928Sjb "bne 2f\n\t" /* exit if not equal */ 592222198Sattilio "stwcx. %4, 0, %2\n\t" /* attempt to store */ 593209975Snwhitehorn #endif 594173928Sjb "bne- 1b\n\t" /* spin if failed */ 595173928Sjb "li %0, 1\n\t" /* success - retval = 1 */ 596173928Sjb "b 3f\n\t" /* we've succeeded */ 597173928Sjb "2:\n\t" 598209975Snwhitehorn #ifdef __powerpc64__ 599222198Sattilio "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 600209975Snwhitehorn #else 601222198Sattilio "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 602209975Snwhitehorn #endif 603173928Sjb "li %0, 0\n\t" /* failure - retval = 0 */ 604173928Sjb "3:\n\t" 605173928Sjb : "=&r" (ret), "=m" (*p) 606173928Sjb : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 607173928Sjb : "cc", "memory"); 608173928Sjb#endif 609173928Sjb 610173928Sjb return (ret); 611173928Sjb} 612173928Sjb 613209975Snwhitehornstatic __inline int 614222198Sattilioatomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) 61570741Sbenno{ 616109479Sgrehan int retval; 61770741Sbenno 618222198Sattilio retval = atomic_cmpset_int(p, cmpval, newval); 619234590Snwhitehorn __ATOMIC_ACQ(); 620109479Sgrehan return (retval); 62170741Sbenno} 62270741Sbenno 623209975Snwhitehornstatic __inline int 624222198Sattilioatomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) 62570741Sbenno{ 626234590Snwhitehorn __ATOMIC_REL(); 627222198Sattilio return (atomic_cmpset_int(p, cmpval, newval)); 62870741Sbenno} 62970741Sbenno 630209975Snwhitehornstatic __inline int 631173928Sjbatomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 632173928Sjb{ 633209975Snwhitehorn u_long retval; 634173928Sjb 635173928Sjb retval = atomic_cmpset_long(p, cmpval, newval); 636234590Snwhitehorn __ATOMIC_ACQ(); 637173928Sjb return (retval); 638173928Sjb} 639173928Sjb 640209975Snwhitehornstatic __inline int 641173928Sjbatomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 642173928Sjb{ 643234590Snwhitehorn __ATOMIC_REL(); 644173928Sjb return (atomic_cmpset_long(p, cmpval, newval)); 645173928Sjb} 646173928Sjb 647222198Sattilio#define atomic_cmpset_32 atomic_cmpset_int 648222198Sattilio#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 649222198Sattilio#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 65070741Sbenno 651209975Snwhitehorn#ifdef __powerpc64__ 652222198Sattilio#define atomic_cmpset_64 atomic_cmpset_long 653222198Sattilio#define atomic_cmpset_acq_64 atomic_cmpset_acq_long 654222198Sattilio#define atomic_cmpset_rel_64 atomic_cmpset_rel_long 655222198Sattilio 656222198Sattilio#define atomic_cmpset_ptr atomic_cmpset_long 657222198Sattilio#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 658222198Sattilio#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 659209975Snwhitehorn#else 660222198Sattilio#define atomic_cmpset_ptr atomic_cmpset_int 661222198Sattilio#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 662222198Sattilio#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 663209975Snwhitehorn#endif 664171334Smarcel 665222198Sattiliostatic __inline u_int 666222198Sattilioatomic_fetchadd_int(volatile u_int *p, u_int v) 667150627Sjhb{ 668222198Sattilio u_int value; 669150627Sjhb 670150627Sjhb do { 671150627Sjhb value = *p; 672222198Sattilio } while (!atomic_cmpset_int(p, value, value + v)); 673150627Sjhb return (value); 674150627Sjhb} 675150627Sjhb 676222198Sattiliostatic __inline u_long 677222198Sattilioatomic_fetchadd_long(volatile u_long *p, u_long v) 678209975Snwhitehorn{ 679222198Sattilio u_long value; 680209975Snwhitehorn 681209975Snwhitehorn do { 682209975Snwhitehorn value = *p; 683209975Snwhitehorn } while (!atomic_cmpset_long(p, value, value + v)); 684209975Snwhitehorn return (value); 685209975Snwhitehorn} 686209975Snwhitehorn 687222198Sattilio#define atomic_fetchadd_32 atomic_fetchadd_int 688222198Sattilio 689222198Sattilio#ifdef __powerpc64__ 690222198Sattilio#define atomic_fetchadd_64 atomic_fetchadd_long 691209975Snwhitehorn#endif 692150627Sjhb 693235931Smarcel#undef __ATOMIC_REL 694235931Smarcel#undef __ATOMIC_ACQ 695235931Smarcel 69670572Sobrien#endif /* ! _MACHINE_ATOMIC_H_ */ 697