atomic.h revision 88617
1/*- 2 * Copyright (c) 1998 Doug Rabson. 3 * Copyright (c) 2001 Jake Burkholder. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 28 * $FreeBSD: head/sys/sparc64/include/atomic.h 88617 2001-12-29 06:48:43Z jake $ 29 */ 30 31#ifndef _MACHINE_ATOMIC_H_ 32#define _MACHINE_ATOMIC_H_ 33 34#include <machine/cpufunc.h> 35 36/* Userland needs different ASI's. */ 37#ifdef _KERNEL 38#define __ASI_ATOMIC ASI_N 39#else 40#define __ASI_ATOMIC ASI_P 41#endif 42 43/* 44 * Various simple arithmetic on memory which is atomic in the presence 45 * of interrupts and multiple processors. See atomic(9) for details. 46 * Note that efficient hardware support exists only for the 32 and 64 47 * bit variants; the 8 and 16 bit versions are not provided and should 48 * not be used in MI code. 49 * 50 * This implementation takes advantage of the fact that the sparc64 51 * cas instruction is both a load and a store. The loop is often coded 52 * as follows: 53 * 54 * do { 55 * expect = *p; 56 * new = expect + 1; 57 * } while (cas(p, expect, new) != expect); 58 * 59 * which performs an unnnecessary load on each iteration that the cas 60 * operation fails. Modified as follows: 61 * 62 * expect = *p; 63 * for (;;) { 64 * new = expect + 1; 65 * result = cas(p, expect, new); 66 * if (result == expect) 67 * break; 68 * expect = result; 69 * } 70 * 71 * the return value of cas is used to avoid the extra reload. At the 72 * time of writing, with gcc version 2.95.3, the branch for the if 73 * statement is predicted incorrectly as not taken, rather than taken. 74 * It is expected that the branch prediction hints available in gcc 3.0, 75 * __builtin_expect, will allow better code to be generated. 76 * 77 * The memory barriers provided by the acq and rel variants are intended 78 * to be sufficient for use of relaxed memory ordering. Due to the 79 * suggested assembly syntax of the membar operands containing a # 80 * character, they cannot be used in macros. The cmask and mmask bits 81 * are hard coded in machine/cpufunc.h and used here through macros. 82 * Hopefully sun will choose not to change the bit numbers. 83 */ 84 85#define itype(sz) u_int ## sz ## _t 86 87#define atomic_cas_32(p, e, s) casa(p, e, s, __ASI_ATOMIC) 88#define atomic_cas_64(p, e, s) casxa(p, e, s, __ASI_ATOMIC) 89 90#define atomic_cas(p, e, s, sz) \ 91 atomic_cas_ ## sz(p, e, s) 92 93#define atomic_cas_acq(p, e, s, sz) ({ \ 94 itype(sz) v; \ 95 v = atomic_cas(p, e, s, sz); \ 96 membar(LoadLoad | LoadStore); \ 97 v; \ 98}) 99 100#define atomic_cas_rel(p, e, s, sz) ({ \ 101 itype(sz) v; \ 102 membar(LoadStore | StoreStore); \ 103 v = atomic_cas(p, e, s, sz); \ 104 v; \ 105}) 106 107#define atomic_op(p, op, v, sz) do { \ 108 itype(sz) e, r, s; \ 109 for (e = *(volatile itype(sz) *)p;; e = r) { \ 110 s = e op v; \ 111 r = atomic_cas_ ## sz(p, e, s); \ 112 if (r == e) \ 113 break; \ 114 } \ 115} while (0) 116 117#define atomic_op_acq(p, op, v, sz) do { \ 118 atomic_op(p, op, v, sz); \ 119 membar(LoadLoad | LoadStore); \ 120} while (0) 121 122#define atomic_op_rel(p, op, v, sz) do { \ 123 membar(LoadStore | StoreStore); \ 124 atomic_op(p, op, v, sz); \ 125} while (0) 126 127#define atomic_load(p, sz) \ 128 atomic_cas(p, 0, 0, sz) 129 130#define atomic_load_acq(p, sz) ({ \ 131 itype(sz) v; \ 132 v = atomic_load(p, sz); \ 133 membar(LoadLoad | LoadStore); \ 134 v; \ 135}) 136 137#define atomic_load_clear(p, sz) ({ \ 138 itype(sz) e, r; \ 139 for (e = *(volatile itype(sz) *)p;; e = r) { \ 140 r = atomic_cas(p, e, 0, sz); \ 141 if (r == e) \ 142 break; \ 143 } \ 144 e; \ 145}) 146 147#define atomic_store(p, v, sz) do { \ 148 itype(sz) e, r; \ 149 for (e = *(volatile itype(sz) *)p;; e = r) { \ 150 r = atomic_cas(p, e, v, sz); \ 151 if (r == e) \ 152 break; \ 153 } \ 154} while (0) 155 156#define atomic_store_rel(p, v, sz) do { \ 157 membar(LoadStore | StoreStore); \ 158 atomic_store(p, v, sz); \ 159} while (0) 160 161#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 162 \ 163static __inline void \ 164atomic_add_ ## name(volatile ptype p, atype v) \ 165{ \ 166 atomic_op(p, +, v, sz); \ 167} \ 168static __inline void \ 169atomic_add_acq_ ## name(volatile ptype p, atype v) \ 170{ \ 171 atomic_op_acq(p, +, v, sz); \ 172} \ 173static __inline void \ 174atomic_add_rel_ ## name(volatile ptype p, atype v) \ 175{ \ 176 atomic_op_rel(p, +, v, sz); \ 177} \ 178 \ 179static __inline void \ 180atomic_clear_ ## name(volatile ptype p, atype v) \ 181{ \ 182 atomic_op(p, &, ~v, sz); \ 183} \ 184static __inline void \ 185atomic_clear_acq_ ## name(volatile ptype p, atype v) \ 186{ \ 187 atomic_op_acq(p, &, ~v, sz); \ 188} \ 189static __inline void \ 190atomic_clear_rel_ ## name(volatile ptype p, atype v) \ 191{ \ 192 atomic_op_rel(p, &, ~v, sz); \ 193} \ 194 \ 195static __inline int \ 196atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 197{ \ 198 return (((vtype)atomic_cas(p, e, s, sz)) == e); \ 199} \ 200static __inline int \ 201atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 202{ \ 203 return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \ 204} \ 205static __inline int \ 206atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 207{ \ 208 return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \ 209} \ 210 \ 211static __inline vtype \ 212atomic_load_ ## name(volatile ptype p) \ 213{ \ 214 return ((vtype)atomic_cas(p, 0, 0, sz)); \ 215} \ 216static __inline vtype \ 217atomic_load_acq_ ## name(volatile ptype p) \ 218{ \ 219 return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \ 220} \ 221 \ 222static __inline vtype \ 223atomic_readandclear_ ## name(volatile ptype p) \ 224{ \ 225 return ((vtype)atomic_load_clear(p, sz)); \ 226} \ 227 \ 228static __inline void \ 229atomic_set_ ## name(volatile ptype p, atype v) \ 230{ \ 231 atomic_op(p, |, v, sz); \ 232} \ 233static __inline void \ 234atomic_set_acq_ ## name(volatile ptype p, atype v) \ 235{ \ 236 atomic_op_acq(p, |, v, sz); \ 237} \ 238static __inline void \ 239atomic_set_rel_ ## name(volatile ptype p, atype v) \ 240{ \ 241 atomic_op_rel(p, |, v, sz); \ 242} \ 243 \ 244static __inline void \ 245atomic_subtract_ ## name(volatile ptype p, atype v) \ 246{ \ 247 atomic_op(p, -, v, sz); \ 248} \ 249static __inline void \ 250atomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 251{ \ 252 atomic_op_acq(p, -, v, sz); \ 253} \ 254static __inline void \ 255atomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 256{ \ 257 atomic_op_rel(p, -, v, sz); \ 258} \ 259 \ 260static __inline void \ 261atomic_store_ ## name(volatile ptype p, vtype v) \ 262{ \ 263 atomic_store(p, v, sz); \ 264} \ 265static __inline void \ 266atomic_store_rel_ ## name(volatile ptype p, vtype v) \ 267{ \ 268 atomic_store_rel(p, v, sz); \ 269} 270 271ATOMIC_GEN(int, int *, int, int, 32); 272ATOMIC_GEN(32, int *, int, int, 32); 273 274ATOMIC_GEN(long, long *, long, long, 64); 275ATOMIC_GEN(64, long *, long, long, 64); 276 277ATOMIC_GEN(ptr, void *, void *, uintptr_t, 64); 278 279#undef __ASI_ATOMIC 280#undef ATOMIC_GEN 281#undef atomic_cas_32 282#undef atomic_cas_64 283#undef atomic_cas 284#undef atomic_cas_acq 285#undef atomic_cas_rel 286#undef atomic_op 287#undef atomic_op_acq 288#undef atomic_op_rel 289#undef atomic_load_acq 290#undef atomic_store_rel 291#undef atomic_load_clear 292 293#endif /* !_MACHINE_ATOMIC_H_ */ 294