atomic.h revision 81334
1/*- 2 * Copyright (c) 2001 Jake Burkholder. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: head/sys/sparc64/include/atomic.h 81334 2001-08-09 02:09:34Z obrien $ 27 */ 28 29#ifndef _MACHINE_ATOMIC_H_ 30#define _MACHINE_ATOMIC_H_ 31 32#include <machine/cpufunc.h> 33 34/* 35 * Various simple arithmetic on memory which is atomic in the presence 36 * of interrupts and multiple processors. See atomic(9) for details. 37 * Note that efficient hardware support exists only for the 32 and 64 38 * bit variants; the 8 and 16 bit versions are not provided and should 39 * not be used in MI code. 40 * 41 * This implementation takes advantage of the fact that the sparc64 42 * cas instruction is both a load and a store. The loop is often coded 43 * as follows: 44 * 45 * do { 46 * expect = *p; 47 * new = expect + 1; 48 * } while (cas(p, expect, new) != expect); 49 * 50 * which performs an unnnecessary load on each iteration that the cas 51 * operation fails. Modified as follows: 52 * 53 * expect = *p; 54 * for (;;) { 55 * new = expect + 1; 56 * result = cas(p, expect, new); 57 * if (result == expect) 58 * break; 59 * expect = result; 60 * } 61 * 62 * the return value of cas is used to avoid the extra reload. At the 63 * time of writing, with gcc version 2.95.3, the branch for the if 64 * statement is predicted incorrectly as not taken, rather than taken. 65 * It is expected that the branch prediction hints available in gcc 3.0, 66 * __builtin_expect, will allow better code to be generated. 67 * 68 * The memory barriers provided by the acq and rel variants are intended 69 * to be sufficient for use of relaxed memory ordering. Due to the 70 * suggested assembly syntax of the membar operands containing a # 71 * character, they cannot be used in macros. The cmask and mmask bits 72 * are hard coded in machine/cpufunc.h and used here through macros. 73 * Hopefully sun will choose not to change the bit numbers. 74 */ 75 76#define itype(sz) u_int ## sz ## _t 77 78#define atomic_cas_32(p, e, s) casa(p, e, s, ASI_N) 79#define atomic_cas_64(p, e, s) casxa(p, e, s, ASI_N) 80 81#define atomic_cas(p, e, s, sz) \ 82 atomic_cas_ ## sz(p, e, s) 83 84#define atomic_cas_acq(p, e, s, sz) ({ \ 85 itype(sz) v; \ 86 v = atomic_cas(p, e, s, sz); \ 87 membar(LoadLoad | LoadStore); \ 88 v; \ 89}) 90 91#define atomic_cas_rel(p, e, s, sz) ({ \ 92 itype(sz) v; \ 93 membar(LoadStore | StoreStore); \ 94 v = atomic_cas(p, e, s, sz); \ 95 v; \ 96}) 97 98#define atomic_op(p, op, v, sz) do { \ 99 itype(sz) e, r, s; \ 100 for (e = *(volatile itype(sz) *)p;; e = r) { \ 101 s = e op v; \ 102 r = atomic_cas_ ## sz(p, e, s); \ 103 if (r == e) \ 104 break; \ 105 } \ 106} while (0) 107 108#define atomic_op_acq(p, op, v, sz) do { \ 109 atomic_op(p, op, v, sz); \ 110 membar(LoadLoad | LoadStore); \ 111} while (0) 112 113#define atomic_op_rel(p, op, v, sz) do { \ 114 membar(LoadStore | StoreStore); \ 115 atomic_op(p, op, v, sz); \ 116} while (0) 117 118#define atomic_load_acq(p, sz) ({ \ 119 itype(sz) v; \ 120 v = atomic_cas_ ## sz(p, 0, 0); \ 121 membar(LoadLoad | LoadStore); \ 122 v; \ 123}) 124 125#define atomic_load_clear(p, sz) ({ \ 126 itype(sz) e, r; \ 127 for (e = *(volatile itype(sz) *)p;; e = r) { \ 128 r = atomic_cas_ ## sz(p, e, 0); \ 129 if (r == e) \ 130 break; \ 131 } \ 132 e; \ 133}) 134 135#define atomic_store_rel(p, v, sz) do { \ 136 itype(sz) e, r; \ 137 membar(LoadStore | StoreStore); \ 138 for (e = *(volatile itype(sz) *)p;; e = r) { \ 139 r = atomic_cas_ ## sz(p, e, v); \ 140 if (r == e) \ 141 break; \ 142 } \ 143} while (0) 144 145#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 146 \ 147static __inline void \ 148atomic_add_ ## name(volatile ptype p, atype v) \ 149{ \ 150 atomic_op(p, +, v, sz); \ 151} \ 152static __inline void \ 153atomic_add_acq_ ## name(volatile ptype p, atype v) \ 154{ \ 155 atomic_op_acq(p, +, v, sz); \ 156} \ 157static __inline void \ 158atomic_add_rel_ ## name(volatile ptype p, atype v) \ 159{ \ 160 atomic_op_rel(p, +, v, sz); \ 161} \ 162 \ 163static __inline void \ 164atomic_clear_ ## name(volatile ptype p, atype v) \ 165{ \ 166 atomic_op(p, &, ~v, sz); \ 167} \ 168static __inline void \ 169atomic_clear_acq_ ## name(volatile ptype p, atype v) \ 170{ \ 171 atomic_op_acq(p, &, ~v, sz); \ 172} \ 173static __inline void \ 174atomic_clear_rel_ ## name(volatile ptype p, atype v) \ 175{ \ 176 atomic_op_rel(p, &, ~v, sz); \ 177} \ 178 \ 179static __inline int \ 180atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 181{ \ 182 return (((vtype)atomic_cas(p, e, s, sz)) == e); \ 183} \ 184static __inline int \ 185atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 186{ \ 187 return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \ 188} \ 189static __inline int \ 190atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 191{ \ 192 return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \ 193} \ 194 \ 195static __inline vtype \ 196atomic_load_acq_ ## name(volatile ptype p) \ 197{ \ 198 return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \ 199} \ 200 \ 201static __inline vtype \ 202atomic_readandclear_ ## name(volatile ptype p) \ 203{ \ 204 return ((vtype)atomic_load_clear(p, sz)); \ 205} \ 206 \ 207static __inline void \ 208atomic_set_ ## name(volatile ptype p, atype v) \ 209{ \ 210 atomic_op(p, |, v, sz); \ 211} \ 212static __inline void \ 213atomic_set_acq_ ## name(volatile ptype p, atype v) \ 214{ \ 215 atomic_op_acq(p, |, v, sz); \ 216} \ 217static __inline void \ 218atomic_set_rel_ ## name(volatile ptype p, atype v) \ 219{ \ 220 atomic_op_rel(p, |, v, sz); \ 221} \ 222 \ 223static __inline void \ 224atomic_subtract_ ## name(volatile ptype p, atype v) \ 225{ \ 226 atomic_op(p, -, v, sz); \ 227} \ 228static __inline void \ 229atomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 230{ \ 231 atomic_op_acq(p, -, v, sz); \ 232} \ 233static __inline void \ 234atomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 235{ \ 236 atomic_op_rel(p, -, v, sz); \ 237} \ 238 \ 239static __inline void \ 240atomic_store_rel_ ## name(volatile ptype p, vtype v) \ 241{ \ 242 atomic_store_rel(p, v, sz); \ 243} 244 245ATOMIC_GEN(int, int *, int, int, 32); 246ATOMIC_GEN(32, int *, int, int, 32); 247 248ATOMIC_GEN(long, long *, long, long, 64); 249ATOMIC_GEN(64, long *, long, long, 64); 250 251ATOMIC_GEN(ptr, void *, void *, uintptr_t, 64); 252 253#undef ATOMIC_GEN 254#undef atomic_cas_32 255#undef atomic_cas_64 256#undef atomic_cas 257#undef atomic_cas_acq 258#undef atomic_cas_rel 259#undef atomic_op 260#undef atomic_op_acq 261#undef atomic_op_rel 262#undef atomic_load_acq 263#undef atomic_store_rel 264#undef atomic_load_clear 265 266#endif /* !_MACHINE_ATOMIC_H_ */ 267