atomic.h revision 82895
1243730Srwatson/*- 2243730Srwatson * Copyright (c) 1998 Doug Rabson. 3243730Srwatson * Copyright (c) 2001 Jake Burkholder. 4243730Srwatson * All rights reserved. 5243730Srwatson * 6243730Srwatson * Redistribution and use in source and binary forms, with or without 7243730Srwatson * modification, are permitted provided that the following conditions 8243730Srwatson * are met: 9243730Srwatson * 1. Redistributions of source code must retain the above copyright 10243730Srwatson * notice, this list of conditions and the following disclaimer. 11243730Srwatson * 2. Redistributions in binary form must reproduce the above copyright 12243730Srwatson * notice, this list of conditions and the following disclaimer in the 13243730Srwatson * documentation and/or other materials provided with the distribution. 14243730Srwatson * 15243730Srwatson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16243730Srwatson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17243730Srwatson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18243730Srwatson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19243730Srwatson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20243730Srwatson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21243730Srwatson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22243730Srwatson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23243730Srwatson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24243730Srwatson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25243730Srwatson * SUCH DAMAGE. 26243730Srwatson * 27243730Srwatson * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 28243730Srwatson * $FreeBSD: head/sys/sparc64/include/atomic.h 82895 2001-09-03 22:03:25Z jake $ 29243730Srwatson */ 30243730Srwatson 31243730Srwatson#ifndef _MACHINE_ATOMIC_H_ 32243730Srwatson#define _MACHINE_ATOMIC_H_ 33243730Srwatson 34243730Srwatson#include <machine/cpufunc.h> 35243730Srwatson 36243730Srwatson/* 37243730Srwatson * Various simple arithmetic on memory which is atomic in the presence 38243730Srwatson * of interrupts and multiple processors. See atomic(9) for details. 39243730Srwatson * Note that efficient hardware support exists only for the 32 and 64 40243730Srwatson * bit variants; the 8 and 16 bit versions are not provided and should 41243730Srwatson * not be used in MI code. 42243730Srwatson * 43243730Srwatson * This implementation takes advantage of the fact that the sparc64 44243730Srwatson * cas instruction is both a load and a store. The loop is often coded 45243730Srwatson * as follows: 46243730Srwatson * 47243730Srwatson * do { 48243730Srwatson * expect = *p; 49243730Srwatson * new = expect + 1; 50243730Srwatson * } while (cas(p, expect, new) != expect); 51243730Srwatson * 52243730Srwatson * which performs an unnnecessary load on each iteration that the cas 53243730Srwatson * operation fails. Modified as follows: 54243730Srwatson * 55243730Srwatson * expect = *p; 56243730Srwatson * for (;;) { 57243730Srwatson * new = expect + 1; 58243730Srwatson * result = cas(p, expect, new); 59243730Srwatson * if (result == expect) 60243730Srwatson * break; 61243730Srwatson * expect = result; 62243730Srwatson * } 63243730Srwatson * 64243730Srwatson * the return value of cas is used to avoid the extra reload. At the 65243730Srwatson * time of writing, with gcc version 2.95.3, the branch for the if 66243730Srwatson * statement is predicted incorrectly as not taken, rather than taken. 67243730Srwatson * It is expected that the branch prediction hints available in gcc 3.0, 68243730Srwatson * __builtin_expect, will allow better code to be generated. 69243730Srwatson * 70243730Srwatson * The memory barriers provided by the acq and rel variants are intended 71243730Srwatson * to be sufficient for use of relaxed memory ordering. Due to the 72243730Srwatson * suggested assembly syntax of the membar operands containing a # 73243730Srwatson * character, they cannot be used in macros. The cmask and mmask bits 74 * are hard coded in machine/cpufunc.h and used here through macros. 75 * Hopefully sun will choose not to change the bit numbers. 76 */ 77 78#define itype(sz) u_int ## sz ## _t 79 80#define atomic_cas_32(p, e, s) casa(p, e, s, ASI_N) 81#define atomic_cas_64(p, e, s) casxa(p, e, s, ASI_N) 82 83#define atomic_cas(p, e, s, sz) \ 84 atomic_cas_ ## sz(p, e, s) 85 86#define atomic_cas_acq(p, e, s, sz) ({ \ 87 itype(sz) v; \ 88 v = atomic_cas(p, e, s, sz); \ 89 membar(LoadLoad | LoadStore); \ 90 v; \ 91}) 92 93#define atomic_cas_rel(p, e, s, sz) ({ \ 94 itype(sz) v; \ 95 membar(LoadStore | StoreStore); \ 96 v = atomic_cas(p, e, s, sz); \ 97 v; \ 98}) 99 100#define atomic_op(p, op, v, sz) do { \ 101 itype(sz) e, r, s; \ 102 for (e = *(volatile itype(sz) *)p;; e = r) { \ 103 s = e op v; \ 104 r = atomic_cas_ ## sz(p, e, s); \ 105 if (r == e) \ 106 break; \ 107 } \ 108} while (0) 109 110#define atomic_op_acq(p, op, v, sz) do { \ 111 atomic_op(p, op, v, sz); \ 112 membar(LoadLoad | LoadStore); \ 113} while (0) 114 115#define atomic_op_rel(p, op, v, sz) do { \ 116 membar(LoadStore | StoreStore); \ 117 atomic_op(p, op, v, sz); \ 118} while (0) 119 120#define atomic_load(p, sz) \ 121 atomic_cas(p, 0, 0, sz) 122 123#define atomic_load_acq(p, sz) ({ \ 124 itype(sz) v; \ 125 v = atomic_load(p, sz); \ 126 membar(LoadLoad | LoadStore); \ 127 v; \ 128}) 129 130#define atomic_load_clear(p, sz) ({ \ 131 itype(sz) e, r; \ 132 for (e = *(volatile itype(sz) *)p;; e = r) { \ 133 r = atomic_cas(p, e, 0, sz); \ 134 if (r == e) \ 135 break; \ 136 } \ 137 e; \ 138}) 139 140#define atomic_store(p, v, sz) do { \ 141 itype(sz) e, r; \ 142 for (e = *(volatile itype(sz) *)p;; e = r) { \ 143 r = atomic_cas(p, e, v, sz); \ 144 if (r == e) \ 145 break; \ 146 } \ 147} while (0) 148 149#define atomic_store_rel(p, v, sz) do { \ 150 membar(LoadStore | StoreStore); \ 151 atomic_store(p, v, sz); \ 152} while (0) 153 154#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 155 \ 156static __inline void \ 157atomic_add_ ## name(volatile ptype p, atype v) \ 158{ \ 159 atomic_op(p, +, v, sz); \ 160} \ 161static __inline void \ 162atomic_add_acq_ ## name(volatile ptype p, atype v) \ 163{ \ 164 atomic_op_acq(p, +, v, sz); \ 165} \ 166static __inline void \ 167atomic_add_rel_ ## name(volatile ptype p, atype v) \ 168{ \ 169 atomic_op_rel(p, +, v, sz); \ 170} \ 171 \ 172static __inline void \ 173atomic_clear_ ## name(volatile ptype p, atype v) \ 174{ \ 175 atomic_op(p, &, ~v, sz); \ 176} \ 177static __inline void \ 178atomic_clear_acq_ ## name(volatile ptype p, atype v) \ 179{ \ 180 atomic_op_acq(p, &, ~v, sz); \ 181} \ 182static __inline void \ 183atomic_clear_rel_ ## name(volatile ptype p, atype v) \ 184{ \ 185 atomic_op_rel(p, &, ~v, sz); \ 186} \ 187 \ 188static __inline int \ 189atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 190{ \ 191 return (((vtype)atomic_cas(p, e, s, sz)) == e); \ 192} \ 193static __inline int \ 194atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 195{ \ 196 return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \ 197} \ 198static __inline int \ 199atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 200{ \ 201 return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \ 202} \ 203 \ 204static __inline vtype \ 205atomic_load_ ## name(volatile ptype p) \ 206{ \ 207 return ((vtype)atomic_cas(p, 0, 0, sz)); \ 208} \ 209static __inline vtype \ 210atomic_load_acq_ ## name(volatile ptype p) \ 211{ \ 212 return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \ 213} \ 214 \ 215static __inline vtype \ 216atomic_readandclear_ ## name(volatile ptype p) \ 217{ \ 218 return ((vtype)atomic_load_clear(p, sz)); \ 219} \ 220 \ 221static __inline void \ 222atomic_set_ ## name(volatile ptype p, atype v) \ 223{ \ 224 atomic_op(p, |, v, sz); \ 225} \ 226static __inline void \ 227atomic_set_acq_ ## name(volatile ptype p, atype v) \ 228{ \ 229 atomic_op_acq(p, |, v, sz); \ 230} \ 231static __inline void \ 232atomic_set_rel_ ## name(volatile ptype p, atype v) \ 233{ \ 234 atomic_op_rel(p, |, v, sz); \ 235} \ 236 \ 237static __inline void \ 238atomic_subtract_ ## name(volatile ptype p, atype v) \ 239{ \ 240 atomic_op(p, -, v, sz); \ 241} \ 242static __inline void \ 243atomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 244{ \ 245 atomic_op_acq(p, -, v, sz); \ 246} \ 247static __inline void \ 248atomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 249{ \ 250 atomic_op_rel(p, -, v, sz); \ 251} \ 252 \ 253static __inline void \ 254atomic_store_ ## name(volatile ptype p, vtype v) \ 255{ \ 256 atomic_store(p, v, sz); \ 257} \ 258static __inline void \ 259atomic_store_rel_ ## name(volatile ptype p, vtype v) \ 260{ \ 261 atomic_store_rel(p, v, sz); \ 262} 263 264ATOMIC_GEN(int, int *, int, int, 32); 265ATOMIC_GEN(32, int *, int, int, 32); 266 267ATOMIC_GEN(long, long *, long, long, 64); 268ATOMIC_GEN(64, long *, long, long, 64); 269 270ATOMIC_GEN(ptr, void *, void *, uintptr_t, 64); 271 272#undef ATOMIC_GEN 273#undef atomic_cas_32 274#undef atomic_cas_64 275#undef atomic_cas 276#undef atomic_cas_acq 277#undef atomic_cas_rel 278#undef atomic_op 279#undef atomic_op_acq 280#undef atomic_op_rel 281#undef atomic_load_acq 282#undef atomic_store_rel 283#undef atomic_load_clear 284 285#endif /* !_MACHINE_ATOMIC_H_ */ 286