atomic.h revision 80709
1238106Sdes/*- 2238106Sdes * Copyright (c) 2001 Jake Burkholder. 3238106Sdes * All rights reserved. 4238106Sdes * 5238106Sdes * Redistribution and use in source and binary forms, with or without 6238106Sdes * modification, are permitted provided that the following conditions 7238106Sdes * are met: 8238106Sdes * 1. Redistributions of source code must retain the above copyright 9238106Sdes * notice, this list of conditions and the following disclaimer. 10238106Sdes * 2. Redistributions in binary form must reproduce the above copyright 11238106Sdes * notice, this list of conditions and the following disclaimer in the 12238106Sdes * documentation and/or other materials provided with the distribution. 13238106Sdes * 14238106Sdes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 15238106Sdes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16238106Sdes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17238106Sdes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 18238106Sdes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19238106Sdes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20238106Sdes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21238106Sdes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22238106Sdes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23238106Sdes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24238106Sdes * SUCH DAMAGE. 25238106Sdes * 26238106Sdes * $FreeBSD: head/sys/sparc64/include/atomic.h 80709 2001-07-31 06:05:05Z jake $ 27238106Sdes */ 28238106Sdes 29238106Sdes#ifndef _MACHINE_ATOMIC_H_ 30238106Sdes#define _MACHINE_ATOMIC_H_ 31238106Sdes 32238106Sdes#include <machine/cpufunc.h> 33238106Sdes 34238106Sdes/* 35238106Sdes * Various simple arithmetic on memory which is atomic in the presence 36238106Sdes * of interrupts and multiple processors. See atomic(9) for details. 37238106Sdes * Note that efficient hardware support exists only for the 32 and 64 38238106Sdes * bit variants; the 8 and 16 bit versions are not provided and should 39238106Sdes * not be used in MI code. 40238106Sdes * 41238106Sdes * This implementation takes advantage of the fact that the sparc64 42238106Sdes * cas instruction is both a load and a store. The loop is often coded 43238106Sdes * as follows: 44238106Sdes * 45238106Sdes * do { 46238106Sdes * expect = *p; 47238106Sdes * new = expect + 1; 48238106Sdes * } while (cas(p, expect, new) != expect); 49238106Sdes * 50238106Sdes * which performs an unnnecessary load on each iteration that the cas 51238106Sdes * operation fails. Modified as follows: 52238106Sdes * 53238106Sdes * expect = *p; 54238106Sdes * for (;;) { 55238106Sdes * new = expect + 1; 56238106Sdes * result = cas(p, expect, new); 57238106Sdes * if (result == expect) 58238106Sdes * break; 59238106Sdes * expect = result; 60238106Sdes * } 61238106Sdes * 62238106Sdes * the return value of cas is used to avoid the extra reload. At the 63238106Sdes * time of writing, with gcc version 2.95.3, the branch for the if 64238106Sdes * statement is predicted incorrectly as not taken, rather than taken. 65238106Sdes * It is expected that the branch prediction hints available in gcc 3.0, 66238106Sdes * __builtin_expect, will allow better code to be generated. 67238106Sdes * 68238106Sdes * The memory barriers provided by the acq and rel variants are intended 69238106Sdes * to be sufficient for use of relaxed memory ordering. Due to the 70238106Sdes * suggested assembly syntax of the membar operands containing a # 71238106Sdes * character, they cannot be used in macros. The cmask and mmask bits 72238106Sdes * are hard coded in machine/cpufunc.h and used here through macros. 73238106Sdes * Hopefully sun will choose not to change the bit numbers. 74238106Sdes */ 75238106Sdes 76238106Sdes#define itype(sz) u_int ## sz ## _t 77238106Sdes 78238106Sdes#define atomic_cas_32(p, e, s) casa(p, e, s, ASI_N) 79238106Sdes#define atomic_cas_64(p, e, s) casxa(p, e, s, ASI_N) 80238106Sdes 81238106Sdes#define atomic_cas(p, e, s, sz) \ 82238106Sdes atomic_cas_ ## sz(p, e, s) 83238106Sdes 84238106Sdes#define atomic_cas_acq(p, e, s, sz) ({ \ 85238106Sdes itype(sz) v; \ 86238106Sdes v = atomic_cas(p, e, s, sz); \ 87238106Sdes membar(LoadLoad | LoadStore); \ 88238106Sdes v; \ 89238106Sdes}) 90238106Sdes 91238106Sdes#define atomic_cas_rel(p, e, s, sz) ({ \ 92238106Sdes itype(sz) v; \ 93238106Sdes membar(LoadStore | StoreStore); \ 94238106Sdes v = atomic_cas(p, e, s, sz); \ 95238106Sdes v; \ 96238106Sdes}) 97238106Sdes 98238106Sdes#define atomic_op(p, op, v, sz) do { \ 99238106Sdes itype(sz) e, r, s; \ 100238106Sdes for (e = *(volatile itype(sz) *)p;; e = r) { \ 101238106Sdes s = e op v; \ 102238106Sdes r = atomic_cas_ ## sz(p, e, s); \ 103238106Sdes if (r == e) \ 104238106Sdes break; \ 105238106Sdes } \ 106238106Sdes} while (0) 107238106Sdes 108238106Sdes#define atomic_op_acq(p, op, v, sz) do { \ 109238106Sdes atomic_op(p, op, v, sz); \ 110238106Sdes membar(LoadLoad | LoadStore); \ 111238106Sdes} while (0) 112238106Sdes 113238106Sdes#define atomic_op_rel(p, op, v, sz) do { \ 114238106Sdes membar(LoadStore | StoreStore); \ 115238106Sdes atomic_op(p, op, v, sz); \ 116238106Sdes} while (0) 117238106Sdes 118238106Sdes#define atomic_load_acq(p, sz) ({ \ 119238106Sdes itype(sz) v; \ 120238106Sdes v = atomic_cas_ ## sz(p, 0, 0); \ 121238106Sdes membar(LoadLoad | LoadStore); \ 122238106Sdes v; \ 123238106Sdes}) 124238106Sdes 125238106Sdes#define atomic_load_clear(p, sz) ({ \ 126238106Sdes itype(sz) e, r; \ 127238106Sdes for (e = *(volatile itype(sz) *)p;; e = r) { \ 128238106Sdes r = atomic_cas_ ## sz(p, e, 0); \ 129238106Sdes if (r == e) \ 130238106Sdes break; \ 131238106Sdes } \ 132238106Sdes e; \ 133238106Sdes}) 134238106Sdes 135238106Sdes#define atomic_store_rel(p, v, sz) do { \ 136238106Sdes itype(sz) e, r; \ 137238106Sdes membar(LoadStore | StoreStore); \ 138238106Sdes for (e = *(volatile itype(sz) *)p;; e = r) { \ 139238106Sdes r = atomic_cas_ ## sz(p, e, v); \ 140238106Sdes if (r == e) \ 141238106Sdes break; \ 142238106Sdes } \ 143238106Sdes} while (0) 144238106Sdes 145238106Sdes#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 146238106Sdes \ 147238106Sdesstatic __inline void \ 148238106Sdesatomic_add_ ## name(volatile ptype p, atype v) \ 149238106Sdes{ \ 150238106Sdes atomic_op(p, +, v, sz); \ 151238106Sdes} \ 152238106Sdesstatic __inline void \ 153238106Sdesatomic_add_acq_ ## name(volatile ptype p, atype v) \ 154238106Sdes{ \ 155238106Sdes atomic_op_acq(p, +, v, sz); \ 156238106Sdes} \ 157238106Sdesstatic __inline void \ 158238106Sdesatomic_add_rel_ ## name(volatile ptype p, atype v) \ 159238106Sdes{ \ 160238106Sdes atomic_op_rel(p, +, v, sz); \ 161238106Sdes} \ 162238106Sdes \ 163238106Sdesstatic __inline void \ 164238106Sdesatomic_clear_ ## name(volatile ptype p, atype v) \ 165238106Sdes{ \ 166238106Sdes atomic_op(p, &, ~v, sz); \ 167238106Sdes} \ 168238106Sdesstatic __inline void \ 169238106Sdesatomic_clear_acq_ ## name(volatile ptype p, atype v) \ 170238106Sdes{ \ 171238106Sdes atomic_op_acq(p, &, ~v, sz); \ 172238106Sdes} \ 173238106Sdesstatic __inline void \ 174238106Sdesatomic_clear_rel_ ## name(volatile ptype p, atype v) \ 175238106Sdes{ \ 176238106Sdes atomic_op_rel(p, &, ~v, sz); \ 177238106Sdes} \ 178238106Sdes \ 179238106Sdesstatic __inline int \ 180238106Sdesatomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 181238106Sdes{ \ 182238106Sdes return (((vtype)atomic_cas(p, e, s, sz)) == e); \ 183238106Sdes} \ 184238106Sdesstatic __inline int \ 185238106Sdesatomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 186238106Sdes{ \ 187238106Sdes return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \ 188238106Sdes} \ 189238106Sdesstatic __inline int \ 190238106Sdesatomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 191238106Sdes{ \ 192238106Sdes return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \ 193238106Sdes} \ 194238106Sdes \ 195238106Sdesstatic __inline vtype \ 196238106Sdesatomic_load_acq_ ## name(volatile ptype p) \ 197238106Sdes{ \ 198238106Sdes return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \ 199238106Sdes} \ 200238106Sdes \ 201238106Sdesstatic __inline vtype \ 202238106Sdesatomic_readandclear_ ## name(volatile ptype p) \ 203238106Sdes{ \ 204238106Sdes return ((vtype)atomic_load_clear(p, sz)); \ 205238106Sdes} \ 206238106Sdes \ 207238106Sdesstatic __inline void \ 208238106Sdesatomic_set_ ## name(volatile ptype p, atype v) \ 209238106Sdes{ \ 210238106Sdes atomic_op(p, |, v, sz); \ 211238106Sdes} \ 212238106Sdesstatic __inline void \ 213238106Sdesatomic_set_acq_ ## name(volatile ptype p, atype v) \ 214238106Sdes{ \ 215238106Sdes atomic_op_acq(p, |, v, sz); \ 216238106Sdes} \ 217238106Sdesstatic __inline void \ 218238106Sdesatomic_set_rel_ ## name(volatile ptype p, atype v) \ 219238106Sdes{ \ 220238106Sdes atomic_op_rel(p, |, v, sz); \ 221238106Sdes} \ 222238106Sdes \ 223238106Sdesstatic __inline void \ 224238106Sdesatomic_subtract_ ## name(volatile ptype p, atype v) \ 225238106Sdes{ \ 226238106Sdes atomic_op(p, -, v, sz); \ 227238106Sdes} \ 228238106Sdesstatic __inline void \ 229238106Sdesatomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 230238106Sdes{ \ 231238106Sdes atomic_op_acq(p, -, v, sz); \ 232238106Sdes} \ 233238106Sdesstatic __inline void \ 234238106Sdesatomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 235238106Sdes{ \ 236238106Sdes atomic_op_rel(p, -, v, sz); \ 237238106Sdes} \ 238238106Sdes \ 239238106Sdesstatic __inline void \ 240238106Sdesatomic_store_rel_ ## name(volatile ptype p, vtype v) \ 241238106Sdes{ \ 242238106Sdes atomic_store_rel(p, v, sz); \ 243238106Sdes} 244238106Sdes 245238106SdesATOMIC_GEN(int, int *, int, int, 32); 246238106SdesATOMIC_GEN(32, int *, int, int, 32); 247238106Sdes 248238106SdesATOMIC_GEN(long, long *, long, long, 64); 249238106SdesATOMIC_GEN(64, long *, long, long, 64); 250238106Sdes 251238106SdesATOMIC_GEN(ptr, void *, void *, uintptr_t, 64); 252238106Sdes 253238106Sdes#undef ATOMIC_GEN 254238106Sdes#undef atomic_cas_32 255238106Sdes#undef atomic_cas_64 256238106Sdes#undef atomic_cas 257238106Sdes#undef atomic_cas_acq 258238106Sdes#undef atomic_cas_rel 259238106Sdes#undef atomic_op 260238106Sdes#undef atomic_op_acq 261238106Sdes#undef atomic_op_rel 262238106Sdes#undef atomic_load_acq 263238106Sdes#undef atomic_store_rel 264238106Sdes#undef atomic_load_clear 265238106Sdes 266238106Sdes#endif /* !_MACHINE_ATOMIC_H_ */ 267238106Sdes