atomic.h revision 82895
180708Sjake/*- 282895Sjake * Copyright (c) 1998 Doug Rabson. 380708Sjake * Copyright (c) 2001 Jake Burkholder. 480708Sjake * All rights reserved. 580708Sjake * 680708Sjake * Redistribution and use in source and binary forms, with or without 780708Sjake * modification, are permitted provided that the following conditions 880708Sjake * are met: 980708Sjake * 1. Redistributions of source code must retain the above copyright 1080708Sjake * notice, this list of conditions and the following disclaimer. 1180708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280708Sjake * notice, this list of conditions and the following disclaimer in the 1380708Sjake * documentation and/or other materials provided with the distribution. 1480708Sjake * 1581334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580708Sjake * SUCH DAMAGE. 2680708Sjake * 2782895Sjake * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 2880708Sjake * $FreeBSD: head/sys/sparc64/include/atomic.h 82895 2001-09-03 22:03:25Z jake $ 2980708Sjake */ 3080708Sjake 3180709Sjake#ifndef _MACHINE_ATOMIC_H_ 3280709Sjake#define _MACHINE_ATOMIC_H_ 3380709Sjake 3480709Sjake#include <machine/cpufunc.h> 3580709Sjake 3680708Sjake/* 3780709Sjake * Various simple arithmetic on memory which is atomic in the presence 3880709Sjake * of interrupts and multiple processors. See atomic(9) for details. 3980709Sjake * Note that efficient hardware support exists only for the 32 and 64 4080709Sjake * bit variants; the 8 and 16 bit versions are not provided and should 4180709Sjake * not be used in MI code. 4280709Sjake * 4380709Sjake * This implementation takes advantage of the fact that the sparc64 4480709Sjake * cas instruction is both a load and a store. The loop is often coded 4580709Sjake * as follows: 4680709Sjake * 4780709Sjake * do { 4880709Sjake * expect = *p; 4980709Sjake * new = expect + 1; 5080709Sjake * } while (cas(p, expect, new) != expect); 5180709Sjake * 5280709Sjake * which performs an unnnecessary load on each iteration that the cas 5380709Sjake * operation fails. Modified as follows: 5480709Sjake * 5580709Sjake * expect = *p; 5680709Sjake * for (;;) { 5780709Sjake * new = expect + 1; 5880709Sjake * result = cas(p, expect, new); 5980709Sjake * if (result == expect) 6080709Sjake * break; 6180709Sjake * expect = result; 6280709Sjake * } 6380709Sjake * 6480709Sjake * the return value of cas is used to avoid the extra reload. At the 6580709Sjake * time of writing, with gcc version 2.95.3, the branch for the if 6680709Sjake * statement is predicted incorrectly as not taken, rather than taken. 6780709Sjake * It is expected that the branch prediction hints available in gcc 3.0, 6880709Sjake * __builtin_expect, will allow better code to be generated. 6980709Sjake * 7080709Sjake * The memory barriers provided by the acq and rel variants are intended 7180709Sjake * to be sufficient for use of relaxed memory ordering. Due to the 7280709Sjake * suggested assembly syntax of the membar operands containing a # 7380709Sjake * character, they cannot be used in macros. The cmask and mmask bits 7480709Sjake * are hard coded in machine/cpufunc.h and used here through macros. 7580709Sjake * Hopefully sun will choose not to change the bit numbers. 7680708Sjake */ 7780708Sjake 7880709Sjake#define itype(sz) u_int ## sz ## _t 7980708Sjake 8080709Sjake#define atomic_cas_32(p, e, s) casa(p, e, s, ASI_N) 8180709Sjake#define atomic_cas_64(p, e, s) casxa(p, e, s, ASI_N) 8280708Sjake 8380709Sjake#define atomic_cas(p, e, s, sz) \ 8480709Sjake atomic_cas_ ## sz(p, e, s) 8580709Sjake 8680709Sjake#define atomic_cas_acq(p, e, s, sz) ({ \ 8780709Sjake itype(sz) v; \ 8880709Sjake v = atomic_cas(p, e, s, sz); \ 8980709Sjake membar(LoadLoad | LoadStore); \ 9080709Sjake v; \ 9180708Sjake}) 9280708Sjake 9380709Sjake#define atomic_cas_rel(p, e, s, sz) ({ \ 9480709Sjake itype(sz) v; \ 9580709Sjake membar(LoadStore | StoreStore); \ 9680709Sjake v = atomic_cas(p, e, s, sz); \ 9780709Sjake v; \ 9880708Sjake}) 9980708Sjake 10080709Sjake#define atomic_op(p, op, v, sz) do { \ 10180709Sjake itype(sz) e, r, s; \ 10280709Sjake for (e = *(volatile itype(sz) *)p;; e = r) { \ 10380709Sjake s = e op v; \ 10480709Sjake r = atomic_cas_ ## sz(p, e, s); \ 10580709Sjake if (r == e) \ 10680709Sjake break; \ 10780708Sjake } \ 10880709Sjake} while (0) 10980708Sjake 11080709Sjake#define atomic_op_acq(p, op, v, sz) do { \ 11180709Sjake atomic_op(p, op, v, sz); \ 11280709Sjake membar(LoadLoad | LoadStore); \ 11380709Sjake} while (0) 11480708Sjake 11580709Sjake#define atomic_op_rel(p, op, v, sz) do { \ 11680709Sjake membar(LoadStore | StoreStore); \ 11780709Sjake atomic_op(p, op, v, sz); \ 11880709Sjake} while (0) 11980708Sjake 12082895Sjake#define atomic_load(p, sz) \ 12182895Sjake atomic_cas(p, 0, 0, sz) 12282895Sjake 12380709Sjake#define atomic_load_acq(p, sz) ({ \ 12480709Sjake itype(sz) v; \ 12582895Sjake v = atomic_load(p, sz); \ 12680709Sjake membar(LoadLoad | LoadStore); \ 12780709Sjake v; \ 12880709Sjake}) 12980708Sjake 13080709Sjake#define atomic_load_clear(p, sz) ({ \ 13180709Sjake itype(sz) e, r; \ 13280709Sjake for (e = *(volatile itype(sz) *)p;; e = r) { \ 13382895Sjake r = atomic_cas(p, e, 0, sz); \ 13480709Sjake if (r == e) \ 13580709Sjake break; \ 13680709Sjake } \ 13780709Sjake e; \ 13880709Sjake}) 13980708Sjake 14082895Sjake#define atomic_store(p, v, sz) do { \ 14180709Sjake itype(sz) e, r; \ 14280709Sjake for (e = *(volatile itype(sz) *)p;; e = r) { \ 14382895Sjake r = atomic_cas(p, e, v, sz); \ 14480709Sjake if (r == e) \ 14580709Sjake break; \ 14680709Sjake } \ 14780709Sjake} while (0) 14880708Sjake 14982895Sjake#define atomic_store_rel(p, v, sz) do { \ 15082895Sjake membar(LoadStore | StoreStore); \ 15182895Sjake atomic_store(p, v, sz); \ 15282895Sjake} while (0) 15382895Sjake 15480709Sjake#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 15580709Sjake \ 15680709Sjakestatic __inline void \ 15780709Sjakeatomic_add_ ## name(volatile ptype p, atype v) \ 15880709Sjake{ \ 15980709Sjake atomic_op(p, +, v, sz); \ 16080709Sjake} \ 16180709Sjakestatic __inline void \ 16280709Sjakeatomic_add_acq_ ## name(volatile ptype p, atype v) \ 16380709Sjake{ \ 16480709Sjake atomic_op_acq(p, +, v, sz); \ 16580709Sjake} \ 16680709Sjakestatic __inline void \ 16780709Sjakeatomic_add_rel_ ## name(volatile ptype p, atype v) \ 16880709Sjake{ \ 16980709Sjake atomic_op_rel(p, +, v, sz); \ 17080709Sjake} \ 17180709Sjake \ 17280709Sjakestatic __inline void \ 17380709Sjakeatomic_clear_ ## name(volatile ptype p, atype v) \ 17480709Sjake{ \ 17580709Sjake atomic_op(p, &, ~v, sz); \ 17680709Sjake} \ 17780709Sjakestatic __inline void \ 17880709Sjakeatomic_clear_acq_ ## name(volatile ptype p, atype v) \ 17980709Sjake{ \ 18080709Sjake atomic_op_acq(p, &, ~v, sz); \ 18180709Sjake} \ 18280709Sjakestatic __inline void \ 18380709Sjakeatomic_clear_rel_ ## name(volatile ptype p, atype v) \ 18480709Sjake{ \ 18580709Sjake atomic_op_rel(p, &, ~v, sz); \ 18680709Sjake} \ 18780709Sjake \ 18880709Sjakestatic __inline int \ 18980709Sjakeatomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 19080709Sjake{ \ 19180709Sjake return (((vtype)atomic_cas(p, e, s, sz)) == e); \ 19280709Sjake} \ 19380709Sjakestatic __inline int \ 19480709Sjakeatomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 19580709Sjake{ \ 19680709Sjake return (((vtype)atomic_cas_acq(p, e, s, sz)) == e); \ 19780709Sjake} \ 19880709Sjakestatic __inline int \ 19980709Sjakeatomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 20080709Sjake{ \ 20180709Sjake return (((vtype)atomic_cas_rel(p, e, s, sz)) == e); \ 20280709Sjake} \ 20380709Sjake \ 20480709Sjakestatic __inline vtype \ 20582895Sjakeatomic_load_ ## name(volatile ptype p) \ 20682895Sjake{ \ 20782895Sjake return ((vtype)atomic_cas(p, 0, 0, sz)); \ 20882895Sjake} \ 20982895Sjakestatic __inline vtype \ 21080709Sjakeatomic_load_acq_ ## name(volatile ptype p) \ 21180709Sjake{ \ 21280709Sjake return ((vtype)atomic_cas_acq(p, 0, 0, sz)); \ 21380709Sjake} \ 21480709Sjake \ 21580709Sjakestatic __inline vtype \ 21680709Sjakeatomic_readandclear_ ## name(volatile ptype p) \ 21780709Sjake{ \ 21880709Sjake return ((vtype)atomic_load_clear(p, sz)); \ 21980709Sjake} \ 22080709Sjake \ 22180709Sjakestatic __inline void \ 22280709Sjakeatomic_set_ ## name(volatile ptype p, atype v) \ 22380709Sjake{ \ 22480709Sjake atomic_op(p, |, v, sz); \ 22580709Sjake} \ 22680709Sjakestatic __inline void \ 22780709Sjakeatomic_set_acq_ ## name(volatile ptype p, atype v) \ 22880709Sjake{ \ 22980709Sjake atomic_op_acq(p, |, v, sz); \ 23080709Sjake} \ 23180709Sjakestatic __inline void \ 23280709Sjakeatomic_set_rel_ ## name(volatile ptype p, atype v) \ 23380709Sjake{ \ 23480709Sjake atomic_op_rel(p, |, v, sz); \ 23580709Sjake} \ 23680709Sjake \ 23780709Sjakestatic __inline void \ 23880709Sjakeatomic_subtract_ ## name(volatile ptype p, atype v) \ 23980709Sjake{ \ 24080709Sjake atomic_op(p, -, v, sz); \ 24180709Sjake} \ 24280709Sjakestatic __inline void \ 24380709Sjakeatomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 24480709Sjake{ \ 24580709Sjake atomic_op_acq(p, -, v, sz); \ 24680709Sjake} \ 24780709Sjakestatic __inline void \ 24880709Sjakeatomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 24980709Sjake{ \ 25080709Sjake atomic_op_rel(p, -, v, sz); \ 25180709Sjake} \ 25280709Sjake \ 25380709Sjakestatic __inline void \ 25482895Sjakeatomic_store_ ## name(volatile ptype p, vtype v) \ 25582895Sjake{ \ 25682895Sjake atomic_store(p, v, sz); \ 25782895Sjake} \ 25882895Sjakestatic __inline void \ 25980709Sjakeatomic_store_rel_ ## name(volatile ptype p, vtype v) \ 26080709Sjake{ \ 26180709Sjake atomic_store_rel(p, v, sz); \ 26280709Sjake} 26380708Sjake 26480709SjakeATOMIC_GEN(int, int *, int, int, 32); 26580709SjakeATOMIC_GEN(32, int *, int, int, 32); 26680708Sjake 26780709SjakeATOMIC_GEN(long, long *, long, long, 64); 26880709SjakeATOMIC_GEN(64, long *, long, long, 64); 26980708Sjake 27080709SjakeATOMIC_GEN(ptr, void *, void *, uintptr_t, 64); 27180708Sjake 27280709Sjake#undef ATOMIC_GEN 27380709Sjake#undef atomic_cas_32 27480709Sjake#undef atomic_cas_64 27580709Sjake#undef atomic_cas 27680709Sjake#undef atomic_cas_acq 27780709Sjake#undef atomic_cas_rel 27880709Sjake#undef atomic_op 27980709Sjake#undef atomic_op_acq 28080709Sjake#undef atomic_op_rel 28180709Sjake#undef atomic_load_acq 28280709Sjake#undef atomic_store_rel 28380709Sjake#undef atomic_load_clear 28480708Sjake 28580708Sjake#endif /* !_MACHINE_ATOMIC_H_ */ 286