180708Sjake/*- 282895Sjake * Copyright (c) 1998 Doug Rabson. 380708Sjake * Copyright (c) 2001 Jake Burkholder. 480708Sjake * All rights reserved. 580708Sjake * 680708Sjake * Redistribution and use in source and binary forms, with or without 780708Sjake * modification, are permitted provided that the following conditions 880708Sjake * are met: 980708Sjake * 1. Redistributions of source code must retain the above copyright 1080708Sjake * notice, this list of conditions and the following disclaimer. 1180708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280708Sjake * notice, this list of conditions and the following disclaimer in the 1380708Sjake * documentation and/or other materials provided with the distribution. 1480708Sjake * 1581334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580708Sjake * SUCH DAMAGE. 2680708Sjake * 2782895Sjake * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 2880708Sjake * $FreeBSD$ 2980708Sjake */ 3080708Sjake 3180709Sjake#ifndef _MACHINE_ATOMIC_H_ 3280709Sjake#define _MACHINE_ATOMIC_H_ 3380709Sjake 3480709Sjake#include <machine/cpufunc.h> 3580709Sjake 36230675Smarius#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") 37230675Smarius#define wmb() mb() 38230675Smarius#define rmb() mb() 39230675Smarius 4088617Sjake/* Userland needs different ASI's. */ 4188617Sjake#ifdef _KERNEL 4288617Sjake#define __ASI_ATOMIC ASI_N 4388617Sjake#else 4488617Sjake#define __ASI_ATOMIC ASI_P 4588617Sjake#endif 4688617Sjake 4780708Sjake/* 4880709Sjake * Various simple arithmetic on memory which is atomic in the presence 4980709Sjake * of interrupts and multiple processors. See atomic(9) for details. 5080709Sjake * Note that efficient hardware support exists only for the 32 and 64 5180709Sjake * bit variants; the 8 and 16 bit versions are not provided and should 5280709Sjake * not be used in MI code. 5380709Sjake * 5480709Sjake * This implementation takes advantage of the fact that the sparc64 5580709Sjake * cas instruction is both a load and a store. The loop is often coded 5680709Sjake * as follows: 5780709Sjake * 5880709Sjake * do { 5980709Sjake * expect = *p; 6080709Sjake * new = expect + 1; 6180709Sjake * } while (cas(p, expect, new) != expect); 6280709Sjake * 6380709Sjake * which performs an unnnecessary load on each iteration that the cas 6480709Sjake * operation fails. Modified as follows: 6580709Sjake * 6680709Sjake * expect = *p; 6780709Sjake * for (;;) { 6880709Sjake * new = expect + 1; 6980709Sjake * result = cas(p, expect, new); 7080709Sjake * if (result == expect) 7180709Sjake * break; 7280709Sjake * expect = result; 7380709Sjake * } 7480709Sjake * 75108153Sjake * the return value of cas is used to avoid the extra reload. 7680709Sjake * 77230673Smarius * We only include a memory barrier in the rel variants as in total store 78230673Smarius * order which we use for running the kernel and all of the userland atomic 79230673Smarius * loads and stores behave as if the were followed by a membar with a mask 80230673Smarius * of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient 81230673Smarius * for use of relaxed memory ordering, the atomic_cas() in the acq variants 82230673Smarius * additionally would have to be followed by a membar #LoadLoad | #LoadStore. 83230673Smarius * Due to the suggested assembly syntax of the membar operands containing a 84230673Smarius * # character, they cannot be used in macros. The cmask and mmask bits thus 8580709Sjake * are hard coded in machine/cpufunc.h and used here through macros. 86230673Smarius * Hopefully the bit numbers won't change in the future. 8780708Sjake */ 8880708Sjake 89129568Smarius#define itype(sz) uint ## sz ## _t 9080708Sjake 91230675Smarius#define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC) 92230675Smarius#define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC) 9380708Sjake 9480709Sjake#define atomic_cas(p, e, s, sz) \ 95230675Smarius atomic_cas_ ## sz((p), (e), (s)) 9680709Sjake 9780709Sjake#define atomic_cas_acq(p, e, s, sz) ({ \ 9880709Sjake itype(sz) v; \ 99230675Smarius v = atomic_cas((p), (e), (s), sz); \ 100254169Smarius __compiler_membar(); \ 10180709Sjake v; \ 10280708Sjake}) 10380708Sjake 10480709Sjake#define atomic_cas_rel(p, e, s, sz) ({ \ 10580709Sjake itype(sz) v; \ 10680709Sjake membar(LoadStore | StoreStore); \ 107230675Smarius v = atomic_cas((p), (e), (s), sz); \ 10880709Sjake v; \ 10980708Sjake}) 11080708Sjake 111108153Sjake#define atomic_op(p, op, v, sz) ({ \ 11280709Sjake itype(sz) e, r, s; \ 113230675Smarius for (e = *(volatile itype(sz) *)(p);; e = r) { \ 114230675Smarius s = e op (v); \ 115230675Smarius r = atomic_cas_ ## sz((p), e, s); \ 11680709Sjake if (r == e) \ 11780709Sjake break; \ 11880708Sjake } \ 119108153Sjake e; \ 120108153Sjake}) 12180708Sjake 122108153Sjake#define atomic_op_acq(p, op, v, sz) ({ \ 123108153Sjake itype(sz) t; \ 124230675Smarius t = atomic_op((p), op, (v), sz); \ 125254169Smarius __compiler_membar(); \ 126108153Sjake t; \ 127108153Sjake}) 12880708Sjake 129108153Sjake#define atomic_op_rel(p, op, v, sz) ({ \ 130108153Sjake itype(sz) t; \ 13180709Sjake membar(LoadStore | StoreStore); \ 132230675Smarius t = atomic_op((p), op, (v), sz); \ 133108153Sjake t; \ 134108153Sjake}) 13580708Sjake 136254170Smarius#define atomic_ld_acq(p, sz) ({ \ 13780709Sjake itype(sz) v; \ 138254170Smarius v = atomic_cas((p), 0, 0, sz); \ 139254169Smarius __compiler_membar(); \ 14080709Sjake v; \ 14180709Sjake}) 14280708Sjake 143254170Smarius#define atomic_ld_clear(p, sz) ({ \ 14480709Sjake itype(sz) e, r; \ 145230675Smarius for (e = *(volatile itype(sz) *)(p);; e = r) { \ 146230675Smarius r = atomic_cas((p), e, 0, sz); \ 14780709Sjake if (r == e) \ 14880709Sjake break; \ 14980709Sjake } \ 15080709Sjake e; \ 15180709Sjake}) 15280708Sjake 153254170Smarius#define atomic_st(p, v, sz) do { \ 15480709Sjake itype(sz) e, r; \ 155230675Smarius for (e = *(volatile itype(sz) *)(p);; e = r) { \ 156230675Smarius r = atomic_cas((p), e, (v), sz); \ 15780709Sjake if (r == e) \ 15880709Sjake break; \ 15980709Sjake } \ 16080709Sjake} while (0) 16180708Sjake 162254170Smarius#define atomic_st_acq(p, v, sz) do { \ 163254170Smarius atomic_st((p), (v), sz); \ 164254170Smarius __compiler_membar(); \ 165254170Smarius} while (0) 166254170Smarius 167254170Smarius#define atomic_st_rel(p, v, sz) do { \ 16882895Sjake membar(LoadStore | StoreStore); \ 169254170Smarius atomic_st((p), (v), sz); \ 17082895Sjake} while (0) 17182895Sjake 17280709Sjake#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 17380709Sjake \ 174108153Sjakestatic __inline vtype \ 17580709Sjakeatomic_add_ ## name(volatile ptype p, atype v) \ 17680709Sjake{ \ 177230675Smarius return ((vtype)atomic_op((p), +, (v), sz)); \ 17880709Sjake} \ 179108153Sjakestatic __inline vtype \ 18080709Sjakeatomic_add_acq_ ## name(volatile ptype p, atype v) \ 18180709Sjake{ \ 182230675Smarius return ((vtype)atomic_op_acq((p), +, (v), sz)); \ 18380709Sjake} \ 184108153Sjakestatic __inline vtype \ 18580709Sjakeatomic_add_rel_ ## name(volatile ptype p, atype v) \ 18680709Sjake{ \ 187230675Smarius return ((vtype)atomic_op_rel((p), +, (v), sz)); \ 18880709Sjake} \ 18980709Sjake \ 190108153Sjakestatic __inline vtype \ 19180709Sjakeatomic_clear_ ## name(volatile ptype p, atype v) \ 19280709Sjake{ \ 193230675Smarius return ((vtype)atomic_op((p), &, ~(v), sz)); \ 19480709Sjake} \ 195108153Sjakestatic __inline vtype \ 19680709Sjakeatomic_clear_acq_ ## name(volatile ptype p, atype v) \ 19780709Sjake{ \ 198230675Smarius return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \ 19980709Sjake} \ 200108153Sjakestatic __inline vtype \ 20180709Sjakeatomic_clear_rel_ ## name(volatile ptype p, atype v) \ 20280709Sjake{ \ 203230675Smarius return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \ 20480709Sjake} \ 20580709Sjake \ 20680709Sjakestatic __inline int \ 20780709Sjakeatomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 20880709Sjake{ \ 209230675Smarius return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \ 21080709Sjake} \ 21180709Sjakestatic __inline int \ 21280709Sjakeatomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 21380709Sjake{ \ 214230675Smarius return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \ 21580709Sjake} \ 21680709Sjakestatic __inline int \ 21780709Sjakeatomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 21880709Sjake{ \ 219230675Smarius return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \ 22080709Sjake} \ 22180709Sjake \ 22280709Sjakestatic __inline vtype \ 22382895Sjakeatomic_load_ ## name(volatile ptype p) \ 22482895Sjake{ \ 225230675Smarius return ((vtype)atomic_cas((p), 0, 0, sz)); \ 22682895Sjake} \ 22782895Sjakestatic __inline vtype \ 22880709Sjakeatomic_load_acq_ ## name(volatile ptype p) \ 22980709Sjake{ \ 230230675Smarius return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \ 23180709Sjake} \ 23280709Sjake \ 23380709Sjakestatic __inline vtype \ 23480709Sjakeatomic_readandclear_ ## name(volatile ptype p) \ 23580709Sjake{ \ 236254170Smarius return ((vtype)atomic_ld_clear((p), sz)); \ 23780709Sjake} \ 23880709Sjake \ 239108153Sjakestatic __inline vtype \ 24080709Sjakeatomic_set_ ## name(volatile ptype p, atype v) \ 24180709Sjake{ \ 242230675Smarius return ((vtype)atomic_op((p), |, (v), sz)); \ 24380709Sjake} \ 244108153Sjakestatic __inline vtype \ 24580709Sjakeatomic_set_acq_ ## name(volatile ptype p, atype v) \ 24680709Sjake{ \ 247230675Smarius return ((vtype)atomic_op_acq((p), |, (v), sz)); \ 24880709Sjake} \ 249108153Sjakestatic __inline vtype \ 25080709Sjakeatomic_set_rel_ ## name(volatile ptype p, atype v) \ 25180709Sjake{ \ 252230675Smarius return ((vtype)atomic_op_rel((p), |, (v), sz)); \ 25380709Sjake} \ 25480709Sjake \ 255108153Sjakestatic __inline vtype \ 25680709Sjakeatomic_subtract_ ## name(volatile ptype p, atype v) \ 25780709Sjake{ \ 258230675Smarius return ((vtype)atomic_op((p), -, (v), sz)); \ 25980709Sjake} \ 260108153Sjakestatic __inline vtype \ 26180709Sjakeatomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 26280709Sjake{ \ 263230675Smarius return ((vtype)atomic_op_acq((p), -, (v), sz)); \ 26480709Sjake} \ 265108153Sjakestatic __inline vtype \ 26680709Sjakeatomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 26780709Sjake{ \ 268230675Smarius return ((vtype)atomic_op_rel((p), -, (v), sz)); \ 26980709Sjake} \ 27080709Sjake \ 27180709Sjakestatic __inline void \ 272254170Smariusatomic_store_acq_ ## name(volatile ptype p, vtype v) \ 27382895Sjake{ \ 274254170Smarius atomic_st_acq((p), (v), sz); \ 27582895Sjake} \ 27682895Sjakestatic __inline void \ 27780709Sjakeatomic_store_rel_ ## name(volatile ptype p, vtype v) \ 27880709Sjake{ \ 279254170Smarius atomic_st_rel((p), (v), sz); \ 28080709Sjake} 28180708Sjake 282129569SmariusATOMIC_GEN(int, u_int *, u_int, u_int, 32); 283129569SmariusATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32); 28480708Sjake 285129569SmariusATOMIC_GEN(long, u_long *, u_long, u_long, 64); 286129569SmariusATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64); 28780708Sjake 288148067SjhbATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64); 28980708Sjake 290150627Sjhb#define atomic_fetchadd_int atomic_add_int 291150627Sjhb#define atomic_fetchadd_32 atomic_add_32 292177373Spjd#define atomic_fetchadd_long atomic_add_long 293150627Sjhb 29480709Sjake#undef ATOMIC_GEN 29580709Sjake#undef atomic_cas 29680709Sjake#undef atomic_cas_acq 29780709Sjake#undef atomic_cas_rel 29880709Sjake#undef atomic_op 29980709Sjake#undef atomic_op_acq 30080709Sjake#undef atomic_op_rel 301254170Smarius#undef atomic_ld_acq 302254170Smarius#undef atomic_ld_clear 303254170Smarius#undef atomic_st 304254170Smarius#undef atomic_st_acq 305254170Smarius#undef atomic_st_rel 30680708Sjake 30780708Sjake#endif /* !_MACHINE_ATOMIC_H_ */ 308