180708Sjake/*- 282895Sjake * Copyright (c) 1998 Doug Rabson. 380708Sjake * Copyright (c) 2001 Jake Burkholder. 480708Sjake * All rights reserved. 580708Sjake * 680708Sjake * Redistribution and use in source and binary forms, with or without 780708Sjake * modification, are permitted provided that the following conditions 880708Sjake * are met: 980708Sjake * 1. Redistributions of source code must retain the above copyright 1080708Sjake * notice, this list of conditions and the following disclaimer. 1180708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280708Sjake * notice, this list of conditions and the following disclaimer in the 1380708Sjake * documentation and/or other materials provided with the distribution. 1480708Sjake * 1581334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580708Sjake * SUCH DAMAGE. 2680708Sjake * 2782895Sjake * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 2880708Sjake * $FreeBSD: releng/11.0/sys/sparc64/include/atomic.h 294539 2016-01-22 00:29:11Z jhb $ 2980708Sjake */ 3080708Sjake 3180709Sjake#ifndef _MACHINE_ATOMIC_H_ 3280709Sjake#define _MACHINE_ATOMIC_H_ 3380709Sjake 3480709Sjake#include <machine/cpufunc.h> 3580709Sjake 36225890Smarius#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") 37225890Smarius#define wmb() mb() 38225890Smarius#define rmb() mb() 39225890Smarius 4088617Sjake/* Userland needs different ASI's. */ 4188617Sjake#ifdef _KERNEL 4288617Sjake#define __ASI_ATOMIC ASI_N 4388617Sjake#else 4488617Sjake#define __ASI_ATOMIC ASI_P 4588617Sjake#endif 4688617Sjake 4780708Sjake/* 4880709Sjake * Various simple arithmetic on memory which is atomic in the presence 4980709Sjake * of interrupts and multiple processors. See atomic(9) for details. 5080709Sjake * Note that efficient hardware support exists only for the 32 and 64 5180709Sjake * bit variants; the 8 and 16 bit versions are not provided and should 5280709Sjake * not be used in MI code. 5380709Sjake * 5480709Sjake * This implementation takes advantage of the fact that the sparc64 5580709Sjake * cas instruction is both a load and a store. The loop is often coded 5680709Sjake * as follows: 5780709Sjake * 5880709Sjake * do { 5980709Sjake * expect = *p; 6080709Sjake * new = expect + 1; 6180709Sjake * } while (cas(p, expect, new) != expect); 6280709Sjake * 6380709Sjake * which performs an unnnecessary load on each iteration that the cas 6480709Sjake * operation fails. Modified as follows: 6580709Sjake * 6680709Sjake * expect = *p; 6780709Sjake * for (;;) { 6880709Sjake * new = expect + 1; 6980709Sjake * result = cas(p, expect, new); 7080709Sjake * if (result == expect) 7180709Sjake * break; 7280709Sjake * expect = result; 7380709Sjake * } 7480709Sjake * 75108153Sjake * the return value of cas is used to avoid the extra reload. 7680709Sjake * 77225889Smarius * We only include a memory barrier in the rel variants as in total store 78225889Smarius * order which we use for running the kernel and all of the userland atomic 79225889Smarius * loads and stores behave as if the were followed by a membar with a mask 80225889Smarius * of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient 81228222Smarius * for use of relaxed memory ordering, the atomic_cas() in the acq variants 82225889Smarius * additionally would have to be followed by a membar #LoadLoad | #LoadStore. 83225889Smarius * Due to the suggested assembly syntax of the membar operands containing a 84225889Smarius * # character, they cannot be used in macros. The cmask and mmask bits thus 8580709Sjake * are hard coded in machine/cpufunc.h and used here through macros. 86225889Smarius * Hopefully the bit numbers won't change in the future. 8780708Sjake */ 8880708Sjake 89129568Smarius#define itype(sz) uint ## sz ## _t 9080708Sjake 91225890Smarius#define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC) 92225890Smarius#define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC) 9380708Sjake 9480709Sjake#define atomic_cas(p, e, s, sz) \ 95225890Smarius atomic_cas_ ## sz((p), (e), (s)) 9680709Sjake 9780709Sjake#define atomic_cas_acq(p, e, s, sz) ({ \ 9880709Sjake itype(sz) v; \ 99225890Smarius v = atomic_cas((p), (e), (s), sz); \ 100241374Sattilio __compiler_membar(); \ 10180709Sjake v; \ 10280708Sjake}) 10380708Sjake 10480709Sjake#define atomic_cas_rel(p, e, s, sz) ({ \ 10580709Sjake itype(sz) v; \ 10680709Sjake membar(LoadStore | StoreStore); \ 107225890Smarius v = atomic_cas((p), (e), (s), sz); \ 10880709Sjake v; \ 10980708Sjake}) 11080708Sjake 111108153Sjake#define atomic_op(p, op, v, sz) ({ \ 11280709Sjake itype(sz) e, r, s; \ 113225890Smarius for (e = *(volatile itype(sz) *)(p);; e = r) { \ 114225890Smarius s = e op (v); \ 115225890Smarius r = atomic_cas_ ## sz((p), e, s); \ 11680709Sjake if (r == e) \ 11780709Sjake break; \ 11880708Sjake } \ 119108153Sjake e; \ 120108153Sjake}) 12180708Sjake 122108153Sjake#define atomic_op_acq(p, op, v, sz) ({ \ 123108153Sjake itype(sz) t; \ 124225890Smarius t = atomic_op((p), op, (v), sz); \ 125241374Sattilio __compiler_membar(); \ 126108153Sjake t; \ 127108153Sjake}) 12880708Sjake 129108153Sjake#define atomic_op_rel(p, op, v, sz) ({ \ 130108153Sjake itype(sz) t; \ 13180709Sjake membar(LoadStore | StoreStore); \ 132225890Smarius t = atomic_op((p), op, (v), sz); \ 133108153Sjake t; \ 134108153Sjake}) 13580708Sjake 136253994Smarius#define atomic_ld_acq(p, sz) ({ \ 13780709Sjake itype(sz) v; \ 138251783Sed v = atomic_cas((p), 0, 0, sz); \ 139241374Sattilio __compiler_membar(); \ 14080709Sjake v; \ 14180709Sjake}) 14280708Sjake 143253994Smarius#define atomic_ld_clear(p, sz) ({ \ 14480709Sjake itype(sz) e, r; \ 145225890Smarius for (e = *(volatile itype(sz) *)(p);; e = r) { \ 146225890Smarius r = atomic_cas((p), e, 0, sz); \ 14780709Sjake if (r == e) \ 14880709Sjake break; \ 14980709Sjake } \ 15080709Sjake e; \ 15180709Sjake}) 15280708Sjake 153253994Smarius#define atomic_st(p, v, sz) do { \ 15480709Sjake itype(sz) e, r; \ 155225890Smarius for (e = *(volatile itype(sz) *)(p);; e = r) { \ 156225890Smarius r = atomic_cas((p), e, (v), sz); \ 15780709Sjake if (r == e) \ 15880709Sjake break; \ 15980709Sjake } \ 16080709Sjake} while (0) 16180708Sjake 162253994Smarius#define atomic_st_acq(p, v, sz) do { \ 163253994Smarius atomic_st((p), (v), sz); \ 164253994Smarius __compiler_membar(); \ 165253994Smarius} while (0) 166253994Smarius 167253994Smarius#define atomic_st_rel(p, v, sz) do { \ 168253994Smarius membar(LoadStore | StoreStore); \ 169253994Smarius atomic_st((p), (v), sz); \ 170253994Smarius} while (0) 171253994Smarius 17280709Sjake#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 17380709Sjake \ 174108153Sjakestatic __inline vtype \ 17580709Sjakeatomic_add_ ## name(volatile ptype p, atype v) \ 17680709Sjake{ \ 177225890Smarius return ((vtype)atomic_op((p), +, (v), sz)); \ 17880709Sjake} \ 179108153Sjakestatic __inline vtype \ 18080709Sjakeatomic_add_acq_ ## name(volatile ptype p, atype v) \ 18180709Sjake{ \ 182225890Smarius return ((vtype)atomic_op_acq((p), +, (v), sz)); \ 18380709Sjake} \ 184108153Sjakestatic __inline vtype \ 18580709Sjakeatomic_add_rel_ ## name(volatile ptype p, atype v) \ 18680709Sjake{ \ 187225890Smarius return ((vtype)atomic_op_rel((p), +, (v), sz)); \ 18880709Sjake} \ 18980709Sjake \ 190108153Sjakestatic __inline vtype \ 19180709Sjakeatomic_clear_ ## name(volatile ptype p, atype v) \ 19280709Sjake{ \ 193225890Smarius return ((vtype)atomic_op((p), &, ~(v), sz)); \ 19480709Sjake} \ 195108153Sjakestatic __inline vtype \ 19680709Sjakeatomic_clear_acq_ ## name(volatile ptype p, atype v) \ 19780709Sjake{ \ 198225890Smarius return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \ 19980709Sjake} \ 200108153Sjakestatic __inline vtype \ 20180709Sjakeatomic_clear_rel_ ## name(volatile ptype p, atype v) \ 20280709Sjake{ \ 203225890Smarius return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \ 20480709Sjake} \ 20580709Sjake \ 20680709Sjakestatic __inline int \ 20780709Sjakeatomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 20880709Sjake{ \ 209225890Smarius return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \ 21080709Sjake} \ 21180709Sjakestatic __inline int \ 21280709Sjakeatomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 21380709Sjake{ \ 214225890Smarius return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \ 21580709Sjake} \ 21680709Sjakestatic __inline int \ 21780709Sjakeatomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 21880709Sjake{ \ 219225890Smarius return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \ 22080709Sjake} \ 22180709Sjake \ 22280709Sjakestatic __inline vtype \ 22382895Sjakeatomic_load_ ## name(volatile ptype p) \ 22482895Sjake{ \ 225225890Smarius return ((vtype)atomic_cas((p), 0, 0, sz)); \ 22682895Sjake} \ 22782895Sjakestatic __inline vtype \ 22880709Sjakeatomic_load_acq_ ## name(volatile ptype p) \ 22980709Sjake{ \ 230225890Smarius return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \ 23180709Sjake} \ 23280709Sjake \ 23380709Sjakestatic __inline vtype \ 23480709Sjakeatomic_readandclear_ ## name(volatile ptype p) \ 23580709Sjake{ \ 236253994Smarius return ((vtype)atomic_ld_clear((p), sz)); \ 23780709Sjake} \ 23880709Sjake \ 239108153Sjakestatic __inline vtype \ 24080709Sjakeatomic_set_ ## name(volatile ptype p, atype v) \ 24180709Sjake{ \ 242225890Smarius return ((vtype)atomic_op((p), |, (v), sz)); \ 24380709Sjake} \ 244108153Sjakestatic __inline vtype \ 24580709Sjakeatomic_set_acq_ ## name(volatile ptype p, atype v) \ 24680709Sjake{ \ 247225890Smarius return ((vtype)atomic_op_acq((p), |, (v), sz)); \ 24880709Sjake} \ 249108153Sjakestatic __inline vtype \ 25080709Sjakeatomic_set_rel_ ## name(volatile ptype p, atype v) \ 25180709Sjake{ \ 252225890Smarius return ((vtype)atomic_op_rel((p), |, (v), sz)); \ 25380709Sjake} \ 25480709Sjake \ 255108153Sjakestatic __inline vtype \ 25680709Sjakeatomic_subtract_ ## name(volatile ptype p, atype v) \ 25780709Sjake{ \ 258225890Smarius return ((vtype)atomic_op((p), -, (v), sz)); \ 25980709Sjake} \ 260108153Sjakestatic __inline vtype \ 26180709Sjakeatomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 26280709Sjake{ \ 263225890Smarius return ((vtype)atomic_op_acq((p), -, (v), sz)); \ 26480709Sjake} \ 265108153Sjakestatic __inline vtype \ 26680709Sjakeatomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 26780709Sjake{ \ 268225890Smarius return ((vtype)atomic_op_rel((p), -, (v), sz)); \ 26980709Sjake} \ 27080709Sjake \ 27180709Sjakestatic __inline void \ 272253994Smariusatomic_store_acq_ ## name(volatile ptype p, vtype v) \ 273253994Smarius{ \ 274253994Smarius atomic_st_acq((p), (v), sz); \ 275253994Smarius} \ 276253994Smariusstatic __inline void \ 27780709Sjakeatomic_store_rel_ ## name(volatile ptype p, vtype v) \ 27880709Sjake{ \ 279253994Smarius atomic_st_rel((p), (v), sz); \ 28080709Sjake} 28180708Sjake 282285283Skibstatic __inline void 283285283Skibatomic_thread_fence_acq(void) 284285283Skib{ 285285283Skib 286285283Skib __compiler_membar(); 287285283Skib} 288285283Skib 289285283Skibstatic __inline void 290285283Skibatomic_thread_fence_rel(void) 291285283Skib{ 292285283Skib 293285283Skib __compiler_membar(); 294285283Skib} 295285283Skib 296285283Skibstatic __inline void 297285283Skibatomic_thread_fence_acq_rel(void) 298285283Skib{ 299285283Skib 300285283Skib __compiler_membar(); 301285283Skib} 302285283Skib 303285283Skibstatic __inline void 304285283Skibatomic_thread_fence_seq_cst(void) 305285283Skib{ 306285283Skib 307285283Skib membar(LoadLoad | LoadStore | StoreStore | StoreLoad); 308285283Skib} 309285283Skib 310285283Skib 311129569SmariusATOMIC_GEN(int, u_int *, u_int, u_int, 32); 312129569SmariusATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32); 31380708Sjake 314129569SmariusATOMIC_GEN(long, u_long *, u_long, u_long, 64); 315129569SmariusATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64); 31680708Sjake 317148067SjhbATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64); 31880708Sjake 319150627Sjhb#define atomic_fetchadd_int atomic_add_int 320150627Sjhb#define atomic_fetchadd_32 atomic_add_32 321177373Spjd#define atomic_fetchadd_long atomic_add_long 322294539Sjhb#define atomic_fetchadd_64 atomic_add_64 323150627Sjhb 32480709Sjake#undef ATOMIC_GEN 32580709Sjake#undef atomic_cas 32680709Sjake#undef atomic_cas_acq 32780709Sjake#undef atomic_cas_rel 32880709Sjake#undef atomic_op 32980709Sjake#undef atomic_op_acq 33080709Sjake#undef atomic_op_rel 331253994Smarius#undef atomic_ld_acq 332253994Smarius#undef atomic_ld_clear 333253994Smarius#undef atomic_st 334253994Smarius#undef atomic_st_acq 335253994Smarius#undef atomic_st_rel 33680708Sjake 33780708Sjake#endif /* !_MACHINE_ATOMIC_H_ */ 338