atomic.h revision 96956
166458Sdfr/*- 266458Sdfr * Copyright (c) 1998 Doug Rabson 366458Sdfr * All rights reserved. 466458Sdfr * 566458Sdfr * Redistribution and use in source and binary forms, with or without 666458Sdfr * modification, are permitted provided that the following conditions 766458Sdfr * are met: 866458Sdfr * 1. Redistributions of source code must retain the above copyright 966458Sdfr * notice, this list of conditions and the following disclaimer. 1066458Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1166458Sdfr * notice, this list of conditions and the following disclaimer in the 1266458Sdfr * documentation and/or other materials provided with the distribution. 1366458Sdfr * 1466458Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1566458Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1666458Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1766458Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1866458Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1966458Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2066458Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2166458Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2266458Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2366458Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2466458Sdfr * SUCH DAMAGE. 2566458Sdfr * 2666458Sdfr * $FreeBSD: head/sys/ia64/include/atomic.h 96956 2002-05-19 20:19:07Z marcel $ 2766458Sdfr */ 2866458Sdfr 2966458Sdfr#ifndef _MACHINE_ATOMIC_H_ 3066458Sdfr#define _MACHINE_ATOMIC_H_ 3166458Sdfr 3266458Sdfr/* 3366458Sdfr * Various simple arithmetic on memory which is atomic in the presence 3466458Sdfr * of interrupts and SMP safe. 3566458Sdfr */ 3666458Sdfr 3766458Sdfr/* 3866458Sdfr * Everything is built out of cmpxchg. 3966458Sdfr */ 4096956Smarcel#define IA64_CMPXCHG(sz, sem, p, cmpval, newval, ret) \ 4166458Sdfr __asm __volatile ( \ 4266458Sdfr "mov ar.ccv=%2;;\n\t" \ 4366458Sdfr "cmpxchg" #sz "." #sem " %0=%4,%3,ar.ccv\n\t" \ 4496956Smarcel : "=r" (ret), "=m" (*p) \ 4596956Smarcel : "r" (cmpval), "r" (newval), "m" (*p) \ 4696956Smarcel : "memory") 4766458Sdfr 4866458Sdfr/* 4966458Sdfr * Some common forms of cmpxch. 5066458Sdfr */ 5166458Sdfrstatic __inline u_int32_t 5266458Sdfria64_cmpxchg_acq_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 5366458Sdfr{ 5496956Smarcel u_int32_t ret; 5596956Smarcel IA64_CMPXCHG(4, acq, p, cmpval, newval, ret); 5696956Smarcel return (ret); 5766458Sdfr} 5866458Sdfr 5966458Sdfrstatic __inline u_int32_t 6066458Sdfria64_cmpxchg_rel_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 6166458Sdfr{ 6296956Smarcel u_int32_t ret; 6396956Smarcel IA64_CMPXCHG(4, rel, p, cmpval, newval, ret); 6496956Smarcel return (ret); 6566458Sdfr} 6666458Sdfr 6766458Sdfrstatic __inline u_int64_t 6866458Sdfria64_cmpxchg_acq_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 6966458Sdfr{ 7096956Smarcel u_int64_t ret; 7196956Smarcel IA64_CMPXCHG(8, acq, p, cmpval, newval, ret); 7296956Smarcel return (ret); 7366458Sdfr} 7466458Sdfr 7566458Sdfrstatic __inline u_int64_t 7666458Sdfria64_cmpxchg_rel_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 7766458Sdfr{ 7896956Smarcel u_int64_t ret; 7996956Smarcel IA64_CMPXCHG(8, rel, p, cmpval, newval, ret); 8096956Smarcel return (ret); 8166458Sdfr} 8266458Sdfr 8367351Sjhb#define ATOMIC_STORE_LOAD(type, width, size) \ 8467351Sjhbstatic __inline u_int##width##_t \ 8567351Sjhbia64_ld_acq_##width(volatile u_int##width##_t* p) \ 8667351Sjhb{ \ 8767522Sdfr u_int##width##_t v; \ 8867351Sjhb \ 8967351Sjhb __asm __volatile ("ld" size ".acq %0=%1" \ 9067522Sdfr : "=r" (v) \ 9167522Sdfr : "m" (*p) \ 9267351Sjhb : "memory"); \ 9367351Sjhb return (v); \ 9467351Sjhb} \ 9567351Sjhb \ 9667351Sjhbstatic __inline u_int##width##_t \ 9767351Sjhbatomic_load_acq_##width(volatile u_int##width##_t* p) \ 9867351Sjhb{ \ 9967522Sdfr u_int##width##_t v; \ 10067351Sjhb \ 10167351Sjhb __asm __volatile ("ld" size ".acq %0=%1" \ 10267522Sdfr : "=r" (v) \ 10367522Sdfr : "m" (*p) \ 10467351Sjhb : "memory"); \ 10567351Sjhb return (v); \ 10667351Sjhb} \ 10767351Sjhb \ 10867351Sjhbstatic __inline u_int##width##_t \ 10967351Sjhbatomic_load_acq_##type(volatile u_int##width##_t* p) \ 11067351Sjhb{ \ 11167522Sdfr u_int##width##_t v; \ 11267351Sjhb \ 11367351Sjhb __asm __volatile ("ld" size ".acq %0=%1" \ 11467522Sdfr : "=r" (v) \ 11567522Sdfr : "m" (*p) \ 11667351Sjhb : "memory"); \ 11767351Sjhb return (v); \ 11867351Sjhb} \ 11967351Sjhb \ 12067351Sjhbstatic __inline void \ 12167351Sjhbia64_st_rel_##width(volatile u_int##width##_t* p, u_int##width##_t v)\ 12267351Sjhb{ \ 12367351Sjhb __asm __volatile ("st" size ".rel %0=%1" \ 12467351Sjhb : "=m" (*p) \ 12567351Sjhb : "r" (v) \ 12667351Sjhb : "memory"); \ 12767351Sjhb} \ 12867351Sjhb \ 12967351Sjhbstatic __inline void \ 13067351Sjhbatomic_store_rel_##width(volatile u_int##width##_t* p, u_int##width##_t v)\ 13167351Sjhb{ \ 13267351Sjhb __asm __volatile ("st" size ".rel %0=%1" \ 13367351Sjhb : "=m" (*p) \ 13467351Sjhb : "r" (v) \ 13567351Sjhb : "memory"); \ 13667351Sjhb} \ 13767351Sjhb \ 13867351Sjhbstatic __inline void \ 13967351Sjhbatomic_store_rel_##type(volatile u_int##width##_t* p, u_int##width##_t v)\ 14067351Sjhb{ \ 14167351Sjhb __asm __volatile ("st" size ".rel %0=%1" \ 14267351Sjhb : "=m" (*p) \ 14367351Sjhb : "r" (v) \ 14467351Sjhb : "memory"); \ 14566458Sdfr} 14666458Sdfr 14767351SjhbATOMIC_STORE_LOAD(char, 8, "1") 14867351SjhbATOMIC_STORE_LOAD(short, 16, "2") 14967351SjhbATOMIC_STORE_LOAD(int, 32, "4") 15067351SjhbATOMIC_STORE_LOAD(long, 64, "8") 15166458Sdfr 15267351Sjhb#undef ATOMIC_STORE_LOAD 15367351Sjhb 15467351Sjhb#define IA64_ATOMIC(sz, type, name, width, op) \ 15566458Sdfr \ 15666458Sdfrstatic __inline void \ 15767351Sjhbatomic_##name##_acq_##width(volatile type *p, type v) \ 15866458Sdfr{ \ 15996956Smarcel type old, ret; \ 16066458Sdfr do { \ 16166458Sdfr old = *p; \ 16296956Smarcel IA64_CMPXCHG(sz, acq, p, old, old op v, ret); \ 16396956Smarcel } while (ret != old); \ 16467351Sjhb} \ 16567351Sjhb \ 16667351Sjhbstatic __inline void \ 16767351Sjhbatomic_##name##_rel_##width(volatile type *p, type v) \ 16867351Sjhb{ \ 16996956Smarcel type old, ret; \ 17067351Sjhb do { \ 17167351Sjhb old = *p; \ 17296956Smarcel IA64_CMPXCHG(sz, rel, p, old, old op v, ret); \ 17396956Smarcel } while (ret != old); \ 17466458Sdfr} 17566458Sdfr 17667351SjhbIA64_ATOMIC(1, u_int8_t, set, 8, |) 17767351SjhbIA64_ATOMIC(2, u_int16_t, set, 16, |) 17867351SjhbIA64_ATOMIC(4, u_int32_t, set, 32, |) 17967351SjhbIA64_ATOMIC(8, u_int64_t, set, 64, |) 18066458Sdfr 18167351SjhbIA64_ATOMIC(1, u_int8_t, clear, 8, &~) 18267351SjhbIA64_ATOMIC(2, u_int16_t, clear, 16, &~) 18367351SjhbIA64_ATOMIC(4, u_int32_t, clear, 32, &~) 18467351SjhbIA64_ATOMIC(8, u_int64_t, clear, 64, &~) 18566458Sdfr 18667351SjhbIA64_ATOMIC(1, u_int8_t, add, 8, +) 18767351SjhbIA64_ATOMIC(2, u_int16_t, add, 16, +) 18867351SjhbIA64_ATOMIC(4, u_int32_t, add, 32, +) 18967351SjhbIA64_ATOMIC(8, u_int64_t, add, 64, +) 19066458Sdfr 19167351SjhbIA64_ATOMIC(1, u_int8_t, subtract, 8, -) 19267351SjhbIA64_ATOMIC(2, u_int16_t, subtract, 16, -) 19367351SjhbIA64_ATOMIC(4, u_int32_t, subtract, 32, -) 19467351SjhbIA64_ATOMIC(8, u_int64_t, subtract, 64, -) 19566458Sdfr 19666458Sdfr#undef IA64_ATOMIC 19766458Sdfr#undef IA64_CMPXCHG 19866458Sdfr 19967351Sjhb#define atomic_set_8 atomic_set_acq_8 20067351Sjhb#define atomic_clear_8 atomic_clear_acq_8 20167351Sjhb#define atomic_add_8 atomic_add_acq_8 20267351Sjhb#define atomic_subtract_8 atomic_subtract_acq_8 20366458Sdfr 20467351Sjhb#define atomic_set_16 atomic_set_acq_16 20567351Sjhb#define atomic_clear_16 atomic_clear_acq_16 20667351Sjhb#define atomic_add_16 atomic_add_acq_16 20767351Sjhb#define atomic_subtract_16 atomic_subtract_acq_16 20866458Sdfr 20967351Sjhb#define atomic_set_32 atomic_set_acq_32 21067351Sjhb#define atomic_clear_32 atomic_clear_acq_32 21167351Sjhb#define atomic_add_32 atomic_add_acq_32 21267351Sjhb#define atomic_subtract_32 atomic_subtract_acq_32 21366458Sdfr 21467351Sjhb#define atomic_set_64 atomic_set_acq_64 21567351Sjhb#define atomic_clear_64 atomic_clear_acq_64 21667351Sjhb#define atomic_add_64 atomic_add_acq_64 21767351Sjhb#define atomic_subtract_64 atomic_subtract_acq_64 21866458Sdfr 21967351Sjhb#define atomic_set_char atomic_set_8 22067351Sjhb#define atomic_clear_char atomic_clear_8 22167351Sjhb#define atomic_add_char atomic_add_8 22267351Sjhb#define atomic_subtract_char atomic_subtract_8 22367351Sjhb#define atomic_set_acq_char atomic_set_acq_8 22467351Sjhb#define atomic_clear_acq_char atomic_clear_acq_8 22567351Sjhb#define atomic_add_acq_char atomic_add_acq_8 22667351Sjhb#define atomic_subtract_acq_char atomic_subtract_acq_8 22767351Sjhb#define atomic_set_rel_char atomic_set_rel_8 22867351Sjhb#define atomic_clear_rel_char atomic_clear_rel_8 22967351Sjhb#define atomic_add_rel_char atomic_add_rel_8 23067351Sjhb#define atomic_subtract_rel_char atomic_subtract_rel_8 23167351Sjhb 23267351Sjhb#define atomic_set_short atomic_set_16 23367351Sjhb#define atomic_clear_short atomic_clear_16 23467351Sjhb#define atomic_add_short atomic_add_16 23567351Sjhb#define atomic_subtract_short atomic_subtract_16 23667351Sjhb#define atomic_set_acq_short atomic_set_acq_16 23767351Sjhb#define atomic_clear_acq_short atomic_clear_acq_16 23867351Sjhb#define atomic_add_acq_short atomic_add_acq_16 23967351Sjhb#define atomic_subtract_acq_short atomic_subtract_acq_16 24067351Sjhb#define atomic_set_rel_short atomic_set_rel_16 24167351Sjhb#define atomic_clear_rel_short atomic_clear_rel_16 24267351Sjhb#define atomic_add_rel_short atomic_add_rel_16 24367351Sjhb#define atomic_subtract_rel_short atomic_subtract_rel_16 24467351Sjhb 24567351Sjhb#define atomic_set_int atomic_set_32 24667351Sjhb#define atomic_clear_int atomic_clear_32 24767351Sjhb#define atomic_add_int atomic_add_32 24867351Sjhb#define atomic_subtract_int atomic_subtract_32 24967351Sjhb#define atomic_set_acq_int atomic_set_acq_32 25067351Sjhb#define atomic_clear_acq_int atomic_clear_acq_32 25167351Sjhb#define atomic_add_acq_int atomic_add_acq_32 25267351Sjhb#define atomic_subtract_acq_int atomic_subtract_acq_32 25367351Sjhb#define atomic_set_rel_int atomic_set_rel_32 25467351Sjhb#define atomic_clear_rel_int atomic_clear_rel_32 25567351Sjhb#define atomic_add_rel_int atomic_add_rel_32 25667351Sjhb#define atomic_subtract_rel_int atomic_subtract_rel_32 25767351Sjhb 25867351Sjhb#define atomic_set_long atomic_set_64 25967351Sjhb#define atomic_clear_long atomic_clear_64 26067351Sjhb#define atomic_add_long atomic_add_64 26167351Sjhb#define atomic_subtract_long atomic_subtract_64 26267351Sjhb#define atomic_set_acq_long atomic_set_acq_64 26367351Sjhb#define atomic_clear_acq_long atomic_clear_acq_64 26467351Sjhb#define atomic_add_acq_long atomic_add_acq_64 26567351Sjhb#define atomic_subtract_acq_long atomic_subtract_acq_64 26667351Sjhb#define atomic_set_rel_long atomic_set_rel_64 26767351Sjhb#define atomic_clear_rel_long atomic_clear_rel_64 26867351Sjhb#define atomic_add_rel_long atomic_add_rel_64 26967351Sjhb#define atomic_subtract_rel_long atomic_subtract_rel_64 27067351Sjhb 27166458Sdfr/* 27266458Sdfr * Atomically compare the value stored at *p with cmpval and if the 27366458Sdfr * two values are equal, update the value of *p with newval. Returns 27466458Sdfr * zero if the compare failed, nonzero otherwise. 27566458Sdfr */ 27666458Sdfrstatic __inline int 27767351Sjhbatomic_cmpset_acq_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 27866458Sdfr{ 27966458Sdfr return ia64_cmpxchg_acq_32(p, cmpval, newval) == cmpval; 28066458Sdfr} 28166458Sdfr 28267351Sjhbstatic __inline int 28367351Sjhbatomic_cmpset_rel_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) 28467351Sjhb{ 28567351Sjhb return ia64_cmpxchg_rel_32(p, cmpval, newval) == cmpval; 28667351Sjhb} 28767351Sjhb 28866458Sdfr/* 28966458Sdfr * Atomically compare the value stored at *p with cmpval and if the 29066458Sdfr * two values are equal, update the value of *p with newval. Returns 29166458Sdfr * zero if the compare failed, nonzero otherwise. 29266458Sdfr */ 29366458Sdfrstatic __inline int 29467351Sjhbatomic_cmpset_acq_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 29566458Sdfr{ 29666458Sdfr return ia64_cmpxchg_acq_64(p, cmpval, newval) == cmpval; 29766458Sdfr} 29866458Sdfr 29967351Sjhbstatic __inline int 30067351Sjhbatomic_cmpset_rel_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) 30167351Sjhb{ 30267351Sjhb return ia64_cmpxchg_rel_64(p, cmpval, newval) == cmpval; 30367351Sjhb} 30467351Sjhb 30567351Sjhb#define atomic_cmpset_32 atomic_cmpset_acq_32 30667351Sjhb#define atomic_cmpset_64 atomic_cmpset_acq_64 30766458Sdfr#define atomic_cmpset_int atomic_cmpset_32 30866458Sdfr#define atomic_cmpset_long atomic_cmpset_64 30967351Sjhb#define atomic_cmpset_acq_int atomic_cmpset_acq_32 31067351Sjhb#define atomic_cmpset_rel_int atomic_cmpset_rel_32 31167351Sjhb#define atomic_cmpset_acq_long atomic_cmpset_acq_64 31267351Sjhb#define atomic_cmpset_rel_long atomic_cmpset_rel_64 31366458Sdfr 31466458Sdfrstatic __inline int 31567351Sjhbatomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) 31666458Sdfr{ 31767351Sjhb return atomic_cmpset_acq_long((volatile u_long *)dst, 31867351Sjhb (u_long)exp, (u_long)src); 31966458Sdfr} 32066458Sdfr 32167351Sjhbstatic __inline int 32267351Sjhbatomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) 32367351Sjhb{ 32467351Sjhb return atomic_cmpset_rel_long((volatile u_long *)dst, 32567351Sjhb (u_long)exp, (u_long)src); 32667351Sjhb} 32767351Sjhb 32867351Sjhb#define atomic_cmpset_ptr atomic_cmpset_acq_ptr 32967351Sjhb 33067351Sjhbstatic __inline void * 33167351Sjhbatomic_load_acq_ptr(volatile void *p) 33267351Sjhb{ 33367351Sjhb return (void *)atomic_load_acq_long((volatile u_long *)p); 33467351Sjhb} 33567351Sjhb 33667351Sjhbstatic __inline void 33767351Sjhbatomic_store_rel_ptr(volatile void *p, void *v) 33867351Sjhb{ 33967351Sjhb atomic_store_rel_long((volatile u_long *)p, (u_long)v); 34067351Sjhb} 34167351Sjhb 34267540Sjhb#define ATOMIC_PTR(NAME) \ 34367540Sjhbstatic __inline void \ 34467540Sjhbatomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 34567540Sjhb{ \ 34667540Sjhb atomic_##NAME##_long((volatile u_long *)p, v); \ 34767540Sjhb} \ 34867540Sjhb \ 34967540Sjhbstatic __inline void \ 35067540Sjhbatomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 35167540Sjhb{ \ 35267540Sjhb atomic_##NAME##_acq_long((volatile u_long *)p, v);\ 35367540Sjhb} \ 35467540Sjhb \ 35567540Sjhbstatic __inline void \ 35667540Sjhbatomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 35767540Sjhb{ \ 35867540Sjhb atomic_##NAME##_rel_long((volatile u_long *)p, v);\ 35967540Sjhb} 36067540Sjhb 36167540SjhbATOMIC_PTR(set) 36267540SjhbATOMIC_PTR(clear) 36367540SjhbATOMIC_PTR(add) 36467540SjhbATOMIC_PTR(subtract) 36567540Sjhb 36667540Sjhb#undef ATOMIC_PTR 36767540Sjhb 36866937Sdfrstatic __inline u_int32_t 36966937Sdfratomic_readandclear_32(volatile u_int32_t* p) 37066937Sdfr{ 37166937Sdfr u_int32_t val; 37266937Sdfr do { 37366937Sdfr val = *p; 37466937Sdfr } while (!atomic_cmpset_32(p, val, 0)); 37566937Sdfr return val; 37666937Sdfr} 37766937Sdfr 37866937Sdfrstatic __inline u_int64_t 37966937Sdfratomic_readandclear_64(volatile u_int64_t* p) 38066937Sdfr{ 38166937Sdfr u_int64_t val; 38266937Sdfr do { 38366937Sdfr val = *p; 38466937Sdfr } while (!atomic_cmpset_64(p, val, 0)); 38566937Sdfr return val; 38666937Sdfr} 38766937Sdfr 38866937Sdfr#define atomic_readandclear_int atomic_readandclear_32 38966937Sdfr#define atomic_readandclear_long atomic_readandclear_64 39066937Sdfr 39166458Sdfr#endif /* ! _MACHINE_ATOMIC_H_ */ 392