atomic.h revision 173970
166458Sdfr/*- 266458Sdfr * Copyright (c) 1998 Doug Rabson 366458Sdfr * All rights reserved. 466458Sdfr * 566458Sdfr * Redistribution and use in source and binary forms, with or without 666458Sdfr * modification, are permitted provided that the following conditions 766458Sdfr * are met: 866458Sdfr * 1. Redistributions of source code must retain the above copyright 966458Sdfr * notice, this list of conditions and the following disclaimer. 1066458Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1166458Sdfr * notice, this list of conditions and the following disclaimer in the 1266458Sdfr * documentation and/or other materials provided with the distribution. 1366458Sdfr * 1466458Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1566458Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1666458Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1766458Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1866458Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1966458Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2066458Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2166458Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2266458Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2366458Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2466458Sdfr * SUCH DAMAGE. 2566458Sdfr * 2666458Sdfr * $FreeBSD: head/sys/ia64/include/atomic.h 173970 2007-11-27 06:34:15Z jasone $ 2766458Sdfr */ 2866458Sdfr 2966458Sdfr#ifndef _MACHINE_ATOMIC_H_ 30135581Smarcel#define _MACHINE_ATOMIC_H_ 3166458Sdfr 3266458Sdfr/* 3366458Sdfr * Various simple arithmetic on memory which is atomic in the presence 3466458Sdfr * of interrupts and SMP safe. 3566458Sdfr */ 3666458Sdfr 3766458Sdfr/* 3866458Sdfr * Everything is built out of cmpxchg. 3966458Sdfr */ 40135581Smarcel#define IA64_CMPXCHG(sz, sem, p, cmpval, newval, ret) \ 41135581Smarcel __asm __volatile ( \ 42135581Smarcel "mov ar.ccv=%2;;\n\t" \ 43135581Smarcel "cmpxchg" #sz "." #sem " %0=%4,%3,ar.ccv\n\t" \ 44135581Smarcel : "=r" (ret), "=m" (*p) \ 45135581Smarcel : "r" (cmpval), "r" (newval), "m" (*p) \ 4696956Smarcel : "memory") 4766458Sdfr 4866458Sdfr/* 4966458Sdfr * Some common forms of cmpxch. 5066458Sdfr */ 51135581Smarcelstatic __inline uint32_t 52135581Smarcelia64_cmpxchg_acq_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 5366458Sdfr{ 54135581Smarcel uint32_t ret; 5596956Smarcel IA64_CMPXCHG(4, acq, p, cmpval, newval, ret); 5696956Smarcel return (ret); 5766458Sdfr} 5866458Sdfr 59135581Smarcelstatic __inline uint32_t 60135581Smarcelia64_cmpxchg_rel_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 6166458Sdfr{ 62135581Smarcel uint32_t ret; 6396956Smarcel IA64_CMPXCHG(4, rel, p, cmpval, newval, ret); 6496956Smarcel return (ret); 6566458Sdfr} 6666458Sdfr 67135581Smarcelstatic __inline uint64_t 68135581Smarcelia64_cmpxchg_acq_64(volatile uint64_t* p, uint64_t cmpval, uint64_t newval) 6966458Sdfr{ 70135581Smarcel uint64_t ret; 7196956Smarcel IA64_CMPXCHG(8, acq, p, cmpval, newval, ret); 7296956Smarcel return (ret); 7366458Sdfr} 7466458Sdfr 75135581Smarcelstatic __inline uint64_t 76135581Smarcelia64_cmpxchg_rel_64(volatile uint64_t* p, uint64_t cmpval, uint64_t newval) 7766458Sdfr{ 78135581Smarcel uint64_t ret; 7996956Smarcel IA64_CMPXCHG(8, rel, p, cmpval, newval, ret); 8096956Smarcel return (ret); 8166458Sdfr} 8266458Sdfr 83135581Smarcel#define ATOMIC_STORE_LOAD(type, width, size) \ 84135581Smarcel static __inline uint##width##_t \ 85135581Smarcel ia64_ld_acq_##width(volatile uint##width##_t* p) \ 86135581Smarcel { \ 87135581Smarcel uint##width##_t v; \ 88135581Smarcel __asm __volatile ("ld" size ".acq %0=%1" : "=r" (v) \ 89135581Smarcel : "m" (*p) : "memory"); \ 90135581Smarcel return (v); \ 91135581Smarcel } \ 92135581Smarcel \ 93135581Smarcel static __inline uint##width##_t \ 94135581Smarcel atomic_load_acq_##width(volatile uint##width##_t* p) \ 95135581Smarcel { \ 96135581Smarcel uint##width##_t v; \ 97135581Smarcel __asm __volatile ("ld" size ".acq %0=%1" : "=r" (v) \ 98135581Smarcel : "m" (*p) : "memory"); \ 99135581Smarcel return (v); \ 100135581Smarcel } \ 101135581Smarcel \ 102135581Smarcel static __inline uint##width##_t \ 103135581Smarcel atomic_load_acq_##type(volatile uint##width##_t* p) \ 104135581Smarcel { \ 105135581Smarcel uint##width##_t v; \ 106135581Smarcel __asm __volatile ("ld" size ".acq %0=%1" : "=r" (v) \ 107135581Smarcel : "m" (*p) : "memory"); \ 108135581Smarcel return (v); \ 109135581Smarcel } \ 110135581Smarcel \ 111135581Smarcel static __inline void \ 112135581Smarcel ia64_st_rel_##width(volatile uint##width##_t* p, uint##width##_t v) \ 113135581Smarcel { \ 114135581Smarcel __asm __volatile ("st" size ".rel %0=%1" : "=m" (*p) \ 115135581Smarcel : "r" (v) : "memory"); \ 116135581Smarcel } \ 117135581Smarcel \ 118135581Smarcel static __inline void \ 119135581Smarcel atomic_store_rel_##width(volatile uint##width##_t* p, \ 120135581Smarcel uint##width##_t v) \ 121135581Smarcel { \ 122135581Smarcel __asm __volatile ("st" size ".rel %0=%1" : "=m" (*p) \ 123135581Smarcel : "r" (v) : "memory"); \ 124135581Smarcel } \ 125135581Smarcel \ 126135581Smarcel static __inline void \ 127135581Smarcel atomic_store_rel_##type(volatile uint##width##_t* p, \ 128135581Smarcel uint##width##_t v) \ 129135581Smarcel { \ 130135581Smarcel __asm __volatile ("st" size ".rel %0=%1" : "=m" (*p) \ 131135581Smarcel : "r" (v) : "memory"); \ 132135581Smarcel } 13366458Sdfr 134135581SmarcelATOMIC_STORE_LOAD(char, 8, "1") 135135581SmarcelATOMIC_STORE_LOAD(short, 16, "2") 136135581SmarcelATOMIC_STORE_LOAD(int, 32, "4") 137135581SmarcelATOMIC_STORE_LOAD(long, 64, "8") 13866458Sdfr 13967351Sjhb#undef ATOMIC_STORE_LOAD 14067351Sjhb 141171662Smarcel#define atomic_load_acq_ptr(p) \ 142171662Smarcel ((void *)atomic_load_acq_64((volatile uint64_t *)p)) 143148067Sjhb 144171662Smarcel#define atomic_store_rel_ptr(p, v) \ 145171662Smarcel atomic_store_rel_64((volatile uint64_t *)p, (uint64_t)v) 146171662Smarcel 147135581Smarcel#define IA64_ATOMIC(sz, type, name, width, op) \ 148135583Smarcel static __inline type \ 149135581Smarcel atomic_##name##_acq_##width(volatile type *p, type v) \ 150135581Smarcel { \ 151135581Smarcel type old, ret; \ 152135581Smarcel do { \ 153135581Smarcel old = *p; \ 154135581Smarcel IA64_CMPXCHG(sz, acq, p, old, old op v, ret); \ 155135581Smarcel } while (ret != old); \ 156135583Smarcel return (old); \ 157135581Smarcel } \ 15866458Sdfr \ 159135583Smarcel static __inline type \ 160135581Smarcel atomic_##name##_rel_##width(volatile type *p, type v) \ 161135581Smarcel { \ 162135581Smarcel type old, ret; \ 163135581Smarcel do { \ 164135581Smarcel old = *p; \ 165135581Smarcel IA64_CMPXCHG(sz, rel, p, old, old op v, ret); \ 166135581Smarcel } while (ret != old); \ 167135583Smarcel return (old); \ 168135581Smarcel } 16966458Sdfr 170135581SmarcelIA64_ATOMIC(1, uint8_t, set, 8, |) 171135581SmarcelIA64_ATOMIC(2, uint16_t, set, 16, |) 172135581SmarcelIA64_ATOMIC(4, uint32_t, set, 32, |) 173135581SmarcelIA64_ATOMIC(8, uint64_t, set, 64, |) 17466458Sdfr 175135581SmarcelIA64_ATOMIC(1, uint8_t, clear, 8, &~) 176135581SmarcelIA64_ATOMIC(2, uint16_t, clear, 16, &~) 177135581SmarcelIA64_ATOMIC(4, uint32_t, clear, 32, &~) 178135581SmarcelIA64_ATOMIC(8, uint64_t, clear, 64, &~) 17966458Sdfr 180135581SmarcelIA64_ATOMIC(1, uint8_t, add, 8, +) 181135581SmarcelIA64_ATOMIC(2, uint16_t, add, 16, +) 182135581SmarcelIA64_ATOMIC(4, uint32_t, add, 32, +) 183135581SmarcelIA64_ATOMIC(8, uint64_t, add, 64, +) 18466458Sdfr 185135581SmarcelIA64_ATOMIC(1, uint8_t, subtract, 8, -) 186135581SmarcelIA64_ATOMIC(2, uint16_t, subtract, 16, -) 187135581SmarcelIA64_ATOMIC(4, uint32_t, subtract, 32, -) 188135581SmarcelIA64_ATOMIC(8, uint64_t, subtract, 64, -) 18966458Sdfr 19066458Sdfr#undef IA64_ATOMIC 19166458Sdfr 192135581Smarcel#define atomic_set_8 atomic_set_acq_8 193135581Smarcel#define atomic_clear_8 atomic_clear_acq_8 194135581Smarcel#define atomic_add_8 atomic_add_acq_8 195135581Smarcel#define atomic_subtract_8 atomic_subtract_acq_8 19666458Sdfr 197135581Smarcel#define atomic_set_16 atomic_set_acq_16 198135581Smarcel#define atomic_clear_16 atomic_clear_acq_16 199135581Smarcel#define atomic_add_16 atomic_add_acq_16 200135581Smarcel#define atomic_subtract_16 atomic_subtract_acq_16 20166458Sdfr 202135581Smarcel#define atomic_set_32 atomic_set_acq_32 203135581Smarcel#define atomic_clear_32 atomic_clear_acq_32 204135581Smarcel#define atomic_add_32 atomic_add_acq_32 205135581Smarcel#define atomic_subtract_32 atomic_subtract_acq_32 20666458Sdfr 207135581Smarcel#define atomic_set_64 atomic_set_acq_64 208135581Smarcel#define atomic_clear_64 atomic_clear_acq_64 209135581Smarcel#define atomic_add_64 atomic_add_acq_64 210135581Smarcel#define atomic_subtract_64 atomic_subtract_acq_64 21166458Sdfr 212135581Smarcel#define atomic_set_char atomic_set_8 213135581Smarcel#define atomic_clear_char atomic_clear_8 214135581Smarcel#define atomic_add_char atomic_add_8 215135581Smarcel#define atomic_subtract_char atomic_subtract_8 216135581Smarcel#define atomic_set_acq_char atomic_set_acq_8 217135581Smarcel#define atomic_clear_acq_char atomic_clear_acq_8 218135581Smarcel#define atomic_add_acq_char atomic_add_acq_8 219135581Smarcel#define atomic_subtract_acq_char atomic_subtract_acq_8 220135581Smarcel#define atomic_set_rel_char atomic_set_rel_8 221135581Smarcel#define atomic_clear_rel_char atomic_clear_rel_8 222135581Smarcel#define atomic_add_rel_char atomic_add_rel_8 223135581Smarcel#define atomic_subtract_rel_char atomic_subtract_rel_8 22467351Sjhb 225135581Smarcel#define atomic_set_short atomic_set_16 226135581Smarcel#define atomic_clear_short atomic_clear_16 227135581Smarcel#define atomic_add_short atomic_add_16 228135581Smarcel#define atomic_subtract_short atomic_subtract_16 229135581Smarcel#define atomic_set_acq_short atomic_set_acq_16 230135581Smarcel#define atomic_clear_acq_short atomic_clear_acq_16 231135581Smarcel#define atomic_add_acq_short atomic_add_acq_16 232135581Smarcel#define atomic_subtract_acq_short atomic_subtract_acq_16 233135581Smarcel#define atomic_set_rel_short atomic_set_rel_16 234135581Smarcel#define atomic_clear_rel_short atomic_clear_rel_16 235135581Smarcel#define atomic_add_rel_short atomic_add_rel_16 236135581Smarcel#define atomic_subtract_rel_short atomic_subtract_rel_16 23767351Sjhb 238135581Smarcel#define atomic_set_int atomic_set_32 239135581Smarcel#define atomic_clear_int atomic_clear_32 240135581Smarcel#define atomic_add_int atomic_add_32 241135581Smarcel#define atomic_subtract_int atomic_subtract_32 242135581Smarcel#define atomic_set_acq_int atomic_set_acq_32 243135581Smarcel#define atomic_clear_acq_int atomic_clear_acq_32 244135581Smarcel#define atomic_add_acq_int atomic_add_acq_32 245135581Smarcel#define atomic_subtract_acq_int atomic_subtract_acq_32 246135581Smarcel#define atomic_set_rel_int atomic_set_rel_32 247135581Smarcel#define atomic_clear_rel_int atomic_clear_rel_32 248135581Smarcel#define atomic_add_rel_int atomic_add_rel_32 249135581Smarcel#define atomic_subtract_rel_int atomic_subtract_rel_32 25067351Sjhb 251135581Smarcel#define atomic_set_long atomic_set_64 252135581Smarcel#define atomic_clear_long atomic_clear_64 253135581Smarcel#define atomic_add_long atomic_add_64 254135581Smarcel#define atomic_subtract_long atomic_subtract_64 255135581Smarcel#define atomic_set_acq_long atomic_set_acq_64 256135581Smarcel#define atomic_clear_acq_long atomic_clear_acq_64 257135581Smarcel#define atomic_add_acq_long atomic_add_acq_64 258135581Smarcel#define atomic_subtract_acq_long atomic_subtract_acq_64 259135581Smarcel#define atomic_set_rel_long atomic_set_rel_64 260135581Smarcel#define atomic_clear_rel_long atomic_clear_rel_64 261135581Smarcel#define atomic_add_rel_long atomic_add_rel_64 262135581Smarcel#define atomic_subtract_rel_long atomic_subtract_rel_64 26367351Sjhb 264171662Smarcel/* XXX Needs casting. */ 265148067Sjhb#define atomic_set_ptr atomic_set_64 266148067Sjhb#define atomic_clear_ptr atomic_clear_64 267148067Sjhb#define atomic_add_ptr atomic_add_64 268148067Sjhb#define atomic_subtract_ptr atomic_subtract_64 269148067Sjhb#define atomic_set_acq_ptr atomic_set_acq_64 270148067Sjhb#define atomic_clear_acq_ptr atomic_clear_acq_64 271148067Sjhb#define atomic_add_acq_ptr atomic_add_acq_64 272148067Sjhb#define atomic_subtract_acq_ptr atomic_subtract_acq_64 273148067Sjhb#define atomic_set_rel_ptr atomic_set_rel_64 274148067Sjhb#define atomic_clear_rel_ptr atomic_clear_rel_64 275148067Sjhb#define atomic_add_rel_ptr atomic_add_rel_64 276148067Sjhb#define atomic_subtract_rel_ptr atomic_subtract_rel_64 277148067Sjhb 278135581Smarcel#undef IA64_CMPXCHG 279135581Smarcel 28066458Sdfr/* 28166458Sdfr * Atomically compare the value stored at *p with cmpval and if the 28266458Sdfr * two values are equal, update the value of *p with newval. Returns 28366458Sdfr * zero if the compare failed, nonzero otherwise. 28466458Sdfr */ 28566458Sdfrstatic __inline int 286135581Smarcelatomic_cmpset_acq_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 28766458Sdfr{ 288135581Smarcel return (ia64_cmpxchg_acq_32(p, cmpval, newval) == cmpval); 28966458Sdfr} 29066458Sdfr 29167351Sjhbstatic __inline int 292135581Smarcelatomic_cmpset_rel_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) 29367351Sjhb{ 294135581Smarcel return (ia64_cmpxchg_rel_32(p, cmpval, newval) == cmpval); 29567351Sjhb} 29667351Sjhb 29766458Sdfr/* 29866458Sdfr * Atomically compare the value stored at *p with cmpval and if the 29966458Sdfr * two values are equal, update the value of *p with newval. Returns 30066458Sdfr * zero if the compare failed, nonzero otherwise. 30166458Sdfr */ 30266458Sdfrstatic __inline int 303135581Smarcelatomic_cmpset_acq_64(volatile uint64_t* p, uint64_t cmpval, uint64_t newval) 30466458Sdfr{ 305135581Smarcel return (ia64_cmpxchg_acq_64(p, cmpval, newval) == cmpval); 30666458Sdfr} 30766458Sdfr 30867351Sjhbstatic __inline int 309135581Smarcelatomic_cmpset_rel_64(volatile uint64_t* p, uint64_t cmpval, uint64_t newval) 31067351Sjhb{ 311135581Smarcel return (ia64_cmpxchg_rel_64(p, cmpval, newval) == cmpval); 31267351Sjhb} 31367351Sjhb 314135581Smarcel#define atomic_cmpset_32 atomic_cmpset_acq_32 315135581Smarcel#define atomic_cmpset_64 atomic_cmpset_acq_64 316135581Smarcel#define atomic_cmpset_int atomic_cmpset_32 317135581Smarcel#define atomic_cmpset_long atomic_cmpset_64 318135581Smarcel#define atomic_cmpset_acq_int atomic_cmpset_acq_32 319135581Smarcel#define atomic_cmpset_rel_int atomic_cmpset_rel_32 320135581Smarcel#define atomic_cmpset_acq_long atomic_cmpset_acq_64 321135581Smarcel#define atomic_cmpset_rel_long atomic_cmpset_rel_64 32266458Sdfr 323171662Smarcel#define atomic_cmpset_acq_ptr(p, o, n) \ 324171662Smarcel (atomic_cmpset_acq_64((volatile uint64_t *)p, (uint64_t)o, (uint64_t)n)) 325171662Smarcel 326171662Smarcel#define atomic_cmpset_ptr atomic_cmpset_acq_ptr 327171662Smarcel 328171662Smarcel#define atomic_cmpset_rel_ptr(p, o, n) \ 329171662Smarcel (atomic_cmpset_rel_64((volatile uint64_t *)p, (uint64_t)o, (uint64_t)n)) 330171662Smarcel 331135581Smarcelstatic __inline uint32_t 332135581Smarcelatomic_readandclear_32(volatile uint32_t* p) 33366937Sdfr{ 334135581Smarcel uint32_t val; 33566937Sdfr do { 33666937Sdfr val = *p; 33766937Sdfr } while (!atomic_cmpset_32(p, val, 0)); 338135581Smarcel return (val); 33966937Sdfr} 34066937Sdfr 341135581Smarcelstatic __inline uint64_t 342135581Smarcelatomic_readandclear_64(volatile uint64_t* p) 34366937Sdfr{ 344135581Smarcel uint64_t val; 34566937Sdfr do { 34666937Sdfr val = *p; 34766937Sdfr } while (!atomic_cmpset_64(p, val, 0)); 348135581Smarcel return (val); 34966937Sdfr} 35066937Sdfr 351135581Smarcel#define atomic_readandclear_int atomic_readandclear_32 352135581Smarcel#define atomic_readandclear_long atomic_readandclear_64 353173970Sjasone#define atomic_readandclear_ptr atomic_readandclear_64 35466937Sdfr 355150627Sjhb/* 356150627Sjhb * Atomically add the value of v to the integer pointed to by p and return 357150627Sjhb * the previous value of *p. 358150627Sjhb * 359150627Sjhb * XXX: Should we use the fetchadd instruction here? 360150627Sjhb */ 361150627Sjhbstatic __inline uint32_t 362150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 363150627Sjhb{ 364150627Sjhb uint32_t value; 365150627Sjhb 366150627Sjhb do { 367150627Sjhb value = *p; 368150627Sjhb } while (!atomic_cmpset_32(p, value, value + v)); 369150627Sjhb return (value); 370150627Sjhb} 371150627Sjhb 372150627Sjhb#define atomic_fetchadd_int atomic_fetchadd_32 373150627Sjhb 37466458Sdfr#endif /* ! _MACHINE_ATOMIC_H_ */ 375