1#ifndef _ATOMIC_H 2#define _ATOMIC_H 3 4#include <stdint.h> 5 6#include "atomic_arch.h" 7 8#ifdef a_ll 9 10#ifndef a_pre_llsc 11#define a_pre_llsc() 12#endif 13 14#ifndef a_post_llsc 15#define a_post_llsc() 16#endif 17 18#ifndef a_cas 19#define a_cas a_cas 20static inline int a_cas(volatile int *p, int t, int s) 21{ 22 int old; 23 a_pre_llsc(); 24 do old = a_ll(p); 25 while (old==t && !a_sc(p, s)); 26 a_post_llsc(); 27 return old; 28} 29#endif 30 31#ifndef a_swap 32#define a_swap a_swap 33static inline int a_swap(volatile int *p, int v) 34{ 35 int old; 36 a_pre_llsc(); 37 do old = a_ll(p); 38 while (!a_sc(p, v)); 39 a_post_llsc(); 40 return old; 41} 42#endif 43 44#ifndef a_fetch_add 45#define a_fetch_add a_fetch_add 46static inline int a_fetch_add(volatile int *p, int v) 47{ 48 int old; 49 a_pre_llsc(); 50 do old = a_ll(p); 51 while (!a_sc(p, (unsigned)old + v)); 52 a_post_llsc(); 53 return old; 54} 55#endif 56 57#ifndef a_fetch_and 58#define a_fetch_and a_fetch_and 59static inline int a_fetch_and(volatile int *p, int v) 60{ 61 int old; 62 a_pre_llsc(); 63 do old = a_ll(p); 64 while (!a_sc(p, old & v)); 65 a_post_llsc(); 66 return old; 67} 68#endif 69 70#ifndef a_fetch_or 71#define a_fetch_or a_fetch_or 72static inline int a_fetch_or(volatile int *p, int v) 73{ 74 int old; 75 a_pre_llsc(); 76 do old = a_ll(p); 77 while (!a_sc(p, old | v)); 78 a_post_llsc(); 79 return old; 80} 81#endif 82 83#endif 84 85#ifdef a_ll_p 86 87#ifndef a_cas_p 88#define a_cas_p a_cas_p 89static inline void *a_cas_p(volatile void *p, void *t, void *s) 90{ 91 void *old; 92 a_pre_llsc(); 93 do old = a_ll_p(p); 94 while (old==t && !a_sc_p(p, s)); 95 a_post_llsc(); 96 return old; 97} 98#endif 99 100#endif 101 102#ifndef a_cas 103#error missing definition of a_cas 104#endif 105 106#ifndef a_swap 107#define a_swap a_swap 108static inline int a_swap(volatile int *p, int v) 109{ 110 int old; 111 do old = *p; 112 while (a_cas(p, old, v) != old); 113 return old; 114} 115#endif 116 117#ifndef a_fetch_add 118#define a_fetch_add a_fetch_add 119static inline int a_fetch_add(volatile int *p, int v) 120{ 121 int old; 122 do old = *p; 123 while (a_cas(p, old, (unsigned)old+v) != old); 124 return old; 125} 126#endif 127 128#ifndef a_fetch_and 129#define a_fetch_and a_fetch_and 130static inline int a_fetch_and(volatile int *p, int v) 131{ 132 int old; 133 do old = *p; 134 while (a_cas(p, old, old&v) != old); 135 return old; 136} 137#endif 138#ifndef a_fetch_or 139#define a_fetch_or a_fetch_or 140static inline int a_fetch_or(volatile int *p, int v) 141{ 142 int old; 143 do old = *p; 144 while (a_cas(p, old, old|v) != old); 145 return old; 146} 147#endif 148 149#ifndef a_and 150#define a_and a_and 151static inline void a_and(volatile int *p, int v) 152{ 153 a_fetch_and(p, v); 154} 155#endif 156 157#ifndef a_or 158#define a_or a_or 159static inline void a_or(volatile int *p, int v) 160{ 161 a_fetch_or(p, v); 162} 163#endif 164 165#ifndef a_inc 166#define a_inc a_inc 167static inline void a_inc(volatile int *p) 168{ 169 a_fetch_add(p, 1); 170} 171#endif 172 173#ifndef a_dec 174#define a_dec a_dec 175static inline void a_dec(volatile int *p) 176{ 177 a_fetch_add(p, -1); 178} 179#endif 180 181#ifndef a_store 182#define a_store a_store 183static inline void a_store(volatile int *p, int v) 184{ 185#ifdef a_barrier 186 a_barrier(); 187 *p = v; 188 a_barrier(); 189#else 190 a_swap(p, v); 191#endif 192} 193#endif 194 195#ifndef a_barrier 196#define a_barrier a_barrier 197static void a_barrier() 198{ 199 volatile int tmp = 0; 200 a_cas(&tmp, 0, 0); 201} 202#endif 203 204#ifndef a_spin 205#define a_spin a_barrier 206#endif 207 208#ifndef a_and_64 209#define a_and_64 a_and_64 210static inline void a_and_64(volatile uint64_t *p, uint64_t v) 211{ 212 union { uint64_t v; uint32_t r[2]; } u = { v }; 213 if (u.r[0]+1) a_and((int *)p, u.r[0]); 214 if (u.r[1]+1) a_and((int *)p+1, u.r[1]); 215} 216#endif 217 218#ifndef a_or_64 219#define a_or_64 a_or_64 220static inline void a_or_64(volatile uint64_t *p, uint64_t v) 221{ 222 union { uint64_t v; uint32_t r[2]; } u = { v }; 223 if (u.r[0]) a_or((int *)p, u.r[0]); 224 if (u.r[1]) a_or((int *)p+1, u.r[1]); 225} 226#endif 227 228#ifndef a_cas_p 229typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1]; 230#define a_cas_p a_cas_p 231static inline void *a_cas_p(volatile void *p, void *t, void *s) 232{ 233 return (void *)a_cas((volatile int *)p, (int)t, (int)s); 234} 235#endif 236 237#ifndef a_or_l 238#define a_or_l a_or_l 239static inline void a_or_l(volatile void *p, long v) 240{ 241 if (sizeof(long) == sizeof(int)) a_or(p, v); 242 else a_or_64(p, v); 243} 244#endif 245 246#ifndef a_crash 247#define a_crash a_crash 248static inline void a_crash() 249{ 250 *(volatile char *)0=0; 251} 252#endif 253 254#ifndef a_ctz_64 255#define a_ctz_64 a_ctz_64 256static inline int a_ctz_64(uint64_t x) 257{ 258 static const char debruijn64[64] = { 259 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, 260 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, 261 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, 262 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 263 }; 264 static const char debruijn32[32] = { 265 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, 266 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 267 }; 268 if (sizeof(long) < 8) { 269 uint32_t y = x; 270 if (!y) { 271 y = x>>32; 272 return 32 + debruijn32[(y&-y)*0x076be629 >> 27]; 273 } 274 return debruijn32[(y&-y)*0x076be629 >> 27]; 275 } 276 return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; 277} 278#endif 279 280#ifndef a_ctz_l 281#define a_ctz_l a_ctz_l 282static inline int a_ctz_l(unsigned long x) 283{ 284 static const char debruijn32[32] = { 285 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, 286 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 287 }; 288 if (sizeof(long) == 8) return a_ctz_64(x); 289 return debruijn32[(x&-x)*0x076be629 >> 27]; 290} 291#endif 292 293#endif 294