1/******************************************************************************/ 2#ifdef JEMALLOC_H_TYPES 3 4#endif /* JEMALLOC_H_TYPES */ 5/******************************************************************************/ 6#ifdef JEMALLOC_H_STRUCTS 7 8#endif /* JEMALLOC_H_STRUCTS */ 9/******************************************************************************/ 10#ifdef JEMALLOC_H_EXTERNS 11 12#define atomic_read_uint64(p) atomic_add_uint64(p, 0) 13#define atomic_read_uint32(p) atomic_add_uint32(p, 0) 14#define atomic_read_z(p) atomic_add_z(p, 0) 15#define atomic_read_u(p) atomic_add_u(p, 0) 16 17#endif /* JEMALLOC_H_EXTERNS */ 18/******************************************************************************/ 19#ifdef JEMALLOC_H_INLINES 20 21#ifndef JEMALLOC_ENABLE_INLINE 22uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); 23uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); 24uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); 25uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); 26size_t atomic_add_z(size_t *p, size_t x); 27size_t atomic_sub_z(size_t *p, size_t x); 28unsigned atomic_add_u(unsigned *p, unsigned x); 29unsigned atomic_sub_u(unsigned *p, unsigned x); 30#endif 31 32#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) 33/******************************************************************************/ 34/* 64-bit operations. */ 35#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) 36# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 37JEMALLOC_INLINE uint64_t 38atomic_add_uint64(uint64_t *p, uint64_t x) 39{ 40 41 return (__sync_add_and_fetch(p, x)); 42} 43 44JEMALLOC_INLINE uint64_t 45atomic_sub_uint64(uint64_t *p, uint64_t x) 46{ 47 48 return (__sync_sub_and_fetch(p, x)); 49} 50#elif (defined(_MSC_VER)) 51JEMALLOC_INLINE uint64_t 52atomic_add_uint64(uint64_t *p, uint64_t x) 53{ 54 55 return (InterlockedExchangeAdd64(p, x)); 56} 57 58JEMALLOC_INLINE uint64_t 59atomic_sub_uint64(uint64_t *p, uint64_t x) 60{ 61 62 return (InterlockedExchangeAdd64(p, -((int64_t)x))); 63} 64#elif (defined(JEMALLOC_OSATOMIC)) 65JEMALLOC_INLINE uint64_t 66atomic_add_uint64(uint64_t *p, uint64_t x) 67{ 68 69 return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); 70} 71 72JEMALLOC_INLINE uint64_t 73atomic_sub_uint64(uint64_t *p, uint64_t x) 74{ 75 76 return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); 77} 78# elif (defined(__amd64__) || defined(__x86_64__)) 79JEMALLOC_INLINE uint64_t 80atomic_add_uint64(uint64_t *p, uint64_t x) 81{ 82 83 asm volatile ( 84 "lock; xaddq %0, %1;" 85 : "+r" (x), "=m" (*p) /* Outputs. */ 86 : "m" (*p) /* Inputs. */ 87 ); 88 89 return (x); 90} 91 92JEMALLOC_INLINE uint64_t 93atomic_sub_uint64(uint64_t *p, uint64_t x) 94{ 95 96 x = (uint64_t)(-(int64_t)x); 97 asm volatile ( 98 "lock; xaddq %0, %1;" 99 : "+r" (x), "=m" (*p) /* Outputs. */ 100 : "m" (*p) /* Inputs. */ 101 ); 102 103 return (x); 104} 105# elif (defined(JEMALLOC_ATOMIC9)) 106JEMALLOC_INLINE uint64_t 107atomic_add_uint64(uint64_t *p, uint64_t x) 108{ 109 110 /* 111 * atomic_fetchadd_64() doesn't exist, but we only ever use this 112 * function on LP64 systems, so atomic_fetchadd_long() will do. 113 */ 114 assert(sizeof(uint64_t) == sizeof(unsigned long)); 115 116 return (atomic_fetchadd_long(p, (unsigned long)x) + x); 117} 118 119JEMALLOC_INLINE uint64_t 120atomic_sub_uint64(uint64_t *p, uint64_t x) 121{ 122 123 assert(sizeof(uint64_t) == sizeof(unsigned long)); 124 125 return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); 126} 127# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) 128JEMALLOC_INLINE uint64_t 129atomic_add_uint64(uint64_t *p, uint64_t x) 130{ 131 132 return (__sync_add_and_fetch(p, x)); 133} 134 135JEMALLOC_INLINE uint64_t 136atomic_sub_uint64(uint64_t *p, uint64_t x) 137{ 138 139 return (__sync_sub_and_fetch(p, x)); 140} 141# else 142# error "Missing implementation for 64-bit atomic operations" 143# endif 144#endif 145 146/******************************************************************************/ 147/* 32-bit operations. */ 148#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 149JEMALLOC_INLINE uint32_t 150atomic_add_uint32(uint32_t *p, uint32_t x) 151{ 152 153 return (__sync_add_and_fetch(p, x)); 154} 155 156JEMALLOC_INLINE uint32_t 157atomic_sub_uint32(uint32_t *p, uint32_t x) 158{ 159 160 return (__sync_sub_and_fetch(p, x)); 161} 162#elif (defined(_MSC_VER)) 163JEMALLOC_INLINE uint32_t 164atomic_add_uint32(uint32_t *p, uint32_t x) 165{ 166 167 return (InterlockedExchangeAdd(p, x)); 168} 169 170JEMALLOC_INLINE uint32_t 171atomic_sub_uint32(uint32_t *p, uint32_t x) 172{ 173 174 return (InterlockedExchangeAdd(p, -((int32_t)x))); 175} 176#elif (defined(JEMALLOC_OSATOMIC)) 177JEMALLOC_INLINE uint32_t 178atomic_add_uint32(uint32_t *p, uint32_t x) 179{ 180 181 return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); 182} 183 184JEMALLOC_INLINE uint32_t 185atomic_sub_uint32(uint32_t *p, uint32_t x) 186{ 187 188 return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); 189} 190#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 191JEMALLOC_INLINE uint32_t 192atomic_add_uint32(uint32_t *p, uint32_t x) 193{ 194 195 asm volatile ( 196 "lock; xaddl %0, %1;" 197 : "+r" (x), "=m" (*p) /* Outputs. */ 198 : "m" (*p) /* Inputs. */ 199 ); 200 201 return (x); 202} 203 204JEMALLOC_INLINE uint32_t 205atomic_sub_uint32(uint32_t *p, uint32_t x) 206{ 207 208 x = (uint32_t)(-(int32_t)x); 209 asm volatile ( 210 "lock; xaddl %0, %1;" 211 : "+r" (x), "=m" (*p) /* Outputs. */ 212 : "m" (*p) /* Inputs. */ 213 ); 214 215 return (x); 216} 217#elif (defined(JEMALLOC_ATOMIC9)) 218JEMALLOC_INLINE uint32_t 219atomic_add_uint32(uint32_t *p, uint32_t x) 220{ 221 222 return (atomic_fetchadd_32(p, x) + x); 223} 224 225JEMALLOC_INLINE uint32_t 226atomic_sub_uint32(uint32_t *p, uint32_t x) 227{ 228 229 return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); 230} 231#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) 232JEMALLOC_INLINE uint32_t 233atomic_add_uint32(uint32_t *p, uint32_t x) 234{ 235 236 return (__sync_add_and_fetch(p, x)); 237} 238 239JEMALLOC_INLINE uint32_t 240atomic_sub_uint32(uint32_t *p, uint32_t x) 241{ 242 243 return (__sync_sub_and_fetch(p, x)); 244} 245#else 246# error "Missing implementation for 32-bit atomic operations" 247#endif 248 249/******************************************************************************/ 250/* size_t operations. */ 251JEMALLOC_INLINE size_t 252atomic_add_z(size_t *p, size_t x) 253{ 254 255#if (LG_SIZEOF_PTR == 3) 256 return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 257#elif (LG_SIZEOF_PTR == 2) 258 return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 259#endif 260} 261 262JEMALLOC_INLINE size_t 263atomic_sub_z(size_t *p, size_t x) 264{ 265 266#if (LG_SIZEOF_PTR == 3) 267 return ((size_t)atomic_add_uint64((uint64_t *)p, 268 (uint64_t)-((int64_t)x))); 269#elif (LG_SIZEOF_PTR == 2) 270 return ((size_t)atomic_add_uint32((uint32_t *)p, 271 (uint32_t)-((int32_t)x))); 272#endif 273} 274 275/******************************************************************************/ 276/* unsigned operations. */ 277JEMALLOC_INLINE unsigned 278atomic_add_u(unsigned *p, unsigned x) 279{ 280 281#if (LG_SIZEOF_INT == 3) 282 return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 283#elif (LG_SIZEOF_INT == 2) 284 return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 285#endif 286} 287 288JEMALLOC_INLINE unsigned 289atomic_sub_u(unsigned *p, unsigned x) 290{ 291 292#if (LG_SIZEOF_INT == 3) 293 return ((unsigned)atomic_add_uint64((uint64_t *)p, 294 (uint64_t)-((int64_t)x))); 295#elif (LG_SIZEOF_INT == 2) 296 return ((unsigned)atomic_add_uint32((uint32_t *)p, 297 (uint32_t)-((int32_t)x))); 298#endif 299} 300/******************************************************************************/ 301#endif 302 303#endif /* JEMALLOC_H_INLINES */ 304/******************************************************************************/ 305