1226048Sobrien//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 268349Sobrien// 3275698Sdelphij// The LLVM Compiler Infrastructure 4175296Sobrien// 568349Sobrien// This file is distributed under the University of Illinois Open Source 6175296Sobrien// License. See LICENSE.TXT for details. 7175296Sobrien// 8175296Sobrien//===----------------------------------------------------------------------===// 968349Sobrien// 10175296Sobrien// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 11175296Sobrien// Not intended for direct inclusion. Include sanitizer_atomic.h. 12175296Sobrien// 13175296Sobrien//===----------------------------------------------------------------------===// 14175296Sobrien 15175296Sobrien#ifndef SANITIZER_ATOMIC_MSVC_H 16175296Sobrien#define SANITIZER_ATOMIC_MSVC_H 17175296Sobrien 18175296Sobrienextern "C" void _ReadWriteBarrier(); 19175296Sobrien#pragma intrinsic(_ReadWriteBarrier) 20175296Sobrienextern "C" void _mm_mfence(); 21175296Sobrien#pragma intrinsic(_mm_mfence) 22175296Sobrienextern "C" void _mm_pause(); 23175296Sobrien#pragma intrinsic(_mm_pause) 24175296Sobrienextern "C" char _InterlockedExchange8( // NOLINT 25175296Sobrien char volatile *Addend, char Value); // NOLINT 26175296Sobrien#pragma intrinsic(_InterlockedExchange8) 27175296Sobrienextern "C" short _InterlockedExchange16( // NOLINT 28175296Sobrien short volatile *Addend, short Value); // NOLINT 29175296Sobrien#pragma intrinsic(_InterlockedExchange16) 30175296Sobrienextern "C" long _InterlockedExchange( // NOLINT 31175296Sobrien long volatile *Addend, long Value); // NOLINT 32175296Sobrien#pragma intrinsic(_InterlockedExchange) 33175296Sobrienextern "C" long _InterlockedExchangeAdd( // NOLINT 34175296Sobrien long volatile * Addend, long Value); // NOLINT 35175296Sobrien#pragma intrinsic(_InterlockedExchangeAdd) 36175296Sobrienextern "C" short _InterlockedCompareExchange16( // NOLINT 37175296Sobrien short volatile *Destination, // NOLINT 38275698Sdelphij short Exchange, short Comparand); // NOLINT 39175296Sobrien#pragma intrinsic(_InterlockedCompareExchange16) 40extern "C" 41long long _InterlockedCompareExchange64( // NOLINT 42 long long volatile *Destination, // NOLINT 43 long long Exchange, long long Comparand); // NOLINT 44#pragma intrinsic(_InterlockedCompareExchange64) 45extern "C" void *_InterlockedCompareExchangePointer( 46 void *volatile *Destination, 47 void *Exchange, void *Comparand); 48#pragma intrinsic(_InterlockedCompareExchangePointer) 49extern "C" 50long __cdecl _InterlockedCompareExchange( // NOLINT 51 long volatile *Destination, // NOLINT 52 long Exchange, long Comparand); // NOLINT 53#pragma intrinsic(_InterlockedCompareExchange) 54 55#ifdef _WIN64 56extern "C" long long _InterlockedExchangeAdd64( // NOLINT 57 long long volatile * Addend, long long Value); // NOLINT 58#pragma intrinsic(_InterlockedExchangeAdd64) 59#endif 60 61namespace __sanitizer { 62 63INLINE void atomic_signal_fence(memory_order) { 64 _ReadWriteBarrier(); 65} 66 67INLINE void atomic_thread_fence(memory_order) { 68 _mm_mfence(); 69} 70 71INLINE void proc_yield(int cnt) { 72 for (int i = 0; i < cnt; i++) 73 _mm_pause(); 74} 75 76template<typename T> 77INLINE typename T::Type atomic_load( 78 const volatile T *a, memory_order mo) { 79 DCHECK(mo & (memory_order_relaxed | memory_order_consume 80 | memory_order_acquire | memory_order_seq_cst)); 81 DCHECK(!((uptr)a % sizeof(*a))); 82 typename T::Type v; 83 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 84 if (mo == memory_order_relaxed) { 85 v = a->val_dont_use; 86 } else { 87 atomic_signal_fence(memory_order_seq_cst); 88 v = a->val_dont_use; 89 atomic_signal_fence(memory_order_seq_cst); 90 } 91 return v; 92} 93 94template<typename T> 95INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 96 DCHECK(mo & (memory_order_relaxed | memory_order_release 97 | memory_order_seq_cst)); 98 DCHECK(!((uptr)a % sizeof(*a))); 99 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 100 if (mo == memory_order_relaxed) { 101 a->val_dont_use = v; 102 } else { 103 atomic_signal_fence(memory_order_seq_cst); 104 a->val_dont_use = v; 105 atomic_signal_fence(memory_order_seq_cst); 106 } 107 if (mo == memory_order_seq_cst) 108 atomic_thread_fence(memory_order_seq_cst); 109} 110 111INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 112 u32 v, memory_order mo) { 113 (void)mo; 114 DCHECK(!((uptr)a % sizeof(*a))); 115 return (u32)_InterlockedExchangeAdd( 116 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 117} 118 119INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 120 uptr v, memory_order mo) { 121 (void)mo; 122 DCHECK(!((uptr)a % sizeof(*a))); 123#ifdef _WIN64 124 return (uptr)_InterlockedExchangeAdd64( 125 (volatile long long*)&a->val_dont_use, (long long)v); // NOLINT 126#else 127 return (uptr)_InterlockedExchangeAdd( 128 (volatile long*)&a->val_dont_use, (long)v); // NOLINT 129#endif 130} 131 132INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 133 u32 v, memory_order mo) { 134 (void)mo; 135 DCHECK(!((uptr)a % sizeof(*a))); 136 return (u32)_InterlockedExchangeAdd( 137 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 138} 139 140INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 141 uptr v, memory_order mo) { 142 (void)mo; 143 DCHECK(!((uptr)a % sizeof(*a))); 144#ifdef _WIN64 145 return (uptr)_InterlockedExchangeAdd64( 146 (volatile long long*)&a->val_dont_use, -(long long)v); // NOLINT 147#else 148 return (uptr)_InterlockedExchangeAdd( 149 (volatile long*)&a->val_dont_use, -(long)v); // NOLINT 150#endif 151} 152 153INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 154 u8 v, memory_order mo) { 155 (void)mo; 156 DCHECK(!((uptr)a % sizeof(*a))); 157 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v); 158} 159 160INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 161 u16 v, memory_order mo) { 162 (void)mo; 163 DCHECK(!((uptr)a % sizeof(*a))); 164 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v); 165} 166 167INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, 168 u32 v, memory_order mo) { 169 (void)mo; 170 DCHECK(!((uptr)a % sizeof(*a))); 171 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); 172} 173 174#ifndef _WIN64 175 176INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 177 u8 *cmp, 178 u8 xchgv, 179 memory_order mo) { 180 (void)mo; 181 DCHECK(!((uptr)a % sizeof(*a))); 182 u8 cmpv = *cmp; 183 u8 prev; 184 __asm { 185 mov al, cmpv 186 mov ecx, a 187 mov dl, xchgv 188 lock cmpxchg [ecx], dl 189 mov prev, al 190 } 191 if (prev == cmpv) 192 return true; 193 *cmp = prev; 194 return false; 195} 196 197#endif 198 199INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 200 uptr *cmp, 201 uptr xchg, 202 memory_order mo) { 203 uptr cmpv = *cmp; 204 uptr prev = (uptr)_InterlockedCompareExchangePointer( 205 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 206 if (prev == cmpv) 207 return true; 208 *cmp = prev; 209 return false; 210} 211 212INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 213 u16 *cmp, 214 u16 xchg, 215 memory_order mo) { 216 u16 cmpv = *cmp; 217 u16 prev = (u16)_InterlockedCompareExchange16( 218 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 219 if (prev == cmpv) 220 return true; 221 *cmp = prev; 222 return false; 223} 224 225INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 226 u32 *cmp, 227 u32 xchg, 228 memory_order mo) { 229 u32 cmpv = *cmp; 230 u32 prev = (u32)_InterlockedCompareExchange( 231 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 232 if (prev == cmpv) 233 return true; 234 *cmp = prev; 235 return false; 236} 237 238INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 239 u64 *cmp, 240 u64 xchg, 241 memory_order mo) { 242 u64 cmpv = *cmp; 243 u64 prev = (u64)_InterlockedCompareExchange64( 244 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 245 if (prev == cmpv) 246 return true; 247 *cmp = prev; 248 return false; 249} 250 251template<typename T> 252INLINE bool atomic_compare_exchange_weak(volatile T *a, 253 typename T::Type *cmp, 254 typename T::Type xchg, 255 memory_order mo) { 256 return atomic_compare_exchange_strong(a, cmp, xchg, mo); 257} 258 259} // namespace __sanitizer 260 261#endif // SANITIZER_ATOMIC_CLANG_H 262