1//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 10// Not intended for direct inclusion. Include sanitizer_atomic.h. 11// 12//===----------------------------------------------------------------------===// 13 14#ifndef SANITIZER_ATOMIC_MSVC_H 15#define SANITIZER_ATOMIC_MSVC_H 16 17extern "C" void _ReadWriteBarrier(); 18#pragma intrinsic(_ReadWriteBarrier) 19extern "C" void _mm_mfence(); 20#pragma intrinsic(_mm_mfence) 21extern "C" void _mm_pause(); 22#pragma intrinsic(_mm_pause) 23extern "C" char _InterlockedExchange8(char volatile *Addend, char Value); 24#pragma intrinsic(_InterlockedExchange8) 25extern "C" short _InterlockedExchange16(short volatile *Addend, short Value); 26#pragma intrinsic(_InterlockedExchange16) 27extern "C" long _InterlockedExchange(long volatile *Addend, long Value); 28#pragma intrinsic(_InterlockedExchange) 29extern "C" long _InterlockedExchangeAdd(long volatile *Addend, long Value); 30#pragma intrinsic(_InterlockedExchangeAdd) 31extern "C" char _InterlockedCompareExchange8(char volatile *Destination, 32 char Exchange, char Comparand); 33#pragma intrinsic(_InterlockedCompareExchange8) 34extern "C" short _InterlockedCompareExchange16(short volatile *Destination, 35 short Exchange, short Comparand); 36#pragma intrinsic(_InterlockedCompareExchange16) 37extern "C" long long _InterlockedCompareExchange64( 38 long long volatile *Destination, long long Exchange, long long Comparand); 39#pragma intrinsic(_InterlockedCompareExchange64) 40extern "C" void *_InterlockedCompareExchangePointer( 41 void *volatile *Destination, 42 void *Exchange, void *Comparand); 43#pragma intrinsic(_InterlockedCompareExchangePointer) 44extern "C" long __cdecl _InterlockedCompareExchange(long volatile *Destination, 45 long Exchange, 46 long Comparand); 47#pragma intrinsic(_InterlockedCompareExchange) 48 49#ifdef _WIN64 50extern "C" long long _InterlockedExchangeAdd64(long long volatile *Addend, 51 long long Value); 52#pragma intrinsic(_InterlockedExchangeAdd64) 53#endif 54 55namespace __sanitizer { 56 57INLINE void atomic_signal_fence(memory_order) { 58 _ReadWriteBarrier(); 59} 60 61INLINE void atomic_thread_fence(memory_order) { 62 _mm_mfence(); 63} 64 65INLINE void proc_yield(int cnt) { 66 for (int i = 0; i < cnt; i++) 67 _mm_pause(); 68} 69 70template<typename T> 71INLINE typename T::Type atomic_load( 72 const volatile T *a, memory_order mo) { 73 DCHECK(mo & (memory_order_relaxed | memory_order_consume 74 | memory_order_acquire | memory_order_seq_cst)); 75 DCHECK(!((uptr)a % sizeof(*a))); 76 typename T::Type v; 77 // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. 78 if (mo == memory_order_relaxed) { 79 v = a->val_dont_use; 80 } else { 81 atomic_signal_fence(memory_order_seq_cst); 82 v = a->val_dont_use; 83 atomic_signal_fence(memory_order_seq_cst); 84 } 85 return v; 86} 87 88template<typename T> 89INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 90 DCHECK(mo & (memory_order_relaxed | memory_order_release 91 | memory_order_seq_cst)); 92 DCHECK(!((uptr)a % sizeof(*a))); 93 // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. 94 if (mo == memory_order_relaxed) { 95 a->val_dont_use = v; 96 } else { 97 atomic_signal_fence(memory_order_seq_cst); 98 a->val_dont_use = v; 99 atomic_signal_fence(memory_order_seq_cst); 100 } 101 if (mo == memory_order_seq_cst) 102 atomic_thread_fence(memory_order_seq_cst); 103} 104 105INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a, 106 u32 v, memory_order mo) { 107 (void)mo; 108 DCHECK(!((uptr)a % sizeof(*a))); 109 return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use, 110 (long)v); 111} 112 113INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a, 114 uptr v, memory_order mo) { 115 (void)mo; 116 DCHECK(!((uptr)a % sizeof(*a))); 117#ifdef _WIN64 118 return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use, 119 (long long)v); 120#else 121 return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use, 122 (long)v); 123#endif 124} 125 126INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a, 127 u32 v, memory_order mo) { 128 (void)mo; 129 DCHECK(!((uptr)a % sizeof(*a))); 130 return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use, 131 -(long)v); 132} 133 134INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a, 135 uptr v, memory_order mo) { 136 (void)mo; 137 DCHECK(!((uptr)a % sizeof(*a))); 138#ifdef _WIN64 139 return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use, 140 -(long long)v); 141#else 142 return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use, 143 -(long)v); 144#endif 145} 146 147INLINE u8 atomic_exchange(volatile atomic_uint8_t *a, 148 u8 v, memory_order mo) { 149 (void)mo; 150 DCHECK(!((uptr)a % sizeof(*a))); 151 return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v); 152} 153 154INLINE u16 atomic_exchange(volatile atomic_uint16_t *a, 155 u16 v, memory_order mo) { 156 (void)mo; 157 DCHECK(!((uptr)a % sizeof(*a))); 158 return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v); 159} 160 161INLINE u32 atomic_exchange(volatile atomic_uint32_t *a, 162 u32 v, memory_order mo) { 163 (void)mo; 164 DCHECK(!((uptr)a % sizeof(*a))); 165 return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v); 166} 167 168INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a, 169 u8 *cmp, 170 u8 xchgv, 171 memory_order mo) { 172 (void)mo; 173 DCHECK(!((uptr)a % sizeof(*a))); 174 u8 cmpv = *cmp; 175#ifdef _WIN64 176 u8 prev = (u8)_InterlockedCompareExchange8( 177 (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv); 178#else 179 u8 prev; 180 __asm { 181 mov al, cmpv 182 mov ecx, a 183 mov dl, xchgv 184 lock cmpxchg [ecx], dl 185 mov prev, al 186 } 187#endif 188 if (prev == cmpv) 189 return true; 190 *cmp = prev; 191 return false; 192} 193 194INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a, 195 uptr *cmp, 196 uptr xchg, 197 memory_order mo) { 198 uptr cmpv = *cmp; 199 uptr prev = (uptr)_InterlockedCompareExchangePointer( 200 (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv); 201 if (prev == cmpv) 202 return true; 203 *cmp = prev; 204 return false; 205} 206 207INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a, 208 u16 *cmp, 209 u16 xchg, 210 memory_order mo) { 211 u16 cmpv = *cmp; 212 u16 prev = (u16)_InterlockedCompareExchange16( 213 (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv); 214 if (prev == cmpv) 215 return true; 216 *cmp = prev; 217 return false; 218} 219 220INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a, 221 u32 *cmp, 222 u32 xchg, 223 memory_order mo) { 224 u32 cmpv = *cmp; 225 u32 prev = (u32)_InterlockedCompareExchange( 226 (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv); 227 if (prev == cmpv) 228 return true; 229 *cmp = prev; 230 return false; 231} 232 233INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a, 234 u64 *cmp, 235 u64 xchg, 236 memory_order mo) { 237 u64 cmpv = *cmp; 238 u64 prev = (u64)_InterlockedCompareExchange64( 239 (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv); 240 if (prev == cmpv) 241 return true; 242 *cmp = prev; 243 return false; 244} 245 246template<typename T> 247INLINE bool atomic_compare_exchange_weak(volatile T *a, 248 typename T::Type *cmp, 249 typename T::Type xchg, 250 memory_order mo) { 251 return atomic_compare_exchange_strong(a, cmp, xchg, mo); 252} 253 254} // namespace __sanitizer 255 256#endif // SANITIZER_ATOMIC_CLANG_H 257