1226048Sobrien//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
268349Sobrien//
3275698Sdelphij//                     The LLVM Compiler Infrastructure
4175296Sobrien//
568349Sobrien// This file is distributed under the University of Illinois Open Source
6175296Sobrien// License. See LICENSE.TXT for details.
7175296Sobrien//
8175296Sobrien//===----------------------------------------------------------------------===//
968349Sobrien//
10175296Sobrien// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
11175296Sobrien// Not intended for direct inclusion. Include sanitizer_atomic.h.
12175296Sobrien//
13175296Sobrien//===----------------------------------------------------------------------===//
14175296Sobrien
15175296Sobrien#ifndef SANITIZER_ATOMIC_MSVC_H
16175296Sobrien#define SANITIZER_ATOMIC_MSVC_H
17175296Sobrien
18175296Sobrienextern "C" void _ReadWriteBarrier();
19175296Sobrien#pragma intrinsic(_ReadWriteBarrier)
20175296Sobrienextern "C" void _mm_mfence();
21175296Sobrien#pragma intrinsic(_mm_mfence)
22175296Sobrienextern "C" void _mm_pause();
23175296Sobrien#pragma intrinsic(_mm_pause)
24175296Sobrienextern "C" char _InterlockedExchange8(   // NOLINT
25175296Sobrien    char volatile *Addend, char Value);  // NOLINT
26175296Sobrien#pragma intrinsic(_InterlockedExchange8)
27175296Sobrienextern "C" short _InterlockedExchange16(   // NOLINT
28175296Sobrien    short volatile *Addend, short Value);  // NOLINT
29175296Sobrien#pragma intrinsic(_InterlockedExchange16)
30175296Sobrienextern "C" long _InterlockedExchange(    // NOLINT
31175296Sobrien    long volatile *Addend, long Value);  // NOLINT
32175296Sobrien#pragma intrinsic(_InterlockedExchange)
33175296Sobrienextern "C" long _InterlockedExchangeAdd(  // NOLINT
34175296Sobrien    long volatile * Addend, long Value);  // NOLINT
35175296Sobrien#pragma intrinsic(_InterlockedExchangeAdd)
36175296Sobrienextern "C" short _InterlockedCompareExchange16(  // NOLINT
37175296Sobrien    short volatile *Destination,                 // NOLINT
38275698Sdelphij    short Exchange, short Comparand);            // NOLINT
39175296Sobrien#pragma intrinsic(_InterlockedCompareExchange16)
40extern "C"
41long long _InterlockedCompareExchange64(  // NOLINT
42    long long volatile *Destination,              // NOLINT
43    long long Exchange, long long Comparand);     // NOLINT
44#pragma intrinsic(_InterlockedCompareExchange64)
45extern "C" void *_InterlockedCompareExchangePointer(
46    void *volatile *Destination,
47    void *Exchange, void *Comparand);
48#pragma intrinsic(_InterlockedCompareExchangePointer)
49extern "C"
50long __cdecl _InterlockedCompareExchange(  // NOLINT
51    long volatile *Destination,            // NOLINT
52    long Exchange, long Comparand);        // NOLINT
53#pragma intrinsic(_InterlockedCompareExchange)
54
55#ifdef _WIN64
56extern "C" long long _InterlockedExchangeAdd64(     // NOLINT
57    long long volatile * Addend, long long Value);  // NOLINT
58#pragma intrinsic(_InterlockedExchangeAdd64)
59#endif
60
61namespace __sanitizer {
62
63INLINE void atomic_signal_fence(memory_order) {
64  _ReadWriteBarrier();
65}
66
67INLINE void atomic_thread_fence(memory_order) {
68  _mm_mfence();
69}
70
71INLINE void proc_yield(int cnt) {
72  for (int i = 0; i < cnt; i++)
73    _mm_pause();
74}
75
76template<typename T>
77INLINE typename T::Type atomic_load(
78    const volatile T *a, memory_order mo) {
79  DCHECK(mo & (memory_order_relaxed | memory_order_consume
80      | memory_order_acquire | memory_order_seq_cst));
81  DCHECK(!((uptr)a % sizeof(*a)));
82  typename T::Type v;
83  // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
84  if (mo == memory_order_relaxed) {
85    v = a->val_dont_use;
86  } else {
87    atomic_signal_fence(memory_order_seq_cst);
88    v = a->val_dont_use;
89    atomic_signal_fence(memory_order_seq_cst);
90  }
91  return v;
92}
93
94template<typename T>
95INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
96  DCHECK(mo & (memory_order_relaxed | memory_order_release
97      | memory_order_seq_cst));
98  DCHECK(!((uptr)a % sizeof(*a)));
99  // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
100  if (mo == memory_order_relaxed) {
101    a->val_dont_use = v;
102  } else {
103    atomic_signal_fence(memory_order_seq_cst);
104    a->val_dont_use = v;
105    atomic_signal_fence(memory_order_seq_cst);
106  }
107  if (mo == memory_order_seq_cst)
108    atomic_thread_fence(memory_order_seq_cst);
109}
110
111INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
112    u32 v, memory_order mo) {
113  (void)mo;
114  DCHECK(!((uptr)a % sizeof(*a)));
115  return (u32)_InterlockedExchangeAdd(
116      (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
117}
118
119INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
120    uptr v, memory_order mo) {
121  (void)mo;
122  DCHECK(!((uptr)a % sizeof(*a)));
123#ifdef _WIN64
124  return (uptr)_InterlockedExchangeAdd64(
125      (volatile long long*)&a->val_dont_use, (long long)v);  // NOLINT
126#else
127  return (uptr)_InterlockedExchangeAdd(
128      (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
129#endif
130}
131
132INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
133    u32 v, memory_order mo) {
134  (void)mo;
135  DCHECK(!((uptr)a % sizeof(*a)));
136  return (u32)_InterlockedExchangeAdd(
137      (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
138}
139
140INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
141    uptr v, memory_order mo) {
142  (void)mo;
143  DCHECK(!((uptr)a % sizeof(*a)));
144#ifdef _WIN64
145  return (uptr)_InterlockedExchangeAdd64(
146      (volatile long long*)&a->val_dont_use, -(long long)v);  // NOLINT
147#else
148  return (uptr)_InterlockedExchangeAdd(
149      (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
150#endif
151}
152
153INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
154    u8 v, memory_order mo) {
155  (void)mo;
156  DCHECK(!((uptr)a % sizeof(*a)));
157  return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
158}
159
160INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
161    u16 v, memory_order mo) {
162  (void)mo;
163  DCHECK(!((uptr)a % sizeof(*a)));
164  return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
165}
166
167INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
168    u32 v, memory_order mo) {
169  (void)mo;
170  DCHECK(!((uptr)a % sizeof(*a)));
171  return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
172}
173
174#ifndef _WIN64
175
176INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
177                                           u8 *cmp,
178                                           u8 xchgv,
179                                           memory_order mo) {
180  (void)mo;
181  DCHECK(!((uptr)a % sizeof(*a)));
182  u8 cmpv = *cmp;
183  u8 prev;
184  __asm {
185    mov al, cmpv
186    mov ecx, a
187    mov dl, xchgv
188    lock cmpxchg [ecx], dl
189    mov prev, al
190  }
191  if (prev == cmpv)
192    return true;
193  *cmp = prev;
194  return false;
195}
196
197#endif
198
199INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
200                                           uptr *cmp,
201                                           uptr xchg,
202                                           memory_order mo) {
203  uptr cmpv = *cmp;
204  uptr prev = (uptr)_InterlockedCompareExchangePointer(
205      (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
206  if (prev == cmpv)
207    return true;
208  *cmp = prev;
209  return false;
210}
211
212INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
213                                           u16 *cmp,
214                                           u16 xchg,
215                                           memory_order mo) {
216  u16 cmpv = *cmp;
217  u16 prev = (u16)_InterlockedCompareExchange16(
218      (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
219  if (prev == cmpv)
220    return true;
221  *cmp = prev;
222  return false;
223}
224
225INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
226                                           u32 *cmp,
227                                           u32 xchg,
228                                           memory_order mo) {
229  u32 cmpv = *cmp;
230  u32 prev = (u32)_InterlockedCompareExchange(
231      (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
232  if (prev == cmpv)
233    return true;
234  *cmp = prev;
235  return false;
236}
237
238INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
239                                           u64 *cmp,
240                                           u64 xchg,
241                                           memory_order mo) {
242  u64 cmpv = *cmp;
243  u64 prev = (u64)_InterlockedCompareExchange64(
244      (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
245  if (prev == cmpv)
246    return true;
247  *cmp = prev;
248  return false;
249}
250
251template<typename T>
252INLINE bool atomic_compare_exchange_weak(volatile T *a,
253                                         typename T::Type *cmp,
254                                         typename T::Type xchg,
255                                         memory_order mo) {
256  return atomic_compare_exchange_strong(a, cmp, xchg, mo);
257}
258
259}  // namespace __sanitizer
260
261#endif  // SANITIZER_ATOMIC_CLANG_H
262