1/*
2 * Copyright (C) 2007, 2008, 2010, 2012, 2013 Apple Inc. All rights reserved.
3 * Copyright (C) 2007 Justin Haygood (jhaygood@reaktix.com)
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1.  Redistributions of source code must retain the above copyright
10 *     notice, this list of conditions and the following disclaimer.
11 * 2.  Redistributions in binary form must reproduce the above copyright
12 *     notice, this list of conditions and the following disclaimer in the
13 *     documentation and/or other materials provided with the distribution.
14 * 3.  Neither the name of Apple Inc. ("Apple") nor the names of
15 *     its contributors may be used to endorse or promote products derived
16 *     from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 *
30 * Note: The implementations of InterlockedIncrement and InterlockedDecrement are based
31 * on atomic_increment and atomic_exchange_and_add from the Boost C++ Library. The license
32 * is virtually identical to the Apple license above but is included here for completeness.
33 *
34 * Boost Software License - Version 1.0 - August 17th, 2003
35 *
36 * Permission is hereby granted, free of charge, to any person or organization
37 * obtaining a copy of the software and accompanying documentation covered by
38 * this license (the "Software") to use, reproduce, display, distribute,
39 * execute, and transmit the Software, and to prepare derivative works of the
40 * Software, and to permit third-parties to whom the Software is furnished to
41 * do so, all subject to the following:
42 *
43 * The copyright notices in the Software and this entire statement, including
44 * the above license grant, this restriction and the following disclaimer,
45 * must be included in all copies of the Software, in whole or in part, and
46 * all derivative works of the Software, unless such copies or derivative
47 * works are solely in the form of machine-executable object code generated by
48 * a source language processor.
49 *
50 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
51 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
52 * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
53 * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
54 * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
55 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
56 * DEALINGS IN THE SOFTWARE.
57 */
58
59#ifndef Atomics_h
60#define Atomics_h
61
62#include <wtf/StdLibExtras.h>
63
64#if OS(WINDOWS)
65#if !COMPILER(GCC)
66extern "C" void _ReadWriteBarrier(void);
67#pragma intrinsic(_ReadWriteBarrier)
68#endif
69#include <windows.h>
70#endif
71
72namespace WTF {
73
74#if OS(WINDOWS)
75inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue)
76{
77#if OS(WINCE)
78    return InterlockedCompareExchange(reinterpret_cast<LONG*>(const_cast<unsigned*>(location)), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected);
79#else
80    return InterlockedCompareExchange(reinterpret_cast<LONG volatile*>(location), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected);
81#endif
82}
83
84inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue)
85{
86    return InterlockedCompareExchangePointer(location, newValue, expected) == expected;
87}
88#else // OS(WINDOWS) --> not windows
89inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue)
90{
91#if ENABLE(COMPARE_AND_SWAP)
92#if CPU(X86) || CPU(X86_64)
93    unsigned char result;
94    asm volatile(
95        "lock; cmpxchgl %3, %2\n\t"
96        "sete %1"
97        : "+a"(expected), "=q"(result), "+m"(*location)
98        : "r"(newValue)
99        : "memory"
100        );
101#elif CPU(ARM_THUMB2)
102    unsigned tmp;
103    unsigned result;
104    asm volatile(
105        "movw %1, #1\n\t"
106        "ldrex %2, %0\n\t"
107        "cmp %3, %2\n\t"
108        "bne.n 0f\n\t"
109        "strex %1, %4, %0\n\t"
110        "0:"
111        : "+Q"(*location), "=&r"(result), "=&r"(tmp)
112        : "r"(expected), "r"(newValue)
113        : "memory");
114    result = !result;
115#elif CPU(ARM64) && COMPILER(GCC)
116    unsigned tmp;
117    unsigned result;
118    asm volatile(
119        "mov %w1, #1\n\t"
120        "ldxr %w2, [%0]\n\t"
121        "cmp %w3, %w2\n\t"
122        "b.ne 0f\n\t"
123        "stxr %w1, %w4, [%0]\n\t"
124        "0:"
125        : "+r"(location), "=&r"(result), "=&r"(tmp)
126        : "r"(expected), "r"(newValue)
127        : "memory");
128    result = !result;
129#elif CPU(ARM64)
130    unsigned tmp;
131    unsigned result;
132    asm volatile(
133        "mov %w1, #1\n\t"
134        "ldxr %w2, %0\n\t"
135        "cmp %w3, %w2\n\t"
136        "b.ne 0f\n\t"
137        "stxr %w1, %w4, %0\n\t"
138        "0:"
139        : "+m"(*location), "=&r"(result), "=&r"(tmp)
140        : "r"(expected), "r"(newValue)
141        : "memory");
142    result = !result;
143#else
144#error "Bad architecture for compare and swap."
145#endif
146    return result;
147#else
148    UNUSED_PARAM(location);
149    UNUSED_PARAM(expected);
150    UNUSED_PARAM(newValue);
151    CRASH();
152    return false;
153#endif
154}
155
156inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue)
157{
158#if ENABLE(COMPARE_AND_SWAP)
159#if CPU(X86_64)
160    bool result;
161    asm volatile(
162        "lock; cmpxchgq %3, %2\n\t"
163        "sete %1"
164        : "+a"(expected), "=q"(result), "+m"(*location)
165        : "r"(newValue)
166        : "memory"
167        );
168    return result;
169#elif CPU(ARM64) && COMPILER(GCC)
170    bool result;
171    void* tmp;
172    asm volatile(
173        "mov %w1, #1\n\t"
174        "ldxr %x2, [%0]\n\t"
175        "cmp %x3, %x2\n\t"
176        "b.ne 0f\n\t"
177        "stxr %w1, %x4, [%0]\n\t"
178        "0:"
179        : "+r"(location), "=&r"(result), "=&r"(tmp)
180        : "r"(expected), "r"(newValue)
181        : "memory");
182    return !result;
183#elif CPU(ARM64)
184    bool result;
185    void* tmp;
186    asm volatile(
187        "mov %w1, #1\n\t"
188        "ldxr %x2, %0\n\t"
189        "cmp %x3, %x2\n\t"
190        "b.ne 0f\n\t"
191        "stxr %w1, %x4, %0\n\t"
192        "0:"
193        : "+m"(*location), "=&r"(result), "=&r"(tmp)
194        : "r"(expected), "r"(newValue)
195        : "memory");
196    return !result;
197#else
198    return weakCompareAndSwap(bitwise_cast<unsigned*>(location), bitwise_cast<unsigned>(expected), bitwise_cast<unsigned>(newValue));
199#endif
200#else // ENABLE(COMPARE_AND_SWAP)
201    UNUSED_PARAM(location);
202    UNUSED_PARAM(expected);
203    UNUSED_PARAM(newValue);
204    CRASH();
205    return 0;
206#endif // ENABLE(COMPARE_AND_SWAP)
207}
208#endif // OS(WINDOWS) (end of the not-windows case)
209
210inline bool weakCompareAndSwapUIntPtr(volatile uintptr_t* location, uintptr_t expected, uintptr_t newValue)
211{
212    return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue));
213}
214
215inline bool weakCompareAndSwapSize(volatile size_t* location, size_t expected, size_t newValue)
216{
217    return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue));
218}
219
220// Just a compiler fence. Has no effect on the hardware, but tells the compiler
221// not to move things around this call. Should not affect the compiler's ability
222// to do things like register allocation and code motion over pure operations.
223inline void compilerFence()
224{
225#if OS(WINDOWS) && !COMPILER(GCC)
226    _ReadWriteBarrier();
227#else
228    asm volatile("" ::: "memory");
229#endif
230}
231
232#if CPU(ARM_THUMB2) || CPU(ARM64)
233
234// Full memory fence. No accesses will float above this, and no accesses will sink
235// below it.
236inline void armV7_dmb()
237{
238    asm volatile("dmb sy" ::: "memory");
239}
240
241// Like the above, but only affects stores.
242inline void armV7_dmb_st()
243{
244    asm volatile("dmb st" ::: "memory");
245}
246
247inline void loadLoadFence() { armV7_dmb(); }
248inline void loadStoreFence() { armV7_dmb(); }
249inline void storeLoadFence() { armV7_dmb(); }
250inline void storeStoreFence() { armV7_dmb_st(); }
251inline void memoryBarrierAfterLock() { armV7_dmb(); }
252inline void memoryBarrierBeforeUnlock() { armV7_dmb(); }
253
254#elif CPU(X86) || CPU(X86_64)
255
256inline void x86_mfence()
257{
258#if OS(WINDOWS)
259    // I think that this does the equivalent of a dummy interlocked instruction,
260    // instead of using the 'mfence' instruction, at least according to MSDN. I
261    // know that it is equivalent for our purposes, but it would be good to
262    // investigate if that is actually better.
263    MemoryBarrier();
264#else
265    asm volatile("mfence" ::: "memory");
266#endif
267}
268
269inline void loadLoadFence() { compilerFence(); }
270inline void loadStoreFence() { compilerFence(); }
271inline void storeLoadFence() { x86_mfence(); }
272inline void storeStoreFence() { compilerFence(); }
273inline void memoryBarrierAfterLock() { compilerFence(); }
274inline void memoryBarrierBeforeUnlock() { compilerFence(); }
275
276#else
277
278inline void loadLoadFence() { compilerFence(); }
279inline void loadStoreFence() { compilerFence(); }
280inline void storeLoadFence() { compilerFence(); }
281inline void storeStoreFence() { compilerFence(); }
282inline void memoryBarrierAfterLock() { compilerFence(); }
283inline void memoryBarrierBeforeUnlock() { compilerFence(); }
284
285#endif
286
287inline bool weakCompareAndSwap(uint8_t* location, uint8_t expected, uint8_t newValue)
288{
289#if ENABLE(COMPARE_AND_SWAP)
290#if !OS(WINDOWS) && (CPU(X86) || CPU(X86_64))
291    unsigned char result;
292    asm volatile(
293        "lock; cmpxchgb %3, %2\n\t"
294        "sete %1"
295        : "+a"(expected), "=q"(result), "+m"(*location)
296        : "q"(newValue)
297        : "memory"
298        );
299    return result;
300#elif OS(WINDOWS) && CPU(X86)
301    // FIXME: We need a 64-bit ASM implementation, but this cannot be inline due to
302    // Microsoft's decision to exclude it from the compiler.
303    bool result = false;
304
305    __asm {
306        mov al, expected
307        mov edx, location
308        mov cl, newValue
309        lock cmpxchg byte ptr[edx], cl
310        setz result
311    }
312
313    return result;
314#else
315    uintptr_t locationValue = bitwise_cast<uintptr_t>(location);
316    uintptr_t alignedLocationValue = locationValue & ~(sizeof(unsigned) - 1);
317    uintptr_t locationOffset = locationValue - alignedLocationValue;
318    ASSERT(locationOffset < sizeof(unsigned));
319    unsigned* alignedLocation = bitwise_cast<unsigned*>(alignedLocationValue);
320    // Make sure that this load is always issued and never optimized away.
321    unsigned oldAlignedValue = *const_cast<volatile unsigned*>(alignedLocation);
322    union {
323        uint8_t bytes[sizeof(unsigned)];
324        unsigned word;
325    } u;
326    u.word = oldAlignedValue;
327    if (u.bytes[locationOffset] != expected)
328        return false;
329    u.bytes[locationOffset] = newValue;
330    unsigned newAlignedValue = u.word;
331    return weakCompareAndSwap(alignedLocation, oldAlignedValue, newAlignedValue);
332#endif
333#else
334    UNUSED_PARAM(location);
335    UNUSED_PARAM(expected);
336    UNUSED_PARAM(newValue);
337    CRASH();
338    return false;
339#endif
340}
341
342} // namespace WTF
343
344#endif // Atomics_h
345