1/* 2 * Copyright (C) 2007, 2008, 2010, 2012, 2013 Apple Inc. All rights reserved. 3 * Copyright (C) 2007 Justin Haygood (jhaygood@reaktix.com) 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Neither the name of Apple Inc. ("Apple") nor the names of 15 * its contributors may be used to endorse or promote products derived 16 * from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY 19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY 22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * 30 * Note: The implementations of InterlockedIncrement and InterlockedDecrement are based 31 * on atomic_increment and atomic_exchange_and_add from the Boost C++ Library. The license 32 * is virtually identical to the Apple license above but is included here for completeness. 33 * 34 * Boost Software License - Version 1.0 - August 17th, 2003 35 * 36 * Permission is hereby granted, free of charge, to any person or organization 37 * obtaining a copy of the software and accompanying documentation covered by 38 * this license (the "Software") to use, reproduce, display, distribute, 39 * execute, and transmit the Software, and to prepare derivative works of the 40 * Software, and to permit third-parties to whom the Software is furnished to 41 * do so, all subject to the following: 42 * 43 * The copyright notices in the Software and this entire statement, including 44 * the above license grant, this restriction and the following disclaimer, 45 * must be included in all copies of the Software, in whole or in part, and 46 * all derivative works of the Software, unless such copies or derivative 47 * works are solely in the form of machine-executable object code generated by 48 * a source language processor. 49 * 50 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 51 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 52 * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 53 * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 54 * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 55 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 56 * DEALINGS IN THE SOFTWARE. 57 */ 58 59#ifndef Atomics_h 60#define Atomics_h 61 62#include <wtf/StdLibExtras.h> 63 64#if OS(WINDOWS) 65#if !COMPILER(GCC) 66extern "C" void _ReadWriteBarrier(void); 67#pragma intrinsic(_ReadWriteBarrier) 68#endif 69#include <windows.h> 70#endif 71 72namespace WTF { 73 74#if OS(WINDOWS) 75inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue) 76{ 77#if OS(WINCE) 78 return InterlockedCompareExchange(reinterpret_cast<LONG*>(const_cast<unsigned*>(location)), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected); 79#else 80 return InterlockedCompareExchange(reinterpret_cast<LONG volatile*>(location), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected); 81#endif 82} 83 84inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue) 85{ 86 return InterlockedCompareExchangePointer(location, newValue, expected) == expected; 87} 88#else // OS(WINDOWS) --> not windows 89inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue) 90{ 91#if ENABLE(COMPARE_AND_SWAP) 92#if CPU(X86) || CPU(X86_64) 93 unsigned char result; 94 asm volatile( 95 "lock; cmpxchgl %3, %2\n\t" 96 "sete %1" 97 : "+a"(expected), "=q"(result), "+m"(*location) 98 : "r"(newValue) 99 : "memory" 100 ); 101#elif CPU(ARM_THUMB2) 102 unsigned tmp; 103 unsigned result; 104 asm volatile( 105 "movw %1, #1\n\t" 106 "ldrex %2, %0\n\t" 107 "cmp %3, %2\n\t" 108 "bne.n 0f\n\t" 109 "strex %1, %4, %0\n\t" 110 "0:" 111 : "+Q"(*location), "=&r"(result), "=&r"(tmp) 112 : "r"(expected), "r"(newValue) 113 : "memory"); 114 result = !result; 115#elif CPU(ARM64) && COMPILER(GCC) 116 unsigned tmp; 117 unsigned result; 118 asm volatile( 119 "mov %w1, #1\n\t" 120 "ldxr %w2, [%0]\n\t" 121 "cmp %w3, %w2\n\t" 122 "b.ne 0f\n\t" 123 "stxr %w1, %w4, [%0]\n\t" 124 "0:" 125 : "+r"(location), "=&r"(result), "=&r"(tmp) 126 : "r"(expected), "r"(newValue) 127 : "memory"); 128 result = !result; 129#elif CPU(ARM64) 130 unsigned tmp; 131 unsigned result; 132 asm volatile( 133 "mov %w1, #1\n\t" 134 "ldxr %w2, %0\n\t" 135 "cmp %w3, %w2\n\t" 136 "b.ne 0f\n\t" 137 "stxr %w1, %w4, %0\n\t" 138 "0:" 139 : "+m"(*location), "=&r"(result), "=&r"(tmp) 140 : "r"(expected), "r"(newValue) 141 : "memory"); 142 result = !result; 143#else 144#error "Bad architecture for compare and swap." 145#endif 146 return result; 147#else 148 UNUSED_PARAM(location); 149 UNUSED_PARAM(expected); 150 UNUSED_PARAM(newValue); 151 CRASH(); 152 return false; 153#endif 154} 155 156inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue) 157{ 158#if ENABLE(COMPARE_AND_SWAP) 159#if CPU(X86_64) 160 bool result; 161 asm volatile( 162 "lock; cmpxchgq %3, %2\n\t" 163 "sete %1" 164 : "+a"(expected), "=q"(result), "+m"(*location) 165 : "r"(newValue) 166 : "memory" 167 ); 168 return result; 169#elif CPU(ARM64) && COMPILER(GCC) 170 bool result; 171 void* tmp; 172 asm volatile( 173 "mov %w1, #1\n\t" 174 "ldxr %x2, [%0]\n\t" 175 "cmp %x3, %x2\n\t" 176 "b.ne 0f\n\t" 177 "stxr %w1, %x4, [%0]\n\t" 178 "0:" 179 : "+r"(location), "=&r"(result), "=&r"(tmp) 180 : "r"(expected), "r"(newValue) 181 : "memory"); 182 return !result; 183#elif CPU(ARM64) 184 bool result; 185 void* tmp; 186 asm volatile( 187 "mov %w1, #1\n\t" 188 "ldxr %x2, %0\n\t" 189 "cmp %x3, %x2\n\t" 190 "b.ne 0f\n\t" 191 "stxr %w1, %x4, %0\n\t" 192 "0:" 193 : "+m"(*location), "=&r"(result), "=&r"(tmp) 194 : "r"(expected), "r"(newValue) 195 : "memory"); 196 return !result; 197#else 198 return weakCompareAndSwap(bitwise_cast<unsigned*>(location), bitwise_cast<unsigned>(expected), bitwise_cast<unsigned>(newValue)); 199#endif 200#else // ENABLE(COMPARE_AND_SWAP) 201 UNUSED_PARAM(location); 202 UNUSED_PARAM(expected); 203 UNUSED_PARAM(newValue); 204 CRASH(); 205 return 0; 206#endif // ENABLE(COMPARE_AND_SWAP) 207} 208#endif // OS(WINDOWS) (end of the not-windows case) 209 210inline bool weakCompareAndSwapUIntPtr(volatile uintptr_t* location, uintptr_t expected, uintptr_t newValue) 211{ 212 return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue)); 213} 214 215inline bool weakCompareAndSwapSize(volatile size_t* location, size_t expected, size_t newValue) 216{ 217 return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue)); 218} 219 220// Just a compiler fence. Has no effect on the hardware, but tells the compiler 221// not to move things around this call. Should not affect the compiler's ability 222// to do things like register allocation and code motion over pure operations. 223inline void compilerFence() 224{ 225#if OS(WINDOWS) && !COMPILER(GCC) 226 _ReadWriteBarrier(); 227#else 228 asm volatile("" ::: "memory"); 229#endif 230} 231 232#if CPU(ARM_THUMB2) || CPU(ARM64) 233 234// Full memory fence. No accesses will float above this, and no accesses will sink 235// below it. 236inline void armV7_dmb() 237{ 238 asm volatile("dmb sy" ::: "memory"); 239} 240 241// Like the above, but only affects stores. 242inline void armV7_dmb_st() 243{ 244 asm volatile("dmb st" ::: "memory"); 245} 246 247inline void loadLoadFence() { armV7_dmb(); } 248inline void loadStoreFence() { armV7_dmb(); } 249inline void storeLoadFence() { armV7_dmb(); } 250inline void storeStoreFence() { armV7_dmb_st(); } 251inline void memoryBarrierAfterLock() { armV7_dmb(); } 252inline void memoryBarrierBeforeUnlock() { armV7_dmb(); } 253 254#elif CPU(X86) || CPU(X86_64) 255 256inline void x86_mfence() 257{ 258#if OS(WINDOWS) 259 // I think that this does the equivalent of a dummy interlocked instruction, 260 // instead of using the 'mfence' instruction, at least according to MSDN. I 261 // know that it is equivalent for our purposes, but it would be good to 262 // investigate if that is actually better. 263 MemoryBarrier(); 264#else 265 asm volatile("mfence" ::: "memory"); 266#endif 267} 268 269inline void loadLoadFence() { compilerFence(); } 270inline void loadStoreFence() { compilerFence(); } 271inline void storeLoadFence() { x86_mfence(); } 272inline void storeStoreFence() { compilerFence(); } 273inline void memoryBarrierAfterLock() { compilerFence(); } 274inline void memoryBarrierBeforeUnlock() { compilerFence(); } 275 276#else 277 278inline void loadLoadFence() { compilerFence(); } 279inline void loadStoreFence() { compilerFence(); } 280inline void storeLoadFence() { compilerFence(); } 281inline void storeStoreFence() { compilerFence(); } 282inline void memoryBarrierAfterLock() { compilerFence(); } 283inline void memoryBarrierBeforeUnlock() { compilerFence(); } 284 285#endif 286 287inline bool weakCompareAndSwap(uint8_t* location, uint8_t expected, uint8_t newValue) 288{ 289#if ENABLE(COMPARE_AND_SWAP) 290#if !OS(WINDOWS) && (CPU(X86) || CPU(X86_64)) 291 unsigned char result; 292 asm volatile( 293 "lock; cmpxchgb %3, %2\n\t" 294 "sete %1" 295 : "+a"(expected), "=q"(result), "+m"(*location) 296 : "q"(newValue) 297 : "memory" 298 ); 299 return result; 300#elif OS(WINDOWS) && CPU(X86) 301 // FIXME: We need a 64-bit ASM implementation, but this cannot be inline due to 302 // Microsoft's decision to exclude it from the compiler. 303 bool result = false; 304 305 __asm { 306 mov al, expected 307 mov edx, location 308 mov cl, newValue 309 lock cmpxchg byte ptr[edx], cl 310 setz result 311 } 312 313 return result; 314#else 315 uintptr_t locationValue = bitwise_cast<uintptr_t>(location); 316 uintptr_t alignedLocationValue = locationValue & ~(sizeof(unsigned) - 1); 317 uintptr_t locationOffset = locationValue - alignedLocationValue; 318 ASSERT(locationOffset < sizeof(unsigned)); 319 unsigned* alignedLocation = bitwise_cast<unsigned*>(alignedLocationValue); 320 // Make sure that this load is always issued and never optimized away. 321 unsigned oldAlignedValue = *const_cast<volatile unsigned*>(alignedLocation); 322 union { 323 uint8_t bytes[sizeof(unsigned)]; 324 unsigned word; 325 } u; 326 u.word = oldAlignedValue; 327 if (u.bytes[locationOffset] != expected) 328 return false; 329 u.bytes[locationOffset] = newValue; 330 unsigned newAlignedValue = u.word; 331 return weakCompareAndSwap(alignedLocation, oldAlignedValue, newAlignedValue); 332#endif 333#else 334 UNUSED_PARAM(location); 335 UNUSED_PARAM(expected); 336 UNUSED_PARAM(newValue); 337 CRASH(); 338 return false; 339#endif 340} 341 342} // namespace WTF 343 344#endif // Atomics_h 345