1//===-- tsan_interface_atomic.cpp -----------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of ThreadSanitizer (TSan), a race detector. 10// 11//===----------------------------------------------------------------------===// 12 13// ThreadSanitizer atomic operations are based on C++11/C1x standards. 14// For background see C++11 standard. A slightly older, publicly 15// available draft of the standard (not entirely up-to-date, but close enough 16// for casual browsing) is available here: 17// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 18// The following page contains more background information: 19// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 20 21#include "sanitizer_common/sanitizer_placement_new.h" 22#include "sanitizer_common/sanitizer_stacktrace.h" 23#include "sanitizer_common/sanitizer_mutex.h" 24#include "tsan_flags.h" 25#include "tsan_interface.h" 26#include "tsan_rtl.h" 27 28using namespace __tsan; 29 30#if !SANITIZER_GO && __TSAN_HAS_INT128 31// Protects emulation of 128-bit atomic operations. 32static StaticSpinMutex mutex128; 33#endif 34 35#if SANITIZER_DEBUG 36static bool IsLoadOrder(morder mo) { 37 return mo == mo_relaxed || mo == mo_consume 38 || mo == mo_acquire || mo == mo_seq_cst; 39} 40 41static bool IsStoreOrder(morder mo) { 42 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; 43} 44#endif 45 46static bool IsReleaseOrder(morder mo) { 47 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; 48} 49 50static bool IsAcquireOrder(morder mo) { 51 return mo == mo_consume || mo == mo_acquire 52 || mo == mo_acq_rel || mo == mo_seq_cst; 53} 54 55static bool IsAcqRelOrder(morder mo) { 56 return mo == mo_acq_rel || mo == mo_seq_cst; 57} 58 59template<typename T> T func_xchg(volatile T *v, T op) { 60 T res = __sync_lock_test_and_set(v, op); 61 // __sync_lock_test_and_set does not contain full barrier. 62 __sync_synchronize(); 63 return res; 64} 65 66template<typename T> T func_add(volatile T *v, T op) { 67 return __sync_fetch_and_add(v, op); 68} 69 70template<typename T> T func_sub(volatile T *v, T op) { 71 return __sync_fetch_and_sub(v, op); 72} 73 74template<typename T> T func_and(volatile T *v, T op) { 75 return __sync_fetch_and_and(v, op); 76} 77 78template<typename T> T func_or(volatile T *v, T op) { 79 return __sync_fetch_and_or(v, op); 80} 81 82template<typename T> T func_xor(volatile T *v, T op) { 83 return __sync_fetch_and_xor(v, op); 84} 85 86template<typename T> T func_nand(volatile T *v, T op) { 87 // clang does not support __sync_fetch_and_nand. 88 T cmp = *v; 89 for (;;) { 90 T newv = ~(cmp & op); 91 T cur = __sync_val_compare_and_swap(v, cmp, newv); 92 if (cmp == cur) 93 return cmp; 94 cmp = cur; 95 } 96} 97 98template<typename T> T func_cas(volatile T *v, T cmp, T xch) { 99 return __sync_val_compare_and_swap(v, cmp, xch); 100} 101 102// clang does not support 128-bit atomic ops. 103// Atomic ops are executed under tsan internal mutex, 104// here we assume that the atomic variables are not accessed 105// from non-instrumented code. 106#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \ 107 && __TSAN_HAS_INT128 108a128 func_xchg(volatile a128 *v, a128 op) { 109 SpinMutexLock lock(&mutex128); 110 a128 cmp = *v; 111 *v = op; 112 return cmp; 113} 114 115a128 func_add(volatile a128 *v, a128 op) { 116 SpinMutexLock lock(&mutex128); 117 a128 cmp = *v; 118 *v = cmp + op; 119 return cmp; 120} 121 122a128 func_sub(volatile a128 *v, a128 op) { 123 SpinMutexLock lock(&mutex128); 124 a128 cmp = *v; 125 *v = cmp - op; 126 return cmp; 127} 128 129a128 func_and(volatile a128 *v, a128 op) { 130 SpinMutexLock lock(&mutex128); 131 a128 cmp = *v; 132 *v = cmp & op; 133 return cmp; 134} 135 136a128 func_or(volatile a128 *v, a128 op) { 137 SpinMutexLock lock(&mutex128); 138 a128 cmp = *v; 139 *v = cmp | op; 140 return cmp; 141} 142 143a128 func_xor(volatile a128 *v, a128 op) { 144 SpinMutexLock lock(&mutex128); 145 a128 cmp = *v; 146 *v = cmp ^ op; 147 return cmp; 148} 149 150a128 func_nand(volatile a128 *v, a128 op) { 151 SpinMutexLock lock(&mutex128); 152 a128 cmp = *v; 153 *v = ~(cmp & op); 154 return cmp; 155} 156 157a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { 158 SpinMutexLock lock(&mutex128); 159 a128 cur = *v; 160 if (cur == cmp) 161 *v = xch; 162 return cur; 163} 164#endif 165 166template <typename T> 167static int AccessSize() { 168 if (sizeof(T) <= 1) 169 return 1; 170 else if (sizeof(T) <= 2) 171 return 2; 172 else if (sizeof(T) <= 4) 173 return 4; 174 else 175 return 8; 176 // For 16-byte atomics we also use 8-byte memory access, 177 // this leads to false negatives only in very obscure cases. 178} 179 180#if !SANITIZER_GO 181static atomic_uint8_t *to_atomic(const volatile a8 *a) { 182 return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a)); 183} 184 185static atomic_uint16_t *to_atomic(const volatile a16 *a) { 186 return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a)); 187} 188#endif 189 190static atomic_uint32_t *to_atomic(const volatile a32 *a) { 191 return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a)); 192} 193 194static atomic_uint64_t *to_atomic(const volatile a64 *a) { 195 return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a)); 196} 197 198static memory_order to_mo(morder mo) { 199 switch (mo) { 200 case mo_relaxed: return memory_order_relaxed; 201 case mo_consume: return memory_order_consume; 202 case mo_acquire: return memory_order_acquire; 203 case mo_release: return memory_order_release; 204 case mo_acq_rel: return memory_order_acq_rel; 205 case mo_seq_cst: return memory_order_seq_cst; 206 } 207 DCHECK(0); 208 return memory_order_seq_cst; 209} 210 211template<typename T> 212static T NoTsanAtomicLoad(const volatile T *a, morder mo) { 213 return atomic_load(to_atomic(a), to_mo(mo)); 214} 215 216#if __TSAN_HAS_INT128 && !SANITIZER_GO 217static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { 218 SpinMutexLock lock(&mutex128); 219 return *a; 220} 221#endif 222 223template <typename T> 224static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { 225 DCHECK(IsLoadOrder(mo)); 226 // This fast-path is critical for performance. 227 // Assume the access is atomic. 228 if (!IsAcquireOrder(mo)) { 229 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), 230 kAccessRead | kAccessAtomic); 231 return NoTsanAtomicLoad(a, mo); 232 } 233 // Don't create sync object if it does not exist yet. For example, an atomic 234 // pointer is initialized to nullptr and then periodically acquire-loaded. 235 T v = NoTsanAtomicLoad(a, mo); 236 SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); 237 if (s) { 238 SlotLocker locker(thr); 239 ReadLock lock(&s->mtx); 240 thr->clock.Acquire(s->clock); 241 // Re-read under sync mutex because we need a consistent snapshot 242 // of the value and the clock we acquire. 243 v = NoTsanAtomicLoad(a, mo); 244 } 245 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead | kAccessAtomic); 246 return v; 247} 248 249template<typename T> 250static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { 251 atomic_store(to_atomic(a), v, to_mo(mo)); 252} 253 254#if __TSAN_HAS_INT128 && !SANITIZER_GO 255static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { 256 SpinMutexLock lock(&mutex128); 257 *a = v; 258} 259#endif 260 261template <typename T> 262static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, 263 morder mo) { 264 DCHECK(IsStoreOrder(mo)); 265 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic); 266 // This fast-path is critical for performance. 267 // Assume the access is atomic. 268 // Strictly saying even relaxed store cuts off release sequence, 269 // so must reset the clock. 270 if (!IsReleaseOrder(mo)) { 271 NoTsanAtomicStore(a, v, mo); 272 return; 273 } 274 SlotLocker locker(thr); 275 { 276 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); 277 Lock lock(&s->mtx); 278 thr->clock.ReleaseStore(&s->clock); 279 NoTsanAtomicStore(a, v, mo); 280 } 281 IncrementEpoch(thr); 282} 283 284template <typename T, T (*F)(volatile T *v, T op)> 285static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { 286 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic); 287 if (LIKELY(mo == mo_relaxed)) 288 return F(a, v); 289 SlotLocker locker(thr); 290 { 291 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); 292 RWLock lock(&s->mtx, IsReleaseOrder(mo)); 293 if (IsAcqRelOrder(mo)) 294 thr->clock.ReleaseAcquire(&s->clock); 295 else if (IsReleaseOrder(mo)) 296 thr->clock.Release(&s->clock); 297 else if (IsAcquireOrder(mo)) 298 thr->clock.Acquire(s->clock); 299 v = F(a, v); 300 } 301 if (IsReleaseOrder(mo)) 302 IncrementEpoch(thr); 303 return v; 304} 305 306template<typename T> 307static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { 308 return func_xchg(a, v); 309} 310 311template<typename T> 312static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { 313 return func_add(a, v); 314} 315 316template<typename T> 317static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { 318 return func_sub(a, v); 319} 320 321template<typename T> 322static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { 323 return func_and(a, v); 324} 325 326template<typename T> 327static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { 328 return func_or(a, v); 329} 330 331template<typename T> 332static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { 333 return func_xor(a, v); 334} 335 336template<typename T> 337static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { 338 return func_nand(a, v); 339} 340 341template<typename T> 342static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, 343 morder mo) { 344 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); 345} 346 347template<typename T> 348static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, 349 morder mo) { 350 return AtomicRMW<T, func_add>(thr, pc, a, v, mo); 351} 352 353template<typename T> 354static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, 355 morder mo) { 356 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); 357} 358 359template<typename T> 360static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, 361 morder mo) { 362 return AtomicRMW<T, func_and>(thr, pc, a, v, mo); 363} 364 365template<typename T> 366static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, 367 morder mo) { 368 return AtomicRMW<T, func_or>(thr, pc, a, v, mo); 369} 370 371template<typename T> 372static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, 373 morder mo) { 374 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); 375} 376 377template<typename T> 378static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, 379 morder mo) { 380 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); 381} 382 383template<typename T> 384static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { 385 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); 386} 387 388#if __TSAN_HAS_INT128 389static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, 390 morder mo, morder fmo) { 391 a128 old = *c; 392 a128 cur = func_cas(a, old, v); 393 if (cur == old) 394 return true; 395 *c = cur; 396 return false; 397} 398#endif 399 400template<typename T> 401static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { 402 NoTsanAtomicCAS(a, &c, v, mo, fmo); 403 return c; 404} 405 406template <typename T> 407static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, 408 morder mo, morder fmo) { 409 // 31.7.2.18: "The failure argument shall not be memory_order_release 410 // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic 411 // (mo_relaxed) when those are used. 412 DCHECK(IsLoadOrder(fmo)); 413 414 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic); 415 if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { 416 T cc = *c; 417 T pr = func_cas(a, cc, v); 418 if (pr == cc) 419 return true; 420 *c = pr; 421 return false; 422 } 423 SlotLocker locker(thr); 424 bool release = IsReleaseOrder(mo); 425 bool success; 426 { 427 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); 428 RWLock lock(&s->mtx, release); 429 T cc = *c; 430 T pr = func_cas(a, cc, v); 431 success = pr == cc; 432 if (!success) { 433 *c = pr; 434 mo = fmo; 435 } 436 if (success && IsAcqRelOrder(mo)) 437 thr->clock.ReleaseAcquire(&s->clock); 438 else if (success && IsReleaseOrder(mo)) 439 thr->clock.Release(&s->clock); 440 else if (IsAcquireOrder(mo)) 441 thr->clock.Acquire(s->clock); 442 } 443 if (success && release) 444 IncrementEpoch(thr); 445 return success; 446} 447 448template<typename T> 449static T AtomicCAS(ThreadState *thr, uptr pc, 450 volatile T *a, T c, T v, morder mo, morder fmo) { 451 AtomicCAS(thr, pc, a, &c, v, mo, fmo); 452 return c; 453} 454 455#if !SANITIZER_GO 456static void NoTsanAtomicFence(morder mo) { 457 __sync_synchronize(); 458} 459 460static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { 461 // FIXME(dvyukov): not implemented. 462 __sync_synchronize(); 463} 464#endif 465 466// Interface functions follow. 467#if !SANITIZER_GO 468 469// C/C++ 470 471static morder convert_morder(morder mo) { 472 if (flags()->force_seq_cst_atomics) 473 return (morder)mo_seq_cst; 474 475 // Filter out additional memory order flags: 476 // MEMMODEL_SYNC = 1 << 15 477 // __ATOMIC_HLE_ACQUIRE = 1 << 16 478 // __ATOMIC_HLE_RELEASE = 1 << 17 479 // 480 // HLE is an optimization, and we pretend that elision always fails. 481 // MEMMODEL_SYNC is used when lowering __sync_ atomics, 482 // since we use __sync_ atomics for actual atomic operations, 483 // we can safely ignore it as well. It also subtly affects semantics, 484 // but we don't model the difference. 485 return (morder)(mo & 0x7fff); 486} 487 488# define ATOMIC_IMPL(func, ...) \ 489 ThreadState *const thr = cur_thread(); \ 490 ProcessPendingSignals(thr); \ 491 if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \ 492 return NoTsanAtomic##func(__VA_ARGS__); \ 493 mo = convert_morder(mo); \ 494 return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__); 495 496extern "C" { 497SANITIZER_INTERFACE_ATTRIBUTE 498a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { 499 ATOMIC_IMPL(Load, a, mo); 500} 501 502SANITIZER_INTERFACE_ATTRIBUTE 503a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { 504 ATOMIC_IMPL(Load, a, mo); 505} 506 507SANITIZER_INTERFACE_ATTRIBUTE 508a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { 509 ATOMIC_IMPL(Load, a, mo); 510} 511 512SANITIZER_INTERFACE_ATTRIBUTE 513a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { 514 ATOMIC_IMPL(Load, a, mo); 515} 516 517#if __TSAN_HAS_INT128 518SANITIZER_INTERFACE_ATTRIBUTE 519a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { 520 ATOMIC_IMPL(Load, a, mo); 521} 522#endif 523 524SANITIZER_INTERFACE_ATTRIBUTE 525void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { 526 ATOMIC_IMPL(Store, a, v, mo); 527} 528 529SANITIZER_INTERFACE_ATTRIBUTE 530void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { 531 ATOMIC_IMPL(Store, a, v, mo); 532} 533 534SANITIZER_INTERFACE_ATTRIBUTE 535void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { 536 ATOMIC_IMPL(Store, a, v, mo); 537} 538 539SANITIZER_INTERFACE_ATTRIBUTE 540void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { 541 ATOMIC_IMPL(Store, a, v, mo); 542} 543 544#if __TSAN_HAS_INT128 545SANITIZER_INTERFACE_ATTRIBUTE 546void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { 547 ATOMIC_IMPL(Store, a, v, mo); 548} 549#endif 550 551SANITIZER_INTERFACE_ATTRIBUTE 552a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { 553 ATOMIC_IMPL(Exchange, a, v, mo); 554} 555 556SANITIZER_INTERFACE_ATTRIBUTE 557a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { 558 ATOMIC_IMPL(Exchange, a, v, mo); 559} 560 561SANITIZER_INTERFACE_ATTRIBUTE 562a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { 563 ATOMIC_IMPL(Exchange, a, v, mo); 564} 565 566SANITIZER_INTERFACE_ATTRIBUTE 567a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { 568 ATOMIC_IMPL(Exchange, a, v, mo); 569} 570 571#if __TSAN_HAS_INT128 572SANITIZER_INTERFACE_ATTRIBUTE 573a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { 574 ATOMIC_IMPL(Exchange, a, v, mo); 575} 576#endif 577 578SANITIZER_INTERFACE_ATTRIBUTE 579a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { 580 ATOMIC_IMPL(FetchAdd, a, v, mo); 581} 582 583SANITIZER_INTERFACE_ATTRIBUTE 584a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { 585 ATOMIC_IMPL(FetchAdd, a, v, mo); 586} 587 588SANITIZER_INTERFACE_ATTRIBUTE 589a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { 590 ATOMIC_IMPL(FetchAdd, a, v, mo); 591} 592 593SANITIZER_INTERFACE_ATTRIBUTE 594a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { 595 ATOMIC_IMPL(FetchAdd, a, v, mo); 596} 597 598#if __TSAN_HAS_INT128 599SANITIZER_INTERFACE_ATTRIBUTE 600a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { 601 ATOMIC_IMPL(FetchAdd, a, v, mo); 602} 603#endif 604 605SANITIZER_INTERFACE_ATTRIBUTE 606a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { 607 ATOMIC_IMPL(FetchSub, a, v, mo); 608} 609 610SANITIZER_INTERFACE_ATTRIBUTE 611a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { 612 ATOMIC_IMPL(FetchSub, a, v, mo); 613} 614 615SANITIZER_INTERFACE_ATTRIBUTE 616a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { 617 ATOMIC_IMPL(FetchSub, a, v, mo); 618} 619 620SANITIZER_INTERFACE_ATTRIBUTE 621a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { 622 ATOMIC_IMPL(FetchSub, a, v, mo); 623} 624 625#if __TSAN_HAS_INT128 626SANITIZER_INTERFACE_ATTRIBUTE 627a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { 628 ATOMIC_IMPL(FetchSub, a, v, mo); 629} 630#endif 631 632SANITIZER_INTERFACE_ATTRIBUTE 633a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { 634 ATOMIC_IMPL(FetchAnd, a, v, mo); 635} 636 637SANITIZER_INTERFACE_ATTRIBUTE 638a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { 639 ATOMIC_IMPL(FetchAnd, a, v, mo); 640} 641 642SANITIZER_INTERFACE_ATTRIBUTE 643a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { 644 ATOMIC_IMPL(FetchAnd, a, v, mo); 645} 646 647SANITIZER_INTERFACE_ATTRIBUTE 648a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { 649 ATOMIC_IMPL(FetchAnd, a, v, mo); 650} 651 652#if __TSAN_HAS_INT128 653SANITIZER_INTERFACE_ATTRIBUTE 654a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { 655 ATOMIC_IMPL(FetchAnd, a, v, mo); 656} 657#endif 658 659SANITIZER_INTERFACE_ATTRIBUTE 660a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { 661 ATOMIC_IMPL(FetchOr, a, v, mo); 662} 663 664SANITIZER_INTERFACE_ATTRIBUTE 665a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { 666 ATOMIC_IMPL(FetchOr, a, v, mo); 667} 668 669SANITIZER_INTERFACE_ATTRIBUTE 670a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { 671 ATOMIC_IMPL(FetchOr, a, v, mo); 672} 673 674SANITIZER_INTERFACE_ATTRIBUTE 675a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { 676 ATOMIC_IMPL(FetchOr, a, v, mo); 677} 678 679#if __TSAN_HAS_INT128 680SANITIZER_INTERFACE_ATTRIBUTE 681a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { 682 ATOMIC_IMPL(FetchOr, a, v, mo); 683} 684#endif 685 686SANITIZER_INTERFACE_ATTRIBUTE 687a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { 688 ATOMIC_IMPL(FetchXor, a, v, mo); 689} 690 691SANITIZER_INTERFACE_ATTRIBUTE 692a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { 693 ATOMIC_IMPL(FetchXor, a, v, mo); 694} 695 696SANITIZER_INTERFACE_ATTRIBUTE 697a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { 698 ATOMIC_IMPL(FetchXor, a, v, mo); 699} 700 701SANITIZER_INTERFACE_ATTRIBUTE 702a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { 703 ATOMIC_IMPL(FetchXor, a, v, mo); 704} 705 706#if __TSAN_HAS_INT128 707SANITIZER_INTERFACE_ATTRIBUTE 708a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { 709 ATOMIC_IMPL(FetchXor, a, v, mo); 710} 711#endif 712 713SANITIZER_INTERFACE_ATTRIBUTE 714a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { 715 ATOMIC_IMPL(FetchNand, a, v, mo); 716} 717 718SANITIZER_INTERFACE_ATTRIBUTE 719a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { 720 ATOMIC_IMPL(FetchNand, a, v, mo); 721} 722 723SANITIZER_INTERFACE_ATTRIBUTE 724a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { 725 ATOMIC_IMPL(FetchNand, a, v, mo); 726} 727 728SANITIZER_INTERFACE_ATTRIBUTE 729a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { 730 ATOMIC_IMPL(FetchNand, a, v, mo); 731} 732 733#if __TSAN_HAS_INT128 734SANITIZER_INTERFACE_ATTRIBUTE 735a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { 736 ATOMIC_IMPL(FetchNand, a, v, mo); 737} 738#endif 739 740SANITIZER_INTERFACE_ATTRIBUTE 741int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, 742 morder mo, morder fmo) { 743 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 744} 745 746SANITIZER_INTERFACE_ATTRIBUTE 747int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, 748 morder mo, morder fmo) { 749 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 750} 751 752SANITIZER_INTERFACE_ATTRIBUTE 753int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, 754 morder mo, morder fmo) { 755 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 756} 757 758SANITIZER_INTERFACE_ATTRIBUTE 759int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, 760 morder mo, morder fmo) { 761 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 762} 763 764#if __TSAN_HAS_INT128 765SANITIZER_INTERFACE_ATTRIBUTE 766int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, 767 morder mo, morder fmo) { 768 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 769} 770#endif 771 772SANITIZER_INTERFACE_ATTRIBUTE 773int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, 774 morder mo, morder fmo) { 775 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 776} 777 778SANITIZER_INTERFACE_ATTRIBUTE 779int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, 780 morder mo, morder fmo) { 781 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 782} 783 784SANITIZER_INTERFACE_ATTRIBUTE 785int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, 786 morder mo, morder fmo) { 787 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 788} 789 790SANITIZER_INTERFACE_ATTRIBUTE 791int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, 792 morder mo, morder fmo) { 793 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 794} 795 796#if __TSAN_HAS_INT128 797SANITIZER_INTERFACE_ATTRIBUTE 798int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, 799 morder mo, morder fmo) { 800 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 801} 802#endif 803 804SANITIZER_INTERFACE_ATTRIBUTE 805a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, 806 morder mo, morder fmo) { 807 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 808} 809 810SANITIZER_INTERFACE_ATTRIBUTE 811a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, 812 morder mo, morder fmo) { 813 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 814} 815 816SANITIZER_INTERFACE_ATTRIBUTE 817a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, 818 morder mo, morder fmo) { 819 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 820} 821 822SANITIZER_INTERFACE_ATTRIBUTE 823a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, 824 morder mo, morder fmo) { 825 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 826} 827 828#if __TSAN_HAS_INT128 829SANITIZER_INTERFACE_ATTRIBUTE 830a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, 831 morder mo, morder fmo) { 832 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); 833} 834#endif 835 836SANITIZER_INTERFACE_ATTRIBUTE 837void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); } 838 839SANITIZER_INTERFACE_ATTRIBUTE 840void __tsan_atomic_signal_fence(morder mo) { 841} 842} // extern "C" 843 844#else // #if !SANITIZER_GO 845 846// Go 847 848# define ATOMIC(func, ...) \ 849 if (thr->ignore_sync) { \ 850 NoTsanAtomic##func(__VA_ARGS__); \ 851 } else { \ 852 FuncEntry(thr, cpc); \ 853 Atomic##func(thr, pc, __VA_ARGS__); \ 854 FuncExit(thr); \ 855 } 856 857# define ATOMIC_RET(func, ret, ...) \ 858 if (thr->ignore_sync) { \ 859 (ret) = NoTsanAtomic##func(__VA_ARGS__); \ 860 } else { \ 861 FuncEntry(thr, cpc); \ 862 (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ 863 FuncExit(thr); \ 864 } 865 866extern "C" { 867SANITIZER_INTERFACE_ATTRIBUTE 868void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 869 ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); 870} 871 872SANITIZER_INTERFACE_ATTRIBUTE 873void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 874 ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); 875} 876 877SANITIZER_INTERFACE_ATTRIBUTE 878void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 879 ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); 880} 881 882SANITIZER_INTERFACE_ATTRIBUTE 883void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 884 ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); 885} 886 887SANITIZER_INTERFACE_ATTRIBUTE 888void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 889 ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); 890} 891 892SANITIZER_INTERFACE_ATTRIBUTE 893void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 894 ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); 895} 896 897SANITIZER_INTERFACE_ATTRIBUTE 898void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 899 ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); 900} 901 902SANITIZER_INTERFACE_ATTRIBUTE 903void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 904 ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); 905} 906 907SANITIZER_INTERFACE_ATTRIBUTE 908void __tsan_go_atomic32_compare_exchange( 909 ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 910 a32 cur = 0; 911 a32 cmp = *(a32*)(a+8); 912 ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); 913 *(bool*)(a+16) = (cur == cmp); 914} 915 916SANITIZER_INTERFACE_ATTRIBUTE 917void __tsan_go_atomic64_compare_exchange( 918 ThreadState *thr, uptr cpc, uptr pc, u8 *a) { 919 a64 cur = 0; 920 a64 cmp = *(a64*)(a+8); 921 ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); 922 *(bool*)(a+24) = (cur == cmp); 923} 924} // extern "C" 925#endif // #if !SANITIZER_GO 926