1//===-- tsan_interceptors_mac.cpp -----------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of ThreadSanitizer (TSan), a race detector. 10// 11// Mac-specific interceptors. 12//===----------------------------------------------------------------------===// 13 14#include "sanitizer_common/sanitizer_platform.h" 15#if SANITIZER_MAC 16 17#include "interception/interception.h" 18#include "tsan_interceptors.h" 19#include "tsan_interface.h" 20#include "tsan_interface_ann.h" 21#include "sanitizer_common/sanitizer_addrhashmap.h" 22 23#include <errno.h> 24#include <libkern/OSAtomic.h> 25#include <objc/objc-sync.h> 26#include <os/lock.h> 27#include <sys/ucontext.h> 28 29#if defined(__has_include) && __has_include(<xpc/xpc.h>) 30#include <xpc/xpc.h> 31#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 32 33typedef long long_t; 34 35extern "C" { 36int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); 37int setcontext(const ucontext_t *ucp); 38} 39 40namespace __tsan { 41 42// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, 43// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are 44// actually aliases of each other, and we cannot have different interceptors for 45// them, because they're actually the same function. Thus, we have to stay 46// conservative and treat the non-barrier versions as mo_acq_rel. 47static const morder kMacOrderBarrier = mo_acq_rel; 48static const morder kMacOrderNonBarrier = mo_acq_rel; 49 50#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 51 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 52 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 53 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ 54 } 55 56#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 57 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 58 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 59 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ 60 } 61 62#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 63 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 64 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 65 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ 66 } 67 68#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 69 mo) \ 70 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 71 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 72 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ 73 } 74 75#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ 76 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 77 kMacOrderNonBarrier) \ 78 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 79 kMacOrderBarrier) \ 80 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ 81 kMacOrderNonBarrier) \ 82 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ 83 kMacOrderBarrier) 84 85#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ 86 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 87 kMacOrderNonBarrier) \ 88 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 89 kMacOrderBarrier) \ 90 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ 91 kMacOrderNonBarrier) \ 92 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ 93 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) 94 95OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, 96 OSATOMIC_INTERCEPTOR_PLUS_X) 97OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, 98 OSATOMIC_INTERCEPTOR_PLUS_1) 99OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, 100 OSATOMIC_INTERCEPTOR_MINUS_1) 101OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, 102 OSATOMIC_INTERCEPTOR) 103OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, 104 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 105OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, 106 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 107 108#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ 109 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ 110 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ 111 return tsan_atomic_f##_compare_exchange_strong( \ 112 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 113 kMacOrderNonBarrier, kMacOrderNonBarrier); \ 114 } \ 115 \ 116 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ 117 t volatile *ptr) { \ 118 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ 119 return tsan_atomic_f##_compare_exchange_strong( \ 120 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 121 kMacOrderBarrier, kMacOrderNonBarrier); \ 122 } 123 124OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) 125OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, 126 long_t) 127OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, 128 void *) 129OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, 130 int32_t) 131OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, 132 int64_t) 133 134#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ 135 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ 136 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ 137 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ 138 char bit = 0x80u >> (n & 7); \ 139 char mask = clear ? ~bit : bit; \ 140 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ 141 return orig_byte & bit; \ 142 } 143 144#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ 145 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ 146 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) 147 148OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) 149OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, 150 true) 151 152TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, 153 size_t offset) { 154 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); 155 __tsan_release(item); 156 REAL(OSAtomicEnqueue)(list, item, offset); 157} 158 159TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { 160 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); 161 void *item = REAL(OSAtomicDequeue)(list, offset); 162 if (item) __tsan_acquire(item); 163 return item; 164} 165 166// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. 167#if !SANITIZER_IOS 168 169TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, 170 size_t offset) { 171 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); 172 __tsan_release(item); 173 REAL(OSAtomicFifoEnqueue)(list, item, offset); 174} 175 176TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, 177 size_t offset) { 178 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); 179 void *item = REAL(OSAtomicFifoDequeue)(list, offset); 180 if (item) __tsan_acquire(item); 181 return item; 182} 183 184#endif 185 186TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { 187 CHECK(!cur_thread()->is_dead); 188 if (!cur_thread()->is_inited) { 189 return REAL(OSSpinLockLock)(lock); 190 } 191 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); 192 REAL(OSSpinLockLock)(lock); 193 Acquire(thr, pc, (uptr)lock); 194} 195 196TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { 197 CHECK(!cur_thread()->is_dead); 198 if (!cur_thread()->is_inited) { 199 return REAL(OSSpinLockTry)(lock); 200 } 201 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); 202 bool result = REAL(OSSpinLockTry)(lock); 203 if (result) 204 Acquire(thr, pc, (uptr)lock); 205 return result; 206} 207 208TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { 209 CHECK(!cur_thread()->is_dead); 210 if (!cur_thread()->is_inited) { 211 return REAL(OSSpinLockUnlock)(lock); 212 } 213 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); 214 Release(thr, pc, (uptr)lock); 215 REAL(OSSpinLockUnlock)(lock); 216} 217 218TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { 219 CHECK(!cur_thread()->is_dead); 220 if (!cur_thread()->is_inited) { 221 return REAL(os_lock_lock)(lock); 222 } 223 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); 224 REAL(os_lock_lock)(lock); 225 Acquire(thr, pc, (uptr)lock); 226} 227 228TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { 229 CHECK(!cur_thread()->is_dead); 230 if (!cur_thread()->is_inited) { 231 return REAL(os_lock_trylock)(lock); 232 } 233 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); 234 bool result = REAL(os_lock_trylock)(lock); 235 if (result) 236 Acquire(thr, pc, (uptr)lock); 237 return result; 238} 239 240TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { 241 CHECK(!cur_thread()->is_dead); 242 if (!cur_thread()->is_inited) { 243 return REAL(os_lock_unlock)(lock); 244 } 245 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); 246 Release(thr, pc, (uptr)lock); 247 REAL(os_lock_unlock)(lock); 248} 249 250TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) { 251 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 252 return REAL(os_unfair_lock_lock)(lock); 253 } 254 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock); 255 REAL(os_unfair_lock_lock)(lock); 256 Acquire(thr, pc, (uptr)lock); 257} 258 259TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock, 260 u32 options) { 261 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 262 return REAL(os_unfair_lock_lock_with_options)(lock, options); 263 } 264 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options); 265 REAL(os_unfair_lock_lock_with_options)(lock, options); 266 Acquire(thr, pc, (uptr)lock); 267} 268 269TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) { 270 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 271 return REAL(os_unfair_lock_trylock)(lock); 272 } 273 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock); 274 bool result = REAL(os_unfair_lock_trylock)(lock); 275 if (result) 276 Acquire(thr, pc, (uptr)lock); 277 return result; 278} 279 280TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) { 281 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 282 return REAL(os_unfair_lock_unlock)(lock); 283 } 284 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock); 285 Release(thr, pc, (uptr)lock); 286 REAL(os_unfair_lock_unlock)(lock); 287} 288 289#if defined(__has_include) && __has_include(<xpc/xpc.h>) 290 291TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, 292 xpc_connection_t connection, xpc_handler_t handler) { 293 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, 294 handler); 295 Release(thr, pc, (uptr)connection); 296 xpc_handler_t new_handler = ^(xpc_object_t object) { 297 { 298 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); 299 Acquire(thr, pc, (uptr)connection); 300 } 301 handler(object); 302 }; 303 REAL(xpc_connection_set_event_handler)(connection, new_handler); 304} 305 306TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, 307 dispatch_block_t barrier) { 308 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); 309 Release(thr, pc, (uptr)connection); 310 dispatch_block_t new_barrier = ^() { 311 { 312 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); 313 Acquire(thr, pc, (uptr)connection); 314 } 315 barrier(); 316 }; 317 REAL(xpc_connection_send_barrier)(connection, new_barrier); 318} 319 320TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, 321 xpc_connection_t connection, xpc_object_t message, 322 dispatch_queue_t replyq, xpc_handler_t handler) { 323 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, 324 message, replyq, handler); 325 Release(thr, pc, (uptr)connection); 326 xpc_handler_t new_handler = ^(xpc_object_t object) { 327 { 328 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); 329 Acquire(thr, pc, (uptr)connection); 330 } 331 handler(object); 332 }; 333 REAL(xpc_connection_send_message_with_reply) 334 (connection, message, replyq, new_handler); 335} 336 337TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { 338 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection); 339 Release(thr, pc, (uptr)connection); 340 REAL(xpc_connection_cancel)(connection); 341} 342 343#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 344 345// Determines whether the Obj-C object pointer is a tagged pointer. Tagged 346// pointers encode the object data directly in their pointer bits and do not 347// have an associated memory allocation. The Obj-C runtime uses tagged pointers 348// to transparently optimize small objects. 349static bool IsTaggedObjCPointer(id obj) { 350 const uptr kPossibleTaggedBits = 0x8000000000000001ull; 351 return ((uptr)obj & kPossibleTaggedBits) != 0; 352} 353 354// Returns an address which can be used to inform TSan about synchronization 355// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid 356// address in the process space. We do a small allocation here to obtain a 357// stable address (the array backing the hash map can change). The memory is 358// never free'd (leaked) and allocation and locking are slow, but this code only 359// runs for @synchronized with tagged pointers, which is very rare. 360static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { 361 typedef AddrHashMap<uptr, 5> Map; 362 static Map Addresses; 363 Map::Handle h(&Addresses, addr); 364 if (h.created()) { 365 ThreadIgnoreBegin(thr, pc); 366 *h = (uptr) user_alloc(thr, pc, /*size=*/1); 367 ThreadIgnoreEnd(thr, pc); 368 } 369 return *h; 370} 371 372// Returns an address on which we can synchronize given an Obj-C object pointer. 373// For normal object pointers, this is just the address of the object in memory. 374// Tagged pointers are not backed by an actual memory allocation, so we need to 375// synthesize a valid address. 376static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { 377 if (IsTaggedObjCPointer(obj)) 378 return GetOrCreateSyncAddress((uptr)obj, thr, pc); 379 return (uptr)obj; 380} 381 382TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { 383 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); 384 if (!obj) return REAL(objc_sync_enter)(obj); 385 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 386 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); 387 int result = REAL(objc_sync_enter)(obj); 388 CHECK_EQ(result, OBJC_SYNC_SUCCESS); 389 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); 390 return result; 391} 392 393TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { 394 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); 395 if (!obj) return REAL(objc_sync_exit)(obj); 396 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 397 MutexUnlock(thr, pc, addr); 398 int result = REAL(objc_sync_exit)(obj); 399 if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); 400 return result; 401} 402 403TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { 404 { 405 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); 406 } 407 // Bacause of swapcontext() semantics we have no option but to copy its 408 // impementation here 409 if (!oucp || !ucp) { 410 errno = EINVAL; 411 return -1; 412 } 413 ThreadState *thr = cur_thread(); 414 const int UCF_SWAPPED = 0x80000000; 415 oucp->uc_onstack &= ~UCF_SWAPPED; 416 thr->ignore_interceptors++; 417 int ret = getcontext(oucp); 418 if (!(oucp->uc_onstack & UCF_SWAPPED)) { 419 thr->ignore_interceptors--; 420 if (!ret) { 421 oucp->uc_onstack |= UCF_SWAPPED; 422 ret = setcontext(ucp); 423 } 424 } 425 return ret; 426} 427 428// On macOS, libc++ is always linked dynamically, so intercepting works the 429// usual way. 430#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 431 432namespace { 433struct fake_shared_weak_count { 434 volatile a64 shared_owners; 435 volatile a64 shared_weak_owners; 436 virtual void _unused_0x0() = 0; 437 virtual void _unused_0x8() = 0; 438 virtual void on_zero_shared() = 0; 439 virtual void _unused_0x18() = 0; 440 virtual void on_zero_shared_weak() = 0; 441}; 442} // namespace 443 444// The following code adds libc++ interceptors for: 445// void __shared_weak_count::__release_shared() _NOEXCEPT; 446// bool __shared_count::__release_shared() _NOEXCEPT; 447// Shared and weak pointers in C++ maintain reference counts via atomics in 448// libc++.dylib, which are TSan-invisible, and this leads to false positives in 449// destructor code. These interceptors re-implements the whole functions so that 450// the mo_acq_rel semantics of the atomic decrement are visible. 451// 452// Unfortunately, the interceptors cannot simply Acquire/Release some sync 453// object and call the original function, because it would have a race between 454// the sync and the destruction of the object. Calling both under a lock will 455// not work because the destructor can invoke this interceptor again (and even 456// in a different thread, so recursive locks don't help). 457 458STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, 459 fake_shared_weak_count *o) { 460 if (!flags()->shared_ptr_interceptor) 461 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); 462 463 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, 464 o); 465 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 466 Acquire(thr, pc, (uptr)&o->shared_owners); 467 o->on_zero_shared(); 468 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == 469 0) { 470 Acquire(thr, pc, (uptr)&o->shared_weak_owners); 471 o->on_zero_shared_weak(); 472 } 473 } 474} 475 476STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv, 477 fake_shared_weak_count *o) { 478 if (!flags()->shared_ptr_interceptor) 479 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o); 480 481 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o); 482 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 483 Acquire(thr, pc, (uptr)&o->shared_owners); 484 o->on_zero_shared(); 485 return true; 486 } 487 return false; 488} 489 490namespace { 491struct call_once_callback_args { 492 void (*orig_func)(void *arg); 493 void *orig_arg; 494 void *flag; 495}; 496 497void call_once_callback_wrapper(void *arg) { 498 call_once_callback_args *new_args = (call_once_callback_args *)arg; 499 new_args->orig_func(new_args->orig_arg); 500 __tsan_release(new_args->flag); 501} 502} // namespace 503 504// This adds a libc++ interceptor for: 505// void __call_once(volatile unsigned long&, void*, void(*)(void*)); 506// C++11 call_once is implemented via an internal function __call_once which is 507// inside libc++.dylib, and the atomic release store inside it is thus 508// TSan-invisible. To avoid false positives, this interceptor wraps the callback 509// function and performs an explicit Release after the user code has run. 510STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, 511 void *arg, void (*func)(void *arg)) { 512 call_once_callback_args new_args = {func, arg, flag}; 513 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, 514 call_once_callback_wrapper); 515} 516 517} // namespace __tsan 518 519#endif // SANITIZER_MAC 520