1//===-- tsan_interceptors.cc ----------------------------------------------===// 2// 3// This file is distributed under the University of Illinois Open Source 4// License. See LICENSE.TXT for details. 5// 6//===----------------------------------------------------------------------===// 7// 8// This file is a part of ThreadSanitizer (TSan), a race detector. 9// 10// FIXME: move as many interceptors as possible into 11// sanitizer_common/sanitizer_common_interceptors.inc 12//===----------------------------------------------------------------------===// 13 14#include "sanitizer_common/sanitizer_atomic.h" 15#include "sanitizer_common/sanitizer_libc.h" 16#include "sanitizer_common/sanitizer_linux.h" 17#include "sanitizer_common/sanitizer_platform_limits_posix.h" 18#include "sanitizer_common/sanitizer_placement_new.h" 19#include "sanitizer_common/sanitizer_stacktrace.h" 20#include "interception/interception.h" 21#include "tsan_interface.h" 22#include "tsan_platform.h" 23#include "tsan_suppressions.h" 24#include "tsan_rtl.h" 25#include "tsan_mman.h" 26#include "tsan_fd.h" 27 28using namespace __tsan; // NOLINT 29 30#if SANITIZER_FREEBSD 31#define __errno_location __error 32#define __libc_malloc __malloc 33#define __libc_realloc __realloc 34#define __libc_calloc __calloc 35#define __libc_free __free 36#define stdout __stdoutp 37#define stderr __stderrp 38#endif 39 40const int kSigCount = 65; 41 42struct my_siginfo_t { 43 // The size is determined by looking at sizeof of real siginfo_t on linux. 44 u64 opaque[128 / sizeof(u64)]; 45}; 46 47struct ucontext_t { 48 // The size is determined by looking at sizeof of real ucontext_t on linux. 49 u64 opaque[936 / sizeof(u64) + 1]; 50}; 51 52extern "C" int pthread_attr_init(void *attr); 53extern "C" int pthread_attr_destroy(void *attr); 54DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) 55extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); 56extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); 57extern "C" int pthread_setspecific(unsigned key, const void *v); 58DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) 59extern "C" int pthread_yield(); 60extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, 61 __sanitizer_sigset_t *oldset); 62// REAL(sigfillset) defined in common interceptors. 63DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) 64DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) 65extern "C" void *pthread_self(); 66extern "C" void _exit(int status); 67extern "C" int *__errno_location(); 68extern "C" int fileno_unlocked(void *stream); 69extern "C" void *__libc_malloc(uptr size); 70extern "C" void *__libc_calloc(uptr size, uptr n); 71extern "C" void *__libc_realloc(void *ptr, uptr size); 72extern "C" void __libc_free(void *ptr); 73#if !SANITIZER_FREEBSD 74extern "C" int mallopt(int param, int value); 75#endif 76extern __sanitizer_FILE *stdout, *stderr; 77const int PTHREAD_MUTEX_RECURSIVE = 1; 78const int PTHREAD_MUTEX_RECURSIVE_NP = 1; 79const int EINVAL = 22; 80const int EBUSY = 16; 81const int EOWNERDEAD = 130; 82const int EPOLL_CTL_ADD = 1; 83const int SIGILL = 4; 84const int SIGABRT = 6; 85const int SIGFPE = 8; 86const int SIGSEGV = 11; 87const int SIGPIPE = 13; 88const int SIGTERM = 15; 89const int SIGBUS = 7; 90const int SIGSYS = 31; 91void *const MAP_FAILED = (void*)-1; 92const int PTHREAD_BARRIER_SERIAL_THREAD = -1; 93const int MAP_FIXED = 0x10; 94typedef long long_t; // NOLINT 95 96// From /usr/include/unistd.h 97# define F_ULOCK 0 /* Unlock a previously locked region. */ 98# define F_LOCK 1 /* Lock a region for exclusive use. */ 99# define F_TLOCK 2 /* Test and lock a region for exclusive use. */ 100# define F_TEST 3 /* Test a region for other processes locks. */ 101 102typedef void (*sighandler_t)(int sig); 103 104#define errno (*__errno_location()) 105 106struct sigaction_t { 107 union { 108 sighandler_t sa_handler; 109 void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx); 110 }; 111#if SANITIZER_FREEBSD 112 int sa_flags; 113 __sanitizer_sigset_t sa_mask; 114#else 115 __sanitizer_sigset_t sa_mask; 116 int sa_flags; 117 void (*sa_restorer)(); 118#endif 119}; 120 121const sighandler_t SIG_DFL = (sighandler_t)0; 122const sighandler_t SIG_IGN = (sighandler_t)1; 123const sighandler_t SIG_ERR = (sighandler_t)-1; 124const int SA_SIGINFO = 4; 125const int SIG_SETMASK = 2; 126 127namespace std { 128struct nothrow_t {}; 129} // namespace std 130 131static sigaction_t sigactions[kSigCount]; 132 133namespace __tsan { 134struct SignalDesc { 135 bool armed; 136 bool sigaction; 137 my_siginfo_t siginfo; 138 ucontext_t ctx; 139}; 140 141struct SignalContext { 142 int int_signal_send; 143 atomic_uintptr_t in_blocking_func; 144 atomic_uintptr_t have_pending_signals; 145 SignalDesc pending_signals[kSigCount]; 146}; 147 148// The object is 64-byte aligned, because we want hot data to be located in 149// a single cache line if possible (it's accessed in every interceptor). 150static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)]; 151static LibIgnore *libignore() { 152 return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]); 153} 154 155void InitializeLibIgnore() { 156 libignore()->Init(*SuppressionContext::Get()); 157 libignore()->OnLibraryLoaded(0); 158} 159 160} // namespace __tsan 161 162static SignalContext *SigCtx(ThreadState *thr) { 163 SignalContext *ctx = (SignalContext*)thr->signal_ctx; 164 if (ctx == 0 && !thr->is_dead) { 165 ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext"); 166 MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); 167 thr->signal_ctx = ctx; 168 } 169 return ctx; 170} 171 172static unsigned g_thread_finalize_key; 173 174class ScopedInterceptor { 175 public: 176 ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); 177 ~ScopedInterceptor(); 178 private: 179 ThreadState *const thr_; 180 const uptr pc_; 181 bool in_ignored_lib_; 182}; 183 184ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, 185 uptr pc) 186 : thr_(thr) 187 , pc_(pc) 188 , in_ignored_lib_(false) { 189 if (!thr_->ignore_interceptors) { 190 Initialize(thr); 191 FuncEntry(thr, pc); 192 } 193 DPrintf("#%d: intercept %s()\n", thr_->tid, fname); 194 if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) { 195 in_ignored_lib_ = true; 196 thr_->in_ignored_lib = true; 197 ThreadIgnoreBegin(thr_, pc_); 198 } 199} 200 201ScopedInterceptor::~ScopedInterceptor() { 202 if (in_ignored_lib_) { 203 thr_->in_ignored_lib = false; 204 ThreadIgnoreEnd(thr_, pc_); 205 } 206 if (!thr_->ignore_interceptors) { 207 ProcessPendingSignals(thr_); 208 FuncExit(thr_); 209 CheckNoLocks(thr_); 210 } 211} 212 213#define SCOPED_INTERCEPTOR_RAW(func, ...) \ 214 ThreadState *thr = cur_thread(); \ 215 const uptr caller_pc = GET_CALLER_PC(); \ 216 ScopedInterceptor si(thr, #func, caller_pc); \ 217 const uptr pc = StackTrace::GetCurrentPc(); \ 218 (void)pc; \ 219/**/ 220 221#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ 222 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 223 if (REAL(func) == 0) { \ 224 Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ 225 Die(); \ 226 } \ 227 if (thr->ignore_interceptors || thr->in_ignored_lib) \ 228 return REAL(func)(__VA_ARGS__); \ 229/**/ 230 231#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) 232#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) 233#if SANITIZER_FREEBSD 234# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) 235#else 236# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) 237#endif 238 239#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) 240 241struct BlockingCall { 242 explicit BlockingCall(ThreadState *thr) 243 : thr(thr) 244 , ctx(SigCtx(thr)) { 245 for (;;) { 246 atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); 247 if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0) 248 break; 249 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 250 ProcessPendingSignals(thr); 251 } 252 // When we are in a "blocking call", we process signals asynchronously 253 // (right when they arrive). In this context we do not expect to be 254 // executing any user/runtime code. The known interceptor sequence when 255 // this is not true is: pthread_join -> munmap(stack). It's fine 256 // to ignore munmap in this case -- we handle stack shadow separately. 257 thr->ignore_interceptors++; 258 } 259 260 ~BlockingCall() { 261 thr->ignore_interceptors--; 262 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); 263 } 264 265 ThreadState *thr; 266 SignalContext *ctx; 267}; 268 269TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { 270 SCOPED_TSAN_INTERCEPTOR(sleep, sec); 271 unsigned res = BLOCK_REAL(sleep)(sec); 272 AfterSleep(thr, pc); 273 return res; 274} 275 276TSAN_INTERCEPTOR(int, usleep, long_t usec) { 277 SCOPED_TSAN_INTERCEPTOR(usleep, usec); 278 int res = BLOCK_REAL(usleep)(usec); 279 AfterSleep(thr, pc); 280 return res; 281} 282 283TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { 284 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); 285 int res = BLOCK_REAL(nanosleep)(req, rem); 286 AfterSleep(thr, pc); 287 return res; 288} 289 290// The sole reason tsan wraps atexit callbacks is to establish synchronization 291// between callback setup and callback execution. 292struct AtExitCtx { 293 void (*f)(); 294 void *arg; 295}; 296 297static void at_exit_wrapper(void *arg) { 298 ThreadState *thr = cur_thread(); 299 uptr pc = 0; 300 Acquire(thr, pc, (uptr)arg); 301 AtExitCtx *ctx = (AtExitCtx*)arg; 302 ((void(*)(void *arg))ctx->f)(ctx->arg); 303 __libc_free(ctx); 304} 305 306static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 307 void *arg, void *dso); 308 309TSAN_INTERCEPTOR(int, atexit, void (*f)()) { 310 if (cur_thread()->in_symbolizer) 311 return 0; 312 // We want to setup the atexit callback even if we are in ignored lib 313 // or after fork. 314 SCOPED_INTERCEPTOR_RAW(atexit, f); 315 return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); 316} 317 318TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { 319 if (cur_thread()->in_symbolizer) 320 return 0; 321 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); 322 return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); 323} 324 325static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), 326 void *arg, void *dso) { 327 AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); 328 ctx->f = f; 329 ctx->arg = arg; 330 Release(thr, pc, (uptr)ctx); 331 // Memory allocation in __cxa_atexit will race with free during exit, 332 // because we do not see synchronization around atexit callback list. 333 ThreadIgnoreBegin(thr, pc); 334 int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso); 335 ThreadIgnoreEnd(thr, pc); 336 return res; 337} 338 339static void on_exit_wrapper(int status, void *arg) { 340 ThreadState *thr = cur_thread(); 341 uptr pc = 0; 342 Acquire(thr, pc, (uptr)arg); 343 AtExitCtx *ctx = (AtExitCtx*)arg; 344 ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); 345 __libc_free(ctx); 346} 347 348TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { 349 if (cur_thread()->in_symbolizer) 350 return 0; 351 SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); 352 AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); 353 ctx->f = (void(*)())f; 354 ctx->arg = arg; 355 Release(thr, pc, (uptr)ctx); 356 // Memory allocation in __cxa_atexit will race with free during exit, 357 // because we do not see synchronization around atexit callback list. 358 ThreadIgnoreBegin(thr, pc); 359 int res = REAL(on_exit)(on_exit_wrapper, ctx); 360 ThreadIgnoreEnd(thr, pc); 361 return res; 362} 363 364// Cleanup old bufs. 365static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { 366 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 367 JmpBuf *buf = &thr->jmp_bufs[i]; 368 if (buf->sp <= sp) { 369 uptr sz = thr->jmp_bufs.Size(); 370 thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1]; 371 thr->jmp_bufs.PopBack(); 372 i--; 373 } 374 } 375} 376 377static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { 378 if (thr->shadow_stack_pos == 0) // called from libc guts during bootstrap 379 return; 380 // Cleanup old bufs. 381 JmpBufGarbageCollect(thr, sp); 382 // Remember the buf. 383 JmpBuf *buf = thr->jmp_bufs.PushBack(); 384 buf->sp = sp; 385 buf->mangled_sp = mangled_sp; 386 buf->shadow_stack_pos = thr->shadow_stack_pos; 387 SignalContext *sctx = SigCtx(thr); 388 buf->int_signal_send = sctx ? sctx->int_signal_send : 0; 389 buf->in_blocking_func = sctx ? 390 atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : 391 false; 392 buf->in_signal_handler = atomic_load(&thr->in_signal_handler, 393 memory_order_relaxed); 394} 395 396static void LongJmp(ThreadState *thr, uptr *env) { 397#if SANITIZER_FREEBSD 398 uptr mangled_sp = env[2]; 399#else 400 uptr mangled_sp = env[6]; 401#endif // SANITIZER_FREEBSD 402 // Find the saved buf by mangled_sp. 403 for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { 404 JmpBuf *buf = &thr->jmp_bufs[i]; 405 if (buf->mangled_sp == mangled_sp) { 406 CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); 407 // Unwind the stack. 408 while (thr->shadow_stack_pos > buf->shadow_stack_pos) 409 FuncExit(thr); 410 SignalContext *sctx = SigCtx(thr); 411 if (sctx) { 412 sctx->int_signal_send = buf->int_signal_send; 413 atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, 414 memory_order_relaxed); 415 } 416 atomic_store(&thr->in_signal_handler, buf->in_signal_handler, 417 memory_order_relaxed); 418 JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp 419 return; 420 } 421 } 422 Printf("ThreadSanitizer: can't find longjmp buf\n"); 423 CHECK(0); 424} 425 426// FIXME: put everything below into a common extern "C" block? 427extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { 428 SetJmp(cur_thread(), sp, mangled_sp); 429} 430 431// Not called. Merely to satisfy TSAN_INTERCEPT(). 432extern "C" SANITIZER_INTERFACE_ATTRIBUTE 433int __interceptor_setjmp(void *env); 434extern "C" int __interceptor_setjmp(void *env) { 435 CHECK(0); 436 return 0; 437} 438 439// FIXME: any reason to have a separate declaration? 440extern "C" SANITIZER_INTERFACE_ATTRIBUTE 441int __interceptor__setjmp(void *env); 442extern "C" int __interceptor__setjmp(void *env) { 443 CHECK(0); 444 return 0; 445} 446 447extern "C" SANITIZER_INTERFACE_ATTRIBUTE 448int __interceptor_sigsetjmp(void *env); 449extern "C" int __interceptor_sigsetjmp(void *env) { 450 CHECK(0); 451 return 0; 452} 453 454extern "C" SANITIZER_INTERFACE_ATTRIBUTE 455int __interceptor___sigsetjmp(void *env); 456extern "C" int __interceptor___sigsetjmp(void *env) { 457 CHECK(0); 458 return 0; 459} 460 461extern "C" int setjmp(void *env); 462extern "C" int _setjmp(void *env); 463extern "C" int sigsetjmp(void *env); 464extern "C" int __sigsetjmp(void *env); 465DEFINE_REAL(int, setjmp, void *env) 466DEFINE_REAL(int, _setjmp, void *env) 467DEFINE_REAL(int, sigsetjmp, void *env) 468DEFINE_REAL(int, __sigsetjmp, void *env) 469 470TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { 471 { 472 SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); 473 } 474 LongJmp(cur_thread(), env); 475 REAL(longjmp)(env, val); 476} 477 478TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { 479 { 480 SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); 481 } 482 LongJmp(cur_thread(), env); 483 REAL(siglongjmp)(env, val); 484} 485 486TSAN_INTERCEPTOR(void*, malloc, uptr size) { 487 if (cur_thread()->in_symbolizer) 488 return __libc_malloc(size); 489 void *p = 0; 490 { 491 SCOPED_INTERCEPTOR_RAW(malloc, size); 492 p = user_alloc(thr, pc, size); 493 } 494 invoke_malloc_hook(p, size); 495 return p; 496} 497 498TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { 499 SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); 500 return user_alloc(thr, pc, sz, align); 501} 502 503TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { 504 if (cur_thread()->in_symbolizer) 505 return __libc_calloc(size, n); 506 if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) 507 return AllocatorReturnNull(); 508 void *p = 0; 509 { 510 SCOPED_INTERCEPTOR_RAW(calloc, size, n); 511 p = user_alloc(thr, pc, n * size); 512 if (p) 513 internal_memset(p, 0, n * size); 514 } 515 invoke_malloc_hook(p, n * size); 516 return p; 517} 518 519TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { 520 if (cur_thread()->in_symbolizer) 521 return __libc_realloc(p, size); 522 if (p) 523 invoke_free_hook(p); 524 { 525 SCOPED_INTERCEPTOR_RAW(realloc, p, size); 526 p = user_realloc(thr, pc, p, size); 527 } 528 invoke_malloc_hook(p, size); 529 return p; 530} 531 532TSAN_INTERCEPTOR(void, free, void *p) { 533 if (p == 0) 534 return; 535 if (cur_thread()->in_symbolizer) 536 return __libc_free(p); 537 invoke_free_hook(p); 538 SCOPED_INTERCEPTOR_RAW(free, p); 539 user_free(thr, pc, p); 540} 541 542TSAN_INTERCEPTOR(void, cfree, void *p) { 543 if (p == 0) 544 return; 545 if (cur_thread()->in_symbolizer) 546 return __libc_free(p); 547 invoke_free_hook(p); 548 SCOPED_INTERCEPTOR_RAW(cfree, p); 549 user_free(thr, pc, p); 550} 551 552TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { 553 SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); 554 return user_alloc_usable_size(p); 555} 556 557#define OPERATOR_NEW_BODY(mangled_name) \ 558 if (cur_thread()->in_symbolizer) \ 559 return __libc_malloc(size); \ 560 void *p = 0; \ 561 { \ 562 SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ 563 p = user_alloc(thr, pc, size); \ 564 } \ 565 invoke_malloc_hook(p, size); \ 566 return p; 567 568SANITIZER_INTERFACE_ATTRIBUTE 569void *operator new(__sanitizer::uptr size); 570void *operator new(__sanitizer::uptr size) { 571 OPERATOR_NEW_BODY(_Znwm); 572} 573 574SANITIZER_INTERFACE_ATTRIBUTE 575void *operator new[](__sanitizer::uptr size); 576void *operator new[](__sanitizer::uptr size) { 577 OPERATOR_NEW_BODY(_Znam); 578} 579 580SANITIZER_INTERFACE_ATTRIBUTE 581void *operator new(__sanitizer::uptr size, std::nothrow_t const&); 582void *operator new(__sanitizer::uptr size, std::nothrow_t const&) { 583 OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t); 584} 585 586SANITIZER_INTERFACE_ATTRIBUTE 587void *operator new[](__sanitizer::uptr size, std::nothrow_t const&); 588void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { 589 OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t); 590} 591 592#define OPERATOR_DELETE_BODY(mangled_name) \ 593 if (ptr == 0) return; \ 594 if (cur_thread()->in_symbolizer) \ 595 return __libc_free(ptr); \ 596 invoke_free_hook(ptr); \ 597 SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ 598 user_free(thr, pc, ptr); 599 600SANITIZER_INTERFACE_ATTRIBUTE 601void operator delete(void *ptr) throw(); 602void operator delete(void *ptr) throw() { 603 OPERATOR_DELETE_BODY(_ZdlPv); 604} 605 606SANITIZER_INTERFACE_ATTRIBUTE 607void operator delete[](void *ptr) throw(); 608void operator delete[](void *ptr) throw() { 609 OPERATOR_DELETE_BODY(_ZdaPv); 610} 611 612SANITIZER_INTERFACE_ATTRIBUTE 613void operator delete(void *ptr, std::nothrow_t const&); 614void operator delete(void *ptr, std::nothrow_t const&) { 615 OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); 616} 617 618SANITIZER_INTERFACE_ATTRIBUTE 619void operator delete[](void *ptr, std::nothrow_t const&); 620void operator delete[](void *ptr, std::nothrow_t const&) { 621 OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); 622} 623 624TSAN_INTERCEPTOR(uptr, strlen, const char *s) { 625 SCOPED_TSAN_INTERCEPTOR(strlen, s); 626 uptr len = internal_strlen(s); 627 MemoryAccessRange(thr, pc, (uptr)s, len + 1, false); 628 return len; 629} 630 631TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) { 632 SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size); 633 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 634 return internal_memset(dst, v, size); 635} 636 637TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { 638 SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size); 639 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 640 MemoryAccessRange(thr, pc, (uptr)src, size, false); 641 return internal_memcpy(dst, src, size); 642} 643 644TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { 645 SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n); 646 int res = 0; 647 uptr len = 0; 648 for (; len < n; len++) { 649 if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len])) 650 break; 651 } 652 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); 653 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); 654 return res; 655} 656 657TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { 658 SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); 659 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 660 MemoryAccessRange(thr, pc, (uptr)src, n, false); 661 return REAL(memmove)(dst, src, n); 662} 663 664TSAN_INTERCEPTOR(char*, strchr, char *s, int c) { 665 SCOPED_TSAN_INTERCEPTOR(strchr, s, c); 666 char *res = REAL(strchr)(s, c); 667 uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1; 668 MemoryAccessRange(thr, pc, (uptr)s, len, false); 669 return res; 670} 671 672TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { 673 SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c); 674 char *res = REAL(strchrnul)(s, c); 675 uptr len = (char*)res - (char*)s + 1; 676 MemoryAccessRange(thr, pc, (uptr)s, len, false); 677 return res; 678} 679 680TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) { 681 SCOPED_TSAN_INTERCEPTOR(strrchr, s, c); 682 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false); 683 return REAL(strrchr)(s, c); 684} 685 686TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT 687 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT 688 uptr srclen = internal_strlen(src); 689 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); 690 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); 691 return REAL(strcpy)(dst, src); // NOLINT 692} 693 694TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { 695 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); 696 uptr srclen = internal_strnlen(src, n); 697 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 698 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); 699 return REAL(strncpy)(dst, src, n); 700} 701 702TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) { 703 SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2); 704 const char *res = REAL(strstr)(s1, s2); 705 uptr len1 = internal_strlen(s1); 706 uptr len2 = internal_strlen(s2); 707 MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false); 708 MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false); 709 return res; 710} 711 712TSAN_INTERCEPTOR(char*, strdup, const char *str) { 713 SCOPED_TSAN_INTERCEPTOR(strdup, str); 714 // strdup will call malloc, so no instrumentation is required here. 715 return REAL(strdup)(str); 716} 717 718static bool fix_mmap_addr(void **addr, long_t sz, int flags) { 719 if (*addr) { 720 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { 721 if (flags & MAP_FIXED) { 722 errno = EINVAL; 723 return false; 724 } else { 725 *addr = 0; 726 } 727 } 728 } 729 return true; 730} 731 732TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, 733 int flags, int fd, unsigned off) { 734 SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off); 735 if (!fix_mmap_addr(&addr, sz, flags)) 736 return MAP_FAILED; 737 void *res = REAL(mmap)(addr, sz, prot, flags, fd, off); 738 if (res != MAP_FAILED) { 739 if (fd > 0) 740 FdAccess(thr, pc, fd); 741 MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); 742 } 743 return res; 744} 745 746#if !SANITIZER_FREEBSD 747TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot, 748 int flags, int fd, u64 off) { 749 SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off); 750 if (!fix_mmap_addr(&addr, sz, flags)) 751 return MAP_FAILED; 752 void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off); 753 if (res != MAP_FAILED) { 754 if (fd > 0) 755 FdAccess(thr, pc, fd); 756 MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); 757 } 758 return res; 759} 760#define TSAN_MAYBE_INTERCEPT_MMAP64 TSAN_INTERCEPT(mmap64) 761#else 762#define TSAN_MAYBE_INTERCEPT_MMAP64 763#endif 764 765TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { 766 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); 767 DontNeedShadowFor((uptr)addr, sz); 768 int res = REAL(munmap)(addr, sz); 769 return res; 770} 771 772#if !SANITIZER_FREEBSD 773TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { 774 SCOPED_INTERCEPTOR_RAW(memalign, align, sz); 775 return user_alloc(thr, pc, sz, align); 776} 777#define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) 778#else 779#define TSAN_MAYBE_INTERCEPT_MEMALIGN 780#endif 781 782TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { 783 SCOPED_INTERCEPTOR_RAW(memalign, align, sz); 784 return user_alloc(thr, pc, sz, align); 785} 786 787TSAN_INTERCEPTOR(void*, valloc, uptr sz) { 788 SCOPED_INTERCEPTOR_RAW(valloc, sz); 789 return user_alloc(thr, pc, sz, GetPageSizeCached()); 790} 791 792#if !SANITIZER_FREEBSD 793TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { 794 SCOPED_INTERCEPTOR_RAW(pvalloc, sz); 795 sz = RoundUp(sz, GetPageSizeCached()); 796 return user_alloc(thr, pc, sz, GetPageSizeCached()); 797} 798#define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) 799#else 800#define TSAN_MAYBE_INTERCEPT_PVALLOC 801#endif 802 803TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { 804 SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); 805 *memptr = user_alloc(thr, pc, sz, align); 806 return 0; 807} 808 809// Used in thread-safe function static initialization. 810extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { 811 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); 812 for (;;) { 813 u32 cmp = atomic_load(g, memory_order_acquire); 814 if (cmp == 0) { 815 if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) 816 return 1; 817 } else if (cmp == 1) { 818 Acquire(thr, pc, (uptr)g); 819 return 0; 820 } else { 821 internal_sched_yield(); 822 } 823 } 824} 825 826extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) { 827 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); 828 Release(thr, pc, (uptr)g); 829 atomic_store(g, 1, memory_order_release); 830} 831 832extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { 833 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); 834 atomic_store(g, 0, memory_order_relaxed); 835} 836 837static void thread_finalize(void *v) { 838 uptr iter = (uptr)v; 839 if (iter > 1) { 840 if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { 841 Printf("ThreadSanitizer: failed to set thread key\n"); 842 Die(); 843 } 844 return; 845 } 846 { 847 ThreadState *thr = cur_thread(); 848 ThreadFinish(thr); 849 SignalContext *sctx = thr->signal_ctx; 850 if (sctx) { 851 thr->signal_ctx = 0; 852 UnmapOrDie(sctx, sizeof(*sctx)); 853 } 854 } 855} 856 857 858struct ThreadParam { 859 void* (*callback)(void *arg); 860 void *param; 861 atomic_uintptr_t tid; 862}; 863 864extern "C" void *__tsan_thread_start_func(void *arg) { 865 ThreadParam *p = (ThreadParam*)arg; 866 void* (*callback)(void *arg) = p->callback; 867 void *param = p->param; 868 int tid = 0; 869 { 870 ThreadState *thr = cur_thread(); 871 // Thread-local state is not initialized yet. 872 ScopedIgnoreInterceptors ignore; 873 ThreadIgnoreBegin(thr, 0); 874 if (pthread_setspecific(g_thread_finalize_key, 875 (void *)kPthreadDestructorIterations)) { 876 Printf("ThreadSanitizer: failed to set thread key\n"); 877 Die(); 878 } 879 ThreadIgnoreEnd(thr, 0); 880 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) 881 pthread_yield(); 882 atomic_store(&p->tid, 0, memory_order_release); 883 ThreadStart(thr, tid, GetTid()); 884 } 885 void *res = callback(param); 886 // Prevent the callback from being tail called, 887 // it mixes up stack traces. 888 volatile int foo = 42; 889 foo++; 890 return res; 891} 892 893TSAN_INTERCEPTOR(int, pthread_create, 894 void *th, void *attr, void *(*callback)(void*), void * param) { 895 SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); 896 if (ctx->after_multithreaded_fork) { 897 if (flags()->die_after_fork) { 898 Report("ThreadSanitizer: starting new threads after multi-threaded " 899 "fork is not supported. Dying (set die_after_fork=0 to override)\n"); 900 Die(); 901 } else { 902 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " 903 "fork is not supported (pid %d). Continuing because of " 904 "die_after_fork=0, but you are on your own\n", internal_getpid()); 905 } 906 } 907 __sanitizer_pthread_attr_t myattr; 908 if (attr == 0) { 909 pthread_attr_init(&myattr); 910 attr = &myattr; 911 } 912 int detached = 0; 913 REAL(pthread_attr_getdetachstate)(attr, &detached); 914 AdjustStackSize(attr); 915 916 ThreadParam p; 917 p.callback = callback; 918 p.param = param; 919 atomic_store(&p.tid, 0, memory_order_relaxed); 920 int res = -1; 921 { 922 // Otherwise we see false positives in pthread stack manipulation. 923 ScopedIgnoreInterceptors ignore; 924 ThreadIgnoreBegin(thr, pc); 925 res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); 926 ThreadIgnoreEnd(thr, pc); 927 } 928 if (res == 0) { 929 int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); 930 CHECK_NE(tid, 0); 931 atomic_store(&p.tid, tid, memory_order_release); 932 while (atomic_load(&p.tid, memory_order_acquire) != 0) 933 pthread_yield(); 934 } 935 if (attr == &myattr) 936 pthread_attr_destroy(&myattr); 937 return res; 938} 939 940TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { 941 SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); 942 int tid = ThreadTid(thr, pc, (uptr)th); 943 ThreadIgnoreBegin(thr, pc); 944 int res = BLOCK_REAL(pthread_join)(th, ret); 945 ThreadIgnoreEnd(thr, pc); 946 if (res == 0) { 947 ThreadJoin(thr, pc, tid); 948 } 949 return res; 950} 951 952TSAN_INTERCEPTOR(int, pthread_detach, void *th) { 953 SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); 954 int tid = ThreadTid(thr, pc, (uptr)th); 955 int res = REAL(pthread_detach)(th); 956 if (res == 0) { 957 ThreadDetach(thr, pc, tid); 958 } 959 return res; 960} 961 962// Problem: 963// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). 964// pthread_cond_t has different size in the different versions. 965// If call new REAL functions for old pthread_cond_t, they will corrupt memory 966// after pthread_cond_t (old cond is smaller). 967// If we call old REAL functions for new pthread_cond_t, we will lose some 968// functionality (e.g. old functions do not support waiting against 969// CLOCK_REALTIME). 970// Proper handling would require to have 2 versions of interceptors as well. 971// But this is messy, in particular requires linker scripts when sanitizer 972// runtime is linked into a shared library. 973// Instead we assume we don't have dynamic libraries built against old 974// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag 975// that allows to work with old libraries (but this mode does not support 976// some features, e.g. pthread_condattr_getpshared). 977static void *init_cond(void *c, bool force = false) { 978 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. 979 // So we allocate additional memory on the side large enough to hold 980 // any pthread_cond_t object. Always call new REAL functions, but pass 981 // the aux object to them. 982 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes 983 // first word of pthread_cond_t to zero. 984 // It's all relevant only for linux. 985 if (!common_flags()->legacy_pthread_cond) 986 return c; 987 atomic_uintptr_t *p = (atomic_uintptr_t*)c; 988 uptr cond = atomic_load(p, memory_order_acquire); 989 if (!force && cond != 0) 990 return (void*)cond; 991 void *newcond = WRAP(malloc)(pthread_cond_t_sz); 992 internal_memset(newcond, 0, pthread_cond_t_sz); 993 if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, 994 memory_order_acq_rel)) 995 return newcond; 996 WRAP(free)(newcond); 997 return (void*)cond; 998} 999 1000struct CondMutexUnlockCtx { 1001 ThreadState *thr; 1002 uptr pc; 1003 void *m; 1004}; 1005 1006static void cond_mutex_unlock(CondMutexUnlockCtx *arg) { 1007 MutexLock(arg->thr, arg->pc, (uptr)arg->m); 1008} 1009 1010INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { 1011 void *cond = init_cond(c, true); 1012 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); 1013 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1014 return REAL(pthread_cond_init)(cond, a); 1015} 1016 1017INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { 1018 void *cond = init_cond(c); 1019 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); 1020 MutexUnlock(thr, pc, (uptr)m); 1021 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1022 CondMutexUnlockCtx arg = {thr, pc, m}; 1023 // This ensures that we handle mutex lock even in case of pthread_cancel. 1024 // See test/tsan/cond_cancel.cc. 1025 int res = call_pthread_cancel_with_cleanup( 1026 (int(*)(void *c, void *m, void *abstime))REAL(pthread_cond_wait), 1027 cond, m, 0, (void(*)(void *arg))cond_mutex_unlock, &arg); 1028 if (res == errno_EOWNERDEAD) 1029 MutexRepair(thr, pc, (uptr)m); 1030 MutexLock(thr, pc, (uptr)m); 1031 return res; 1032} 1033 1034INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { 1035 void *cond = init_cond(c); 1036 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); 1037 MutexUnlock(thr, pc, (uptr)m); 1038 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1039 CondMutexUnlockCtx arg = {thr, pc, m}; 1040 // This ensures that we handle mutex lock even in case of pthread_cancel. 1041 // See test/tsan/cond_cancel.cc. 1042 int res = call_pthread_cancel_with_cleanup( 1043 REAL(pthread_cond_timedwait), cond, m, abstime, 1044 (void(*)(void *arg))cond_mutex_unlock, &arg); 1045 if (res == errno_EOWNERDEAD) 1046 MutexRepair(thr, pc, (uptr)m); 1047 MutexLock(thr, pc, (uptr)m); 1048 return res; 1049} 1050 1051INTERCEPTOR(int, pthread_cond_signal, void *c) { 1052 void *cond = init_cond(c); 1053 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); 1054 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1055 return REAL(pthread_cond_signal)(cond); 1056} 1057 1058INTERCEPTOR(int, pthread_cond_broadcast, void *c) { 1059 void *cond = init_cond(c); 1060 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); 1061 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); 1062 return REAL(pthread_cond_broadcast)(cond); 1063} 1064 1065INTERCEPTOR(int, pthread_cond_destroy, void *c) { 1066 void *cond = init_cond(c); 1067 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); 1068 MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); 1069 int res = REAL(pthread_cond_destroy)(cond); 1070 if (common_flags()->legacy_pthread_cond) { 1071 // Free our aux cond and zero the pointer to not leave dangling pointers. 1072 WRAP(free)(cond); 1073 atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); 1074 } 1075 return res; 1076} 1077 1078TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { 1079 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); 1080 int res = REAL(pthread_mutex_init)(m, a); 1081 if (res == 0) { 1082 bool recursive = false; 1083 if (a) { 1084 int type = 0; 1085 if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) 1086 recursive = (type == PTHREAD_MUTEX_RECURSIVE 1087 || type == PTHREAD_MUTEX_RECURSIVE_NP); 1088 } 1089 MutexCreate(thr, pc, (uptr)m, false, recursive, false); 1090 } 1091 return res; 1092} 1093 1094TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { 1095 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); 1096 int res = REAL(pthread_mutex_destroy)(m); 1097 if (res == 0 || res == EBUSY) { 1098 MutexDestroy(thr, pc, (uptr)m); 1099 } 1100 return res; 1101} 1102 1103TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { 1104 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); 1105 int res = REAL(pthread_mutex_trylock)(m); 1106 if (res == EOWNERDEAD) 1107 MutexRepair(thr, pc, (uptr)m); 1108 if (res == 0 || res == EOWNERDEAD) 1109 MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); 1110 return res; 1111} 1112 1113TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { 1114 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); 1115 int res = REAL(pthread_mutex_timedlock)(m, abstime); 1116 if (res == 0) { 1117 MutexLock(thr, pc, (uptr)m); 1118 } 1119 return res; 1120} 1121 1122TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { 1123 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); 1124 int res = REAL(pthread_spin_init)(m, pshared); 1125 if (res == 0) { 1126 MutexCreate(thr, pc, (uptr)m, false, false, false); 1127 } 1128 return res; 1129} 1130 1131TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { 1132 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); 1133 int res = REAL(pthread_spin_destroy)(m); 1134 if (res == 0) { 1135 MutexDestroy(thr, pc, (uptr)m); 1136 } 1137 return res; 1138} 1139 1140TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { 1141 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); 1142 int res = REAL(pthread_spin_lock)(m); 1143 if (res == 0) { 1144 MutexLock(thr, pc, (uptr)m); 1145 } 1146 return res; 1147} 1148 1149TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { 1150 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); 1151 int res = REAL(pthread_spin_trylock)(m); 1152 if (res == 0) { 1153 MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); 1154 } 1155 return res; 1156} 1157 1158TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { 1159 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); 1160 MutexUnlock(thr, pc, (uptr)m); 1161 int res = REAL(pthread_spin_unlock)(m); 1162 return res; 1163} 1164 1165TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { 1166 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); 1167 int res = REAL(pthread_rwlock_init)(m, a); 1168 if (res == 0) { 1169 MutexCreate(thr, pc, (uptr)m, true, false, false); 1170 } 1171 return res; 1172} 1173 1174TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { 1175 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); 1176 int res = REAL(pthread_rwlock_destroy)(m); 1177 if (res == 0) { 1178 MutexDestroy(thr, pc, (uptr)m); 1179 } 1180 return res; 1181} 1182 1183TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { 1184 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); 1185 int res = REAL(pthread_rwlock_rdlock)(m); 1186 if (res == 0) { 1187 MutexReadLock(thr, pc, (uptr)m); 1188 } 1189 return res; 1190} 1191 1192TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { 1193 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); 1194 int res = REAL(pthread_rwlock_tryrdlock)(m); 1195 if (res == 0) { 1196 MutexReadLock(thr, pc, (uptr)m, /*try_lock=*/true); 1197 } 1198 return res; 1199} 1200 1201TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { 1202 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); 1203 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); 1204 if (res == 0) { 1205 MutexReadLock(thr, pc, (uptr)m); 1206 } 1207 return res; 1208} 1209 1210TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { 1211 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); 1212 int res = REAL(pthread_rwlock_wrlock)(m); 1213 if (res == 0) { 1214 MutexLock(thr, pc, (uptr)m); 1215 } 1216 return res; 1217} 1218 1219TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { 1220 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); 1221 int res = REAL(pthread_rwlock_trywrlock)(m); 1222 if (res == 0) { 1223 MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); 1224 } 1225 return res; 1226} 1227 1228TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { 1229 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); 1230 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); 1231 if (res == 0) { 1232 MutexLock(thr, pc, (uptr)m); 1233 } 1234 return res; 1235} 1236 1237TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { 1238 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); 1239 MutexReadOrWriteUnlock(thr, pc, (uptr)m); 1240 int res = REAL(pthread_rwlock_unlock)(m); 1241 return res; 1242} 1243 1244TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { 1245 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); 1246 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1247 int res = REAL(pthread_barrier_init)(b, a, count); 1248 return res; 1249} 1250 1251TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { 1252 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); 1253 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); 1254 int res = REAL(pthread_barrier_destroy)(b); 1255 return res; 1256} 1257 1258TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { 1259 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); 1260 Release(thr, pc, (uptr)b); 1261 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1262 int res = REAL(pthread_barrier_wait)(b); 1263 MemoryRead(thr, pc, (uptr)b, kSizeLog1); 1264 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { 1265 Acquire(thr, pc, (uptr)b); 1266 } 1267 return res; 1268} 1269 1270TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { 1271 SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); 1272 if (o == 0 || f == 0) 1273 return EINVAL; 1274 atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o); 1275 u32 v = atomic_load(a, memory_order_acquire); 1276 if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, 1277 memory_order_relaxed)) { 1278 (*f)(); 1279 if (!thr->in_ignored_lib) 1280 Release(thr, pc, (uptr)o); 1281 atomic_store(a, 2, memory_order_release); 1282 } else { 1283 while (v != 2) { 1284 pthread_yield(); 1285 v = atomic_load(a, memory_order_acquire); 1286 } 1287 if (!thr->in_ignored_lib) 1288 Acquire(thr, pc, (uptr)o); 1289 } 1290 return 0; 1291} 1292 1293TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) { 1294 SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value); 1295 int res = REAL(sem_init)(s, pshared, value); 1296 return res; 1297} 1298 1299TSAN_INTERCEPTOR(int, sem_destroy, void *s) { 1300 SCOPED_TSAN_INTERCEPTOR(sem_destroy, s); 1301 int res = REAL(sem_destroy)(s); 1302 return res; 1303} 1304 1305TSAN_INTERCEPTOR(int, sem_wait, void *s) { 1306 SCOPED_TSAN_INTERCEPTOR(sem_wait, s); 1307 int res = BLOCK_REAL(sem_wait)(s); 1308 if (res == 0) { 1309 Acquire(thr, pc, (uptr)s); 1310 } 1311 return res; 1312} 1313 1314TSAN_INTERCEPTOR(int, sem_trywait, void *s) { 1315 SCOPED_TSAN_INTERCEPTOR(sem_trywait, s); 1316 int res = BLOCK_REAL(sem_trywait)(s); 1317 if (res == 0) { 1318 Acquire(thr, pc, (uptr)s); 1319 } 1320 return res; 1321} 1322 1323TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) { 1324 SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime); 1325 int res = BLOCK_REAL(sem_timedwait)(s, abstime); 1326 if (res == 0) { 1327 Acquire(thr, pc, (uptr)s); 1328 } 1329 return res; 1330} 1331 1332TSAN_INTERCEPTOR(int, sem_post, void *s) { 1333 SCOPED_TSAN_INTERCEPTOR(sem_post, s); 1334 Release(thr, pc, (uptr)s); 1335 int res = REAL(sem_post)(s); 1336 return res; 1337} 1338 1339TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) { 1340 SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval); 1341 int res = REAL(sem_getvalue)(s, sval); 1342 if (res == 0) { 1343 Acquire(thr, pc, (uptr)s); 1344 } 1345 return res; 1346} 1347 1348#if !SANITIZER_FREEBSD 1349TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { 1350 SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf); 1351 return REAL(__xstat)(version, path, buf); 1352} 1353#define TSAN_MAYBE_INTERCEPT___XSTAT TSAN_INTERCEPT(__xstat) 1354#else 1355#define TSAN_MAYBE_INTERCEPT___XSTAT 1356#endif 1357 1358TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) { 1359#if SANITIZER_FREEBSD 1360 SCOPED_TSAN_INTERCEPTOR(stat, path, buf); 1361 return REAL(stat)(path, buf); 1362#else 1363 SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf); 1364 return REAL(__xstat)(0, path, buf); 1365#endif 1366} 1367 1368#if !SANITIZER_FREEBSD 1369TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { 1370 SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf); 1371 return REAL(__xstat64)(version, path, buf); 1372} 1373#define TSAN_MAYBE_INTERCEPT___XSTAT64 TSAN_INTERCEPT(__xstat64) 1374#else 1375#define TSAN_MAYBE_INTERCEPT___XSTAT64 1376#endif 1377 1378#if !SANITIZER_FREEBSD 1379TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) { 1380 SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf); 1381 return REAL(__xstat64)(0, path, buf); 1382} 1383#define TSAN_MAYBE_INTERCEPT_STAT64 TSAN_INTERCEPT(stat64) 1384#else 1385#define TSAN_MAYBE_INTERCEPT_STAT64 1386#endif 1387 1388#if !SANITIZER_FREEBSD 1389TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { 1390 SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf); 1391 return REAL(__lxstat)(version, path, buf); 1392} 1393#define TSAN_MAYBE_INTERCEPT___LXSTAT TSAN_INTERCEPT(__lxstat) 1394#else 1395#define TSAN_MAYBE_INTERCEPT___LXSTAT 1396#endif 1397 1398TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) { 1399#if SANITIZER_FREEBSD 1400 SCOPED_TSAN_INTERCEPTOR(lstat, path, buf); 1401 return REAL(lstat)(path, buf); 1402#else 1403 SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf); 1404 return REAL(__lxstat)(0, path, buf); 1405#endif 1406} 1407 1408#if !SANITIZER_FREEBSD 1409TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { 1410 SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf); 1411 return REAL(__lxstat64)(version, path, buf); 1412} 1413#define TSAN_MAYBE_INTERCEPT___LXSTAT64 TSAN_INTERCEPT(__lxstat64) 1414#else 1415#define TSAN_MAYBE_INTERCEPT___LXSTAT64 1416#endif 1417 1418#if !SANITIZER_FREEBSD 1419TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) { 1420 SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf); 1421 return REAL(__lxstat64)(0, path, buf); 1422} 1423#define TSAN_MAYBE_INTERCEPT_LSTAT64 TSAN_INTERCEPT(lstat64) 1424#else 1425#define TSAN_MAYBE_INTERCEPT_LSTAT64 1426#endif 1427 1428#if !SANITIZER_FREEBSD 1429TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { 1430 SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); 1431 if (fd > 0) 1432 FdAccess(thr, pc, fd); 1433 return REAL(__fxstat)(version, fd, buf); 1434} 1435#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) 1436#else 1437#define TSAN_MAYBE_INTERCEPT___FXSTAT 1438#endif 1439 1440TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { 1441#if SANITIZER_FREEBSD 1442 SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); 1443 if (fd > 0) 1444 FdAccess(thr, pc, fd); 1445 return REAL(fstat)(fd, buf); 1446#else 1447 SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); 1448 if (fd > 0) 1449 FdAccess(thr, pc, fd); 1450 return REAL(__fxstat)(0, fd, buf); 1451#endif 1452} 1453 1454#if !SANITIZER_FREEBSD 1455TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { 1456 SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); 1457 if (fd > 0) 1458 FdAccess(thr, pc, fd); 1459 return REAL(__fxstat64)(version, fd, buf); 1460} 1461#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) 1462#else 1463#define TSAN_MAYBE_INTERCEPT___FXSTAT64 1464#endif 1465 1466#if !SANITIZER_FREEBSD 1467TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { 1468 SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); 1469 if (fd > 0) 1470 FdAccess(thr, pc, fd); 1471 return REAL(__fxstat64)(0, fd, buf); 1472} 1473#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) 1474#else 1475#define TSAN_MAYBE_INTERCEPT_FSTAT64 1476#endif 1477 1478TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { 1479 SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); 1480 int fd = REAL(open)(name, flags, mode); 1481 if (fd >= 0) 1482 FdFileCreate(thr, pc, fd); 1483 return fd; 1484} 1485 1486#if !SANITIZER_FREEBSD 1487TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { 1488 SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); 1489 int fd = REAL(open64)(name, flags, mode); 1490 if (fd >= 0) 1491 FdFileCreate(thr, pc, fd); 1492 return fd; 1493} 1494#define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) 1495#else 1496#define TSAN_MAYBE_INTERCEPT_OPEN64 1497#endif 1498 1499TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { 1500 SCOPED_TSAN_INTERCEPTOR(creat, name, mode); 1501 int fd = REAL(creat)(name, mode); 1502 if (fd >= 0) 1503 FdFileCreate(thr, pc, fd); 1504 return fd; 1505} 1506 1507#if !SANITIZER_FREEBSD 1508TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { 1509 SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); 1510 int fd = REAL(creat64)(name, mode); 1511 if (fd >= 0) 1512 FdFileCreate(thr, pc, fd); 1513 return fd; 1514} 1515#define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) 1516#else 1517#define TSAN_MAYBE_INTERCEPT_CREAT64 1518#endif 1519 1520TSAN_INTERCEPTOR(int, dup, int oldfd) { 1521 SCOPED_TSAN_INTERCEPTOR(dup, oldfd); 1522 int newfd = REAL(dup)(oldfd); 1523 if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) 1524 FdDup(thr, pc, oldfd, newfd); 1525 return newfd; 1526} 1527 1528TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { 1529 SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); 1530 int newfd2 = REAL(dup2)(oldfd, newfd); 1531 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1532 FdDup(thr, pc, oldfd, newfd2); 1533 return newfd2; 1534} 1535 1536TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { 1537 SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); 1538 int newfd2 = REAL(dup3)(oldfd, newfd, flags); 1539 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) 1540 FdDup(thr, pc, oldfd, newfd2); 1541 return newfd2; 1542} 1543 1544#if !SANITIZER_FREEBSD 1545TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { 1546 SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); 1547 int fd = REAL(eventfd)(initval, flags); 1548 if (fd >= 0) 1549 FdEventCreate(thr, pc, fd); 1550 return fd; 1551} 1552#define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) 1553#else 1554#define TSAN_MAYBE_INTERCEPT_EVENTFD 1555#endif 1556 1557#if !SANITIZER_FREEBSD 1558TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { 1559 SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); 1560 if (fd >= 0) 1561 FdClose(thr, pc, fd); 1562 fd = REAL(signalfd)(fd, mask, flags); 1563 if (fd >= 0) 1564 FdSignalCreate(thr, pc, fd); 1565 return fd; 1566} 1567#define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) 1568#else 1569#define TSAN_MAYBE_INTERCEPT_SIGNALFD 1570#endif 1571 1572#if !SANITIZER_FREEBSD 1573TSAN_INTERCEPTOR(int, inotify_init, int fake) { 1574 SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); 1575 int fd = REAL(inotify_init)(fake); 1576 if (fd >= 0) 1577 FdInotifyCreate(thr, pc, fd); 1578 return fd; 1579} 1580#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) 1581#else 1582#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT 1583#endif 1584 1585#if !SANITIZER_FREEBSD 1586TSAN_INTERCEPTOR(int, inotify_init1, int flags) { 1587 SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); 1588 int fd = REAL(inotify_init1)(flags); 1589 if (fd >= 0) 1590 FdInotifyCreate(thr, pc, fd); 1591 return fd; 1592} 1593#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) 1594#else 1595#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 1596#endif 1597 1598TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { 1599 SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); 1600 int fd = REAL(socket)(domain, type, protocol); 1601 if (fd >= 0) 1602 FdSocketCreate(thr, pc, fd); 1603 return fd; 1604} 1605 1606TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { 1607 SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); 1608 int res = REAL(socketpair)(domain, type, protocol, fd); 1609 if (res == 0 && fd[0] >= 0 && fd[1] >= 0) 1610 FdPipeCreate(thr, pc, fd[0], fd[1]); 1611 return res; 1612} 1613 1614TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { 1615 SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); 1616 FdSocketConnecting(thr, pc, fd); 1617 int res = REAL(connect)(fd, addr, addrlen); 1618 if (res == 0 && fd >= 0) 1619 FdSocketConnect(thr, pc, fd); 1620 return res; 1621} 1622 1623TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { 1624 SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); 1625 int res = REAL(bind)(fd, addr, addrlen); 1626 if (fd > 0 && res == 0) 1627 FdAccess(thr, pc, fd); 1628 return res; 1629} 1630 1631TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { 1632 SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); 1633 int res = REAL(listen)(fd, backlog); 1634 if (fd > 0 && res == 0) 1635 FdAccess(thr, pc, fd); 1636 return res; 1637} 1638 1639#if !SANITIZER_FREEBSD 1640TSAN_INTERCEPTOR(int, epoll_create, int size) { 1641 SCOPED_TSAN_INTERCEPTOR(epoll_create, size); 1642 int fd = REAL(epoll_create)(size); 1643 if (fd >= 0) 1644 FdPollCreate(thr, pc, fd); 1645 return fd; 1646} 1647#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE TSAN_INTERCEPT(epoll_create) 1648#else 1649#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE 1650#endif 1651 1652#if !SANITIZER_FREEBSD 1653TSAN_INTERCEPTOR(int, epoll_create1, int flags) { 1654 SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); 1655 int fd = REAL(epoll_create1)(flags); 1656 if (fd >= 0) 1657 FdPollCreate(thr, pc, fd); 1658 return fd; 1659} 1660#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 TSAN_INTERCEPT(epoll_create1) 1661#else 1662#define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 1663#endif 1664 1665TSAN_INTERCEPTOR(int, close, int fd) { 1666 SCOPED_TSAN_INTERCEPTOR(close, fd); 1667 if (fd >= 0) 1668 FdClose(thr, pc, fd); 1669 return REAL(close)(fd); 1670} 1671 1672#if !SANITIZER_FREEBSD 1673TSAN_INTERCEPTOR(int, __close, int fd) { 1674 SCOPED_TSAN_INTERCEPTOR(__close, fd); 1675 if (fd >= 0) 1676 FdClose(thr, pc, fd); 1677 return REAL(__close)(fd); 1678} 1679#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) 1680#else 1681#define TSAN_MAYBE_INTERCEPT___CLOSE 1682#endif 1683 1684// glibc guts 1685#if !SANITIZER_FREEBSD 1686TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { 1687 SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); 1688 int fds[64]; 1689 int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); 1690 for (int i = 0; i < cnt; i++) { 1691 if (fds[i] > 0) 1692 FdClose(thr, pc, fds[i]); 1693 } 1694 REAL(__res_iclose)(state, free_addr); 1695} 1696#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) 1697#else 1698#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE 1699#endif 1700 1701TSAN_INTERCEPTOR(int, pipe, int *pipefd) { 1702 SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); 1703 int res = REAL(pipe)(pipefd); 1704 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1705 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1706 return res; 1707} 1708 1709TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { 1710 SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); 1711 int res = REAL(pipe2)(pipefd, flags); 1712 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) 1713 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); 1714 return res; 1715} 1716 1717TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { 1718 SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); 1719 if (fd >= 0) { 1720 FdAccess(thr, pc, fd); 1721 FdRelease(thr, pc, fd); 1722 } 1723 int res = REAL(send)(fd, buf, len, flags); 1724 return res; 1725} 1726 1727TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { 1728 SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags); 1729 if (fd >= 0) { 1730 FdAccess(thr, pc, fd); 1731 FdRelease(thr, pc, fd); 1732 } 1733 int res = REAL(sendmsg)(fd, msg, flags); 1734 return res; 1735} 1736 1737TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { 1738 SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags); 1739 if (fd >= 0) 1740 FdAccess(thr, pc, fd); 1741 int res = REAL(recv)(fd, buf, len, flags); 1742 if (res >= 0 && fd >= 0) { 1743 FdAcquire(thr, pc, fd); 1744 } 1745 return res; 1746} 1747 1748TSAN_INTERCEPTOR(int, unlink, char *path) { 1749 SCOPED_TSAN_INTERCEPTOR(unlink, path); 1750 Release(thr, pc, File2addr(path)); 1751 int res = REAL(unlink)(path); 1752 return res; 1753} 1754 1755TSAN_INTERCEPTOR(void*, tmpfile, int fake) { 1756 SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); 1757 void *res = REAL(tmpfile)(fake); 1758 if (res) { 1759 int fd = fileno_unlocked(res); 1760 if (fd >= 0) 1761 FdFileCreate(thr, pc, fd); 1762 } 1763 return res; 1764} 1765 1766#if !SANITIZER_FREEBSD 1767TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { 1768 SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); 1769 void *res = REAL(tmpfile64)(fake); 1770 if (res) { 1771 int fd = fileno_unlocked(res); 1772 if (fd >= 0) 1773 FdFileCreate(thr, pc, fd); 1774 } 1775 return res; 1776} 1777#define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) 1778#else 1779#define TSAN_MAYBE_INTERCEPT_TMPFILE64 1780#endif 1781 1782TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { 1783 // libc file streams can call user-supplied functions, see fopencookie. 1784 { 1785 SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); 1786 MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); 1787 } 1788 return REAL(fread)(ptr, size, nmemb, f); 1789} 1790 1791TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { 1792 // libc file streams can call user-supplied functions, see fopencookie. 1793 { 1794 SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); 1795 MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); 1796 } 1797 return REAL(fwrite)(p, size, nmemb, f); 1798} 1799 1800TSAN_INTERCEPTOR(void, abort, int fake) { 1801 SCOPED_TSAN_INTERCEPTOR(abort, fake); 1802 REAL(fflush)(0); 1803 REAL(abort)(fake); 1804} 1805 1806TSAN_INTERCEPTOR(int, puts, const char *s) { 1807 SCOPED_TSAN_INTERCEPTOR(puts, s); 1808 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false); 1809 return REAL(puts)(s); 1810} 1811 1812TSAN_INTERCEPTOR(int, rmdir, char *path) { 1813 SCOPED_TSAN_INTERCEPTOR(rmdir, path); 1814 Release(thr, pc, Dir2addr(path)); 1815 int res = REAL(rmdir)(path); 1816 return res; 1817} 1818 1819TSAN_INTERCEPTOR(void*, opendir, char *path) { 1820 SCOPED_TSAN_INTERCEPTOR(opendir, path); 1821 void *res = REAL(opendir)(path); 1822 if (res != 0) 1823 Acquire(thr, pc, Dir2addr(path)); 1824 return res; 1825} 1826 1827#if !SANITIZER_FREEBSD 1828TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { 1829 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); 1830 if (epfd >= 0) 1831 FdAccess(thr, pc, epfd); 1832 if (epfd >= 0 && fd >= 0) 1833 FdAccess(thr, pc, fd); 1834 if (op == EPOLL_CTL_ADD && epfd >= 0) 1835 FdRelease(thr, pc, epfd); 1836 int res = REAL(epoll_ctl)(epfd, op, fd, ev); 1837 return res; 1838} 1839#define TSAN_MAYBE_INTERCEPT_EPOLL_CTL TSAN_INTERCEPT(epoll_ctl) 1840#else 1841#define TSAN_MAYBE_INTERCEPT_EPOLL_CTL 1842#endif 1843 1844#if !SANITIZER_FREEBSD 1845TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { 1846 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); 1847 if (epfd >= 0) 1848 FdAccess(thr, pc, epfd); 1849 int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); 1850 if (res > 0 && epfd >= 0) 1851 FdAcquire(thr, pc, epfd); 1852 return res; 1853} 1854#define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT TSAN_INTERCEPT(epoll_wait) 1855#else 1856#define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT 1857#endif 1858 1859namespace __tsan { 1860 1861static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, 1862 bool sigact, int sig, my_siginfo_t *info, void *uctx) { 1863 if (acquire) 1864 Acquire(thr, 0, (uptr)&sigactions[sig]); 1865 // Ensure that the handler does not spoil errno. 1866 const int saved_errno = errno; 1867 errno = 99; 1868 // Need to remember pc before the call, because the handler can reset it. 1869 uptr pc = sigact ? 1870 (uptr)sigactions[sig].sa_sigaction : 1871 (uptr)sigactions[sig].sa_handler; 1872 pc += 1; // return address is expected, OutputReport() will undo this 1873 if (sigact) 1874 sigactions[sig].sa_sigaction(sig, info, uctx); 1875 else 1876 sigactions[sig].sa_handler(sig); 1877 // We do not detect errno spoiling for SIGTERM, 1878 // because some SIGTERM handlers do spoil errno but reraise SIGTERM, 1879 // tsan reports false positive in such case. 1880 // It's difficult to properly detect this situation (reraise), 1881 // because in async signal processing case (when handler is called directly 1882 // from rtl_generic_sighandler) we have not yet received the reraised 1883 // signal; and it looks too fragile to intercept all ways to reraise a signal. 1884 if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { 1885 VarSizeStackTrace stack; 1886 ObtainCurrentStack(thr, pc, &stack); 1887 ThreadRegistryLock l(ctx->thread_registry); 1888 ScopedReport rep(ReportTypeErrnoInSignal); 1889 if (!IsFiredSuppression(ctx, rep, stack)) { 1890 rep.AddStack(stack, true); 1891 OutputReport(thr, rep); 1892 } 1893 } 1894 errno = saved_errno; 1895} 1896 1897void ProcessPendingSignals(ThreadState *thr) { 1898 SignalContext *sctx = SigCtx(thr); 1899 if (sctx == 0 || 1900 atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0) 1901 return; 1902 atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); 1903 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 1904 // These are too big for stack. 1905 static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; 1906 REAL(sigfillset)(&emptyset); 1907 pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); 1908 for (int sig = 0; sig < kSigCount; sig++) { 1909 SignalDesc *signal = &sctx->pending_signals[sig]; 1910 if (signal->armed) { 1911 signal->armed = false; 1912 if (sigactions[sig].sa_handler != SIG_DFL 1913 && sigactions[sig].sa_handler != SIG_IGN) { 1914 CallUserSignalHandler(thr, false, true, signal->sigaction, 1915 sig, &signal->siginfo, &signal->ctx); 1916 } 1917 } 1918 } 1919 pthread_sigmask(SIG_SETMASK, &oldset, 0); 1920 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 1921} 1922 1923} // namespace __tsan 1924 1925static bool is_sync_signal(SignalContext *sctx, int sig) { 1926 return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || 1927 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || 1928 // If we are sending signal to ourselves, we must process it now. 1929 (sctx && sig == sctx->int_signal_send); 1930} 1931 1932void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, 1933 my_siginfo_t *info, void *ctx) { 1934 ThreadState *thr = cur_thread(); 1935 SignalContext *sctx = SigCtx(thr); 1936 if (sig < 0 || sig >= kSigCount) { 1937 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); 1938 return; 1939 } 1940 // Don't mess with synchronous signals. 1941 const bool sync = is_sync_signal(sctx, sig); 1942 if (sync || 1943 // If we are in blocking function, we can safely process it now 1944 // (but check if we are in a recursive interceptor, 1945 // i.e. pthread_join()->munmap()). 1946 (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { 1947 atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); 1948 if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { 1949 // We ignore interceptors in blocking functions, 1950 // temporary enbled them again while we are calling user function. 1951 int const i = thr->ignore_interceptors; 1952 thr->ignore_interceptors = 0; 1953 atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); 1954 CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx); 1955 thr->ignore_interceptors = i; 1956 atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); 1957 } else { 1958 // Be very conservative with when we do acquire in this case. 1959 // It's unsafe to do acquire in async handlers, because ThreadState 1960 // can be in inconsistent state. 1961 // SIGSYS looks relatively safe -- it's synchronous and can actually 1962 // need some global state. 1963 bool acq = (sig == SIGSYS); 1964 CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx); 1965 } 1966 atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); 1967 return; 1968 } 1969 1970 if (sctx == 0) 1971 return; 1972 SignalDesc *signal = &sctx->pending_signals[sig]; 1973 if (signal->armed == false) { 1974 signal->armed = true; 1975 signal->sigaction = sigact; 1976 if (info) 1977 internal_memcpy(&signal->siginfo, info, sizeof(*info)); 1978 if (ctx) 1979 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); 1980 atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed); 1981 } 1982} 1983 1984static void rtl_sighandler(int sig) { 1985 rtl_generic_sighandler(false, sig, 0, 0); 1986} 1987 1988static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { 1989 rtl_generic_sighandler(true, sig, info, ctx); 1990} 1991 1992TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { 1993 SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old); 1994 if (old) 1995 internal_memcpy(old, &sigactions[sig], sizeof(*old)); 1996 if (act == 0) 1997 return 0; 1998 internal_memcpy(&sigactions[sig], act, sizeof(*act)); 1999 sigaction_t newact; 2000 internal_memcpy(&newact, act, sizeof(newact)); 2001 REAL(sigfillset)(&newact.sa_mask); 2002 if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { 2003 if (newact.sa_flags & SA_SIGINFO) 2004 newact.sa_sigaction = rtl_sigaction; 2005 else 2006 newact.sa_handler = rtl_sighandler; 2007 } 2008 ReleaseStore(thr, pc, (uptr)&sigactions[sig]); 2009 int res = REAL(sigaction)(sig, &newact, 0); 2010 return res; 2011} 2012 2013TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { 2014 sigaction_t act; 2015 act.sa_handler = h; 2016 REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask)); 2017 act.sa_flags = 0; 2018 sigaction_t old; 2019 int res = sigaction(sig, &act, &old); 2020 if (res) 2021 return SIG_ERR; 2022 return old.sa_handler; 2023} 2024 2025TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { 2026 SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); 2027 return REAL(sigsuspend)(mask); 2028} 2029 2030TSAN_INTERCEPTOR(int, raise, int sig) { 2031 SCOPED_TSAN_INTERCEPTOR(raise, sig); 2032 SignalContext *sctx = SigCtx(thr); 2033 CHECK_NE(sctx, 0); 2034 int prev = sctx->int_signal_send; 2035 sctx->int_signal_send = sig; 2036 int res = REAL(raise)(sig); 2037 CHECK_EQ(sctx->int_signal_send, sig); 2038 sctx->int_signal_send = prev; 2039 return res; 2040} 2041 2042TSAN_INTERCEPTOR(int, kill, int pid, int sig) { 2043 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); 2044 SignalContext *sctx = SigCtx(thr); 2045 CHECK_NE(sctx, 0); 2046 int prev = sctx->int_signal_send; 2047 if (pid == (int)internal_getpid()) { 2048 sctx->int_signal_send = sig; 2049 } 2050 int res = REAL(kill)(pid, sig); 2051 if (pid == (int)internal_getpid()) { 2052 CHECK_EQ(sctx->int_signal_send, sig); 2053 sctx->int_signal_send = prev; 2054 } 2055 return res; 2056} 2057 2058TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { 2059 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); 2060 SignalContext *sctx = SigCtx(thr); 2061 CHECK_NE(sctx, 0); 2062 int prev = sctx->int_signal_send; 2063 if (tid == pthread_self()) { 2064 sctx->int_signal_send = sig; 2065 } 2066 int res = REAL(pthread_kill)(tid, sig); 2067 if (tid == pthread_self()) { 2068 CHECK_EQ(sctx->int_signal_send, sig); 2069 sctx->int_signal_send = prev; 2070 } 2071 return res; 2072} 2073 2074TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { 2075 SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); 2076 // It's intercepted merely to process pending signals. 2077 return REAL(gettimeofday)(tv, tz); 2078} 2079 2080TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, 2081 void *hints, void *rv) { 2082 SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); 2083 // We miss atomic synchronization in getaddrinfo, 2084 // and can report false race between malloc and free 2085 // inside of getaddrinfo. So ignore memory accesses. 2086 ThreadIgnoreBegin(thr, pc); 2087 int res = REAL(getaddrinfo)(node, service, hints, rv); 2088 ThreadIgnoreEnd(thr, pc); 2089 return res; 2090} 2091 2092TSAN_INTERCEPTOR(int, fork, int fake) { 2093 if (cur_thread()->in_symbolizer) 2094 return REAL(fork)(fake); 2095 SCOPED_INTERCEPTOR_RAW(fork, fake); 2096 ForkBefore(thr, pc); 2097 int pid = REAL(fork)(fake); 2098 if (pid == 0) { 2099 // child 2100 ForkChildAfter(thr, pc); 2101 FdOnFork(thr, pc); 2102 } else if (pid > 0) { 2103 // parent 2104 ForkParentAfter(thr, pc); 2105 } else { 2106 // error 2107 ForkParentAfter(thr, pc); 2108 } 2109 return pid; 2110} 2111 2112TSAN_INTERCEPTOR(int, vfork, int fake) { 2113 // Some programs (e.g. openjdk) call close for all file descriptors 2114 // in the child process. Under tsan it leads to false positives, because 2115 // address space is shared, so the parent process also thinks that 2116 // the descriptors are closed (while they are actually not). 2117 // This leads to false positives due to missed synchronization. 2118 // Strictly saying this is undefined behavior, because vfork child is not 2119 // allowed to call any functions other than exec/exit. But this is what 2120 // openjdk does, so we want to handle it. 2121 // We could disable interceptors in the child process. But it's not possible 2122 // to simply intercept and wrap vfork, because vfork child is not allowed 2123 // to return from the function that calls vfork, and that's exactly what 2124 // we would do. So this would require some assembly trickery as well. 2125 // Instead we simply turn vfork into fork. 2126 return WRAP(fork)(fake); 2127} 2128 2129static int OnExit(ThreadState *thr) { 2130 int status = Finalize(thr); 2131 REAL(fflush)(0); 2132 return status; 2133} 2134 2135struct TsanInterceptorContext { 2136 ThreadState *thr; 2137 const uptr caller_pc; 2138 const uptr pc; 2139}; 2140 2141static void HandleRecvmsg(ThreadState *thr, uptr pc, 2142 __sanitizer_msghdr *msg) { 2143 int fds[64]; 2144 int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); 2145 for (int i = 0; i < cnt; i++) 2146 FdEventCreate(thr, pc, fds[i]); 2147} 2148 2149#include "sanitizer_common/sanitizer_platform_interceptors.h" 2150// Causes interceptor recursion (getaddrinfo() and fopen()) 2151#undef SANITIZER_INTERCEPT_GETADDRINFO 2152// There interceptors do not seem to be strictly necessary for tsan. 2153// But we see cases where the interceptors consume 70% of execution time. 2154// Memory blocks passed to fgetgrent_r are "written to" by tsan several times. 2155// First, there is some recursion (getgrnam_r calls fgetgrent_r), and each 2156// function "writes to" the buffer. Then, the same memory is "written to" 2157// twice, first as buf and then as pwbufp (both of them refer to the same 2158// addresses). 2159#undef SANITIZER_INTERCEPT_GETPWENT 2160#undef SANITIZER_INTERCEPT_GETPWENT_R 2161#undef SANITIZER_INTERCEPT_FGETPWENT 2162#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS 2163#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS 2164 2165#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) 2166 2167#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ 2168 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ 2169 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ 2170 true) 2171 2172#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ 2173 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ 2174 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ 2175 false) 2176 2177#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ 2178 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ 2179 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 2180 ctx = (void *)&_ctx; \ 2181 (void) ctx; 2182 2183#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ 2184 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 2185 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ 2186 ctx = (void *)&_ctx; \ 2187 (void) ctx; 2188 2189#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ 2190 Acquire(thr, pc, File2addr(path)); \ 2191 if (file) { \ 2192 int fd = fileno_unlocked(file); \ 2193 if (fd >= 0) FdFileCreate(thr, pc, fd); \ 2194 } 2195 2196#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ 2197 if (file) { \ 2198 int fd = fileno_unlocked(file); \ 2199 if (fd >= 0) FdClose(thr, pc, fd); \ 2200 } 2201 2202#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res) \ 2203 libignore()->OnLibraryLoaded(filename) 2204 2205#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ 2206 libignore()->OnLibraryUnloaded() 2207 2208#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ 2209 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2210 2211#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ 2212 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2213 2214#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ 2215 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) 2216 2217#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ 2218 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) 2219 2220#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ 2221 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) 2222 2223#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ 2224 __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name) 2225 2226#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) 2227 2228#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ 2229 OnExit(((TsanInterceptorContext *) ctx)->thr) 2230 2231#define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \ 2232 MutexLock(((TsanInterceptorContext *)ctx)->thr, \ 2233 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2234 2235#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ 2236 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ 2237 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2238 2239#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ 2240 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ 2241 ((TsanInterceptorContext *)ctx)->pc, (uptr)m) 2242 2243#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ 2244 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ 2245 ((TsanInterceptorContext *)ctx)->pc, msg) 2246 2247#include "sanitizer_common/sanitizer_common_interceptors.inc" 2248 2249#define TSAN_SYSCALL() \ 2250 ThreadState *thr = cur_thread(); \ 2251 if (thr->ignore_interceptors) \ 2252 return; \ 2253 ScopedSyscall scoped_syscall(thr) \ 2254/**/ 2255 2256struct ScopedSyscall { 2257 ThreadState *thr; 2258 2259 explicit ScopedSyscall(ThreadState *thr) 2260 : thr(thr) { 2261 Initialize(thr); 2262 } 2263 2264 ~ScopedSyscall() { 2265 ProcessPendingSignals(thr); 2266 } 2267}; 2268 2269static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { 2270 TSAN_SYSCALL(); 2271 MemoryAccessRange(thr, pc, p, s, write); 2272} 2273 2274static void syscall_acquire(uptr pc, uptr addr) { 2275 TSAN_SYSCALL(); 2276 Acquire(thr, pc, addr); 2277 DPrintf("syscall_acquire(%p)\n", addr); 2278} 2279 2280static void syscall_release(uptr pc, uptr addr) { 2281 TSAN_SYSCALL(); 2282 DPrintf("syscall_release(%p)\n", addr); 2283 Release(thr, pc, addr); 2284} 2285 2286static void syscall_fd_close(uptr pc, int fd) { 2287 TSAN_SYSCALL(); 2288 FdClose(thr, pc, fd); 2289} 2290 2291static USED void syscall_fd_acquire(uptr pc, int fd) { 2292 TSAN_SYSCALL(); 2293 FdAcquire(thr, pc, fd); 2294 DPrintf("syscall_fd_acquire(%p)\n", fd); 2295} 2296 2297static USED void syscall_fd_release(uptr pc, int fd) { 2298 TSAN_SYSCALL(); 2299 DPrintf("syscall_fd_release(%p)\n", fd); 2300 FdRelease(thr, pc, fd); 2301} 2302 2303static void syscall_pre_fork(uptr pc) { 2304 TSAN_SYSCALL(); 2305 ForkBefore(thr, pc); 2306} 2307 2308static void syscall_post_fork(uptr pc, int pid) { 2309 TSAN_SYSCALL(); 2310 if (pid == 0) { 2311 // child 2312 ForkChildAfter(thr, pc); 2313 FdOnFork(thr, pc); 2314 } else if (pid > 0) { 2315 // parent 2316 ForkParentAfter(thr, pc); 2317 } else { 2318 // error 2319 ForkParentAfter(thr, pc); 2320 } 2321} 2322 2323#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ 2324 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) 2325 2326#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ 2327 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) 2328 2329#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ 2330 do { \ 2331 (void)(p); \ 2332 (void)(s); \ 2333 } while (false) 2334 2335#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ 2336 do { \ 2337 (void)(p); \ 2338 (void)(s); \ 2339 } while (false) 2340 2341#define COMMON_SYSCALL_ACQUIRE(addr) \ 2342 syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) 2343 2344#define COMMON_SYSCALL_RELEASE(addr) \ 2345 syscall_release(GET_CALLER_PC(), (uptr)(addr)) 2346 2347#define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) 2348 2349#define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) 2350 2351#define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) 2352 2353#define COMMON_SYSCALL_PRE_FORK() \ 2354 syscall_pre_fork(GET_CALLER_PC()) 2355 2356#define COMMON_SYSCALL_POST_FORK(res) \ 2357 syscall_post_fork(GET_CALLER_PC(), res) 2358 2359#include "sanitizer_common/sanitizer_common_syscalls.inc" 2360 2361namespace __tsan { 2362 2363static void finalize(void *arg) { 2364 ThreadState *thr = cur_thread(); 2365 int status = Finalize(thr); 2366 // Make sure the output is not lost. 2367 // Flushing all the streams here may freeze the process if a child thread is 2368 // performing file stream operations at the same time. 2369 REAL(fflush)(stdout); 2370 REAL(fflush)(stderr); 2371 if (status) 2372 REAL(_exit)(status); 2373} 2374 2375static void unreachable() { 2376 Report("FATAL: ThreadSanitizer: unreachable called\n"); 2377 Die(); 2378} 2379 2380void InitializeInterceptors() { 2381 // We need to setup it early, because functions like dlsym() can call it. 2382 REAL(memset) = internal_memset; 2383 REAL(memcpy) = internal_memcpy; 2384 REAL(memcmp) = internal_memcmp; 2385 2386 // Instruct libc malloc to consume less memory. 2387#if !SANITIZER_FREEBSD 2388 mallopt(1, 0); // M_MXFAST 2389 mallopt(-3, 32*1024); // M_MMAP_THRESHOLD 2390#endif 2391 2392 InitializeCommonInterceptors(); 2393 2394 // We can not use TSAN_INTERCEPT to get setjmp addr, 2395 // because it does &setjmp and setjmp is not present in some versions of libc. 2396 using __interception::GetRealFunctionAddress; 2397 GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0); 2398 GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); 2399 GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0); 2400 GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); 2401 2402 TSAN_INTERCEPT(longjmp); 2403 TSAN_INTERCEPT(siglongjmp); 2404 2405 TSAN_INTERCEPT(malloc); 2406 TSAN_INTERCEPT(__libc_memalign); 2407 TSAN_INTERCEPT(calloc); 2408 TSAN_INTERCEPT(realloc); 2409 TSAN_INTERCEPT(free); 2410 TSAN_INTERCEPT(cfree); 2411 TSAN_INTERCEPT(mmap); 2412 TSAN_MAYBE_INTERCEPT_MMAP64; 2413 TSAN_INTERCEPT(munmap); 2414 TSAN_MAYBE_INTERCEPT_MEMALIGN; 2415 TSAN_INTERCEPT(valloc); 2416 TSAN_MAYBE_INTERCEPT_PVALLOC; 2417 TSAN_INTERCEPT(posix_memalign); 2418 2419 TSAN_INTERCEPT(strlen); 2420 TSAN_INTERCEPT(memset); 2421 TSAN_INTERCEPT(memcpy); 2422 TSAN_INTERCEPT(memmove); 2423 TSAN_INTERCEPT(memcmp); 2424 TSAN_INTERCEPT(strchr); 2425 TSAN_INTERCEPT(strchrnul); 2426 TSAN_INTERCEPT(strrchr); 2427 TSAN_INTERCEPT(strcpy); // NOLINT 2428 TSAN_INTERCEPT(strncpy); 2429 TSAN_INTERCEPT(strstr); 2430 TSAN_INTERCEPT(strdup); 2431 2432 TSAN_INTERCEPT(pthread_create); 2433 TSAN_INTERCEPT(pthread_join); 2434 TSAN_INTERCEPT(pthread_detach); 2435 2436 TSAN_INTERCEPT_VER(pthread_cond_init, "GLIBC_2.3.2"); 2437 TSAN_INTERCEPT_VER(pthread_cond_signal, "GLIBC_2.3.2"); 2438 TSAN_INTERCEPT_VER(pthread_cond_broadcast, "GLIBC_2.3.2"); 2439 TSAN_INTERCEPT_VER(pthread_cond_wait, "GLIBC_2.3.2"); 2440 TSAN_INTERCEPT_VER(pthread_cond_timedwait, "GLIBC_2.3.2"); 2441 TSAN_INTERCEPT_VER(pthread_cond_destroy, "GLIBC_2.3.2"); 2442 2443 TSAN_INTERCEPT(pthread_mutex_init); 2444 TSAN_INTERCEPT(pthread_mutex_destroy); 2445 TSAN_INTERCEPT(pthread_mutex_trylock); 2446 TSAN_INTERCEPT(pthread_mutex_timedlock); 2447 2448 TSAN_INTERCEPT(pthread_spin_init); 2449 TSAN_INTERCEPT(pthread_spin_destroy); 2450 TSAN_INTERCEPT(pthread_spin_lock); 2451 TSAN_INTERCEPT(pthread_spin_trylock); 2452 TSAN_INTERCEPT(pthread_spin_unlock); 2453 2454 TSAN_INTERCEPT(pthread_rwlock_init); 2455 TSAN_INTERCEPT(pthread_rwlock_destroy); 2456 TSAN_INTERCEPT(pthread_rwlock_rdlock); 2457 TSAN_INTERCEPT(pthread_rwlock_tryrdlock); 2458 TSAN_INTERCEPT(pthread_rwlock_timedrdlock); 2459 TSAN_INTERCEPT(pthread_rwlock_wrlock); 2460 TSAN_INTERCEPT(pthread_rwlock_trywrlock); 2461 TSAN_INTERCEPT(pthread_rwlock_timedwrlock); 2462 TSAN_INTERCEPT(pthread_rwlock_unlock); 2463 2464 TSAN_INTERCEPT(pthread_barrier_init); 2465 TSAN_INTERCEPT(pthread_barrier_destroy); 2466 TSAN_INTERCEPT(pthread_barrier_wait); 2467 2468 TSAN_INTERCEPT(pthread_once); 2469 2470 TSAN_INTERCEPT(sem_init); 2471 TSAN_INTERCEPT(sem_destroy); 2472 TSAN_INTERCEPT(sem_wait); 2473 TSAN_INTERCEPT(sem_trywait); 2474 TSAN_INTERCEPT(sem_timedwait); 2475 TSAN_INTERCEPT(sem_post); 2476 TSAN_INTERCEPT(sem_getvalue); 2477 2478 TSAN_INTERCEPT(stat); 2479 TSAN_MAYBE_INTERCEPT___XSTAT; 2480 TSAN_MAYBE_INTERCEPT_STAT64; 2481 TSAN_MAYBE_INTERCEPT___XSTAT64; 2482 TSAN_INTERCEPT(lstat); 2483 TSAN_MAYBE_INTERCEPT___LXSTAT; 2484 TSAN_MAYBE_INTERCEPT_LSTAT64; 2485 TSAN_MAYBE_INTERCEPT___LXSTAT64; 2486 TSAN_INTERCEPT(fstat); 2487 TSAN_MAYBE_INTERCEPT___FXSTAT; 2488 TSAN_MAYBE_INTERCEPT_FSTAT64; 2489 TSAN_MAYBE_INTERCEPT___FXSTAT64; 2490 TSAN_INTERCEPT(open); 2491 TSAN_MAYBE_INTERCEPT_OPEN64; 2492 TSAN_INTERCEPT(creat); 2493 TSAN_MAYBE_INTERCEPT_CREAT64; 2494 TSAN_INTERCEPT(dup); 2495 TSAN_INTERCEPT(dup2); 2496 TSAN_INTERCEPT(dup3); 2497 TSAN_MAYBE_INTERCEPT_EVENTFD; 2498 TSAN_MAYBE_INTERCEPT_SIGNALFD; 2499 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; 2500 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; 2501 TSAN_INTERCEPT(socket); 2502 TSAN_INTERCEPT(socketpair); 2503 TSAN_INTERCEPT(connect); 2504 TSAN_INTERCEPT(bind); 2505 TSAN_INTERCEPT(listen); 2506 TSAN_MAYBE_INTERCEPT_EPOLL_CREATE; 2507 TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1; 2508 TSAN_INTERCEPT(close); 2509 TSAN_MAYBE_INTERCEPT___CLOSE; 2510 TSAN_MAYBE_INTERCEPT___RES_ICLOSE; 2511 TSAN_INTERCEPT(pipe); 2512 TSAN_INTERCEPT(pipe2); 2513 2514 TSAN_INTERCEPT(send); 2515 TSAN_INTERCEPT(sendmsg); 2516 TSAN_INTERCEPT(recv); 2517 2518 TSAN_INTERCEPT(unlink); 2519 TSAN_INTERCEPT(tmpfile); 2520 TSAN_MAYBE_INTERCEPT_TMPFILE64; 2521 TSAN_INTERCEPT(fread); 2522 TSAN_INTERCEPT(fwrite); 2523 TSAN_INTERCEPT(abort); 2524 TSAN_INTERCEPT(puts); 2525 TSAN_INTERCEPT(rmdir); 2526 TSAN_INTERCEPT(opendir); 2527 2528 TSAN_MAYBE_INTERCEPT_EPOLL_CTL; 2529 TSAN_MAYBE_INTERCEPT_EPOLL_WAIT; 2530 2531 TSAN_INTERCEPT(sigaction); 2532 TSAN_INTERCEPT(signal); 2533 TSAN_INTERCEPT(sigsuspend); 2534 TSAN_INTERCEPT(raise); 2535 TSAN_INTERCEPT(kill); 2536 TSAN_INTERCEPT(pthread_kill); 2537 TSAN_INTERCEPT(sleep); 2538 TSAN_INTERCEPT(usleep); 2539 TSAN_INTERCEPT(nanosleep); 2540 TSAN_INTERCEPT(gettimeofday); 2541 TSAN_INTERCEPT(getaddrinfo); 2542 2543 TSAN_INTERCEPT(fork); 2544 TSAN_INTERCEPT(vfork); 2545 TSAN_INTERCEPT(on_exit); 2546 TSAN_INTERCEPT(__cxa_atexit); 2547 TSAN_INTERCEPT(_exit); 2548 2549 // Need to setup it, because interceptors check that the function is resolved. 2550 // But atexit is emitted directly into the module, so can't be resolved. 2551 REAL(atexit) = (int(*)(void(*)()))unreachable; 2552 if (REAL(__cxa_atexit)(&finalize, 0, 0)) { 2553 Printf("ThreadSanitizer: failed to setup atexit callback\n"); 2554 Die(); 2555 } 2556 2557 if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { 2558 Printf("ThreadSanitizer: failed to create thread key\n"); 2559 Die(); 2560 } 2561 2562 FdInit(); 2563} 2564 2565void *internal_start_thread(void(*func)(void *arg), void *arg) { 2566 // Start the thread with signals blocked, otherwise it can steal user signals. 2567 __sanitizer_sigset_t set, old; 2568 internal_sigfillset(&set); 2569 internal_sigprocmask(SIG_SETMASK, &set, &old); 2570 void *th; 2571 REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg); 2572 internal_sigprocmask(SIG_SETMASK, &old, 0); 2573 return th; 2574} 2575 2576void internal_join_thread(void *th) { 2577 REAL(pthread_join)(th, 0); 2578} 2579 2580} // namespace __tsan 2581