1//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// FIXME: move as many interceptors as possible into
12// sanitizer_common/sanitizer_common_interceptors.inc
13//===----------------------------------------------------------------------===//
14
15#include "sanitizer_common/sanitizer_atomic.h"
16#include "sanitizer_common/sanitizer_errno.h"
17#include "sanitizer_common/sanitizer_libc.h"
18#include "sanitizer_common/sanitizer_linux.h"
19#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
20#include "sanitizer_common/sanitizer_platform_limits_posix.h"
21#include "sanitizer_common/sanitizer_placement_new.h"
22#include "sanitizer_common/sanitizer_posix.h"
23#include "sanitizer_common/sanitizer_stacktrace.h"
24#include "sanitizer_common/sanitizer_tls_get_addr.h"
25#include "interception/interception.h"
26#include "tsan_interceptors.h"
27#include "tsan_interface.h"
28#include "tsan_platform.h"
29#include "tsan_suppressions.h"
30#include "tsan_rtl.h"
31#include "tsan_mman.h"
32#include "tsan_fd.h"
33
34#include <stdarg.h>
35
36using namespace __tsan;
37
38#if SANITIZER_FREEBSD || SANITIZER_MAC
39#define stdout __stdoutp
40#define stderr __stderrp
41#endif
42
43#if SANITIZER_NETBSD
44#define dirfd(dirp) (*(int *)(dirp))
45#define fileno_unlocked(fp)              \
46  (((__sanitizer_FILE *)fp)->_file == -1 \
47       ? -1                              \
48       : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file))
49
50#define stdout ((__sanitizer_FILE*)&__sF[1])
51#define stderr ((__sanitizer_FILE*)&__sF[2])
52
53#define nanosleep __nanosleep50
54#define vfork __vfork14
55#endif
56
57#ifdef __mips__
58const int kSigCount = 129;
59#else
60const int kSigCount = 65;
61#endif
62
63#ifdef __mips__
64struct ucontext_t {
65  u64 opaque[768 / sizeof(u64) + 1];
66};
67#else
68struct ucontext_t {
69  // The size is determined by looking at sizeof of real ucontext_t on linux.
70  u64 opaque[936 / sizeof(u64) + 1];
71};
72#endif
73
74#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
75    defined(__s390x__)
76#define PTHREAD_ABI_BASE  "GLIBC_2.3.2"
77#elif defined(__aarch64__) || SANITIZER_PPC64V2
78#define PTHREAD_ABI_BASE  "GLIBC_2.17"
79#endif
80
81extern "C" int pthread_attr_init(void *attr);
82extern "C" int pthread_attr_destroy(void *attr);
83DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
84extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
85extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void),
86                              void (*child)(void));
87extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
88extern "C" int pthread_setspecific(unsigned key, const void *v);
89DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
90DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
91DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
92DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
93extern "C" void *pthread_self();
94extern "C" void _exit(int status);
95#if !SANITIZER_NETBSD
96extern "C" int fileno_unlocked(void *stream);
97extern "C" int dirfd(void *dirp);
98#endif
99#if SANITIZER_NETBSD
100extern __sanitizer_FILE __sF[];
101#else
102extern __sanitizer_FILE *stdout, *stderr;
103#endif
104#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
105const int PTHREAD_MUTEX_RECURSIVE = 1;
106const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
107#else
108const int PTHREAD_MUTEX_RECURSIVE = 2;
109const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
110#endif
111#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
112const int EPOLL_CTL_ADD = 1;
113#endif
114const int SIGILL = 4;
115const int SIGTRAP = 5;
116const int SIGABRT = 6;
117const int SIGFPE = 8;
118const int SIGSEGV = 11;
119const int SIGPIPE = 13;
120const int SIGTERM = 15;
121#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
122const int SIGBUS = 10;
123const int SIGSYS = 12;
124#else
125const int SIGBUS = 7;
126const int SIGSYS = 31;
127#endif
128void *const MAP_FAILED = (void*)-1;
129#if SANITIZER_NETBSD
130const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
131#elif !SANITIZER_MAC
132const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
133#endif
134const int MAP_FIXED = 0x10;
135typedef long long_t;
136typedef __sanitizer::u16 mode_t;
137
138// From /usr/include/unistd.h
139# define F_ULOCK 0      /* Unlock a previously locked region.  */
140# define F_LOCK  1      /* Lock a region for exclusive use.  */
141# define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
142# define F_TEST  3      /* Test a region for other processes locks.  */
143
144#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
145const int SA_SIGINFO = 0x40;
146const int SIG_SETMASK = 3;
147#elif defined(__mips__)
148const int SA_SIGINFO = 8;
149const int SIG_SETMASK = 3;
150#else
151const int SA_SIGINFO = 4;
152const int SIG_SETMASK = 2;
153#endif
154
155#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
156  (!cur_thread_init()->is_inited)
157
158namespace __tsan {
159struct SignalDesc {
160  bool armed;
161  __sanitizer_siginfo siginfo;
162  ucontext_t ctx;
163};
164
165struct ThreadSignalContext {
166  int int_signal_send;
167  atomic_uintptr_t in_blocking_func;
168  SignalDesc pending_signals[kSigCount];
169  // emptyset and oldset are too big for stack.
170  __sanitizer_sigset_t emptyset;
171  __sanitizer_sigset_t oldset;
172};
173
174// The sole reason tsan wraps atexit callbacks is to establish synchronization
175// between callback setup and callback execution.
176struct AtExitCtx {
177  void (*f)();
178  void *arg;
179};
180
181// InterceptorContext holds all global data required for interceptors.
182// It's explicitly constructed in InitializeInterceptors with placement new
183// and is never destroyed. This allows usage of members with non-trivial
184// constructors and destructors.
185struct InterceptorContext {
186  // The object is 64-byte aligned, because we want hot data to be located
187  // in a single cache line if possible (it's accessed in every interceptor).
188  ALIGNED(64) LibIgnore libignore;
189  __sanitizer_sigaction sigactions[kSigCount];
190#if !SANITIZER_MAC && !SANITIZER_NETBSD
191  unsigned finalize_key;
192#endif
193
194  Mutex atexit_mu;
195  Vector<struct AtExitCtx *> AtExitStack;
196
197  InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {}
198};
199
200static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];
201InterceptorContext *interceptor_ctx() {
202  return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);
203}
204
205LibIgnore *libignore() {
206  return &interceptor_ctx()->libignore;
207}
208
209void InitializeLibIgnore() {
210  const SuppressionContext &supp = *Suppressions();
211  const uptr n = supp.SuppressionCount();
212  for (uptr i = 0; i < n; i++) {
213    const Suppression *s = supp.SuppressionAt(i);
214    if (0 == internal_strcmp(s->type, kSuppressionLib))
215      libignore()->AddIgnoredLibrary(s->templ);
216  }
217  if (flags()->ignore_noninstrumented_modules)
218    libignore()->IgnoreNoninstrumentedModules(true);
219  libignore()->OnLibraryLoaded(0);
220}
221
222// The following two hooks can be used by for cooperative scheduling when
223// locking.
224#ifdef TSAN_EXTERNAL_HOOKS
225void OnPotentiallyBlockingRegionBegin();
226void OnPotentiallyBlockingRegionEnd();
227#else
228SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}
229SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
230#endif
231
232}  // namespace __tsan
233
234static ThreadSignalContext *SigCtx(ThreadState *thr) {
235  ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx;
236  if (ctx == 0 && !thr->is_dead) {
237    ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext");
238    MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
239    thr->signal_ctx = ctx;
240  }
241  return ctx;
242}
243
244ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
245                                     uptr pc)
246    : thr_(thr), in_ignored_lib_(false), ignoring_(false) {
247  LazyInitialize(thr);
248  if (!thr_->is_inited) return;
249  if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
250  DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
251  ignoring_ =
252      !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses ||
253                                libignore()->IsIgnored(pc, &in_ignored_lib_));
254  EnableIgnores();
255}
256
257ScopedInterceptor::~ScopedInterceptor() {
258  if (!thr_->is_inited) return;
259  DisableIgnores();
260  if (!thr_->ignore_interceptors) {
261    ProcessPendingSignals(thr_);
262    FuncExit(thr_);
263    CheckedMutex::CheckNoLocks();
264  }
265}
266
267NOINLINE
268void ScopedInterceptor::EnableIgnoresImpl() {
269  ThreadIgnoreBegin(thr_, 0);
270  if (flags()->ignore_noninstrumented_modules)
271    thr_->suppress_reports++;
272  if (in_ignored_lib_) {
273    DCHECK(!thr_->in_ignored_lib);
274    thr_->in_ignored_lib = true;
275  }
276}
277
278NOINLINE
279void ScopedInterceptor::DisableIgnoresImpl() {
280  ThreadIgnoreEnd(thr_);
281  if (flags()->ignore_noninstrumented_modules)
282    thr_->suppress_reports--;
283  if (in_ignored_lib_) {
284    DCHECK(thr_->in_ignored_lib);
285    thr_->in_ignored_lib = false;
286  }
287}
288
289#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
290#if SANITIZER_FREEBSD
291# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
292# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
293# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
294#elif SANITIZER_NETBSD
295# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
296# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
297         INTERCEPT_FUNCTION(__libc_##func)
298# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
299         INTERCEPT_FUNCTION(__libc_thr_##func)
300#else
301# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
302# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
303# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
304#endif
305
306#define READ_STRING_OF_LEN(thr, pc, s, len, n)                 \
307  MemoryAccessRange((thr), (pc), (uptr)(s),                         \
308    common_flags()->strict_string_checks ? (len) + 1 : (n), false)
309
310#define READ_STRING(thr, pc, s, n)                             \
311    READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n))
312
313#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
314
315struct BlockingCall {
316  explicit BlockingCall(ThreadState *thr)
317      : thr(thr)
318      , ctx(SigCtx(thr)) {
319    for (;;) {
320      atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed);
321      if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0)
322        break;
323      atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
324      ProcessPendingSignals(thr);
325    }
326    // When we are in a "blocking call", we process signals asynchronously
327    // (right when they arrive). In this context we do not expect to be
328    // executing any user/runtime code. The known interceptor sequence when
329    // this is not true is: pthread_join -> munmap(stack). It's fine
330    // to ignore munmap in this case -- we handle stack shadow separately.
331    thr->ignore_interceptors++;
332  }
333
334  ~BlockingCall() {
335    thr->ignore_interceptors--;
336    atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
337  }
338
339  ThreadState *thr;
340  ThreadSignalContext *ctx;
341};
342
343TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
344  SCOPED_TSAN_INTERCEPTOR(sleep, sec);
345  unsigned res = BLOCK_REAL(sleep)(sec);
346  AfterSleep(thr, pc);
347  return res;
348}
349
350TSAN_INTERCEPTOR(int, usleep, long_t usec) {
351  SCOPED_TSAN_INTERCEPTOR(usleep, usec);
352  int res = BLOCK_REAL(usleep)(usec);
353  AfterSleep(thr, pc);
354  return res;
355}
356
357TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
358  SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
359  int res = BLOCK_REAL(nanosleep)(req, rem);
360  AfterSleep(thr, pc);
361  return res;
362}
363
364TSAN_INTERCEPTOR(int, pause, int fake) {
365  SCOPED_TSAN_INTERCEPTOR(pause, fake);
366  return BLOCK_REAL(pause)(fake);
367}
368
369static void at_exit_wrapper() {
370  AtExitCtx *ctx;
371  {
372    // Ensure thread-safety.
373    Lock l(&interceptor_ctx()->atexit_mu);
374
375    // Pop AtExitCtx from the top of the stack of callback functions
376    uptr element = interceptor_ctx()->AtExitStack.Size() - 1;
377    ctx = interceptor_ctx()->AtExitStack[element];
378    interceptor_ctx()->AtExitStack.PopBack();
379  }
380
381  Acquire(cur_thread(), (uptr)0, (uptr)ctx);
382  ((void(*)())ctx->f)();
383  Free(ctx);
384}
385
386static void cxa_at_exit_wrapper(void *arg) {
387  Acquire(cur_thread(), 0, (uptr)arg);
388  AtExitCtx *ctx = (AtExitCtx*)arg;
389  ((void(*)(void *arg))ctx->f)(ctx->arg);
390  Free(ctx);
391}
392
393static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
394      void *arg, void *dso);
395
396#if !SANITIZER_ANDROID
397TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
398  if (in_symbolizer())
399    return 0;
400  // We want to setup the atexit callback even if we are in ignored lib
401  // or after fork.
402  SCOPED_INTERCEPTOR_RAW(atexit, f);
403  return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0);
404}
405#endif
406
407TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
408  if (in_symbolizer())
409    return 0;
410  SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
411  return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
412}
413
414static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
415      void *arg, void *dso) {
416  auto *ctx = New<AtExitCtx>();
417  ctx->f = f;
418  ctx->arg = arg;
419  Release(thr, pc, (uptr)ctx);
420  // Memory allocation in __cxa_atexit will race with free during exit,
421  // because we do not see synchronization around atexit callback list.
422  ThreadIgnoreBegin(thr, pc);
423  int res;
424  if (!dso) {
425    // NetBSD does not preserve the 2nd argument if dso is equal to 0
426    // Store ctx in a local stack-like structure
427
428    // Ensure thread-safety.
429    Lock l(&interceptor_ctx()->atexit_mu);
430    // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail
431    // due to atexit_mu held on exit from the calloc interceptor.
432    ScopedIgnoreInterceptors ignore;
433
434    res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0);
435    // Push AtExitCtx on the top of the stack of callback functions
436    if (!res) {
437      interceptor_ctx()->AtExitStack.PushBack(ctx);
438    }
439  } else {
440    res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso);
441  }
442  ThreadIgnoreEnd(thr);
443  return res;
444}
445
446#if !SANITIZER_MAC && !SANITIZER_NETBSD
447static void on_exit_wrapper(int status, void *arg) {
448  ThreadState *thr = cur_thread();
449  uptr pc = 0;
450  Acquire(thr, pc, (uptr)arg);
451  AtExitCtx *ctx = (AtExitCtx*)arg;
452  ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg);
453  Free(ctx);
454}
455
456TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
457  if (in_symbolizer())
458    return 0;
459  SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
460  auto *ctx = New<AtExitCtx>();
461  ctx->f = (void(*)())f;
462  ctx->arg = arg;
463  Release(thr, pc, (uptr)ctx);
464  // Memory allocation in __cxa_atexit will race with free during exit,
465  // because we do not see synchronization around atexit callback list.
466  ThreadIgnoreBegin(thr, pc);
467  int res = REAL(on_exit)(on_exit_wrapper, ctx);
468  ThreadIgnoreEnd(thr);
469  return res;
470}
471#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
472#else
473#define TSAN_MAYBE_INTERCEPT_ON_EXIT
474#endif
475
476// Cleanup old bufs.
477static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
478  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
479    JmpBuf *buf = &thr->jmp_bufs[i];
480    if (buf->sp <= sp) {
481      uptr sz = thr->jmp_bufs.Size();
482      internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf));
483      thr->jmp_bufs.PopBack();
484      i--;
485    }
486  }
487}
488
489static void SetJmp(ThreadState *thr, uptr sp) {
490  if (!thr->is_inited)  // called from libc guts during bootstrap
491    return;
492  // Cleanup old bufs.
493  JmpBufGarbageCollect(thr, sp);
494  // Remember the buf.
495  JmpBuf *buf = thr->jmp_bufs.PushBack();
496  buf->sp = sp;
497  buf->shadow_stack_pos = thr->shadow_stack_pos;
498  ThreadSignalContext *sctx = SigCtx(thr);
499  buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
500  buf->in_blocking_func = sctx ?
501      atomic_load(&sctx->in_blocking_func, memory_order_relaxed) :
502      false;
503  buf->in_signal_handler = atomic_load(&thr->in_signal_handler,
504      memory_order_relaxed);
505}
506
507static void LongJmp(ThreadState *thr, uptr *env) {
508  uptr sp = ExtractLongJmpSp(env);
509  // Find the saved buf with matching sp.
510  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
511    JmpBuf *buf = &thr->jmp_bufs[i];
512    if (buf->sp == sp) {
513      CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
514      // Unwind the stack.
515      while (thr->shadow_stack_pos > buf->shadow_stack_pos)
516        FuncExit(thr);
517      ThreadSignalContext *sctx = SigCtx(thr);
518      if (sctx) {
519        sctx->int_signal_send = buf->int_signal_send;
520        atomic_store(&sctx->in_blocking_func, buf->in_blocking_func,
521            memory_order_relaxed);
522      }
523      atomic_store(&thr->in_signal_handler, buf->in_signal_handler,
524          memory_order_relaxed);
525      JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
526      return;
527    }
528  }
529  Printf("ThreadSanitizer: can't find longjmp buf\n");
530  CHECK(0);
531}
532
533// FIXME: put everything below into a common extern "C" block?
534extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); }
535
536#if SANITIZER_MAC
537TSAN_INTERCEPTOR(int, setjmp, void *env);
538TSAN_INTERCEPTOR(int, _setjmp, void *env);
539TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
540#else  // SANITIZER_MAC
541
542#if SANITIZER_NETBSD
543#define setjmp_symname __setjmp14
544#define sigsetjmp_symname __sigsetjmp14
545#else
546#define setjmp_symname setjmp
547#define sigsetjmp_symname sigsetjmp
548#endif
549
550#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
551#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
552#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
553#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
554
555#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
556#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
557
558// Not called.  Merely to satisfy TSAN_INTERCEPT().
559extern "C" SANITIZER_INTERFACE_ATTRIBUTE
560int TSAN_INTERCEPTOR_SETJMP(void *env);
561extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) {
562  CHECK(0);
563  return 0;
564}
565
566// FIXME: any reason to have a separate declaration?
567extern "C" SANITIZER_INTERFACE_ATTRIBUTE
568int __interceptor__setjmp(void *env);
569extern "C" int __interceptor__setjmp(void *env) {
570  CHECK(0);
571  return 0;
572}
573
574extern "C" SANITIZER_INTERFACE_ATTRIBUTE
575int TSAN_INTERCEPTOR_SIGSETJMP(void *env);
576extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) {
577  CHECK(0);
578  return 0;
579}
580
581#if !SANITIZER_NETBSD
582extern "C" SANITIZER_INTERFACE_ATTRIBUTE
583int __interceptor___sigsetjmp(void *env);
584extern "C" int __interceptor___sigsetjmp(void *env) {
585  CHECK(0);
586  return 0;
587}
588#endif
589
590extern "C" int setjmp_symname(void *env);
591extern "C" int _setjmp(void *env);
592extern "C" int sigsetjmp_symname(void *env);
593#if !SANITIZER_NETBSD
594extern "C" int __sigsetjmp(void *env);
595#endif
596DEFINE_REAL(int, setjmp_symname, void *env)
597DEFINE_REAL(int, _setjmp, void *env)
598DEFINE_REAL(int, sigsetjmp_symname, void *env)
599#if !SANITIZER_NETBSD
600DEFINE_REAL(int, __sigsetjmp, void *env)
601#endif
602#endif  // SANITIZER_MAC
603
604#if SANITIZER_NETBSD
605#define longjmp_symname __longjmp14
606#define siglongjmp_symname __siglongjmp14
607#else
608#define longjmp_symname longjmp
609#define siglongjmp_symname siglongjmp
610#endif
611
612TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) {
613  // Note: if we call REAL(longjmp) in the context of ScopedInterceptor,
614  // bad things will happen. We will jump over ScopedInterceptor dtor and can
615  // leave thr->in_ignored_lib set.
616  {
617    SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val);
618  }
619  LongJmp(cur_thread(), env);
620  REAL(longjmp_symname)(env, val);
621}
622
623TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) {
624  {
625    SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val);
626  }
627  LongJmp(cur_thread(), env);
628  REAL(siglongjmp_symname)(env, val);
629}
630
631#if SANITIZER_NETBSD
632TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
633  {
634    SCOPED_INTERCEPTOR_RAW(_longjmp, env, val);
635  }
636  LongJmp(cur_thread(), env);
637  REAL(_longjmp)(env, val);
638}
639#endif
640
641#if !SANITIZER_MAC
642TSAN_INTERCEPTOR(void*, malloc, uptr size) {
643  if (in_symbolizer())
644    return InternalAlloc(size);
645  void *p = 0;
646  {
647    SCOPED_INTERCEPTOR_RAW(malloc, size);
648    p = user_alloc(thr, pc, size);
649  }
650  invoke_malloc_hook(p, size);
651  return p;
652}
653
654// In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept
655// __libc_memalign so that (1) we can detect races (2) free will not be called
656// on libc internally allocated blocks.
657TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
658  SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz);
659  return user_memalign(thr, pc, align, sz);
660}
661
662TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
663  if (in_symbolizer())
664    return InternalCalloc(size, n);
665  void *p = 0;
666  {
667    SCOPED_INTERCEPTOR_RAW(calloc, size, n);
668    p = user_calloc(thr, pc, size, n);
669  }
670  invoke_malloc_hook(p, n * size);
671  return p;
672}
673
674TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
675  if (in_symbolizer())
676    return InternalRealloc(p, size);
677  if (p)
678    invoke_free_hook(p);
679  {
680    SCOPED_INTERCEPTOR_RAW(realloc, p, size);
681    p = user_realloc(thr, pc, p, size);
682  }
683  invoke_malloc_hook(p, size);
684  return p;
685}
686
687TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) {
688  if (in_symbolizer())
689    return InternalReallocArray(p, size, n);
690  if (p)
691    invoke_free_hook(p);
692  {
693    SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n);
694    p = user_reallocarray(thr, pc, p, size, n);
695  }
696  invoke_malloc_hook(p, size);
697  return p;
698}
699
700TSAN_INTERCEPTOR(void, free, void *p) {
701  if (p == 0)
702    return;
703  if (in_symbolizer())
704    return InternalFree(p);
705  invoke_free_hook(p);
706  SCOPED_INTERCEPTOR_RAW(free, p);
707  user_free(thr, pc, p);
708}
709
710TSAN_INTERCEPTOR(void, cfree, void *p) {
711  if (p == 0)
712    return;
713  if (in_symbolizer())
714    return InternalFree(p);
715  invoke_free_hook(p);
716  SCOPED_INTERCEPTOR_RAW(cfree, p);
717  user_free(thr, pc, p);
718}
719
720TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
721  SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
722  return user_alloc_usable_size(p);
723}
724#endif
725
726TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) {
727  SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);
728  uptr srclen = internal_strlen(src);
729  MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
730  MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
731  return REAL(strcpy)(dst, src);
732}
733
734TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
735  SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
736  uptr srclen = internal_strnlen(src, n);
737  MemoryAccessRange(thr, pc, (uptr)dst, n, true);
738  MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
739  return REAL(strncpy)(dst, src, n);
740}
741
742TSAN_INTERCEPTOR(char*, strdup, const char *str) {
743  SCOPED_TSAN_INTERCEPTOR(strdup, str);
744  // strdup will call malloc, so no instrumentation is required here.
745  return REAL(strdup)(str);
746}
747
748// Zero out addr if it points into shadow memory and was provided as a hint
749// only, i.e., MAP_FIXED is not set.
750static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
751  if (*addr) {
752    if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
753      if (flags & MAP_FIXED) {
754        errno = errno_EINVAL;
755        return false;
756      } else {
757        *addr = 0;
758      }
759    }
760  }
761  return true;
762}
763
764template <class Mmap>
765static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap,
766                              void *addr, SIZE_T sz, int prot, int flags,
767                              int fd, OFF64_T off) {
768  if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED;
769  void *res = real_mmap(addr, sz, prot, flags, fd, off);
770  if (res != MAP_FAILED) {
771    if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) {
772      Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n",
773             addr, (void*)sz, res);
774      Die();
775    }
776    if (fd > 0) FdAccess(thr, pc, fd);
777    MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz);
778  }
779  return res;
780}
781
782TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
783  SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
784  UnmapShadow(thr, (uptr)addr, sz);
785  int res = REAL(munmap)(addr, sz);
786  return res;
787}
788
789#if SANITIZER_LINUX
790TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
791  SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
792  return user_memalign(thr, pc, align, sz);
793}
794#define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
795#else
796#define TSAN_MAYBE_INTERCEPT_MEMALIGN
797#endif
798
799#if !SANITIZER_MAC
800TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
801  if (in_symbolizer())
802    return InternalAlloc(sz, nullptr, align);
803  SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz);
804  return user_aligned_alloc(thr, pc, align, sz);
805}
806
807TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
808  if (in_symbolizer())
809    return InternalAlloc(sz, nullptr, GetPageSizeCached());
810  SCOPED_INTERCEPTOR_RAW(valloc, sz);
811  return user_valloc(thr, pc, sz);
812}
813#endif
814
815#if SANITIZER_LINUX
816TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
817  if (in_symbolizer()) {
818    uptr PageSize = GetPageSizeCached();
819    sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
820    return InternalAlloc(sz, nullptr, PageSize);
821  }
822  SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
823  return user_pvalloc(thr, pc, sz);
824}
825#define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
826#else
827#define TSAN_MAYBE_INTERCEPT_PVALLOC
828#endif
829
830#if !SANITIZER_MAC
831TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
832  if (in_symbolizer()) {
833    void *p = InternalAlloc(sz, nullptr, align);
834    if (!p)
835      return errno_ENOMEM;
836    *memptr = p;
837    return 0;
838  }
839  SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
840  return user_posix_memalign(thr, pc, memptr, align, sz);
841}
842#endif
843
844// Both __cxa_guard_acquire and pthread_once 0-initialize
845// the object initially. pthread_once does not have any
846// other ABI requirements. __cxa_guard_acquire assumes
847// that any non-0 value in the first byte means that
848// initialization is completed. Contents of the remaining
849// bytes are up to us.
850constexpr u32 kGuardInit = 0;
851constexpr u32 kGuardDone = 1;
852constexpr u32 kGuardRunning = 1 << 16;
853constexpr u32 kGuardWaiter = 1 << 17;
854
855static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g,
856                         bool blocking_hooks = true) {
857  if (blocking_hooks)
858    OnPotentiallyBlockingRegionBegin();
859  auto on_exit = at_scope_exit([blocking_hooks] {
860    if (blocking_hooks)
861      OnPotentiallyBlockingRegionEnd();
862  });
863
864  for (;;) {
865    u32 cmp = atomic_load(g, memory_order_acquire);
866    if (cmp == kGuardInit) {
867      if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning,
868                                         memory_order_relaxed))
869        return 1;
870    } else if (cmp == kGuardDone) {
871      if (!thr->in_ignored_lib)
872        Acquire(thr, pc, (uptr)g);
873      return 0;
874    } else {
875      if ((cmp & kGuardWaiter) ||
876          atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter,
877                                         memory_order_relaxed))
878        FutexWait(g, cmp | kGuardWaiter);
879    }
880  }
881}
882
883static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g) {
884  if (!thr->in_ignored_lib)
885    Release(thr, pc, (uptr)g);
886  u32 old = atomic_exchange(g, kGuardDone, memory_order_release);
887  if (old & kGuardWaiter)
888    FutexWake(g, 1 << 30);
889}
890
891// __cxa_guard_acquire and friends need to be intercepted in a special way -
892// regular interceptors will break statically-linked libstdc++. Linux
893// interceptors are especially defined as weak functions (so that they don't
894// cause link errors when user defines them as well). So they silently
895// auto-disable themselves when such symbol is already present in the binary. If
896// we link libstdc++ statically, it will bring own __cxa_guard_acquire which
897// will silently replace our interceptor.  That's why on Linux we simply export
898// these interceptors with INTERFACE_ATTRIBUTE.
899// On OS X, we don't support statically linking, so we just use a regular
900// interceptor.
901#if SANITIZER_MAC
902#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
903#else
904#define STDCXX_INTERCEPTOR(rettype, name, ...) \
905  extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__)
906#endif
907
908// Used in thread-safe function static initialization.
909STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {
910  SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
911  return guard_acquire(thr, pc, g);
912}
913
914STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
915  SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
916  guard_release(thr, pc, g);
917}
918
919STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
920  SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
921  atomic_store(g, kGuardInit, memory_order_relaxed);
922}
923
924namespace __tsan {
925void DestroyThreadState() {
926  ThreadState *thr = cur_thread();
927  Processor *proc = thr->proc();
928  ThreadFinish(thr);
929  ProcUnwire(proc, thr);
930  ProcDestroy(proc);
931  DTLS_Destroy();
932  cur_thread_finalize();
933}
934
935void PlatformCleanUpThreadState(ThreadState *thr) {
936  ThreadSignalContext *sctx = thr->signal_ctx;
937  if (sctx) {
938    thr->signal_ctx = 0;
939    UnmapOrDie(sctx, sizeof(*sctx));
940  }
941}
942}  // namespace __tsan
943
944#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
945static void thread_finalize(void *v) {
946  uptr iter = (uptr)v;
947  if (iter > 1) {
948    if (pthread_setspecific(interceptor_ctx()->finalize_key,
949        (void*)(iter - 1))) {
950      Printf("ThreadSanitizer: failed to set thread key\n");
951      Die();
952    }
953    return;
954  }
955  DestroyThreadState();
956}
957#endif
958
959
960struct ThreadParam {
961  void* (*callback)(void *arg);
962  void *param;
963  Tid tid;
964  Semaphore created;
965  Semaphore started;
966};
967
968extern "C" void *__tsan_thread_start_func(void *arg) {
969  ThreadParam *p = (ThreadParam*)arg;
970  void* (*callback)(void *arg) = p->callback;
971  void *param = p->param;
972  {
973    ThreadState *thr = cur_thread_init();
974    // Thread-local state is not initialized yet.
975    ScopedIgnoreInterceptors ignore;
976#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
977    ThreadIgnoreBegin(thr, 0);
978    if (pthread_setspecific(interceptor_ctx()->finalize_key,
979                            (void *)GetPthreadDestructorIterations())) {
980      Printf("ThreadSanitizer: failed to set thread key\n");
981      Die();
982    }
983    ThreadIgnoreEnd(thr);
984#endif
985    p->created.Wait();
986    Processor *proc = ProcCreate();
987    ProcWire(proc, thr);
988    ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular);
989    p->started.Post();
990  }
991  void *res = callback(param);
992  // Prevent the callback from being tail called,
993  // it mixes up stack traces.
994  volatile int foo = 42;
995  foo++;
996  return res;
997}
998
999TSAN_INTERCEPTOR(int, pthread_create,
1000    void *th, void *attr, void *(*callback)(void*), void * param) {
1001  SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
1002
1003  MaybeSpawnBackgroundThread();
1004
1005  if (ctx->after_multithreaded_fork) {
1006    if (flags()->die_after_fork) {
1007      Report("ThreadSanitizer: starting new threads after multi-threaded "
1008          "fork is not supported. Dying (set die_after_fork=0 to override)\n");
1009      Die();
1010    } else {
1011      VPrintf(1,
1012              "ThreadSanitizer: starting new threads after multi-threaded "
1013              "fork is not supported (pid %lu). Continuing because of "
1014              "die_after_fork=0, but you are on your own\n",
1015              internal_getpid());
1016    }
1017  }
1018  __sanitizer_pthread_attr_t myattr;
1019  if (attr == 0) {
1020    pthread_attr_init(&myattr);
1021    attr = &myattr;
1022  }
1023  int detached = 0;
1024  REAL(pthread_attr_getdetachstate)(attr, &detached);
1025  AdjustStackSize(attr);
1026
1027  ThreadParam p;
1028  p.callback = callback;
1029  p.param = param;
1030  p.tid = kMainTid;
1031  int res = -1;
1032  {
1033    // Otherwise we see false positives in pthread stack manipulation.
1034    ScopedIgnoreInterceptors ignore;
1035    ThreadIgnoreBegin(thr, pc);
1036    res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
1037    ThreadIgnoreEnd(thr);
1038  }
1039  if (res == 0) {
1040    p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached));
1041    CHECK_NE(p.tid, kMainTid);
1042    // Synchronization on p.tid serves two purposes:
1043    // 1. ThreadCreate must finish before the new thread starts.
1044    //    Otherwise the new thread can call pthread_detach, but the pthread_t
1045    //    identifier is not yet registered in ThreadRegistry by ThreadCreate.
1046    // 2. ThreadStart must finish before this thread continues.
1047    //    Otherwise, this thread can call pthread_detach and reset thr->sync
1048    //    before the new thread got a chance to acquire from it in ThreadStart.
1049    p.created.Post();
1050    p.started.Wait();
1051  }
1052  if (attr == &myattr)
1053    pthread_attr_destroy(&myattr);
1054  return res;
1055}
1056
1057TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
1058  SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
1059  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1060  ThreadIgnoreBegin(thr, pc);
1061  int res = BLOCK_REAL(pthread_join)(th, ret);
1062  ThreadIgnoreEnd(thr);
1063  if (res == 0) {
1064    ThreadJoin(thr, pc, tid);
1065  }
1066  return res;
1067}
1068
1069DEFINE_REAL_PTHREAD_FUNCTIONS
1070
1071TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1072  SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
1073  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1074  int res = REAL(pthread_detach)(th);
1075  if (res == 0) {
1076    ThreadDetach(thr, pc, tid);
1077  }
1078  return res;
1079}
1080
1081TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
1082  {
1083    SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
1084#if !SANITIZER_MAC && !SANITIZER_ANDROID
1085    CHECK_EQ(thr, reinterpret_cast<char *>((reinterpret_cast<uptr>(cur_thread_placeholder) + SANITIZER_CACHE_LINE_SIZE - 1) & ~static_cast<uptr>(SANITIZER_CACHE_LINE_SIZE - 1)));
1086#endif
1087  }
1088  REAL(pthread_exit)(retval);
1089}
1090
1091#if SANITIZER_LINUX
1092TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) {
1093  SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);
1094  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1095  ThreadIgnoreBegin(thr, pc);
1096  int res = REAL(pthread_tryjoin_np)(th, ret);
1097  ThreadIgnoreEnd(thr);
1098  if (res == 0)
1099    ThreadJoin(thr, pc, tid);
1100  else
1101    ThreadNotJoined(thr, pc, tid, (uptr)th);
1102  return res;
1103}
1104
1105TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret,
1106                 const struct timespec *abstime) {
1107  SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);
1108  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1109  ThreadIgnoreBegin(thr, pc);
1110  int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);
1111  ThreadIgnoreEnd(thr);
1112  if (res == 0)
1113    ThreadJoin(thr, pc, tid);
1114  else
1115    ThreadNotJoined(thr, pc, tid, (uptr)th);
1116  return res;
1117}
1118#endif
1119
1120// Problem:
1121// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
1122// pthread_cond_t has different size in the different versions.
1123// If call new REAL functions for old pthread_cond_t, they will corrupt memory
1124// after pthread_cond_t (old cond is smaller).
1125// If we call old REAL functions for new pthread_cond_t, we will lose  some
1126// functionality (e.g. old functions do not support waiting against
1127// CLOCK_REALTIME).
1128// Proper handling would require to have 2 versions of interceptors as well.
1129// But this is messy, in particular requires linker scripts when sanitizer
1130// runtime is linked into a shared library.
1131// Instead we assume we don't have dynamic libraries built against old
1132// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
1133// that allows to work with old libraries (but this mode does not support
1134// some features, e.g. pthread_condattr_getpshared).
1135static void *init_cond(void *c, bool force = false) {
1136  // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
1137  // So we allocate additional memory on the side large enough to hold
1138  // any pthread_cond_t object. Always call new REAL functions, but pass
1139  // the aux object to them.
1140  // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
1141  // first word of pthread_cond_t to zero.
1142  // It's all relevant only for linux.
1143  if (!common_flags()->legacy_pthread_cond)
1144    return c;
1145  atomic_uintptr_t *p = (atomic_uintptr_t*)c;
1146  uptr cond = atomic_load(p, memory_order_acquire);
1147  if (!force && cond != 0)
1148    return (void*)cond;
1149  void *newcond = WRAP(malloc)(pthread_cond_t_sz);
1150  internal_memset(newcond, 0, pthread_cond_t_sz);
1151  if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
1152      memory_order_acq_rel))
1153    return newcond;
1154  WRAP(free)(newcond);
1155  return (void*)cond;
1156}
1157
1158namespace {
1159
1160template <class Fn>
1161struct CondMutexUnlockCtx {
1162  ScopedInterceptor *si;
1163  ThreadState *thr;
1164  uptr pc;
1165  void *m;
1166  void *c;
1167  const Fn &fn;
1168
1169  int Cancel() const { return fn(); }
1170  void Unlock() const;
1171};
1172
1173template <class Fn>
1174void CondMutexUnlockCtx<Fn>::Unlock() const {
1175  // pthread_cond_wait interceptor has enabled async signal delivery
1176  // (see BlockingCall below). Disable async signals since we are running
1177  // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
1178  // since the thread is cancelled, so we have to manually execute them
1179  // (the thread still can run some user code due to pthread_cleanup_push).
1180  ThreadSignalContext *ctx = SigCtx(thr);
1181  CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
1182  atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
1183  MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
1184  // Undo BlockingCall ctor effects.
1185  thr->ignore_interceptors--;
1186  si->~ScopedInterceptor();
1187}
1188}  // namespace
1189
1190INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
1191  void *cond = init_cond(c, true);
1192  SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a);
1193  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
1194  return REAL(pthread_cond_init)(cond, a);
1195}
1196
1197template <class Fn>
1198int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn,
1199              void *c, void *m) {
1200  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
1201  MutexUnlock(thr, pc, (uptr)m);
1202  int res = 0;
1203  // This ensures that we handle mutex lock even in case of pthread_cancel.
1204  // See test/tsan/cond_cancel.cpp.
1205  {
1206    // Enable signal delivery while the thread is blocked.
1207    BlockingCall bc(thr);
1208    CondMutexUnlockCtx<Fn> arg = {si, thr, pc, m, c, fn};
1209    res = call_pthread_cancel_with_cleanup(
1210        [](void *arg) -> int {
1211          return ((const CondMutexUnlockCtx<Fn> *)arg)->Cancel();
1212        },
1213        [](void *arg) { ((const CondMutexUnlockCtx<Fn> *)arg)->Unlock(); },
1214        &arg);
1215  }
1216  if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m);
1217  MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
1218  return res;
1219}
1220
1221INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
1222  void *cond = init_cond(c);
1223  SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m);
1224  return cond_wait(
1225      thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond,
1226      m);
1227}
1228
1229INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
1230  void *cond = init_cond(c);
1231  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime);
1232  return cond_wait(
1233      thr, pc, &si,
1234      [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond,
1235      m);
1236}
1237
1238#if SANITIZER_LINUX
1239INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m,
1240            __sanitizer_clockid_t clock, void *abstime) {
1241  void *cond = init_cond(c);
1242  SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime);
1243  return cond_wait(
1244      thr, pc, &si,
1245      [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); },
1246      cond, m);
1247}
1248#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait)
1249#else
1250#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
1251#endif
1252
1253#if SANITIZER_MAC
1254INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m,
1255            void *reltime) {
1256  void *cond = init_cond(c);
1257  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime);
1258  return cond_wait(
1259      thr, pc, &si,
1260      [=]() {
1261        return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime);
1262      },
1263      cond, m);
1264}
1265#endif
1266
1267INTERCEPTOR(int, pthread_cond_signal, void *c) {
1268  void *cond = init_cond(c);
1269  SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond);
1270  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
1271  return REAL(pthread_cond_signal)(cond);
1272}
1273
1274INTERCEPTOR(int, pthread_cond_broadcast, void *c) {
1275  void *cond = init_cond(c);
1276  SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond);
1277  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
1278  return REAL(pthread_cond_broadcast)(cond);
1279}
1280
1281INTERCEPTOR(int, pthread_cond_destroy, void *c) {
1282  void *cond = init_cond(c);
1283  SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond);
1284  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
1285  int res = REAL(pthread_cond_destroy)(cond);
1286  if (common_flags()->legacy_pthread_cond) {
1287    // Free our aux cond and zero the pointer to not leave dangling pointers.
1288    WRAP(free)(cond);
1289    atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed);
1290  }
1291  return res;
1292}
1293
1294TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
1295  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
1296  int res = REAL(pthread_mutex_init)(m, a);
1297  if (res == 0) {
1298    u32 flagz = 0;
1299    if (a) {
1300      int type = 0;
1301      if (REAL(pthread_mutexattr_gettype)(a, &type) == 0)
1302        if (type == PTHREAD_MUTEX_RECURSIVE ||
1303            type == PTHREAD_MUTEX_RECURSIVE_NP)
1304          flagz |= MutexFlagWriteReentrant;
1305    }
1306    MutexCreate(thr, pc, (uptr)m, flagz);
1307  }
1308  return res;
1309}
1310
1311TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1312  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
1313  int res = REAL(pthread_mutex_destroy)(m);
1314  if (res == 0 || res == errno_EBUSY) {
1315    MutexDestroy(thr, pc, (uptr)m);
1316  }
1317  return res;
1318}
1319
1320TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1321  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
1322  int res = REAL(pthread_mutex_trylock)(m);
1323  if (res == errno_EOWNERDEAD)
1324    MutexRepair(thr, pc, (uptr)m);
1325  if (res == 0 || res == errno_EOWNERDEAD)
1326    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
1327  return res;
1328}
1329
1330#if !SANITIZER_MAC
1331TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
1332  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
1333  int res = REAL(pthread_mutex_timedlock)(m, abstime);
1334  if (res == 0) {
1335    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
1336  }
1337  return res;
1338}
1339#endif
1340
1341#if !SANITIZER_MAC
1342TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
1343  SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
1344  int res = REAL(pthread_spin_init)(m, pshared);
1345  if (res == 0) {
1346    MutexCreate(thr, pc, (uptr)m);
1347  }
1348  return res;
1349}
1350
1351TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
1352  SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
1353  int res = REAL(pthread_spin_destroy)(m);
1354  if (res == 0) {
1355    MutexDestroy(thr, pc, (uptr)m);
1356  }
1357  return res;
1358}
1359
1360TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
1361  SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
1362  MutexPreLock(thr, pc, (uptr)m);
1363  int res = REAL(pthread_spin_lock)(m);
1364  if (res == 0) {
1365    MutexPostLock(thr, pc, (uptr)m);
1366  }
1367  return res;
1368}
1369
1370TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
1371  SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
1372  int res = REAL(pthread_spin_trylock)(m);
1373  if (res == 0) {
1374    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
1375  }
1376  return res;
1377}
1378
1379TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
1380  SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
1381  MutexUnlock(thr, pc, (uptr)m);
1382  int res = REAL(pthread_spin_unlock)(m);
1383  return res;
1384}
1385#endif
1386
1387TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
1388  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
1389  int res = REAL(pthread_rwlock_init)(m, a);
1390  if (res == 0) {
1391    MutexCreate(thr, pc, (uptr)m);
1392  }
1393  return res;
1394}
1395
1396TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
1397  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
1398  int res = REAL(pthread_rwlock_destroy)(m);
1399  if (res == 0) {
1400    MutexDestroy(thr, pc, (uptr)m);
1401  }
1402  return res;
1403}
1404
1405TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
1406  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
1407  MutexPreReadLock(thr, pc, (uptr)m);
1408  int res = REAL(pthread_rwlock_rdlock)(m);
1409  if (res == 0) {
1410    MutexPostReadLock(thr, pc, (uptr)m);
1411  }
1412  return res;
1413}
1414
1415TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1416  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
1417  int res = REAL(pthread_rwlock_tryrdlock)(m);
1418  if (res == 0) {
1419    MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock);
1420  }
1421  return res;
1422}
1423
1424#if !SANITIZER_MAC
1425TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
1426  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
1427  int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
1428  if (res == 0) {
1429    MutexPostReadLock(thr, pc, (uptr)m);
1430  }
1431  return res;
1432}
1433#endif
1434
1435TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1436  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1437  MutexPreLock(thr, pc, (uptr)m);
1438  int res = REAL(pthread_rwlock_wrlock)(m);
1439  if (res == 0) {
1440    MutexPostLock(thr, pc, (uptr)m);
1441  }
1442  return res;
1443}
1444
1445TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1446  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
1447  int res = REAL(pthread_rwlock_trywrlock)(m);
1448  if (res == 0) {
1449    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
1450  }
1451  return res;
1452}
1453
1454#if !SANITIZER_MAC
1455TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
1456  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
1457  int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
1458  if (res == 0) {
1459    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
1460  }
1461  return res;
1462}
1463#endif
1464
1465TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1466  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
1467  MutexReadOrWriteUnlock(thr, pc, (uptr)m);
1468  int res = REAL(pthread_rwlock_unlock)(m);
1469  return res;
1470}
1471
1472#if !SANITIZER_MAC
1473TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
1474  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
1475  MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
1476  int res = REAL(pthread_barrier_init)(b, a, count);
1477  return res;
1478}
1479
1480TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1481  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
1482  MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
1483  int res = REAL(pthread_barrier_destroy)(b);
1484  return res;
1485}
1486
1487TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1488  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
1489  Release(thr, pc, (uptr)b);
1490  MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
1491  int res = REAL(pthread_barrier_wait)(b);
1492  MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
1493  if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
1494    Acquire(thr, pc, (uptr)b);
1495  }
1496  return res;
1497}
1498#endif
1499
1500TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
1501  SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
1502  if (o == 0 || f == 0)
1503    return errno_EINVAL;
1504  atomic_uint32_t *a;
1505
1506  if (SANITIZER_MAC)
1507    a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t)));
1508  else if (SANITIZER_NETBSD)
1509    a = static_cast<atomic_uint32_t*>
1510          ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz));
1511  else
1512    a = static_cast<atomic_uint32_t*>(o);
1513
1514  // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks
1515  // result in crashes due to too little stack space.
1516  if (guard_acquire(thr, pc, a, !SANITIZER_MAC)) {
1517    (*f)();
1518    guard_release(thr, pc, a);
1519  }
1520  return 0;
1521}
1522
1523#if SANITIZER_LINUX && !SANITIZER_ANDROID
1524TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1525  SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
1526  if (fd > 0)
1527    FdAccess(thr, pc, fd);
1528  return REAL(__fxstat)(version, fd, buf);
1529}
1530#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
1531#else
1532#define TSAN_MAYBE_INTERCEPT___FXSTAT
1533#endif
1534
1535TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1536#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
1537  SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
1538  if (fd > 0)
1539    FdAccess(thr, pc, fd);
1540  return REAL(fstat)(fd, buf);
1541#else
1542  SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1543  if (fd > 0)
1544    FdAccess(thr, pc, fd);
1545  return REAL(__fxstat)(0, fd, buf);
1546#endif
1547}
1548
1549#if SANITIZER_LINUX && !SANITIZER_ANDROID
1550TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1551  SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1552  if (fd > 0)
1553    FdAccess(thr, pc, fd);
1554  return REAL(__fxstat64)(version, fd, buf);
1555}
1556#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
1557#else
1558#define TSAN_MAYBE_INTERCEPT___FXSTAT64
1559#endif
1560
1561#if SANITIZER_LINUX && !SANITIZER_ANDROID
1562TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1563  SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
1564  if (fd > 0)
1565    FdAccess(thr, pc, fd);
1566  return REAL(__fxstat64)(0, fd, buf);
1567}
1568#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
1569#else
1570#define TSAN_MAYBE_INTERCEPT_FSTAT64
1571#endif
1572
1573TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
1574  va_list ap;
1575  va_start(ap, oflag);
1576  mode_t mode = va_arg(ap, int);
1577  va_end(ap);
1578  SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);
1579  READ_STRING(thr, pc, name, 0);
1580  int fd = REAL(open)(name, oflag, mode);
1581  if (fd >= 0)
1582    FdFileCreate(thr, pc, fd);
1583  return fd;
1584}
1585
1586#if SANITIZER_LINUX
1587TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) {
1588  va_list ap;
1589  va_start(ap, oflag);
1590  mode_t mode = va_arg(ap, int);
1591  va_end(ap);
1592  SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode);
1593  READ_STRING(thr, pc, name, 0);
1594  int fd = REAL(open64)(name, oflag, mode);
1595  if (fd >= 0)
1596    FdFileCreate(thr, pc, fd);
1597  return fd;
1598}
1599#define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64)
1600#else
1601#define TSAN_MAYBE_INTERCEPT_OPEN64
1602#endif
1603
1604TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
1605  SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
1606  READ_STRING(thr, pc, name, 0);
1607  int fd = REAL(creat)(name, mode);
1608  if (fd >= 0)
1609    FdFileCreate(thr, pc, fd);
1610  return fd;
1611}
1612
1613#if SANITIZER_LINUX
1614TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
1615  SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
1616  READ_STRING(thr, pc, name, 0);
1617  int fd = REAL(creat64)(name, mode);
1618  if (fd >= 0)
1619    FdFileCreate(thr, pc, fd);
1620  return fd;
1621}
1622#define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64)
1623#else
1624#define TSAN_MAYBE_INTERCEPT_CREAT64
1625#endif
1626
1627TSAN_INTERCEPTOR(int, dup, int oldfd) {
1628  SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
1629  int newfd = REAL(dup)(oldfd);
1630  if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
1631    FdDup(thr, pc, oldfd, newfd, true);
1632  return newfd;
1633}
1634
1635TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1636  SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
1637  int newfd2 = REAL(dup2)(oldfd, newfd);
1638  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1639    FdDup(thr, pc, oldfd, newfd2, false);
1640  return newfd2;
1641}
1642
1643#if !SANITIZER_MAC
1644TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1645  SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1646  int newfd2 = REAL(dup3)(oldfd, newfd, flags);
1647  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1648    FdDup(thr, pc, oldfd, newfd2, false);
1649  return newfd2;
1650}
1651#endif
1652
1653#if SANITIZER_LINUX
1654TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1655  SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
1656  int fd = REAL(eventfd)(initval, flags);
1657  if (fd >= 0)
1658    FdEventCreate(thr, pc, fd);
1659  return fd;
1660}
1661#define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd)
1662#else
1663#define TSAN_MAYBE_INTERCEPT_EVENTFD
1664#endif
1665
1666#if SANITIZER_LINUX
1667TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1668  SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
1669  if (fd >= 0)
1670    FdClose(thr, pc, fd);
1671  fd = REAL(signalfd)(fd, mask, flags);
1672  if (fd >= 0)
1673    FdSignalCreate(thr, pc, fd);
1674  return fd;
1675}
1676#define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd)
1677#else
1678#define TSAN_MAYBE_INTERCEPT_SIGNALFD
1679#endif
1680
1681#if SANITIZER_LINUX
1682TSAN_INTERCEPTOR(int, inotify_init, int fake) {
1683  SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
1684  int fd = REAL(inotify_init)(fake);
1685  if (fd >= 0)
1686    FdInotifyCreate(thr, pc, fd);
1687  return fd;
1688}
1689#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init)
1690#else
1691#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
1692#endif
1693
1694#if SANITIZER_LINUX
1695TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
1696  SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
1697  int fd = REAL(inotify_init1)(flags);
1698  if (fd >= 0)
1699    FdInotifyCreate(thr, pc, fd);
1700  return fd;
1701}
1702#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1)
1703#else
1704#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
1705#endif
1706
1707TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
1708  SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
1709  int fd = REAL(socket)(domain, type, protocol);
1710  if (fd >= 0)
1711    FdSocketCreate(thr, pc, fd);
1712  return fd;
1713}
1714
1715TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
1716  SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
1717  int res = REAL(socketpair)(domain, type, protocol, fd);
1718  if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
1719    FdPipeCreate(thr, pc, fd[0], fd[1]);
1720  return res;
1721}
1722
1723TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
1724  SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
1725  FdSocketConnecting(thr, pc, fd);
1726  int res = REAL(connect)(fd, addr, addrlen);
1727  if (res == 0 && fd >= 0)
1728    FdSocketConnect(thr, pc, fd);
1729  return res;
1730}
1731
1732TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
1733  SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
1734  int res = REAL(bind)(fd, addr, addrlen);
1735  if (fd > 0 && res == 0)
1736    FdAccess(thr, pc, fd);
1737  return res;
1738}
1739
1740TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
1741  SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
1742  int res = REAL(listen)(fd, backlog);
1743  if (fd > 0 && res == 0)
1744    FdAccess(thr, pc, fd);
1745  return res;
1746}
1747
1748TSAN_INTERCEPTOR(int, close, int fd) {
1749  SCOPED_TSAN_INTERCEPTOR(close, fd);
1750  if (fd >= 0)
1751    FdClose(thr, pc, fd);
1752  return REAL(close)(fd);
1753}
1754
1755#if SANITIZER_LINUX
1756TSAN_INTERCEPTOR(int, __close, int fd) {
1757  SCOPED_TSAN_INTERCEPTOR(__close, fd);
1758  if (fd >= 0)
1759    FdClose(thr, pc, fd);
1760  return REAL(__close)(fd);
1761}
1762#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
1763#else
1764#define TSAN_MAYBE_INTERCEPT___CLOSE
1765#endif
1766
1767// glibc guts
1768#if SANITIZER_LINUX && !SANITIZER_ANDROID
1769TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
1770  SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
1771  int fds[64];
1772  int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
1773  for (int i = 0; i < cnt; i++) {
1774    if (fds[i] > 0)
1775      FdClose(thr, pc, fds[i]);
1776  }
1777  REAL(__res_iclose)(state, free_addr);
1778}
1779#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
1780#else
1781#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE
1782#endif
1783
1784TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1785  SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
1786  int res = REAL(pipe)(pipefd);
1787  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1788    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1789  return res;
1790}
1791
1792#if !SANITIZER_MAC
1793TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1794  SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1795  int res = REAL(pipe2)(pipefd, flags);
1796  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1797    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1798  return res;
1799}
1800#endif
1801
1802TSAN_INTERCEPTOR(int, unlink, char *path) {
1803  SCOPED_TSAN_INTERCEPTOR(unlink, path);
1804  Release(thr, pc, File2addr(path));
1805  int res = REAL(unlink)(path);
1806  return res;
1807}
1808
1809TSAN_INTERCEPTOR(void*, tmpfile, int fake) {
1810  SCOPED_TSAN_INTERCEPTOR(tmpfile, fake);
1811  void *res = REAL(tmpfile)(fake);
1812  if (res) {
1813    int fd = fileno_unlocked(res);
1814    if (fd >= 0)
1815      FdFileCreate(thr, pc, fd);
1816  }
1817  return res;
1818}
1819
1820#if SANITIZER_LINUX
1821TSAN_INTERCEPTOR(void*, tmpfile64, int fake) {
1822  SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake);
1823  void *res = REAL(tmpfile64)(fake);
1824  if (res) {
1825    int fd = fileno_unlocked(res);
1826    if (fd >= 0)
1827      FdFileCreate(thr, pc, fd);
1828  }
1829  return res;
1830}
1831#define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64)
1832#else
1833#define TSAN_MAYBE_INTERCEPT_TMPFILE64
1834#endif
1835
1836static void FlushStreams() {
1837  // Flushing all the streams here may freeze the process if a child thread is
1838  // performing file stream operations at the same time.
1839  REAL(fflush)(stdout);
1840  REAL(fflush)(stderr);
1841}
1842
1843TSAN_INTERCEPTOR(void, abort, int fake) {
1844  SCOPED_TSAN_INTERCEPTOR(abort, fake);
1845  FlushStreams();
1846  REAL(abort)(fake);
1847}
1848
1849TSAN_INTERCEPTOR(int, rmdir, char *path) {
1850  SCOPED_TSAN_INTERCEPTOR(rmdir, path);
1851  Release(thr, pc, Dir2addr(path));
1852  int res = REAL(rmdir)(path);
1853  return res;
1854}
1855
1856TSAN_INTERCEPTOR(int, closedir, void *dirp) {
1857  SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
1858  if (dirp) {
1859    int fd = dirfd(dirp);
1860    FdClose(thr, pc, fd);
1861  }
1862  return REAL(closedir)(dirp);
1863}
1864
1865#if SANITIZER_LINUX
1866TSAN_INTERCEPTOR(int, epoll_create, int size) {
1867  SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
1868  int fd = REAL(epoll_create)(size);
1869  if (fd >= 0)
1870    FdPollCreate(thr, pc, fd);
1871  return fd;
1872}
1873
1874TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
1875  SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
1876  int fd = REAL(epoll_create1)(flags);
1877  if (fd >= 0)
1878    FdPollCreate(thr, pc, fd);
1879  return fd;
1880}
1881
1882TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1883  SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
1884  if (epfd >= 0)
1885    FdAccess(thr, pc, epfd);
1886  if (epfd >= 0 && fd >= 0)
1887    FdAccess(thr, pc, fd);
1888  if (op == EPOLL_CTL_ADD && epfd >= 0)
1889    FdRelease(thr, pc, epfd);
1890  int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1891  return res;
1892}
1893
1894TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
1895  SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
1896  if (epfd >= 0)
1897    FdAccess(thr, pc, epfd);
1898  int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
1899  if (res > 0 && epfd >= 0)
1900    FdAcquire(thr, pc, epfd);
1901  return res;
1902}
1903
1904TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout,
1905                 void *sigmask) {
1906  SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask);
1907  if (epfd >= 0)
1908    FdAccess(thr, pc, epfd);
1909  int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask);
1910  if (res > 0 && epfd >= 0)
1911    FdAcquire(thr, pc, epfd);
1912  return res;
1913}
1914
1915#define TSAN_MAYBE_INTERCEPT_EPOLL \
1916    TSAN_INTERCEPT(epoll_create); \
1917    TSAN_INTERCEPT(epoll_create1); \
1918    TSAN_INTERCEPT(epoll_ctl); \
1919    TSAN_INTERCEPT(epoll_wait); \
1920    TSAN_INTERCEPT(epoll_pwait)
1921#else
1922#define TSAN_MAYBE_INTERCEPT_EPOLL
1923#endif
1924
1925// The following functions are intercepted merely to process pending signals.
1926// If program blocks signal X, we must deliver the signal before the function
1927// returns. Similarly, if program unblocks a signal (or returns from sigsuspend)
1928// it's better to deliver the signal straight away.
1929TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
1930  SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
1931  return REAL(sigsuspend)(mask);
1932}
1933
1934TSAN_INTERCEPTOR(int, sigblock, int mask) {
1935  SCOPED_TSAN_INTERCEPTOR(sigblock, mask);
1936  return REAL(sigblock)(mask);
1937}
1938
1939TSAN_INTERCEPTOR(int, sigsetmask, int mask) {
1940  SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask);
1941  return REAL(sigsetmask)(mask);
1942}
1943
1944TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,
1945    __sanitizer_sigset_t *oldset) {
1946  SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset);
1947  return REAL(pthread_sigmask)(how, set, oldset);
1948}
1949
1950namespace __tsan {
1951
1952static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) {
1953  VarSizeStackTrace stack;
1954  // StackTrace::GetNestInstructionPc(pc) is used because return address is
1955  // expected, OutputReport() will undo this.
1956  ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
1957  ThreadRegistryLock l(&ctx->thread_registry);
1958  ScopedReport rep(ReportTypeErrnoInSignal);
1959  if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
1960    rep.AddStack(stack, true);
1961    OutputReport(thr, rep);
1962  }
1963}
1964
1965static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1966                                  int sig, __sanitizer_siginfo *info,
1967                                  void *uctx) {
1968  __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
1969  if (acquire)
1970    Acquire(thr, 0, (uptr)&sigactions[sig]);
1971  // Signals are generally asynchronous, so if we receive a signals when
1972  // ignores are enabled we should disable ignores. This is critical for sync
1973  // and interceptors, because otherwise we can miss synchronization and report
1974  // false races.
1975  int ignore_reads_and_writes = thr->ignore_reads_and_writes;
1976  int ignore_interceptors = thr->ignore_interceptors;
1977  int ignore_sync = thr->ignore_sync;
1978  // For symbolizer we only process SIGSEGVs synchronously
1979  // (bug in symbolizer or in tsan). But we want to reset
1980  // in_symbolizer to fail gracefully. Symbolizer and user code
1981  // use different memory allocators, so if we don't reset
1982  // in_symbolizer we can get memory allocated with one being
1983  // feed with another, which can cause more crashes.
1984  int in_symbolizer = thr->in_symbolizer;
1985  if (!ctx->after_multithreaded_fork) {
1986    thr->ignore_reads_and_writes = 0;
1987    thr->fast_state.ClearIgnoreBit();
1988    thr->ignore_interceptors = 0;
1989    thr->ignore_sync = 0;
1990    thr->in_symbolizer = 0;
1991  }
1992  // Ensure that the handler does not spoil errno.
1993  const int saved_errno = errno;
1994  errno = 99;
1995  // This code races with sigaction. Be careful to not read sa_sigaction twice.
1996  // Also need to remember pc for reporting before the call,
1997  // because the handler can reset it.
1998  volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO)
1999                         ? (uptr)sigactions[sig].sigaction
2000                         : (uptr)sigactions[sig].handler;
2001  if (pc != sig_dfl && pc != sig_ign) {
2002    // The callback can be either sa_handler or sa_sigaction.
2003    // They have different signatures, but we assume that passing
2004    // additional arguments to sa_handler works and is harmless.
2005    ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
2006  }
2007  if (!ctx->after_multithreaded_fork) {
2008    thr->ignore_reads_and_writes = ignore_reads_and_writes;
2009    if (ignore_reads_and_writes)
2010      thr->fast_state.SetIgnoreBit();
2011    thr->ignore_interceptors = ignore_interceptors;
2012    thr->ignore_sync = ignore_sync;
2013    thr->in_symbolizer = in_symbolizer;
2014  }
2015  // We do not detect errno spoiling for SIGTERM,
2016  // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
2017  // tsan reports false positive in such case.
2018  // It's difficult to properly detect this situation (reraise),
2019  // because in async signal processing case (when handler is called directly
2020  // from rtl_generic_sighandler) we have not yet received the reraised
2021  // signal; and it looks too fragile to intercept all ways to reraise a signal.
2022  if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&
2023      errno != 99)
2024    ReportErrnoSpoiling(thr, pc);
2025  errno = saved_errno;
2026}
2027
2028void ProcessPendingSignalsImpl(ThreadState *thr) {
2029  atomic_store(&thr->pending_signals, 0, memory_order_relaxed);
2030  ThreadSignalContext *sctx = SigCtx(thr);
2031  if (sctx == 0)
2032    return;
2033  atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
2034  internal_sigfillset(&sctx->emptyset);
2035  int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);
2036  CHECK_EQ(res, 0);
2037  for (int sig = 0; sig < kSigCount; sig++) {
2038    SignalDesc *signal = &sctx->pending_signals[sig];
2039    if (signal->armed) {
2040      signal->armed = false;
2041      CallUserSignalHandler(thr, false, true, sig, &signal->siginfo,
2042                            &signal->ctx);
2043    }
2044  }
2045  res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);
2046  CHECK_EQ(res, 0);
2047  atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
2048}
2049
2050}  // namespace __tsan
2051
2052static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
2053  return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP ||
2054         sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
2055         // If we are sending signal to ourselves, we must process it now.
2056         (sctx && sig == sctx->int_signal_send);
2057}
2058
2059void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) {
2060  ThreadState *thr = cur_thread_init();
2061  ThreadSignalContext *sctx = SigCtx(thr);
2062  if (sig < 0 || sig >= kSigCount) {
2063    VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
2064    return;
2065  }
2066  // Don't mess with synchronous signals.
2067  const bool sync = is_sync_signal(sctx, sig);
2068  if (sync ||
2069      // If we are in blocking function, we can safely process it now
2070      // (but check if we are in a recursive interceptor,
2071      // i.e. pthread_join()->munmap()).
2072      (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) {
2073    atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
2074    if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) {
2075      atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed);
2076      CallUserSignalHandler(thr, sync, true, sig, info, ctx);
2077      atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed);
2078    } else {
2079      // Be very conservative with when we do acquire in this case.
2080      // It's unsafe to do acquire in async handlers, because ThreadState
2081      // can be in inconsistent state.
2082      // SIGSYS looks relatively safe -- it's synchronous and can actually
2083      // need some global state.
2084      bool acq = (sig == SIGSYS);
2085      CallUserSignalHandler(thr, sync, acq, sig, info, ctx);
2086    }
2087    atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
2088    return;
2089  }
2090
2091  if (sctx == 0)
2092    return;
2093  SignalDesc *signal = &sctx->pending_signals[sig];
2094  if (signal->armed == false) {
2095    signal->armed = true;
2096    internal_memcpy(&signal->siginfo, info, sizeof(*info));
2097    internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
2098    atomic_store(&thr->pending_signals, 1, memory_order_relaxed);
2099  }
2100}
2101
2102TSAN_INTERCEPTOR(int, raise, int sig) {
2103  SCOPED_TSAN_INTERCEPTOR(raise, sig);
2104  ThreadSignalContext *sctx = SigCtx(thr);
2105  CHECK_NE(sctx, 0);
2106  int prev = sctx->int_signal_send;
2107  sctx->int_signal_send = sig;
2108  int res = REAL(raise)(sig);
2109  CHECK_EQ(sctx->int_signal_send, sig);
2110  sctx->int_signal_send = prev;
2111  return res;
2112}
2113
2114TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
2115  SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
2116  ThreadSignalContext *sctx = SigCtx(thr);
2117  CHECK_NE(sctx, 0);
2118  int prev = sctx->int_signal_send;
2119  if (pid == (int)internal_getpid()) {
2120    sctx->int_signal_send = sig;
2121  }
2122  int res = REAL(kill)(pid, sig);
2123  if (pid == (int)internal_getpid()) {
2124    CHECK_EQ(sctx->int_signal_send, sig);
2125    sctx->int_signal_send = prev;
2126  }
2127  return res;
2128}
2129
2130TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
2131  SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
2132  ThreadSignalContext *sctx = SigCtx(thr);
2133  CHECK_NE(sctx, 0);
2134  int prev = sctx->int_signal_send;
2135  if (tid == pthread_self()) {
2136    sctx->int_signal_send = sig;
2137  }
2138  int res = REAL(pthread_kill)(tid, sig);
2139  if (tid == pthread_self()) {
2140    CHECK_EQ(sctx->int_signal_send, sig);
2141    sctx->int_signal_send = prev;
2142  }
2143  return res;
2144}
2145
2146TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
2147  SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
2148  // It's intercepted merely to process pending signals.
2149  return REAL(gettimeofday)(tv, tz);
2150}
2151
2152TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
2153    void *hints, void *rv) {
2154  SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
2155  // We miss atomic synchronization in getaddrinfo,
2156  // and can report false race between malloc and free
2157  // inside of getaddrinfo. So ignore memory accesses.
2158  ThreadIgnoreBegin(thr, pc);
2159  int res = REAL(getaddrinfo)(node, service, hints, rv);
2160  ThreadIgnoreEnd(thr);
2161  return res;
2162}
2163
2164TSAN_INTERCEPTOR(int, fork, int fake) {
2165  if (in_symbolizer())
2166    return REAL(fork)(fake);
2167  SCOPED_INTERCEPTOR_RAW(fork, fake);
2168  return REAL(fork)(fake);
2169}
2170
2171void atfork_prepare() {
2172  if (in_symbolizer())
2173    return;
2174  ThreadState *thr = cur_thread();
2175  const uptr pc = StackTrace::GetCurrentPc();
2176  ForkBefore(thr, pc);
2177}
2178
2179void atfork_parent() {
2180  if (in_symbolizer())
2181    return;
2182  ThreadState *thr = cur_thread();
2183  const uptr pc = StackTrace::GetCurrentPc();
2184  ForkParentAfter(thr, pc);
2185}
2186
2187void atfork_child() {
2188  if (in_symbolizer())
2189    return;
2190  ThreadState *thr = cur_thread();
2191  const uptr pc = StackTrace::GetCurrentPc();
2192  ForkChildAfter(thr, pc, true);
2193  FdOnFork(thr, pc);
2194}
2195
2196TSAN_INTERCEPTOR(int, vfork, int fake) {
2197  // Some programs (e.g. openjdk) call close for all file descriptors
2198  // in the child process. Under tsan it leads to false positives, because
2199  // address space is shared, so the parent process also thinks that
2200  // the descriptors are closed (while they are actually not).
2201  // This leads to false positives due to missed synchronization.
2202  // Strictly saying this is undefined behavior, because vfork child is not
2203  // allowed to call any functions other than exec/exit. But this is what
2204  // openjdk does, so we want to handle it.
2205  // We could disable interceptors in the child process. But it's not possible
2206  // to simply intercept and wrap vfork, because vfork child is not allowed
2207  // to return from the function that calls vfork, and that's exactly what
2208  // we would do. So this would require some assembly trickery as well.
2209  // Instead we simply turn vfork into fork.
2210  return WRAP(fork)(fake);
2211}
2212
2213#if SANITIZER_LINUX
2214TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags,
2215                 void *arg, int *parent_tid, void *tls, pid_t *child_tid) {
2216  SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
2217                         child_tid);
2218  struct Arg {
2219    int (*fn)(void *);
2220    void *arg;
2221  };
2222  auto wrapper = +[](void *p) -> int {
2223    auto *thr = cur_thread();
2224    uptr pc = GET_CURRENT_PC();
2225    // Start the background thread for fork, but not for clone.
2226    // For fork we did this always and it's known to work (or user code has
2227    // adopted). But if we do this for the new clone interceptor some code
2228    // (sandbox2) fails. So model we used to do for years and don't start the
2229    // background thread after clone.
2230    ForkChildAfter(thr, pc, false);
2231    FdOnFork(thr, pc);
2232    auto *arg = static_cast<Arg *>(p);
2233    return arg->fn(arg->arg);
2234  };
2235  ForkBefore(thr, pc);
2236  Arg arg_wrapper = {fn, arg};
2237  int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls,
2238                        child_tid);
2239  ForkParentAfter(thr, pc);
2240  return pid;
2241}
2242#endif
2243
2244#if !SANITIZER_MAC && !SANITIZER_ANDROID
2245typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size,
2246                                    void *data);
2247struct dl_iterate_phdr_data {
2248  ThreadState *thr;
2249  uptr pc;
2250  dl_iterate_phdr_cb_t cb;
2251  void *data;
2252};
2253
2254static bool IsAppNotRodata(uptr addr) {
2255  return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata;
2256}
2257
2258static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
2259                              void *data) {
2260  dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data;
2261  // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later
2262  // accessible in dl_iterate_phdr callback. But we don't see synchronization
2263  // inside of dynamic linker, so we "unpoison" it here in order to not
2264  // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough
2265  // because some libc functions call __libc_dlopen.
2266  if (info && IsAppNotRodata((uptr)info->dlpi_name))
2267    MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name,
2268                     internal_strlen(info->dlpi_name));
2269  int res = cbdata->cb(info, size, cbdata->data);
2270  // Perform the check one more time in case info->dlpi_name was overwritten
2271  // by user callback.
2272  if (info && IsAppNotRodata((uptr)info->dlpi_name))
2273    MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name,
2274                     internal_strlen(info->dlpi_name));
2275  return res;
2276}
2277
2278TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) {
2279  SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data);
2280  dl_iterate_phdr_data cbdata;
2281  cbdata.thr = thr;
2282  cbdata.pc = pc;
2283  cbdata.cb = cb;
2284  cbdata.data = data;
2285  int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata);
2286  return res;
2287}
2288#endif
2289
2290static int OnExit(ThreadState *thr) {
2291  int status = Finalize(thr);
2292  FlushStreams();
2293  return status;
2294}
2295
2296struct TsanInterceptorContext {
2297  ThreadState *thr;
2298  const uptr pc;
2299};
2300
2301#if !SANITIZER_MAC
2302static void HandleRecvmsg(ThreadState *thr, uptr pc,
2303    __sanitizer_msghdr *msg) {
2304  int fds[64];
2305  int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
2306  for (int i = 0; i < cnt; i++)
2307    FdEventCreate(thr, pc, fds[i]);
2308}
2309#endif
2310
2311#include "sanitizer_common/sanitizer_platform_interceptors.h"
2312// Causes interceptor recursion (getaddrinfo() and fopen())
2313#undef SANITIZER_INTERCEPT_GETADDRINFO
2314// We define our own.
2315#if SANITIZER_INTERCEPT_TLS_GET_ADDR
2316#define NEED_TLS_GET_ADDR
2317#endif
2318#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
2319#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
2320#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
2321
2322#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
2323#define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                          \
2324  INTERCEPT_FUNCTION_VER(name, ver)
2325#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
2326  (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name))
2327
2328#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
2329  MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
2330                    ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2331                    true)
2332
2333#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
2334  MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
2335                    ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2336                    false)
2337
2338#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
2339  SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);    \
2340  TsanInterceptorContext _ctx = {thr, pc};       \
2341  ctx = (void *)&_ctx;                           \
2342  (void)ctx;
2343
2344#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2345  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
2346  TsanInterceptorContext _ctx = {thr, pc};                \
2347  ctx = (void *)&_ctx;                                    \
2348  (void)ctx;
2349
2350#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2351  if (path)                                           \
2352    Acquire(thr, pc, File2addr(path));                \
2353  if (file) {                                         \
2354    int fd = fileno_unlocked(file);                   \
2355    if (fd >= 0) FdFileCreate(thr, pc, fd);           \
2356  }
2357
2358#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2359  if (file) {                                    \
2360    int fd = fileno_unlocked(file);              \
2361    if (fd >= 0) FdClose(thr, pc, fd);           \
2362  }
2363
2364#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
2365  libignore()->OnLibraryLoaded(filename)
2366
2367#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2368  libignore()->OnLibraryUnloaded()
2369
2370#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
2371  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
2372
2373#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \
2374  Release(((TsanInterceptorContext *) ctx)->thr, pc, u)
2375
2376#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
2377  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path))
2378
2379#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
2380  FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2381
2382#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
2383  FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2384
2385#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
2386  FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2387
2388#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
2389  FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
2390
2391#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2392  ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2393
2394#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2395  __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name)
2396
2397#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2398
2399#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2400  OnExit(((TsanInterceptorContext *) ctx)->thr)
2401
2402#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
2403  MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
2404            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2405
2406#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
2407  MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
2408            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2409
2410#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2411  MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2412            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2413
2414#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2415  MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2416            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2417
2418#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
2419  MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
2420                     ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2421
2422#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd,  \
2423                                     off)                                   \
2424  do {                                                                      \
2425    return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \
2426                            off);                                           \
2427  } while (false)
2428
2429#if !SANITIZER_MAC
2430#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2431  HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2432      ((TsanInterceptorContext *)ctx)->pc, msg)
2433#endif
2434
2435#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
2436  if (TsanThread *t = GetCurrentThread()) {                                    \
2437    *begin = t->tls_begin();                                                   \
2438    *end = t->tls_end();                                                       \
2439  } else {                                                                     \
2440    *begin = *end = 0;                                                         \
2441  }
2442
2443#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \
2444  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START()
2445
2446#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \
2447  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END()
2448
2449#include "sanitizer_common/sanitizer_common_interceptors.inc"
2450
2451static int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2452                          __sanitizer_sigaction *old);
2453static __sanitizer_sighandler_ptr signal_impl(int sig,
2454                                              __sanitizer_sighandler_ptr h);
2455
2456#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
2457  { return sigaction_impl(signo, act, oldact); }
2458
2459#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
2460  { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
2461
2462#include "sanitizer_common/sanitizer_signal_interceptors.inc"
2463
2464int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2465                   __sanitizer_sigaction *old) {
2466  // Note: if we call REAL(sigaction) directly for any reason without proxying
2467  // the signal handler through sighandler, very bad things will happen.
2468  // The handler will run synchronously and corrupt tsan per-thread state.
2469  SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
2470  if (sig <= 0 || sig >= kSigCount) {
2471    errno = errno_EINVAL;
2472    return -1;
2473  }
2474  __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
2475  __sanitizer_sigaction old_stored;
2476  if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored));
2477  __sanitizer_sigaction newact;
2478  if (act) {
2479    // Copy act into sigactions[sig].
2480    // Can't use struct copy, because compiler can emit call to memcpy.
2481    // Can't use internal_memcpy, because it copies byte-by-byte,
2482    // and signal handler reads the handler concurrently. It it can read
2483    // some bytes from old value and some bytes from new value.
2484    // Use volatile to prevent insertion of memcpy.
2485    sigactions[sig].handler =
2486        *(volatile __sanitizer_sighandler_ptr const *)&act->handler;
2487    sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags;
2488    internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
2489                    sizeof(sigactions[sig].sa_mask));
2490#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
2491    sigactions[sig].sa_restorer = act->sa_restorer;
2492#endif
2493    internal_memcpy(&newact, act, sizeof(newact));
2494    internal_sigfillset(&newact.sa_mask);
2495    if ((act->sa_flags & SA_SIGINFO) ||
2496        ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) {
2497      newact.sa_flags |= SA_SIGINFO;
2498      newact.sigaction = sighandler;
2499    }
2500    ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
2501    act = &newact;
2502  }
2503  int res = REAL(sigaction)(sig, act, old);
2504  if (res == 0 && old && old->sigaction == sighandler)
2505    internal_memcpy(old, &old_stored, sizeof(*old));
2506  return res;
2507}
2508
2509static __sanitizer_sighandler_ptr signal_impl(int sig,
2510                                              __sanitizer_sighandler_ptr h) {
2511  __sanitizer_sigaction act;
2512  act.handler = h;
2513  internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask));
2514  act.sa_flags = 0;
2515  __sanitizer_sigaction old;
2516  int res = sigaction_symname(sig, &act, &old);
2517  if (res) return (__sanitizer_sighandler_ptr)sig_err;
2518  return old.handler;
2519}
2520
2521#define TSAN_SYSCALL()             \
2522  ThreadState *thr = cur_thread(); \
2523  if (thr->ignore_interceptors)    \
2524    return;                        \
2525  ScopedSyscall scoped_syscall(thr)
2526
2527struct ScopedSyscall {
2528  ThreadState *thr;
2529
2530  explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); }
2531
2532  ~ScopedSyscall() {
2533    ProcessPendingSignals(thr);
2534  }
2535};
2536
2537#if !SANITIZER_FREEBSD && !SANITIZER_MAC
2538static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2539  TSAN_SYSCALL();
2540  MemoryAccessRange(thr, pc, p, s, write);
2541}
2542
2543static USED void syscall_acquire(uptr pc, uptr addr) {
2544  TSAN_SYSCALL();
2545  Acquire(thr, pc, addr);
2546  DPrintf("syscall_acquire(0x%zx))\n", addr);
2547}
2548
2549static USED void syscall_release(uptr pc, uptr addr) {
2550  TSAN_SYSCALL();
2551  DPrintf("syscall_release(0x%zx)\n", addr);
2552  Release(thr, pc, addr);
2553}
2554
2555static void syscall_fd_close(uptr pc, int fd) {
2556  TSAN_SYSCALL();
2557  FdClose(thr, pc, fd);
2558}
2559
2560static USED void syscall_fd_acquire(uptr pc, int fd) {
2561  TSAN_SYSCALL();
2562  FdAcquire(thr, pc, fd);
2563  DPrintf("syscall_fd_acquire(%d)\n", fd);
2564}
2565
2566static USED void syscall_fd_release(uptr pc, int fd) {
2567  TSAN_SYSCALL();
2568  DPrintf("syscall_fd_release(%d)\n", fd);
2569  FdRelease(thr, pc, fd);
2570}
2571
2572static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }
2573
2574static void syscall_post_fork(uptr pc, int pid) {
2575  ThreadState *thr = cur_thread();
2576  if (pid == 0) {
2577    // child
2578    ForkChildAfter(thr, pc, true);
2579    FdOnFork(thr, pc);
2580  } else if (pid > 0) {
2581    // parent
2582    ForkParentAfter(thr, pc);
2583  } else {
2584    // error
2585    ForkParentAfter(thr, pc);
2586  }
2587}
2588#endif
2589
2590#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
2591  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
2592
2593#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
2594  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
2595
2596#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
2597  do {                                       \
2598    (void)(p);                               \
2599    (void)(s);                               \
2600  } while (false)
2601
2602#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
2603  do {                                        \
2604    (void)(p);                                \
2605    (void)(s);                                \
2606  } while (false)
2607
2608#define COMMON_SYSCALL_ACQUIRE(addr) \
2609    syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
2610
2611#define COMMON_SYSCALL_RELEASE(addr) \
2612    syscall_release(GET_CALLER_PC(), (uptr)(addr))
2613
2614#define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
2615
2616#define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
2617
2618#define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
2619
2620#define COMMON_SYSCALL_PRE_FORK() \
2621  syscall_pre_fork(GET_CALLER_PC())
2622
2623#define COMMON_SYSCALL_POST_FORK(res) \
2624  syscall_post_fork(GET_CALLER_PC(), res)
2625
2626#include "sanitizer_common/sanitizer_common_syscalls.inc"
2627#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
2628
2629#ifdef NEED_TLS_GET_ADDR
2630
2631static void handle_tls_addr(void *arg, void *res) {
2632  ThreadState *thr = cur_thread();
2633  if (!thr)
2634    return;
2635  DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
2636                                        thr->tls_addr + thr->tls_size);
2637  if (!dtv)
2638    return;
2639  // New DTLS block has been allocated.
2640  MemoryResetRange(thr, 0, dtv->beg, dtv->size);
2641}
2642
2643#if !SANITIZER_S390
2644// Define own interceptor instead of sanitizer_common's for three reasons:
2645// 1. It must not process pending signals.
2646//    Signal handlers may contain MOVDQA instruction (see below).
2647// 2. It must be as simple as possible to not contain MOVDQA.
2648// 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which
2649//    is empty for tsan (meant only for msan).
2650// Note: __tls_get_addr can be called with mis-aligned stack due to:
2651// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
2652// So the interceptor must work with mis-aligned stack, in particular, does not
2653// execute MOVDQA with stack addresses.
2654TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
2655  void *res = REAL(__tls_get_addr)(arg);
2656  handle_tls_addr(arg, res);
2657  return res;
2658}
2659#else // SANITIZER_S390
2660TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
2661  uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset));
2662  char *tp = static_cast<char *>(__builtin_thread_pointer());
2663  handle_tls_addr(arg, res + tp);
2664  return res;
2665}
2666#endif
2667#endif
2668
2669#if SANITIZER_NETBSD
2670TSAN_INTERCEPTOR(void, _lwp_exit) {
2671  SCOPED_TSAN_INTERCEPTOR(_lwp_exit);
2672  DestroyThreadState();
2673  REAL(_lwp_exit)();
2674}
2675#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
2676#else
2677#define TSAN_MAYBE_INTERCEPT__LWP_EXIT
2678#endif
2679
2680#if SANITIZER_FREEBSD
2681TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {
2682  SCOPED_TSAN_INTERCEPTOR(thr_exit, state);
2683  DestroyThreadState();
2684  REAL(thr_exit(state));
2685}
2686#define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
2687#else
2688#define TSAN_MAYBE_INTERCEPT_THR_EXIT
2689#endif
2690
2691TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a)
2692TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)
2693TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)
2694TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m)
2695TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)
2696TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a)
2697TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)
2698TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)
2699TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a)
2700TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)
2701TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)
2702TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m)
2703TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m)
2704TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m)
2705TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m)
2706TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)())
2707TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b,
2708  void *c)
2709
2710namespace __tsan {
2711
2712static void finalize(void *arg) {
2713  ThreadState *thr = cur_thread();
2714  int status = Finalize(thr);
2715  // Make sure the output is not lost.
2716  FlushStreams();
2717  if (status)
2718    Die();
2719}
2720
2721#if !SANITIZER_MAC && !SANITIZER_ANDROID
2722static void unreachable() {
2723  Report("FATAL: ThreadSanitizer: unreachable called\n");
2724  Die();
2725}
2726#endif
2727
2728// Define default implementation since interception of libdispatch  is optional.
2729SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
2730
2731void InitializeInterceptors() {
2732#if !SANITIZER_MAC
2733  // We need to setup it early, because functions like dlsym() can call it.
2734  REAL(memset) = internal_memset;
2735  REAL(memcpy) = internal_memcpy;
2736#endif
2737
2738  new(interceptor_ctx()) InterceptorContext();
2739
2740  InitializeCommonInterceptors();
2741  InitializeSignalInterceptors();
2742  InitializeLibdispatchInterceptors();
2743
2744#if !SANITIZER_MAC
2745  // We can not use TSAN_INTERCEPT to get setjmp addr,
2746  // because it does &setjmp and setjmp is not present in some versions of libc.
2747  using __interception::InterceptFunction;
2748  InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0);
2749  InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
2750  InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0,
2751                    0);
2752#if !SANITIZER_NETBSD
2753  InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
2754#endif
2755#endif
2756
2757  TSAN_INTERCEPT(longjmp_symname);
2758  TSAN_INTERCEPT(siglongjmp_symname);
2759#if SANITIZER_NETBSD
2760  TSAN_INTERCEPT(_longjmp);
2761#endif
2762
2763  TSAN_INTERCEPT(malloc);
2764  TSAN_INTERCEPT(__libc_memalign);
2765  TSAN_INTERCEPT(calloc);
2766  TSAN_INTERCEPT(realloc);
2767  TSAN_INTERCEPT(reallocarray);
2768  TSAN_INTERCEPT(free);
2769  TSAN_INTERCEPT(cfree);
2770  TSAN_INTERCEPT(munmap);
2771  TSAN_MAYBE_INTERCEPT_MEMALIGN;
2772  TSAN_INTERCEPT(valloc);
2773  TSAN_MAYBE_INTERCEPT_PVALLOC;
2774  TSAN_INTERCEPT(posix_memalign);
2775
2776  TSAN_INTERCEPT(strcpy);
2777  TSAN_INTERCEPT(strncpy);
2778  TSAN_INTERCEPT(strdup);
2779
2780  TSAN_INTERCEPT(pthread_create);
2781  TSAN_INTERCEPT(pthread_join);
2782  TSAN_INTERCEPT(pthread_detach);
2783  TSAN_INTERCEPT(pthread_exit);
2784  #if SANITIZER_LINUX
2785  TSAN_INTERCEPT(pthread_tryjoin_np);
2786  TSAN_INTERCEPT(pthread_timedjoin_np);
2787  #endif
2788
2789  TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE);
2790  TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE);
2791  TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE);
2792  TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE);
2793  TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE);
2794  TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE);
2795
2796  TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT;
2797
2798  TSAN_INTERCEPT(pthread_mutex_init);
2799  TSAN_INTERCEPT(pthread_mutex_destroy);
2800  TSAN_INTERCEPT(pthread_mutex_trylock);
2801  TSAN_INTERCEPT(pthread_mutex_timedlock);
2802
2803  TSAN_INTERCEPT(pthread_spin_init);
2804  TSAN_INTERCEPT(pthread_spin_destroy);
2805  TSAN_INTERCEPT(pthread_spin_lock);
2806  TSAN_INTERCEPT(pthread_spin_trylock);
2807  TSAN_INTERCEPT(pthread_spin_unlock);
2808
2809  TSAN_INTERCEPT(pthread_rwlock_init);
2810  TSAN_INTERCEPT(pthread_rwlock_destroy);
2811  TSAN_INTERCEPT(pthread_rwlock_rdlock);
2812  TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
2813  TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
2814  TSAN_INTERCEPT(pthread_rwlock_wrlock);
2815  TSAN_INTERCEPT(pthread_rwlock_trywrlock);
2816  TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
2817  TSAN_INTERCEPT(pthread_rwlock_unlock);
2818
2819  TSAN_INTERCEPT(pthread_barrier_init);
2820  TSAN_INTERCEPT(pthread_barrier_destroy);
2821  TSAN_INTERCEPT(pthread_barrier_wait);
2822
2823  TSAN_INTERCEPT(pthread_once);
2824
2825  TSAN_INTERCEPT(fstat);
2826  TSAN_MAYBE_INTERCEPT___FXSTAT;
2827  TSAN_MAYBE_INTERCEPT_FSTAT64;
2828  TSAN_MAYBE_INTERCEPT___FXSTAT64;
2829  TSAN_INTERCEPT(open);
2830  TSAN_MAYBE_INTERCEPT_OPEN64;
2831  TSAN_INTERCEPT(creat);
2832  TSAN_MAYBE_INTERCEPT_CREAT64;
2833  TSAN_INTERCEPT(dup);
2834  TSAN_INTERCEPT(dup2);
2835  TSAN_INTERCEPT(dup3);
2836  TSAN_MAYBE_INTERCEPT_EVENTFD;
2837  TSAN_MAYBE_INTERCEPT_SIGNALFD;
2838  TSAN_MAYBE_INTERCEPT_INOTIFY_INIT;
2839  TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1;
2840  TSAN_INTERCEPT(socket);
2841  TSAN_INTERCEPT(socketpair);
2842  TSAN_INTERCEPT(connect);
2843  TSAN_INTERCEPT(bind);
2844  TSAN_INTERCEPT(listen);
2845  TSAN_MAYBE_INTERCEPT_EPOLL;
2846  TSAN_INTERCEPT(close);
2847  TSAN_MAYBE_INTERCEPT___CLOSE;
2848  TSAN_MAYBE_INTERCEPT___RES_ICLOSE;
2849  TSAN_INTERCEPT(pipe);
2850  TSAN_INTERCEPT(pipe2);
2851
2852  TSAN_INTERCEPT(unlink);
2853  TSAN_INTERCEPT(tmpfile);
2854  TSAN_MAYBE_INTERCEPT_TMPFILE64;
2855  TSAN_INTERCEPT(abort);
2856  TSAN_INTERCEPT(rmdir);
2857  TSAN_INTERCEPT(closedir);
2858
2859  TSAN_INTERCEPT(sigsuspend);
2860  TSAN_INTERCEPT(sigblock);
2861  TSAN_INTERCEPT(sigsetmask);
2862  TSAN_INTERCEPT(pthread_sigmask);
2863  TSAN_INTERCEPT(raise);
2864  TSAN_INTERCEPT(kill);
2865  TSAN_INTERCEPT(pthread_kill);
2866  TSAN_INTERCEPT(sleep);
2867  TSAN_INTERCEPT(usleep);
2868  TSAN_INTERCEPT(nanosleep);
2869  TSAN_INTERCEPT(pause);
2870  TSAN_INTERCEPT(gettimeofday);
2871  TSAN_INTERCEPT(getaddrinfo);
2872
2873  TSAN_INTERCEPT(fork);
2874  TSAN_INTERCEPT(vfork);
2875#if SANITIZER_LINUX
2876  TSAN_INTERCEPT(clone);
2877#endif
2878#if !SANITIZER_ANDROID
2879  TSAN_INTERCEPT(dl_iterate_phdr);
2880#endif
2881  TSAN_MAYBE_INTERCEPT_ON_EXIT;
2882  TSAN_INTERCEPT(__cxa_atexit);
2883  TSAN_INTERCEPT(_exit);
2884
2885#ifdef NEED_TLS_GET_ADDR
2886#if !SANITIZER_S390
2887  TSAN_INTERCEPT(__tls_get_addr);
2888#else
2889  TSAN_INTERCEPT(__tls_get_addr_internal);
2890  TSAN_INTERCEPT(__tls_get_offset);
2891#endif
2892#endif
2893
2894  TSAN_MAYBE_INTERCEPT__LWP_EXIT;
2895  TSAN_MAYBE_INTERCEPT_THR_EXIT;
2896
2897#if !SANITIZER_MAC && !SANITIZER_ANDROID
2898  // Need to setup it, because interceptors check that the function is resolved.
2899  // But atexit is emitted directly into the module, so can't be resolved.
2900  REAL(atexit) = (int(*)(void(*)()))unreachable;
2901#endif
2902
2903  if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
2904    Printf("ThreadSanitizer: failed to setup atexit callback\n");
2905    Die();
2906  }
2907  if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) {
2908    Printf("ThreadSanitizer: failed to setup atfork callbacks\n");
2909    Die();
2910  }
2911
2912#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
2913  if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
2914    Printf("ThreadSanitizer: failed to create thread key\n");
2915    Die();
2916  }
2917#endif
2918
2919  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
2920  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
2921  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
2922  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait);
2923  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
2924  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
2925  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
2926  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
2927  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
2928  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
2929  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
2930  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock);
2931  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock);
2932  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock);
2933  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock);
2934  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once);
2935  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask);
2936
2937  FdInit();
2938}
2939
2940}  // namespace __tsan
2941
2942// Invisible barrier for tests.
2943// There were several unsuccessful iterations for this functionality:
2944// 1. Initially it was implemented in user code using
2945//    REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on
2946//    MacOS. Futexes are linux-specific for this matter.
2947// 2. Then we switched to atomics+usleep(10). But usleep produced parasitic
2948//    "as-if synchronized via sleep" messages in reports which failed some
2949//    output tests.
2950// 3. Then we switched to atomics+sched_yield. But this produced tons of tsan-
2951//    visible events, which lead to "failed to restore stack trace" failures.
2952// Note that no_sanitize_thread attribute does not turn off atomic interception
2953// so attaching it to the function defined in user code does not help.
2954// That's why we now have what we have.
2955constexpr u32 kBarrierThreadBits = 10;
2956constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits;
2957
2958extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(
2959    atomic_uint32_t *barrier, u32 num_threads) {
2960  if (num_threads >= kBarrierThreads) {
2961    Printf("barrier_init: count is too large (%d)\n", num_threads);
2962    Die();
2963  }
2964  // kBarrierThreadBits lsb is thread count,
2965  // the remaining are count of entered threads.
2966  atomic_store(barrier, num_threads, memory_order_relaxed);
2967}
2968
2969static u32 barrier_epoch(u32 value) {
2970  return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1));
2971}
2972
2973extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
2974    atomic_uint32_t *barrier) {
2975  u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed);
2976  u32 old_epoch = barrier_epoch(old);
2977  if (barrier_epoch(old + kBarrierThreads) != old_epoch) {
2978    FutexWake(barrier, (1 << 30));
2979    return;
2980  }
2981  for (;;) {
2982    u32 cur = atomic_load(barrier, memory_order_relaxed);
2983    if (barrier_epoch(cur) != old_epoch)
2984      return;
2985    FutexWait(barrier, cur);
2986  }
2987}
2988