1//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer (TSan), a race detector.
11//
12// Main internal TSan header file.
13//
14// Ground rules:
15//   - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
16//     function-scope locals)
17//   - All functions/classes/etc reside in namespace __tsan, except for those
18//     declared in tsan_interface.h.
19//   - Platform-specific files should be used instead of ifdefs (*).
20//   - No system headers included in header files (*).
21//   - Platform specific headres included only into platform-specific files (*).
22//
23//  (*) Except when inlining is critical for performance.
24//===----------------------------------------------------------------------===//
25
26#ifndef TSAN_RTL_H
27#define TSAN_RTL_H
28
29#include "sanitizer_common/sanitizer_allocator.h"
30#include "sanitizer_common/sanitizer_allocator_internal.h"
31#include "sanitizer_common/sanitizer_asm.h"
32#include "sanitizer_common/sanitizer_common.h"
33#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
34#include "sanitizer_common/sanitizer_libignore.h"
35#include "sanitizer_common/sanitizer_suppressions.h"
36#include "sanitizer_common/sanitizer_thread_registry.h"
37#include "sanitizer_common/sanitizer_vector.h"
38#include "tsan_clock.h"
39#include "tsan_defs.h"
40#include "tsan_flags.h"
41#include "tsan_mman.h"
42#include "tsan_sync.h"
43#include "tsan_trace.h"
44#include "tsan_report.h"
45#include "tsan_platform.h"
46#include "tsan_mutexset.h"
47#include "tsan_ignoreset.h"
48#include "tsan_stack_trace.h"
49
50#if SANITIZER_WORDSIZE != 64
51# error "ThreadSanitizer is supported only on 64-bit platforms"
52#endif
53
54namespace __tsan {
55
56#if !SANITIZER_GO
57struct MapUnmapCallback;
58#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
59static const uptr kAllocatorRegionSizeLog = 20;
60static const uptr kAllocatorNumRegions =
61    SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog;
62using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12,
63                                LocalAddressSpaceView, MapUnmapCallback>;
64struct AP32 {
65  static const uptr kSpaceBeg = 0;
66  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
67  static const uptr kMetadataSize = 0;
68  typedef __sanitizer::CompactSizeClassMap SizeClassMap;
69  static const uptr kRegionSizeLog = kAllocatorRegionSizeLog;
70  using AddressSpaceView = LocalAddressSpaceView;
71  using ByteMap = __tsan::ByteMap;
72  typedef __tsan::MapUnmapCallback MapUnmapCallback;
73  static const uptr kFlags = 0;
74};
75typedef SizeClassAllocator32<AP32> PrimaryAllocator;
76#else
77struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
78  static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
79  static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
80  static const uptr kMetadataSize = 0;
81  typedef DefaultSizeClassMap SizeClassMap;
82  typedef __tsan::MapUnmapCallback MapUnmapCallback;
83  static const uptr kFlags = 0;
84  using AddressSpaceView = LocalAddressSpaceView;
85};
86typedef SizeClassAllocator64<AP64> PrimaryAllocator;
87#endif
88typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
89typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator;
90typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
91    SecondaryAllocator> Allocator;
92Allocator *allocator();
93#endif
94
95void TsanCheckFailed(const char *file, int line, const char *cond,
96                     u64 v1, u64 v2);
97
98const u64 kShadowRodata = (u64)-1;  // .rodata shadow marker
99
100// FastState (from most significant bit):
101//   ignore          : 1
102//   tid             : kTidBits
103//   unused          : -
104//   history_size    : 3
105//   epoch           : kClkBits
106class FastState {
107 public:
108  FastState(u64 tid, u64 epoch) {
109    x_ = tid << kTidShift;
110    x_ |= epoch;
111    DCHECK_EQ(tid, this->tid());
112    DCHECK_EQ(epoch, this->epoch());
113    DCHECK_EQ(GetIgnoreBit(), false);
114  }
115
116  explicit FastState(u64 x)
117      : x_(x) {
118  }
119
120  u64 raw() const {
121    return x_;
122  }
123
124  u64 tid() const {
125    u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
126    return res;
127  }
128
129  u64 TidWithIgnore() const {
130    u64 res = x_ >> kTidShift;
131    return res;
132  }
133
134  u64 epoch() const {
135    u64 res = x_ & ((1ull << kClkBits) - 1);
136    return res;
137  }
138
139  void IncrementEpoch() {
140    u64 old_epoch = epoch();
141    x_ += 1;
142    DCHECK_EQ(old_epoch + 1, epoch());
143    (void)old_epoch;
144  }
145
146  void SetIgnoreBit() { x_ |= kIgnoreBit; }
147  void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
148  bool GetIgnoreBit() const { return (s64)x_ < 0; }
149
150  void SetHistorySize(int hs) {
151    CHECK_GE(hs, 0);
152    CHECK_LE(hs, 7);
153    x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
154  }
155
156  ALWAYS_INLINE
157  int GetHistorySize() const {
158    return (int)((x_ >> kHistoryShift) & kHistoryMask);
159  }
160
161  void ClearHistorySize() {
162    SetHistorySize(0);
163  }
164
165  ALWAYS_INLINE
166  u64 GetTracePos() const {
167    const int hs = GetHistorySize();
168    // When hs == 0, the trace consists of 2 parts.
169    const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
170    return epoch() & mask;
171  }
172
173 private:
174  friend class Shadow;
175  static const int kTidShift = 64 - kTidBits - 1;
176  static const u64 kIgnoreBit = 1ull << 63;
177  static const u64 kFreedBit = 1ull << 63;
178  static const u64 kHistoryShift = kClkBits;
179  static const u64 kHistoryMask = 7;
180  u64 x_;
181};
182
183// Shadow (from most significant bit):
184//   freed           : 1
185//   tid             : kTidBits
186//   is_atomic       : 1
187//   is_read         : 1
188//   size_log        : 2
189//   addr0           : 3
190//   epoch           : kClkBits
191class Shadow : public FastState {
192 public:
193  explicit Shadow(u64 x)
194      : FastState(x) {
195  }
196
197  explicit Shadow(const FastState &s)
198      : FastState(s.x_) {
199    ClearHistorySize();
200  }
201
202  void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
203    DCHECK_EQ((x_ >> kClkBits) & 31, 0);
204    DCHECK_LE(addr0, 7);
205    DCHECK_LE(kAccessSizeLog, 3);
206    x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
207    DCHECK_EQ(kAccessSizeLog, size_log());
208    DCHECK_EQ(addr0, this->addr0());
209  }
210
211  void SetWrite(unsigned kAccessIsWrite) {
212    DCHECK_EQ(x_ & kReadBit, 0);
213    if (!kAccessIsWrite)
214      x_ |= kReadBit;
215    DCHECK_EQ(kAccessIsWrite, IsWrite());
216  }
217
218  void SetAtomic(bool kIsAtomic) {
219    DCHECK(!IsAtomic());
220    if (kIsAtomic)
221      x_ |= kAtomicBit;
222    DCHECK_EQ(IsAtomic(), kIsAtomic);
223  }
224
225  bool IsAtomic() const {
226    return x_ & kAtomicBit;
227  }
228
229  bool IsZero() const {
230    return x_ == 0;
231  }
232
233  static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
234    u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
235    DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
236    return shifted_xor == 0;
237  }
238
239  static ALWAYS_INLINE
240  bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
241    u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
242    return masked_xor == 0;
243  }
244
245  static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
246      unsigned kS2AccessSize) {
247    bool res = false;
248    u64 diff = s1.addr0() - s2.addr0();
249    if ((s64)diff < 0) {  // s1.addr0 < s2.addr0  // NOLINT
250      // if (s1.addr0() + size1) > s2.addr0()) return true;
251      if (s1.size() > -diff)
252        res = true;
253    } else {
254      // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
255      if (kS2AccessSize > diff)
256        res = true;
257    }
258    DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
259    DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
260    return res;
261  }
262
263  u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
264  u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
265  bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
266  bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
267
268  // The idea behind the freed bit is as follows.
269  // When the memory is freed (or otherwise unaccessible) we write to the shadow
270  // values with tid/epoch related to the free and the freed bit set.
271  // During memory accesses processing the freed bit is considered
272  // as msb of tid. So any access races with shadow with freed bit set
273  // (it is as if write from a thread with which we never synchronized before).
274  // This allows us to detect accesses to freed memory w/o additional
275  // overheads in memory access processing and at the same time restore
276  // tid/epoch of free.
277  void MarkAsFreed() {
278     x_ |= kFreedBit;
279  }
280
281  bool IsFreed() const {
282    return x_ & kFreedBit;
283  }
284
285  bool GetFreedAndReset() {
286    bool res = x_ & kFreedBit;
287    x_ &= ~kFreedBit;
288    return res;
289  }
290
291  bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
292    bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
293        | (u64(kIsAtomic) << kAtomicShift));
294    DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
295    return v;
296  }
297
298  bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
299    bool v = ((x_ >> kReadShift) & 3)
300        <= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
301    DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
302        (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
303    return v;
304  }
305
306  bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
307    bool v = ((x_ >> kReadShift) & 3)
308        >= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
309    DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
310        (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
311    return v;
312  }
313
314 private:
315  static const u64 kReadShift   = 5 + kClkBits;
316  static const u64 kReadBit     = 1ull << kReadShift;
317  static const u64 kAtomicShift = 6 + kClkBits;
318  static const u64 kAtomicBit   = 1ull << kAtomicShift;
319
320  u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
321
322  static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
323    if (s1.addr0() == s2.addr0()) return true;
324    if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
325      return true;
326    if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
327      return true;
328    return false;
329  }
330};
331
332struct ThreadSignalContext;
333
334struct JmpBuf {
335  uptr sp;
336  uptr mangled_sp;
337  int int_signal_send;
338  bool in_blocking_func;
339  uptr in_signal_handler;
340  uptr *shadow_stack_pos;
341};
342
343// A Processor represents a physical thread, or a P for Go.
344// It is used to store internal resources like allocate cache, and does not
345// participate in race-detection logic (invisible to end user).
346// In C++ it is tied to an OS thread just like ThreadState, however ideally
347// it should be tied to a CPU (this way we will have fewer allocator caches).
348// In Go it is tied to a P, so there are significantly fewer Processor's than
349// ThreadState's (which are tied to Gs).
350// A ThreadState must be wired with a Processor to handle events.
351struct Processor {
352  ThreadState *thr; // currently wired thread, or nullptr
353#if !SANITIZER_GO
354  AllocatorCache alloc_cache;
355  InternalAllocatorCache internal_alloc_cache;
356#endif
357  DenseSlabAllocCache block_cache;
358  DenseSlabAllocCache sync_cache;
359  DenseSlabAllocCache clock_cache;
360  DDPhysicalThread *dd_pt;
361};
362
363#if !SANITIZER_GO
364// ScopedGlobalProcessor temporary setups a global processor for the current
365// thread, if it does not have one. Intended for interceptors that can run
366// at the very thread end, when we already destroyed the thread processor.
367struct ScopedGlobalProcessor {
368  ScopedGlobalProcessor();
369  ~ScopedGlobalProcessor();
370};
371#endif
372
373// This struct is stored in TLS.
374struct ThreadState {
375  FastState fast_state;
376  // Synch epoch represents the threads's epoch before the last synchronization
377  // action. It allows to reduce number of shadow state updates.
378  // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
379  // if we are processing write to X from the same thread at epoch=200,
380  // we do nothing, because both writes happen in the same 'synch epoch'.
381  // That is, if another memory access does not race with the former write,
382  // it does not race with the latter as well.
383  // QUESTION: can we can squeeze this into ThreadState::Fast?
384  // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
385  // taken by epoch between synchs.
386  // This way we can save one load from tls.
387  u64 fast_synch_epoch;
388  // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
389  // We do not distinguish beteween ignoring reads and writes
390  // for better performance.
391  int ignore_reads_and_writes;
392  int ignore_sync;
393  int suppress_reports;
394  // Go does not support ignores.
395#if !SANITIZER_GO
396  IgnoreSet mop_ignore_set;
397  IgnoreSet sync_ignore_set;
398#endif
399  // C/C++ uses fixed size shadow stack embed into Trace.
400  // Go uses malloc-allocated shadow stack with dynamic size.
401  uptr *shadow_stack;
402  uptr *shadow_stack_end;
403  uptr *shadow_stack_pos;
404  u64 *racy_shadow_addr;
405  u64 racy_state[2];
406  MutexSet mset;
407  ThreadClock clock;
408#if !SANITIZER_GO
409  Vector<JmpBuf> jmp_bufs;
410  int ignore_interceptors;
411#endif
412#if TSAN_COLLECT_STATS
413  u64 stat[StatCnt];
414#endif
415  const int tid;
416  const int unique_id;
417  bool in_symbolizer;
418  bool in_ignored_lib;
419  bool is_inited;
420  bool is_dead;
421  bool is_freeing;
422  bool is_vptr_access;
423  const uptr stk_addr;
424  const uptr stk_size;
425  const uptr tls_addr;
426  const uptr tls_size;
427  ThreadContext *tctx;
428
429#if SANITIZER_DEBUG && !SANITIZER_GO
430  InternalDeadlockDetector internal_deadlock_detector;
431#endif
432  DDLogicalThread *dd_lt;
433
434  // Current wired Processor, or nullptr. Required to handle any events.
435  Processor *proc1;
436#if !SANITIZER_GO
437  Processor *proc() { return proc1; }
438#else
439  Processor *proc();
440#endif
441
442  atomic_uintptr_t in_signal_handler;
443  ThreadSignalContext *signal_ctx;
444
445#if !SANITIZER_GO
446  u32 last_sleep_stack_id;
447  ThreadClock last_sleep_clock;
448#endif
449
450  // Set in regions of runtime that must be signal-safe and fork-safe.
451  // If set, malloc must not be called.
452  int nomalloc;
453
454  const ReportDesc *current_report;
455
456  explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
457                       unsigned reuse_count,
458                       uptr stk_addr, uptr stk_size,
459                       uptr tls_addr, uptr tls_size);
460};
461
462#if !SANITIZER_GO
463#if SANITIZER_MAC || SANITIZER_ANDROID
464ThreadState *cur_thread();
465void cur_thread_finalize();
466#else
467__attribute__((tls_model("initial-exec")))
468extern THREADLOCAL char cur_thread_placeholder[];
469INLINE ThreadState *cur_thread() {
470  return reinterpret_cast<ThreadState *>(&cur_thread_placeholder);
471}
472INLINE void cur_thread_finalize() { }
473#endif  // SANITIZER_MAC || SANITIZER_ANDROID
474#endif  // SANITIZER_GO
475
476class ThreadContext : public ThreadContextBase {
477 public:
478  explicit ThreadContext(int tid);
479  ~ThreadContext();
480  ThreadState *thr;
481  u32 creation_stack_id;
482  SyncClock sync;
483  // Epoch at which the thread had started.
484  // If we see an event from the thread stamped by an older epoch,
485  // the event is from a dead thread that shared tid with this thread.
486  u64 epoch0;
487  u64 epoch1;
488
489  // Override superclass callbacks.
490  void OnDead() override;
491  void OnJoined(void *arg) override;
492  void OnFinished() override;
493  void OnStarted(void *arg) override;
494  void OnCreated(void *arg) override;
495  void OnReset() override;
496  void OnDetached(void *arg) override;
497};
498
499struct RacyStacks {
500  MD5Hash hash[2];
501  bool operator==(const RacyStacks &other) const {
502    if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
503      return true;
504    if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
505      return true;
506    return false;
507  }
508};
509
510struct RacyAddress {
511  uptr addr_min;
512  uptr addr_max;
513};
514
515struct FiredSuppression {
516  ReportType type;
517  uptr pc_or_addr;
518  Suppression *supp;
519};
520
521struct Context {
522  Context();
523
524  bool initialized;
525#if !SANITIZER_GO
526  bool after_multithreaded_fork;
527#endif
528
529  MetaMap metamap;
530
531  Mutex report_mtx;
532  int nreported;
533  int nmissed_expected;
534  atomic_uint64_t last_symbolize_time_ns;
535
536  void *background_thread;
537  atomic_uint32_t stop_background_thread;
538
539  ThreadRegistry *thread_registry;
540
541  Mutex racy_mtx;
542  Vector<RacyStacks> racy_stacks;
543  Vector<RacyAddress> racy_addresses;
544  // Number of fired suppressions may be large enough.
545  Mutex fired_suppressions_mtx;
546  InternalMmapVector<FiredSuppression> fired_suppressions;
547  DDetector *dd;
548
549  ClockAlloc clock_alloc;
550
551  Flags flags;
552
553  u64 stat[StatCnt];
554  u64 int_alloc_cnt[MBlockTypeCount];
555  u64 int_alloc_siz[MBlockTypeCount];
556};
557
558extern Context *ctx;  // The one and the only global runtime context.
559
560ALWAYS_INLINE Flags *flags() {
561  return &ctx->flags;
562}
563
564struct ScopedIgnoreInterceptors {
565  ScopedIgnoreInterceptors() {
566#if !SANITIZER_GO
567    cur_thread()->ignore_interceptors++;
568#endif
569  }
570
571  ~ScopedIgnoreInterceptors() {
572#if !SANITIZER_GO
573    cur_thread()->ignore_interceptors--;
574#endif
575  }
576};
577
578const char *GetObjectTypeFromTag(uptr tag);
579const char *GetReportHeaderFromTag(uptr tag);
580uptr TagFromShadowStackFrame(uptr pc);
581
582class ScopedReportBase {
583 public:
584  void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
585                       const MutexSet *mset);
586  void AddStack(StackTrace stack, bool suppressable = false);
587  void AddThread(const ThreadContext *tctx, bool suppressable = false);
588  void AddThread(int unique_tid, bool suppressable = false);
589  void AddUniqueTid(int unique_tid);
590  void AddMutex(const SyncVar *s);
591  u64 AddMutex(u64 id);
592  void AddLocation(uptr addr, uptr size);
593  void AddSleep(u32 stack_id);
594  void SetCount(int count);
595
596  const ReportDesc *GetReport() const;
597
598 protected:
599  ScopedReportBase(ReportType typ, uptr tag);
600  ~ScopedReportBase();
601
602 private:
603  ReportDesc *rep_;
604  // Symbolizer makes lots of intercepted calls. If we try to process them,
605  // at best it will cause deadlocks on internal mutexes.
606  ScopedIgnoreInterceptors ignore_interceptors_;
607
608  void AddDeadMutex(u64 id);
609
610  ScopedReportBase(const ScopedReportBase &) = delete;
611  void operator=(const ScopedReportBase &) = delete;
612};
613
614class ScopedReport : public ScopedReportBase {
615 public:
616  explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
617  ~ScopedReport();
618
619 private:
620  ScopedErrorReportLock lock_;
621};
622
623ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
624void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
625                  MutexSet *mset, uptr *tag = nullptr);
626
627// The stack could look like:
628//   <start> | <main> | <foo> | tag | <bar>
629// This will extract the tag and keep:
630//   <start> | <main> | <foo> | <bar>
631template<typename StackTraceTy>
632void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
633  if (stack->size < 2) return;
634  uptr possible_tag_pc = stack->trace[stack->size - 2];
635  uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
636  if (possible_tag == kExternalTagNone) return;
637  stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
638  stack->size -= 1;
639  if (tag) *tag = possible_tag;
640}
641
642template<typename StackTraceTy>
643void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
644                        uptr *tag = nullptr) {
645  uptr size = thr->shadow_stack_pos - thr->shadow_stack;
646  uptr start = 0;
647  if (size + !!toppc > kStackTraceMax) {
648    start = size + !!toppc - kStackTraceMax;
649    size = kStackTraceMax - !!toppc;
650  }
651  stack->Init(&thr->shadow_stack[start], size, toppc);
652  ExtractTagFromStack(stack, tag);
653}
654
655#define GET_STACK_TRACE_FATAL(thr, pc) \
656  VarSizeStackTrace stack; \
657  ObtainCurrentStack(thr, pc, &stack); \
658  stack.ReverseOrder();
659
660#if TSAN_COLLECT_STATS
661void StatAggregate(u64 *dst, u64 *src);
662void StatOutput(u64 *stat);
663#endif
664
665void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) {
666#if TSAN_COLLECT_STATS
667  thr->stat[typ] += n;
668#endif
669}
670void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) {
671#if TSAN_COLLECT_STATS
672  thr->stat[typ] = n;
673#endif
674}
675
676void MapShadow(uptr addr, uptr size);
677void MapThreadTrace(uptr addr, uptr size, const char *name);
678void DontNeedShadowFor(uptr addr, uptr size);
679void InitializeShadowMemory();
680void InitializeInterceptors();
681void InitializeLibIgnore();
682void InitializeDynamicAnnotations();
683
684void ForkBefore(ThreadState *thr, uptr pc);
685void ForkParentAfter(ThreadState *thr, uptr pc);
686void ForkChildAfter(ThreadState *thr, uptr pc);
687
688void ReportRace(ThreadState *thr);
689bool OutputReport(ThreadState *thr, const ScopedReport &srep);
690bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
691bool IsExpectedReport(uptr addr, uptr size);
692void PrintMatchedBenignRaces();
693
694#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
695# define DPrintf Printf
696#else
697# define DPrintf(...)
698#endif
699
700#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
701# define DPrintf2 Printf
702#else
703# define DPrintf2(...)
704#endif
705
706u32 CurrentStackId(ThreadState *thr, uptr pc);
707ReportStack *SymbolizeStackId(u32 stack_id);
708void PrintCurrentStack(ThreadState *thr, uptr pc);
709void PrintCurrentStackSlow(uptr pc);  // uses libunwind
710
711void Initialize(ThreadState *thr);
712void MaybeSpawnBackgroundThread();
713int Finalize(ThreadState *thr);
714
715void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
716void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
717
718void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
719    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
720void MemoryAccessImpl(ThreadState *thr, uptr addr,
721    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
722    u64 *shadow_mem, Shadow cur);
723void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
724    uptr size, bool is_write);
725void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr,
726    uptr size, uptr step, bool is_write);
727void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
728    int size, bool kAccessIsWrite, bool kIsAtomic);
729
730const int kSizeLog1 = 0;
731const int kSizeLog2 = 1;
732const int kSizeLog4 = 2;
733const int kSizeLog8 = 3;
734
735void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc,
736                                     uptr addr, int kAccessSizeLog) {
737  MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false);
738}
739
740void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc,
741                                      uptr addr, int kAccessSizeLog) {
742  MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false);
743}
744
745void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc,
746                                           uptr addr, int kAccessSizeLog) {
747  MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true);
748}
749
750void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
751                                            uptr addr, int kAccessSizeLog) {
752  MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
753}
754
755void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
756void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
757void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
758
759void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
760void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
761void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
762void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
763
764void FuncEntry(ThreadState *thr, uptr pc);
765void FuncExit(ThreadState *thr);
766
767int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
768void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread);
769void ThreadFinish(ThreadState *thr);
770int ThreadTid(ThreadState *thr, uptr pc, uptr uid);
771void ThreadJoin(ThreadState *thr, uptr pc, int tid);
772void ThreadDetach(ThreadState *thr, uptr pc, int tid);
773void ThreadFinalize(ThreadState *thr);
774void ThreadSetName(ThreadState *thr, const char *name);
775int ThreadCount(ThreadState *thr);
776void ProcessPendingSignals(ThreadState *thr);
777void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid);
778
779Processor *ProcCreate();
780void ProcDestroy(Processor *proc);
781void ProcWire(Processor *proc, ThreadState *thr);
782void ProcUnwire(Processor *proc, ThreadState *thr);
783
784// Note: the parameter is called flagz, because flags is already taken
785// by the global function that returns flags.
786void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
787void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
788void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
789void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
790    int rec = 1);
791int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
792void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
793void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
794void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
795void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
796void MutexRepair(ThreadState *thr, uptr pc, uptr addr);  // call on EOWNERDEAD
797void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
798
799void Acquire(ThreadState *thr, uptr pc, uptr addr);
800// AcquireGlobal synchronizes the current thread with all other threads.
801// In terms of happens-before relation, it draws a HB edge from all threads
802// (where they happen to execute right now) to the current thread. We use it to
803// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
804// right before executing finalizers. This provides a coarse, but simple
805// approximation of the actual required synchronization.
806void AcquireGlobal(ThreadState *thr, uptr pc);
807void Release(ThreadState *thr, uptr pc, uptr addr);
808void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
809void AfterSleep(ThreadState *thr, uptr pc);
810void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
811void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
812void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
813void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
814
815// The hacky call uses custom calling convention and an assembly thunk.
816// It is considerably faster that a normal call for the caller
817// if it is not executed (it is intended for slow paths from hot functions).
818// The trick is that the call preserves all registers and the compiler
819// does not treat it as a call.
820// If it does not work for you, use normal call.
821#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
822// The caller may not create the stack frame for itself at all,
823// so we create a reserve stack frame for it (1024b must be enough).
824#define HACKY_CALL(f) \
825  __asm__ __volatile__("sub $1024, %%rsp;" \
826                       CFI_INL_ADJUST_CFA_OFFSET(1024) \
827                       ".hidden " #f "_thunk;" \
828                       "call " #f "_thunk;" \
829                       "add $1024, %%rsp;" \
830                       CFI_INL_ADJUST_CFA_OFFSET(-1024) \
831                       ::: "memory", "cc");
832#else
833#define HACKY_CALL(f) f()
834#endif
835
836void TraceSwitch(ThreadState *thr);
837uptr TraceTopPC(ThreadState *thr);
838uptr TraceSize();
839uptr TraceParts();
840Trace *ThreadTrace(int tid);
841
842extern "C" void __tsan_trace_switch();
843void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
844                                        EventType typ, u64 addr) {
845  if (!kCollectHistory)
846    return;
847  DCHECK_GE((int)typ, 0);
848  DCHECK_LE((int)typ, 7);
849  DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
850  StatInc(thr, StatEvents);
851  u64 pos = fs.GetTracePos();
852  if (UNLIKELY((pos % kTracePartSize) == 0)) {
853#if !SANITIZER_GO
854    HACKY_CALL(__tsan_trace_switch);
855#else
856    TraceSwitch(thr);
857#endif
858  }
859  Event *trace = (Event*)GetThreadTrace(fs.tid());
860  Event *evp = &trace[pos];
861  Event ev = (u64)addr | ((u64)typ << kEventPCBits);
862  *evp = ev;
863}
864
865#if !SANITIZER_GO
866uptr ALWAYS_INLINE HeapEnd() {
867  return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
868}
869#endif
870
871}  // namespace __tsan
872
873#endif  // TSAN_RTL_H
874