1//===-- tsan_mman.cpp -----------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11//===----------------------------------------------------------------------===//
12#include "sanitizer_common/sanitizer_allocator_checks.h"
13#include "sanitizer_common/sanitizer_allocator_interface.h"
14#include "sanitizer_common/sanitizer_allocator_report.h"
15#include "sanitizer_common/sanitizer_common.h"
16#include "sanitizer_common/sanitizer_errno.h"
17#include "sanitizer_common/sanitizer_placement_new.h"
18#include "tsan_mman.h"
19#include "tsan_rtl.h"
20#include "tsan_report.h"
21#include "tsan_flags.h"
22
23// May be overriden by front-end.
24SANITIZER_WEAK_DEFAULT_IMPL
25void __sanitizer_malloc_hook(void *ptr, uptr size) {
26  (void)ptr;
27  (void)size;
28}
29
30SANITIZER_WEAK_DEFAULT_IMPL
31void __sanitizer_free_hook(void *ptr) {
32  (void)ptr;
33}
34
35namespace __tsan {
36
37struct MapUnmapCallback {
38  void OnMap(uptr p, uptr size) const { }
39  void OnUnmap(uptr p, uptr size) const {
40    // We are about to unmap a chunk of user memory.
41    // Mark the corresponding shadow memory as not needed.
42    DontNeedShadowFor(p, size);
43    // Mark the corresponding meta shadow memory as not needed.
44    // Note the block does not contain any meta info at this point
45    // (this happens after free).
46    const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
47    const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
48    // Block came from LargeMmapAllocator, so must be large.
49    // We rely on this in the calculations below.
50    CHECK_GE(size, 2 * kPageSize);
51    uptr diff = RoundUp(p, kPageSize) - p;
52    if (diff != 0) {
53      p += diff;
54      size -= diff;
55    }
56    diff = p + size - RoundDown(p + size, kPageSize);
57    if (diff != 0)
58      size -= diff;
59    uptr p_meta = (uptr)MemToMeta(p);
60    ReleaseMemoryPagesToOS(p_meta, p_meta + size / kMetaRatio);
61  }
62};
63
64static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
65Allocator *allocator() {
66  return reinterpret_cast<Allocator*>(&allocator_placeholder);
67}
68
69struct GlobalProc {
70  Mutex mtx;
71  Processor *proc;
72
73  GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}
74};
75
76static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
77GlobalProc *global_proc() {
78  return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
79}
80
81ScopedGlobalProcessor::ScopedGlobalProcessor() {
82  GlobalProc *gp = global_proc();
83  ThreadState *thr = cur_thread();
84  if (thr->proc())
85    return;
86  // If we don't have a proc, use the global one.
87  // There are currently only two known case where this path is triggered:
88  //   __interceptor_free
89  //   __nptl_deallocate_tsd
90  //   start_thread
91  //   clone
92  // and:
93  //   ResetRange
94  //   __interceptor_munmap
95  //   __deallocate_stack
96  //   start_thread
97  //   clone
98  // Ideally, we destroy thread state (and unwire proc) when a thread actually
99  // exits (i.e. when we join/wait it). Then we would not need the global proc
100  gp->mtx.Lock();
101  ProcWire(gp->proc, thr);
102}
103
104ScopedGlobalProcessor::~ScopedGlobalProcessor() {
105  GlobalProc *gp = global_proc();
106  ThreadState *thr = cur_thread();
107  if (thr->proc() != gp->proc)
108    return;
109  ProcUnwire(gp->proc, thr);
110  gp->mtx.Unlock();
111}
112
113static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
114static uptr max_user_defined_malloc_size;
115
116void InitializeAllocator() {
117  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
118  allocator()->Init(common_flags()->allocator_release_to_os_interval_ms);
119  max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
120                                     ? common_flags()->max_allocation_size_mb
121                                           << 20
122                                     : kMaxAllowedMallocSize;
123}
124
125void InitializeAllocatorLate() {
126  new(global_proc()) GlobalProc();
127}
128
129void AllocatorProcStart(Processor *proc) {
130  allocator()->InitCache(&proc->alloc_cache);
131  internal_allocator()->InitCache(&proc->internal_alloc_cache);
132}
133
134void AllocatorProcFinish(Processor *proc) {
135  allocator()->DestroyCache(&proc->alloc_cache);
136  internal_allocator()->DestroyCache(&proc->internal_alloc_cache);
137}
138
139void AllocatorPrintStats() {
140  allocator()->PrintStats();
141}
142
143static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
144  if (atomic_load_relaxed(&thr->in_signal_handler) == 0 ||
145      !ShouldReport(thr, ReportTypeSignalUnsafe))
146    return;
147  VarSizeStackTrace stack;
148  ObtainCurrentStack(thr, pc, &stack);
149  if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
150    return;
151  ThreadRegistryLock l(&ctx->thread_registry);
152  ScopedReport rep(ReportTypeSignalUnsafe);
153  rep.AddStack(stack, true);
154  OutputReport(thr, rep);
155}
156
157
158void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
159                          bool signal) {
160  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize ||
161      sz > max_user_defined_malloc_size) {
162    if (AllocatorMayReturnNull())
163      return nullptr;
164    uptr malloc_limit =
165        Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
166    GET_STACK_TRACE_FATAL(thr, pc);
167    ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
168  }
169  void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
170  if (UNLIKELY(!p)) {
171    SetAllocatorOutOfMemory();
172    if (AllocatorMayReturnNull())
173      return nullptr;
174    GET_STACK_TRACE_FATAL(thr, pc);
175    ReportOutOfMemory(sz, &stack);
176  }
177  if (ctx && ctx->initialized)
178    OnUserAlloc(thr, pc, (uptr)p, sz, true);
179  if (signal)
180    SignalUnsafeCall(thr, pc);
181  return p;
182}
183
184void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
185  ScopedGlobalProcessor sgp;
186  if (ctx && ctx->initialized)
187    OnUserFree(thr, pc, (uptr)p, true);
188  allocator()->Deallocate(&thr->proc()->alloc_cache, p);
189  if (signal)
190    SignalUnsafeCall(thr, pc);
191}
192
193void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
194  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment));
195}
196
197void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
198  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
199    if (AllocatorMayReturnNull())
200      return SetErrnoOnNull(nullptr);
201    GET_STACK_TRACE_FATAL(thr, pc);
202    ReportCallocOverflow(n, size, &stack);
203  }
204  void *p = user_alloc_internal(thr, pc, n * size);
205  if (p)
206    internal_memset(p, 0, n * size);
207  return SetErrnoOnNull(p);
208}
209
210void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) {
211  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
212    if (AllocatorMayReturnNull())
213      return SetErrnoOnNull(nullptr);
214    GET_STACK_TRACE_FATAL(thr, pc);
215    ReportReallocArrayOverflow(size, n, &stack);
216  }
217  return user_realloc(thr, pc, p, size * n);
218}
219
220void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
221  DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p);
222  ctx->metamap.AllocBlock(thr, pc, p, sz);
223  if (write && thr->ignore_reads_and_writes == 0)
224    MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
225  else
226    MemoryResetRange(thr, pc, (uptr)p, sz);
227}
228
229void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
230  CHECK_NE(p, (void*)0);
231  uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
232  DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz);
233  if (write && thr->ignore_reads_and_writes == 0)
234    MemoryRangeFreed(thr, pc, (uptr)p, sz);
235}
236
237void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
238  // FIXME: Handle "shrinking" more efficiently,
239  // it seems that some software actually does this.
240  if (!p)
241    return SetErrnoOnNull(user_alloc_internal(thr, pc, sz));
242  if (!sz) {
243    user_free(thr, pc, p);
244    return nullptr;
245  }
246  void *new_p = user_alloc_internal(thr, pc, sz);
247  if (new_p) {
248    uptr old_sz = user_alloc_usable_size(p);
249    internal_memcpy(new_p, p, min(old_sz, sz));
250    user_free(thr, pc, p);
251  }
252  return SetErrnoOnNull(new_p);
253}
254
255void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
256  if (UNLIKELY(!IsPowerOfTwo(align))) {
257    errno = errno_EINVAL;
258    if (AllocatorMayReturnNull())
259      return nullptr;
260    GET_STACK_TRACE_FATAL(thr, pc);
261    ReportInvalidAllocationAlignment(align, &stack);
262  }
263  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
264}
265
266int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
267                        uptr sz) {
268  if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
269    if (AllocatorMayReturnNull())
270      return errno_EINVAL;
271    GET_STACK_TRACE_FATAL(thr, pc);
272    ReportInvalidPosixMemalignAlignment(align, &stack);
273  }
274  void *ptr = user_alloc_internal(thr, pc, sz, align);
275  if (UNLIKELY(!ptr))
276    // OOM error is already taken care of by user_alloc_internal.
277    return errno_ENOMEM;
278  CHECK(IsAligned((uptr)ptr, align));
279  *memptr = ptr;
280  return 0;
281}
282
283void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
284  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
285    errno = errno_EINVAL;
286    if (AllocatorMayReturnNull())
287      return nullptr;
288    GET_STACK_TRACE_FATAL(thr, pc);
289    ReportInvalidAlignedAllocAlignment(sz, align, &stack);
290  }
291  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
292}
293
294void *user_valloc(ThreadState *thr, uptr pc, uptr sz) {
295  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached()));
296}
297
298void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
299  uptr PageSize = GetPageSizeCached();
300  if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
301    errno = errno_ENOMEM;
302    if (AllocatorMayReturnNull())
303      return nullptr;
304    GET_STACK_TRACE_FATAL(thr, pc);
305    ReportPvallocOverflow(sz, &stack);
306  }
307  // pvalloc(0) should allocate one page.
308  sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
309  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
310}
311
312uptr user_alloc_usable_size(const void *p) {
313  if (p == 0)
314    return 0;
315  MBlock *b = ctx->metamap.GetBlock((uptr)p);
316  if (!b)
317    return 0;  // Not a valid pointer.
318  if (b->siz == 0)
319    return 1;  // Zero-sized allocations are actually 1 byte.
320  return b->siz;
321}
322
323void invoke_malloc_hook(void *ptr, uptr size) {
324  ThreadState *thr = cur_thread();
325  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
326    return;
327  __sanitizer_malloc_hook(ptr, size);
328  RunMallocHooks(ptr, size);
329}
330
331void invoke_free_hook(void *ptr) {
332  ThreadState *thr = cur_thread();
333  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
334    return;
335  __sanitizer_free_hook(ptr);
336  RunFreeHooks(ptr);
337}
338
339void *Alloc(uptr sz) {
340  ThreadState *thr = cur_thread();
341  if (thr->nomalloc) {
342    thr->nomalloc = 0;  // CHECK calls internal_malloc().
343    CHECK(0);
344  }
345  return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
346}
347
348void FreeImpl(void *p) {
349  ThreadState *thr = cur_thread();
350  if (thr->nomalloc) {
351    thr->nomalloc = 0;  // CHECK calls internal_malloc().
352    CHECK(0);
353  }
354  InternalFree(p, &thr->proc()->internal_alloc_cache);
355}
356
357}  // namespace __tsan
358
359using namespace __tsan;
360
361extern "C" {
362uptr __sanitizer_get_current_allocated_bytes() {
363  uptr stats[AllocatorStatCount];
364  allocator()->GetStats(stats);
365  return stats[AllocatorStatAllocated];
366}
367
368uptr __sanitizer_get_heap_size() {
369  uptr stats[AllocatorStatCount];
370  allocator()->GetStats(stats);
371  return stats[AllocatorStatMapped];
372}
373
374uptr __sanitizer_get_free_bytes() {
375  return 1;
376}
377
378uptr __sanitizer_get_unmapped_bytes() {
379  return 1;
380}
381
382uptr __sanitizer_get_estimated_allocated_size(uptr size) {
383  return size;
384}
385
386int __sanitizer_get_ownership(const void *p) {
387  return allocator()->GetBlockBegin(p) != 0;
388}
389
390uptr __sanitizer_get_allocated_size(const void *p) {
391  return user_alloc_usable_size(p);
392}
393
394void __tsan_on_thread_idle() {
395  ThreadState *thr = cur_thread();
396  thr->clock.ResetCached(&thr->proc()->clock_cache);
397  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
398  allocator()->SwallowCache(&thr->proc()->alloc_cache);
399  internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
400  ctx->metamap.OnProcIdle(thr->proc());
401}
402}  // extern "C"
403