1//=-- lsan_allocator.cpp --------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of LeakSanitizer.
10// See lsan_allocator.h for details.
11//
12//===----------------------------------------------------------------------===//
13
14#include "lsan_allocator.h"
15
16#include "sanitizer_common/sanitizer_allocator.h"
17#include "sanitizer_common/sanitizer_allocator_checks.h"
18#include "sanitizer_common/sanitizer_allocator_interface.h"
19#include "sanitizer_common/sanitizer_allocator_report.h"
20#include "sanitizer_common/sanitizer_errno.h"
21#include "sanitizer_common/sanitizer_internal_defs.h"
22#include "sanitizer_common/sanitizer_stackdepot.h"
23#include "sanitizer_common/sanitizer_stacktrace.h"
24#include "lsan_common.h"
25
26extern "C" void *memset(void *ptr, int value, uptr num);
27
28namespace __lsan {
29#if defined(__i386__) || defined(__arm__)
30static const uptr kMaxAllowedMallocSize = 1UL << 30;
31#elif defined(__mips64) || defined(__aarch64__)
32static const uptr kMaxAllowedMallocSize = 4UL << 30;
33#else
34static const uptr kMaxAllowedMallocSize = 8UL << 30;
35#endif
36
37static Allocator allocator;
38
39static uptr max_malloc_size;
40
41void InitializeAllocator() {
42  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
43  allocator.InitLinkerInitialized(
44      common_flags()->allocator_release_to_os_interval_ms);
45  if (common_flags()->max_allocation_size_mb)
46    max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
47                          kMaxAllowedMallocSize);
48  else
49    max_malloc_size = kMaxAllowedMallocSize;
50}
51
52void AllocatorThreadFinish() {
53  allocator.SwallowCache(GetAllocatorCache());
54}
55
56static ChunkMetadata *Metadata(const void *p) {
57  return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p));
58}
59
60static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) {
61  if (!p) return;
62  ChunkMetadata *m = Metadata(p);
63  CHECK(m);
64  m->tag = DisabledInThisThread() ? kIgnored : kDirectlyLeaked;
65  m->stack_trace_id = StackDepotPut(stack);
66  m->requested_size = size;
67  atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed);
68}
69
70static void RegisterDeallocation(void *p) {
71  if (!p) return;
72  ChunkMetadata *m = Metadata(p);
73  CHECK(m);
74  atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed);
75}
76
77static void *ReportAllocationSizeTooBig(uptr size, const StackTrace &stack) {
78  if (AllocatorMayReturnNull()) {
79    Report("WARNING: LeakSanitizer failed to allocate 0x%zx bytes\n", size);
80    return nullptr;
81  }
82  ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
83}
84
85void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
86               bool cleared) {
87  if (size == 0)
88    size = 1;
89  if (size > max_malloc_size)
90    return ReportAllocationSizeTooBig(size, stack);
91  void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
92  if (UNLIKELY(!p)) {
93    SetAllocatorOutOfMemory();
94    if (AllocatorMayReturnNull())
95      return nullptr;
96    ReportOutOfMemory(size, &stack);
97  }
98  // Do not rely on the allocator to clear the memory (it's slow).
99  if (cleared && allocator.FromPrimary(p))
100    memset(p, 0, size);
101  RegisterAllocation(stack, p, size);
102  if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size);
103  RunMallocHooks(p, size);
104  return p;
105}
106
107static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) {
108  if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
109    if (AllocatorMayReturnNull())
110      return nullptr;
111    ReportCallocOverflow(nmemb, size, &stack);
112  }
113  size *= nmemb;
114  return Allocate(stack, size, 1, true);
115}
116
117void Deallocate(void *p) {
118  if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
119  RunFreeHooks(p);
120  RegisterDeallocation(p);
121  allocator.Deallocate(GetAllocatorCache(), p);
122}
123
124void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
125                 uptr alignment) {
126  if (new_size > max_malloc_size) {
127    ReportAllocationSizeTooBig(new_size, stack);
128    return nullptr;
129  }
130  RegisterDeallocation(p);
131  void *new_p =
132      allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
133  if (new_p)
134    RegisterAllocation(stack, new_p, new_size);
135  else if (new_size != 0)
136    RegisterAllocation(stack, p, new_size);
137  return new_p;
138}
139
140void GetAllocatorCacheRange(uptr *begin, uptr *end) {
141  *begin = (uptr)GetAllocatorCache();
142  *end = *begin + sizeof(AllocatorCache);
143}
144
145uptr GetMallocUsableSize(const void *p) {
146  ChunkMetadata *m = Metadata(p);
147  if (!m) return 0;
148  return m->requested_size;
149}
150
151int lsan_posix_memalign(void **memptr, uptr alignment, uptr size,
152                        const StackTrace &stack) {
153  if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
154    if (AllocatorMayReturnNull())
155      return errno_EINVAL;
156    ReportInvalidPosixMemalignAlignment(alignment, &stack);
157  }
158  void *ptr = Allocate(stack, size, alignment, kAlwaysClearMemory);
159  if (UNLIKELY(!ptr))
160    // OOM error is already taken care of by Allocate.
161    return errno_ENOMEM;
162  CHECK(IsAligned((uptr)ptr, alignment));
163  *memptr = ptr;
164  return 0;
165}
166
167void *lsan_aligned_alloc(uptr alignment, uptr size, const StackTrace &stack) {
168  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
169    errno = errno_EINVAL;
170    if (AllocatorMayReturnNull())
171      return nullptr;
172    ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
173  }
174  return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory));
175}
176
177void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) {
178  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
179    errno = errno_EINVAL;
180    if (AllocatorMayReturnNull())
181      return nullptr;
182    ReportInvalidAllocationAlignment(alignment, &stack);
183  }
184  return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory));
185}
186
187void *lsan_malloc(uptr size, const StackTrace &stack) {
188  return SetErrnoOnNull(Allocate(stack, size, 1, kAlwaysClearMemory));
189}
190
191void lsan_free(void *p) {
192  Deallocate(p);
193}
194
195void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
196  return SetErrnoOnNull(Reallocate(stack, p, size, 1));
197}
198
199void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size,
200                        const StackTrace &stack) {
201  if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
202    errno = errno_ENOMEM;
203    if (AllocatorMayReturnNull())
204      return nullptr;
205    ReportReallocArrayOverflow(nmemb, size, &stack);
206  }
207  return lsan_realloc(ptr, nmemb * size, stack);
208}
209
210void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
211  return SetErrnoOnNull(Calloc(nmemb, size, stack));
212}
213
214void *lsan_valloc(uptr size, const StackTrace &stack) {
215  return SetErrnoOnNull(
216      Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory));
217}
218
219void *lsan_pvalloc(uptr size, const StackTrace &stack) {
220  uptr PageSize = GetPageSizeCached();
221  if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
222    errno = errno_ENOMEM;
223    if (AllocatorMayReturnNull())
224      return nullptr;
225    ReportPvallocOverflow(size, &stack);
226  }
227  // pvalloc(0) should allocate one page.
228  size = size ? RoundUpTo(size, PageSize) : PageSize;
229  return SetErrnoOnNull(Allocate(stack, size, PageSize, kAlwaysClearMemory));
230}
231
232uptr lsan_mz_size(const void *p) {
233  return GetMallocUsableSize(p);
234}
235
236///// Interface to the common LSan module. /////
237
238void LockAllocator() {
239  allocator.ForceLock();
240}
241
242void UnlockAllocator() {
243  allocator.ForceUnlock();
244}
245
246void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
247  *begin = (uptr)&allocator;
248  *end = *begin + sizeof(allocator);
249}
250
251uptr PointsIntoChunk(void* p) {
252  uptr addr = reinterpret_cast<uptr>(p);
253  uptr chunk = reinterpret_cast<uptr>(allocator.GetBlockBeginFastLocked(p));
254  if (!chunk) return 0;
255  // LargeMmapAllocator considers pointers to the meta-region of a chunk to be
256  // valid, but we don't want that.
257  if (addr < chunk) return 0;
258  ChunkMetadata *m = Metadata(reinterpret_cast<void *>(chunk));
259  CHECK(m);
260  if (!m->allocated)
261    return 0;
262  if (addr < chunk + m->requested_size)
263    return chunk;
264  if (IsSpecialCaseOfOperatorNew0(chunk, m->requested_size, addr))
265    return chunk;
266  return 0;
267}
268
269uptr GetUserBegin(uptr chunk) {
270  return chunk;
271}
272
273LsanMetadata::LsanMetadata(uptr chunk) {
274  metadata_ = Metadata(reinterpret_cast<void *>(chunk));
275  CHECK(metadata_);
276}
277
278bool LsanMetadata::allocated() const {
279  return reinterpret_cast<ChunkMetadata *>(metadata_)->allocated;
280}
281
282ChunkTag LsanMetadata::tag() const {
283  return reinterpret_cast<ChunkMetadata *>(metadata_)->tag;
284}
285
286void LsanMetadata::set_tag(ChunkTag value) {
287  reinterpret_cast<ChunkMetadata *>(metadata_)->tag = value;
288}
289
290uptr LsanMetadata::requested_size() const {
291  return reinterpret_cast<ChunkMetadata *>(metadata_)->requested_size;
292}
293
294u32 LsanMetadata::stack_trace_id() const {
295  return reinterpret_cast<ChunkMetadata *>(metadata_)->stack_trace_id;
296}
297
298void ForEachChunk(ForEachChunkCallback callback, void *arg) {
299  allocator.ForEachChunk(callback, arg);
300}
301
302IgnoreObjectResult IgnoreObjectLocked(const void *p) {
303  void *chunk = allocator.GetBlockBegin(p);
304  if (!chunk || p < chunk) return kIgnoreObjectInvalid;
305  ChunkMetadata *m = Metadata(chunk);
306  CHECK(m);
307  if (m->allocated && (uptr)p < (uptr)chunk + m->requested_size) {
308    if (m->tag == kIgnored)
309      return kIgnoreObjectAlreadyIgnored;
310    m->tag = kIgnored;
311    return kIgnoreObjectSuccess;
312  } else {
313    return kIgnoreObjectInvalid;
314  }
315}
316
317void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
318  // This function can be used to treat memory reachable from `tctx` as live.
319  // This is useful for threads that have been created but not yet started.
320
321  // This is currently a no-op because the LSan `pthread_create()` interceptor
322  // blocks until the child thread starts which keeps the thread's `arg` pointer
323  // live.
324}
325
326} // namespace __lsan
327
328using namespace __lsan;
329
330extern "C" {
331SANITIZER_INTERFACE_ATTRIBUTE
332uptr __sanitizer_get_current_allocated_bytes() {
333  uptr stats[AllocatorStatCount];
334  allocator.GetStats(stats);
335  return stats[AllocatorStatAllocated];
336}
337
338SANITIZER_INTERFACE_ATTRIBUTE
339uptr __sanitizer_get_heap_size() {
340  uptr stats[AllocatorStatCount];
341  allocator.GetStats(stats);
342  return stats[AllocatorStatMapped];
343}
344
345SANITIZER_INTERFACE_ATTRIBUTE
346uptr __sanitizer_get_free_bytes() { return 0; }
347
348SANITIZER_INTERFACE_ATTRIBUTE
349uptr __sanitizer_get_unmapped_bytes() { return 0; }
350
351SANITIZER_INTERFACE_ATTRIBUTE
352uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
353
354SANITIZER_INTERFACE_ATTRIBUTE
355int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; }
356
357SANITIZER_INTERFACE_ATTRIBUTE
358uptr __sanitizer_get_allocated_size(const void *p) {
359  return GetMallocUsableSize(p);
360}
361
362#if !SANITIZER_SUPPORTS_WEAK_HOOKS
363// Provide default (no-op) implementation of malloc hooks.
364SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
365void __sanitizer_malloc_hook(void *ptr, uptr size) {
366  (void)ptr;
367  (void)size;
368}
369SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
370void __sanitizer_free_hook(void *ptr) {
371  (void)ptr;
372}
373#endif
374} // extern "C"
375