1//=-- lsan_allocator.cpp --------------------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of LeakSanitizer. 10// See lsan_allocator.h for details. 11// 12//===----------------------------------------------------------------------===// 13 14#include "lsan_allocator.h" 15 16#include "sanitizer_common/sanitizer_allocator.h" 17#include "sanitizer_common/sanitizer_allocator_checks.h" 18#include "sanitizer_common/sanitizer_allocator_interface.h" 19#include "sanitizer_common/sanitizer_allocator_report.h" 20#include "sanitizer_common/sanitizer_errno.h" 21#include "sanitizer_common/sanitizer_internal_defs.h" 22#include "sanitizer_common/sanitizer_stackdepot.h" 23#include "sanitizer_common/sanitizer_stacktrace.h" 24#include "lsan_common.h" 25 26extern "C" void *memset(void *ptr, int value, uptr num); 27 28namespace __lsan { 29#if defined(__i386__) || defined(__arm__) 30static const uptr kMaxAllowedMallocSize = 1UL << 30; 31#elif defined(__mips64) || defined(__aarch64__) 32static const uptr kMaxAllowedMallocSize = 4UL << 30; 33#else 34static const uptr kMaxAllowedMallocSize = 8UL << 30; 35#endif 36 37static Allocator allocator; 38 39static uptr max_malloc_size; 40 41void InitializeAllocator() { 42 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); 43 allocator.InitLinkerInitialized( 44 common_flags()->allocator_release_to_os_interval_ms); 45 if (common_flags()->max_allocation_size_mb) 46 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20, 47 kMaxAllowedMallocSize); 48 else 49 max_malloc_size = kMaxAllowedMallocSize; 50} 51 52void AllocatorThreadFinish() { 53 allocator.SwallowCache(GetAllocatorCache()); 54} 55 56static ChunkMetadata *Metadata(const void *p) { 57 return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p)); 58} 59 60static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) { 61 if (!p) return; 62 ChunkMetadata *m = Metadata(p); 63 CHECK(m); 64 m->tag = DisabledInThisThread() ? kIgnored : kDirectlyLeaked; 65 m->stack_trace_id = StackDepotPut(stack); 66 m->requested_size = size; 67 atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed); 68} 69 70static void RegisterDeallocation(void *p) { 71 if (!p) return; 72 ChunkMetadata *m = Metadata(p); 73 CHECK(m); 74 atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed); 75} 76 77static void *ReportAllocationSizeTooBig(uptr size, const StackTrace &stack) { 78 if (AllocatorMayReturnNull()) { 79 Report("WARNING: LeakSanitizer failed to allocate 0x%zx bytes\n", size); 80 return nullptr; 81 } 82 ReportAllocationSizeTooBig(size, max_malloc_size, &stack); 83} 84 85void *Allocate(const StackTrace &stack, uptr size, uptr alignment, 86 bool cleared) { 87 if (size == 0) 88 size = 1; 89 if (size > max_malloc_size) 90 return ReportAllocationSizeTooBig(size, stack); 91 void *p = allocator.Allocate(GetAllocatorCache(), size, alignment); 92 if (UNLIKELY(!p)) { 93 SetAllocatorOutOfMemory(); 94 if (AllocatorMayReturnNull()) 95 return nullptr; 96 ReportOutOfMemory(size, &stack); 97 } 98 // Do not rely on the allocator to clear the memory (it's slow). 99 if (cleared && allocator.FromPrimary(p)) 100 memset(p, 0, size); 101 RegisterAllocation(stack, p, size); 102 if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size); 103 RunMallocHooks(p, size); 104 return p; 105} 106 107static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) { 108 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 109 if (AllocatorMayReturnNull()) 110 return nullptr; 111 ReportCallocOverflow(nmemb, size, &stack); 112 } 113 size *= nmemb; 114 return Allocate(stack, size, 1, true); 115} 116 117void Deallocate(void *p) { 118 if (&__sanitizer_free_hook) __sanitizer_free_hook(p); 119 RunFreeHooks(p); 120 RegisterDeallocation(p); 121 allocator.Deallocate(GetAllocatorCache(), p); 122} 123 124void *Reallocate(const StackTrace &stack, void *p, uptr new_size, 125 uptr alignment) { 126 if (new_size > max_malloc_size) { 127 ReportAllocationSizeTooBig(new_size, stack); 128 return nullptr; 129 } 130 RegisterDeallocation(p); 131 void *new_p = 132 allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment); 133 if (new_p) 134 RegisterAllocation(stack, new_p, new_size); 135 else if (new_size != 0) 136 RegisterAllocation(stack, p, new_size); 137 return new_p; 138} 139 140void GetAllocatorCacheRange(uptr *begin, uptr *end) { 141 *begin = (uptr)GetAllocatorCache(); 142 *end = *begin + sizeof(AllocatorCache); 143} 144 145uptr GetMallocUsableSize(const void *p) { 146 ChunkMetadata *m = Metadata(p); 147 if (!m) return 0; 148 return m->requested_size; 149} 150 151int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, 152 const StackTrace &stack) { 153 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { 154 if (AllocatorMayReturnNull()) 155 return errno_EINVAL; 156 ReportInvalidPosixMemalignAlignment(alignment, &stack); 157 } 158 void *ptr = Allocate(stack, size, alignment, kAlwaysClearMemory); 159 if (UNLIKELY(!ptr)) 160 // OOM error is already taken care of by Allocate. 161 return errno_ENOMEM; 162 CHECK(IsAligned((uptr)ptr, alignment)); 163 *memptr = ptr; 164 return 0; 165} 166 167void *lsan_aligned_alloc(uptr alignment, uptr size, const StackTrace &stack) { 168 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { 169 errno = errno_EINVAL; 170 if (AllocatorMayReturnNull()) 171 return nullptr; 172 ReportInvalidAlignedAllocAlignment(size, alignment, &stack); 173 } 174 return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory)); 175} 176 177void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) { 178 if (UNLIKELY(!IsPowerOfTwo(alignment))) { 179 errno = errno_EINVAL; 180 if (AllocatorMayReturnNull()) 181 return nullptr; 182 ReportInvalidAllocationAlignment(alignment, &stack); 183 } 184 return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory)); 185} 186 187void *lsan_malloc(uptr size, const StackTrace &stack) { 188 return SetErrnoOnNull(Allocate(stack, size, 1, kAlwaysClearMemory)); 189} 190 191void lsan_free(void *p) { 192 Deallocate(p); 193} 194 195void *lsan_realloc(void *p, uptr size, const StackTrace &stack) { 196 return SetErrnoOnNull(Reallocate(stack, p, size, 1)); 197} 198 199void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size, 200 const StackTrace &stack) { 201 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 202 errno = errno_ENOMEM; 203 if (AllocatorMayReturnNull()) 204 return nullptr; 205 ReportReallocArrayOverflow(nmemb, size, &stack); 206 } 207 return lsan_realloc(ptr, nmemb * size, stack); 208} 209 210void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) { 211 return SetErrnoOnNull(Calloc(nmemb, size, stack)); 212} 213 214void *lsan_valloc(uptr size, const StackTrace &stack) { 215 return SetErrnoOnNull( 216 Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory)); 217} 218 219void *lsan_pvalloc(uptr size, const StackTrace &stack) { 220 uptr PageSize = GetPageSizeCached(); 221 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { 222 errno = errno_ENOMEM; 223 if (AllocatorMayReturnNull()) 224 return nullptr; 225 ReportPvallocOverflow(size, &stack); 226 } 227 // pvalloc(0) should allocate one page. 228 size = size ? RoundUpTo(size, PageSize) : PageSize; 229 return SetErrnoOnNull(Allocate(stack, size, PageSize, kAlwaysClearMemory)); 230} 231 232uptr lsan_mz_size(const void *p) { 233 return GetMallocUsableSize(p); 234} 235 236///// Interface to the common LSan module. ///// 237 238void LockAllocator() { 239 allocator.ForceLock(); 240} 241 242void UnlockAllocator() { 243 allocator.ForceUnlock(); 244} 245 246void GetAllocatorGlobalRange(uptr *begin, uptr *end) { 247 *begin = (uptr)&allocator; 248 *end = *begin + sizeof(allocator); 249} 250 251uptr PointsIntoChunk(void* p) { 252 uptr addr = reinterpret_cast<uptr>(p); 253 uptr chunk = reinterpret_cast<uptr>(allocator.GetBlockBeginFastLocked(p)); 254 if (!chunk) return 0; 255 // LargeMmapAllocator considers pointers to the meta-region of a chunk to be 256 // valid, but we don't want that. 257 if (addr < chunk) return 0; 258 ChunkMetadata *m = Metadata(reinterpret_cast<void *>(chunk)); 259 CHECK(m); 260 if (!m->allocated) 261 return 0; 262 if (addr < chunk + m->requested_size) 263 return chunk; 264 if (IsSpecialCaseOfOperatorNew0(chunk, m->requested_size, addr)) 265 return chunk; 266 return 0; 267} 268 269uptr GetUserBegin(uptr chunk) { 270 return chunk; 271} 272 273LsanMetadata::LsanMetadata(uptr chunk) { 274 metadata_ = Metadata(reinterpret_cast<void *>(chunk)); 275 CHECK(metadata_); 276} 277 278bool LsanMetadata::allocated() const { 279 return reinterpret_cast<ChunkMetadata *>(metadata_)->allocated; 280} 281 282ChunkTag LsanMetadata::tag() const { 283 return reinterpret_cast<ChunkMetadata *>(metadata_)->tag; 284} 285 286void LsanMetadata::set_tag(ChunkTag value) { 287 reinterpret_cast<ChunkMetadata *>(metadata_)->tag = value; 288} 289 290uptr LsanMetadata::requested_size() const { 291 return reinterpret_cast<ChunkMetadata *>(metadata_)->requested_size; 292} 293 294u32 LsanMetadata::stack_trace_id() const { 295 return reinterpret_cast<ChunkMetadata *>(metadata_)->stack_trace_id; 296} 297 298void ForEachChunk(ForEachChunkCallback callback, void *arg) { 299 allocator.ForEachChunk(callback, arg); 300} 301 302IgnoreObjectResult IgnoreObjectLocked(const void *p) { 303 void *chunk = allocator.GetBlockBegin(p); 304 if (!chunk || p < chunk) return kIgnoreObjectInvalid; 305 ChunkMetadata *m = Metadata(chunk); 306 CHECK(m); 307 if (m->allocated && (uptr)p < (uptr)chunk + m->requested_size) { 308 if (m->tag == kIgnored) 309 return kIgnoreObjectAlreadyIgnored; 310 m->tag = kIgnored; 311 return kIgnoreObjectSuccess; 312 } else { 313 return kIgnoreObjectInvalid; 314 } 315} 316 317void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) { 318 // This function can be used to treat memory reachable from `tctx` as live. 319 // This is useful for threads that have been created but not yet started. 320 321 // This is currently a no-op because the LSan `pthread_create()` interceptor 322 // blocks until the child thread starts which keeps the thread's `arg` pointer 323 // live. 324} 325 326} // namespace __lsan 327 328using namespace __lsan; 329 330extern "C" { 331SANITIZER_INTERFACE_ATTRIBUTE 332uptr __sanitizer_get_current_allocated_bytes() { 333 uptr stats[AllocatorStatCount]; 334 allocator.GetStats(stats); 335 return stats[AllocatorStatAllocated]; 336} 337 338SANITIZER_INTERFACE_ATTRIBUTE 339uptr __sanitizer_get_heap_size() { 340 uptr stats[AllocatorStatCount]; 341 allocator.GetStats(stats); 342 return stats[AllocatorStatMapped]; 343} 344 345SANITIZER_INTERFACE_ATTRIBUTE 346uptr __sanitizer_get_free_bytes() { return 0; } 347 348SANITIZER_INTERFACE_ATTRIBUTE 349uptr __sanitizer_get_unmapped_bytes() { return 0; } 350 351SANITIZER_INTERFACE_ATTRIBUTE 352uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } 353 354SANITIZER_INTERFACE_ATTRIBUTE 355int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; } 356 357SANITIZER_INTERFACE_ATTRIBUTE 358uptr __sanitizer_get_allocated_size(const void *p) { 359 return GetMallocUsableSize(p); 360} 361 362#if !SANITIZER_SUPPORTS_WEAK_HOOKS 363// Provide default (no-op) implementation of malloc hooks. 364SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 365void __sanitizer_malloc_hook(void *ptr, uptr size) { 366 (void)ptr; 367 (void)size; 368} 369SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 370void __sanitizer_free_hook(void *ptr) { 371 (void)ptr; 372} 373#endif 374} // extern "C" 375