1//===-- sanitizer_allocator.cpp -------------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is shared between AddressSanitizer and ThreadSanitizer 10// run-time libraries. 11// This allocator is used inside run-times. 12//===----------------------------------------------------------------------===// 13 14#include "sanitizer_allocator.h" 15 16#include "sanitizer_allocator_checks.h" 17#include "sanitizer_allocator_internal.h" 18#include "sanitizer_atomic.h" 19#include "sanitizer_common.h" 20#include "sanitizer_platform.h" 21 22namespace __sanitizer { 23 24// Default allocator names. 25const char *PrimaryAllocatorName = "SizeClassAllocator"; 26const char *SecondaryAllocatorName = "LargeMmapAllocator"; 27 28static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; 29static atomic_uint8_t internal_allocator_initialized; 30static StaticSpinMutex internal_alloc_init_mu; 31 32static InternalAllocatorCache internal_allocator_cache; 33static StaticSpinMutex internal_allocator_cache_mu; 34 35InternalAllocator *internal_allocator() { 36 InternalAllocator *internal_allocator_instance = 37 reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder); 38 if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) { 39 SpinMutexLock l(&internal_alloc_init_mu); 40 if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) == 41 0) { 42 internal_allocator_instance->Init(kReleaseToOSIntervalNever); 43 atomic_store(&internal_allocator_initialized, 1, memory_order_release); 44 } 45 } 46 return internal_allocator_instance; 47} 48 49static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, 50 uptr alignment) { 51 if (alignment == 0) alignment = 8; 52 if (cache == 0) { 53 SpinMutexLock l(&internal_allocator_cache_mu); 54 return internal_allocator()->Allocate(&internal_allocator_cache, size, 55 alignment); 56 } 57 return internal_allocator()->Allocate(cache, size, alignment); 58} 59 60static void *RawInternalRealloc(void *ptr, uptr size, 61 InternalAllocatorCache *cache) { 62 uptr alignment = 8; 63 if (cache == 0) { 64 SpinMutexLock l(&internal_allocator_cache_mu); 65 return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, 66 size, alignment); 67 } 68 return internal_allocator()->Reallocate(cache, ptr, size, alignment); 69} 70 71static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { 72 if (!cache) { 73 SpinMutexLock l(&internal_allocator_cache_mu); 74 return internal_allocator()->Deallocate(&internal_allocator_cache, ptr); 75 } 76 internal_allocator()->Deallocate(cache, ptr); 77} 78 79static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) { 80 SetAllocatorOutOfMemory(); 81 Report("FATAL: %s: internal allocator is out of memory trying to allocate " 82 "0x%zx bytes\n", SanitizerToolName, requested_size); 83 Die(); 84} 85 86void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { 87 void *p = RawInternalAlloc(size, cache, alignment); 88 if (UNLIKELY(!p)) 89 ReportInternalAllocatorOutOfMemory(size); 90 return p; 91} 92 93void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { 94 void *p = RawInternalRealloc(addr, size, cache); 95 if (UNLIKELY(!p)) 96 ReportInternalAllocatorOutOfMemory(size); 97 return p; 98} 99 100void *InternalReallocArray(void *addr, uptr count, uptr size, 101 InternalAllocatorCache *cache) { 102 if (UNLIKELY(CheckForCallocOverflow(count, size))) { 103 Report( 104 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) " 105 "cannot be represented in type size_t\n", 106 SanitizerToolName, count, size); 107 Die(); 108 } 109 return InternalRealloc(addr, count * size, cache); 110} 111 112void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { 113 if (UNLIKELY(CheckForCallocOverflow(count, size))) { 114 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) " 115 "cannot be represented in type size_t\n", SanitizerToolName, count, 116 size); 117 Die(); 118 } 119 void *p = InternalAlloc(count * size, cache); 120 if (LIKELY(p)) 121 internal_memset(p, 0, count * size); 122 return p; 123} 124 125void InternalFree(void *addr, InternalAllocatorCache *cache) { 126 RawInternalFree(addr, cache); 127} 128 129void InternalAllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { 130 internal_allocator_cache_mu.Lock(); 131 internal_allocator()->ForceLock(); 132} 133 134void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { 135 internal_allocator()->ForceUnlock(); 136 internal_allocator_cache_mu.Unlock(); 137} 138 139// LowLevelAllocator 140constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; 141static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; 142static LowLevelAllocateCallback low_level_alloc_callback; 143 144void *LowLevelAllocator::Allocate(uptr size) { 145 // Align allocation size. 146 size = RoundUpTo(size, low_level_alloc_min_alignment); 147 if (allocated_end_ - allocated_current_ < (sptr)size) { 148 uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached()); 149 allocated_current_ = 150 (char*)MmapOrDie(size_to_allocate, __func__); 151 allocated_end_ = allocated_current_ + size_to_allocate; 152 if (low_level_alloc_callback) { 153 low_level_alloc_callback((uptr)allocated_current_, 154 size_to_allocate); 155 } 156 } 157 CHECK(allocated_end_ - allocated_current_ >= (sptr)size); 158 void *res = allocated_current_; 159 allocated_current_ += size; 160 return res; 161} 162 163void SetLowLevelAllocateMinAlignment(uptr alignment) { 164 CHECK(IsPowerOfTwo(alignment)); 165 low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment); 166} 167 168void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { 169 low_level_alloc_callback = callback; 170} 171 172// Allocator's OOM and other errors handling support. 173 174static atomic_uint8_t allocator_out_of_memory = {0}; 175static atomic_uint8_t allocator_may_return_null = {0}; 176 177bool IsAllocatorOutOfMemory() { 178 return atomic_load_relaxed(&allocator_out_of_memory); 179} 180 181void SetAllocatorOutOfMemory() { 182 atomic_store_relaxed(&allocator_out_of_memory, 1); 183} 184 185bool AllocatorMayReturnNull() { 186 return atomic_load(&allocator_may_return_null, memory_order_relaxed); 187} 188 189void SetAllocatorMayReturnNull(bool may_return_null) { 190 atomic_store(&allocator_may_return_null, may_return_null, 191 memory_order_relaxed); 192} 193 194void PrintHintAllocatorCannotReturnNull() { 195 Report("HINT: if you don't care about these errors you may set " 196 "allocator_may_return_null=1\n"); 197} 198 199static atomic_uint8_t rss_limit_exceeded; 200 201bool IsRssLimitExceeded() { 202 return atomic_load(&rss_limit_exceeded, memory_order_relaxed); 203} 204 205void SetRssLimitExceeded(bool limit_exceeded) { 206 atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed); 207} 208 209} // namespace __sanitizer 210