1351282Sdim//===-- combined.h ----------------------------------------------*- C++ -*-===// 2351282Sdim// 3351282Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4351282Sdim// See https://llvm.org/LICENSE.txt for license information. 5351282Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6351282Sdim// 7351282Sdim//===----------------------------------------------------------------------===// 8351282Sdim 9351282Sdim#ifndef SCUDO_COMBINED_H_ 10351282Sdim#define SCUDO_COMBINED_H_ 11351282Sdim 12351282Sdim#include "chunk.h" 13351282Sdim#include "common.h" 14351282Sdim#include "flags.h" 15351282Sdim#include "flags_parser.h" 16351282Sdim#include "interface.h" 17351282Sdim#include "local_cache.h" 18351282Sdim#include "quarantine.h" 19351282Sdim#include "report.h" 20351282Sdim#include "secondary.h" 21360784Sdim#include "string_utils.h" 22351282Sdim#include "tsd.h" 23351282Sdim 24360784Sdim#ifdef GWP_ASAN_HOOKS 25360784Sdim#include "gwp_asan/guarded_pool_allocator.h" 26360784Sdim// GWP-ASan is declared here in order to avoid indirect call overhead. It's also 27360784Sdim// instantiated outside of the Allocator class, as the allocator is only 28360784Sdim// zero-initialised. GWP-ASan requires constant initialisation, and the Scudo 29360784Sdim// allocator doesn't have a constexpr constructor (see discussion here: 30360784Sdim// https://reviews.llvm.org/D69265#inline-624315). 31360784Sdimstatic gwp_asan::GuardedPoolAllocator GuardedAlloc; 32360784Sdim#endif // GWP_ASAN_HOOKS 33360784Sdim 34360784Sdimextern "C" inline void EmptyCallback() {} 35360784Sdim 36351282Sdimnamespace scudo { 37351282Sdim 38360784Sdimtemplate <class Params, void (*PostInitCallback)(void) = EmptyCallback> 39360784Sdimclass Allocator { 40351282Sdimpublic: 41351282Sdim using PrimaryT = typename Params::Primary; 42351282Sdim using CacheT = typename PrimaryT::CacheT; 43360784Sdim typedef Allocator<Params, PostInitCallback> ThisT; 44351282Sdim typedef typename Params::template TSDRegistryT<ThisT> TSDRegistryT; 45351282Sdim 46360784Sdim void callPostInitCallback() { 47360784Sdim static pthread_once_t OnceControl = PTHREAD_ONCE_INIT; 48360784Sdim pthread_once(&OnceControl, PostInitCallback); 49360784Sdim } 50360784Sdim 51351282Sdim struct QuarantineCallback { 52351282Sdim explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache) 53351282Sdim : Allocator(Instance), Cache(LocalCache) {} 54351282Sdim 55351282Sdim // Chunk recycling function, returns a quarantined chunk to the backend, 56351282Sdim // first making sure it hasn't been tampered with. 57351282Sdim void recycle(void *Ptr) { 58351282Sdim Chunk::UnpackedHeader Header; 59351282Sdim Chunk::loadHeader(Allocator.Cookie, Ptr, &Header); 60351282Sdim if (UNLIKELY(Header.State != Chunk::State::Quarantined)) 61351282Sdim reportInvalidChunkState(AllocatorAction::Recycling, Ptr); 62351282Sdim 63351282Sdim Chunk::UnpackedHeader NewHeader = Header; 64351282Sdim NewHeader.State = Chunk::State::Available; 65351282Sdim Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); 66351282Sdim 67351282Sdim void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader); 68360784Sdim const uptr ClassId = NewHeader.ClassId; 69360784Sdim if (LIKELY(ClassId)) 70351282Sdim Cache.deallocate(ClassId, BlockBegin); 71351282Sdim else 72351282Sdim Allocator.Secondary.deallocate(BlockBegin); 73351282Sdim } 74351282Sdim 75351282Sdim // We take a shortcut when allocating a quarantine batch by working with the 76351282Sdim // appropriate class ID instead of using Size. The compiler should optimize 77351282Sdim // the class ID computation and work with the associated cache directly. 78351282Sdim void *allocate(UNUSED uptr Size) { 79351282Sdim const uptr QuarantineClassId = SizeClassMap::getClassIdBySize( 80351282Sdim sizeof(QuarantineBatch) + Chunk::getHeaderSize()); 81351282Sdim void *Ptr = Cache.allocate(QuarantineClassId); 82351282Sdim // Quarantine batch allocation failure is fatal. 83351282Sdim if (UNLIKELY(!Ptr)) 84351282Sdim reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId)); 85351282Sdim 86351282Sdim Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) + 87351282Sdim Chunk::getHeaderSize()); 88351282Sdim Chunk::UnpackedHeader Header = {}; 89351282Sdim Header.ClassId = QuarantineClassId & Chunk::ClassIdMask; 90351282Sdim Header.SizeOrUnusedBytes = sizeof(QuarantineBatch); 91351282Sdim Header.State = Chunk::State::Allocated; 92351282Sdim Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); 93351282Sdim 94351282Sdim return Ptr; 95351282Sdim } 96351282Sdim 97351282Sdim void deallocate(void *Ptr) { 98351282Sdim const uptr QuarantineClassId = SizeClassMap::getClassIdBySize( 99351282Sdim sizeof(QuarantineBatch) + Chunk::getHeaderSize()); 100351282Sdim Chunk::UnpackedHeader Header; 101351282Sdim Chunk::loadHeader(Allocator.Cookie, Ptr, &Header); 102351282Sdim 103351282Sdim if (UNLIKELY(Header.State != Chunk::State::Allocated)) 104351282Sdim reportInvalidChunkState(AllocatorAction::Deallocating, Ptr); 105351282Sdim DCHECK_EQ(Header.ClassId, QuarantineClassId); 106351282Sdim DCHECK_EQ(Header.Offset, 0); 107351282Sdim DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch)); 108351282Sdim 109351282Sdim Chunk::UnpackedHeader NewHeader = Header; 110351282Sdim NewHeader.State = Chunk::State::Available; 111351282Sdim Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); 112351282Sdim Cache.deallocate(QuarantineClassId, 113351282Sdim reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) - 114351282Sdim Chunk::getHeaderSize())); 115351282Sdim } 116351282Sdim 117351282Sdim private: 118351282Sdim ThisT &Allocator; 119351282Sdim CacheT &Cache; 120351282Sdim }; 121351282Sdim 122351282Sdim typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT; 123351282Sdim typedef typename QuarantineT::CacheT QuarantineCacheT; 124351282Sdim 125351282Sdim void initLinkerInitialized() { 126351282Sdim performSanityChecks(); 127351282Sdim 128351282Sdim // Check if hardware CRC32 is supported in the binary and by the platform, 129351282Sdim // if so, opt for the CRC32 hardware version of the checksum. 130351282Sdim if (&computeHardwareCRC32 && hasHardwareCRC32()) 131351282Sdim HashAlgorithm = Checksum::HardwareCRC32; 132351282Sdim 133351282Sdim if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie)))) 134351282Sdim Cookie = static_cast<u32>(getMonotonicTime() ^ 135351282Sdim (reinterpret_cast<uptr>(this) >> 4)); 136351282Sdim 137351282Sdim initFlags(); 138351282Sdim reportUnrecognizedFlags(); 139351282Sdim 140351282Sdim // Store some flags locally. 141351282Sdim Options.MayReturnNull = getFlags()->may_return_null; 142351282Sdim Options.ZeroContents = getFlags()->zero_contents; 143351282Sdim Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch; 144351282Sdim Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch; 145360784Sdim Options.QuarantineMaxChunkSize = 146360784Sdim static_cast<u32>(getFlags()->quarantine_max_chunk_size); 147351282Sdim 148351282Sdim Stats.initLinkerInitialized(); 149351282Sdim Primary.initLinkerInitialized(getFlags()->release_to_os_interval_ms); 150351282Sdim Secondary.initLinkerInitialized(&Stats); 151351282Sdim 152360784Sdim Quarantine.init( 153360784Sdim static_cast<uptr>(getFlags()->quarantine_size_kb << 10), 154360784Sdim static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10)); 155360784Sdim 156360784Sdim#ifdef GWP_ASAN_HOOKS 157360784Sdim gwp_asan::options::Options Opt; 158360784Sdim Opt.Enabled = getFlags()->GWP_ASAN_Enabled; 159360784Sdim // Bear in mind - Scudo has its own alignment guarantees that are strictly 160360784Sdim // enforced. Scudo exposes the same allocation function for everything from 161360784Sdim // malloc() to posix_memalign, so in general this flag goes unused, as Scudo 162360784Sdim // will always ask GWP-ASan for an aligned amount of bytes. 163360784Sdim Opt.PerfectlyRightAlign = getFlags()->GWP_ASAN_PerfectlyRightAlign; 164360784Sdim Opt.MaxSimultaneousAllocations = 165360784Sdim getFlags()->GWP_ASAN_MaxSimultaneousAllocations; 166360784Sdim Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate; 167360784Sdim Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers; 168360784Sdim Opt.Printf = Printf; 169360784Sdim GuardedAlloc.init(Opt); 170360784Sdim#endif // GWP_ASAN_HOOKS 171351282Sdim } 172351282Sdim 173351282Sdim void reset() { memset(this, 0, sizeof(*this)); } 174351282Sdim 175351282Sdim void unmapTestOnly() { 176351282Sdim TSDRegistry.unmapTestOnly(); 177351282Sdim Primary.unmapTestOnly(); 178351282Sdim } 179351282Sdim 180351282Sdim TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } 181351282Sdim 182360784Sdim // The Cache must be provided zero-initialized. 183360784Sdim void initCache(CacheT *Cache) { 184360784Sdim Cache->initLinkerInitialized(&Stats, &Primary); 185360784Sdim } 186351282Sdim 187351282Sdim // Release the resources used by a TSD, which involves: 188351282Sdim // - draining the local quarantine cache to the global quarantine; 189351282Sdim // - releasing the cached pointers back to the Primary; 190351282Sdim // - unlinking the local stats from the global ones (destroying the cache does 191351282Sdim // the last two items). 192351282Sdim void commitBack(TSD<ThisT> *TSD) { 193351282Sdim Quarantine.drain(&TSD->QuarantineCache, 194351282Sdim QuarantineCallback(*this, TSD->Cache)); 195351282Sdim TSD->Cache.destroy(&Stats); 196351282Sdim } 197351282Sdim 198351282Sdim NOINLINE void *allocate(uptr Size, Chunk::Origin Origin, 199351282Sdim uptr Alignment = MinAlignment, 200351282Sdim bool ZeroContents = false) { 201351282Sdim initThreadMaybe(); 202351282Sdim 203360784Sdim#ifdef GWP_ASAN_HOOKS 204360784Sdim if (UNLIKELY(GuardedAlloc.shouldSample())) { 205360784Sdim if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment))) 206360784Sdim return Ptr; 207360784Sdim } 208360784Sdim#endif // GWP_ASAN_HOOKS 209360784Sdim 210360784Sdim ZeroContents |= static_cast<bool>(Options.ZeroContents); 211360784Sdim 212351282Sdim if (UNLIKELY(Alignment > MaxAlignment)) { 213351282Sdim if (Options.MayReturnNull) 214351282Sdim return nullptr; 215351282Sdim reportAlignmentTooBig(Alignment, MaxAlignment); 216351282Sdim } 217360784Sdim if (Alignment < MinAlignment) 218351282Sdim Alignment = MinAlignment; 219351282Sdim 220351282Sdim // If the requested size happens to be 0 (more common than you might think), 221360784Sdim // allocate MinAlignment bytes on top of the header. Then add the extra 222360784Sdim // bytes required to fulfill the alignment requirements: we allocate enough 223360784Sdim // to be sure that there will be an address in the block that will satisfy 224360784Sdim // the alignment. 225351282Sdim const uptr NeededSize = 226360784Sdim roundUpTo(Size, MinAlignment) + 227360784Sdim ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize()); 228351282Sdim 229351282Sdim // Takes care of extravagantly large sizes as well as integer overflows. 230360784Sdim static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, ""); 231360784Sdim if (UNLIKELY(Size >= MaxAllowedMallocSize)) { 232351282Sdim if (Options.MayReturnNull) 233351282Sdim return nullptr; 234351282Sdim reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize); 235351282Sdim } 236360784Sdim DCHECK_LE(Size, NeededSize); 237351282Sdim 238351282Sdim void *Block; 239351282Sdim uptr ClassId; 240360784Sdim uptr BlockEnd; 241360784Sdim if (LIKELY(PrimaryT::canAllocate(NeededSize))) { 242351282Sdim ClassId = SizeClassMap::getClassIdBySize(NeededSize); 243360784Sdim DCHECK_NE(ClassId, 0U); 244351282Sdim bool UnlockRequired; 245351282Sdim auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); 246351282Sdim Block = TSD->Cache.allocate(ClassId); 247351282Sdim if (UnlockRequired) 248351282Sdim TSD->unlock(); 249351282Sdim } else { 250351282Sdim ClassId = 0; 251360784Sdim Block = 252360784Sdim Secondary.allocate(NeededSize, Alignment, &BlockEnd, ZeroContents); 253351282Sdim } 254351282Sdim 255351282Sdim if (UNLIKELY(!Block)) { 256351282Sdim if (Options.MayReturnNull) 257351282Sdim return nullptr; 258351282Sdim reportOutOfMemory(NeededSize); 259351282Sdim } 260351282Sdim 261360784Sdim // We only need to zero the contents for Primary backed allocations. This 262360784Sdim // condition is not necessarily unlikely, but since memset is costly, we 263360784Sdim // might as well mark it as such. 264360784Sdim if (UNLIKELY(ZeroContents && ClassId)) 265351282Sdim memset(Block, 0, PrimaryT::getSizeByClassId(ClassId)); 266351282Sdim 267360784Sdim const uptr UnalignedUserPtr = 268360784Sdim reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(); 269360784Sdim const uptr UserPtr = roundUpTo(UnalignedUserPtr, Alignment); 270360784Sdim 271351282Sdim Chunk::UnpackedHeader Header = {}; 272360784Sdim if (UNLIKELY(UnalignedUserPtr != UserPtr)) { 273360784Sdim const uptr Offset = UserPtr - UnalignedUserPtr; 274360784Sdim DCHECK_GE(Offset, 2 * sizeof(u32)); 275351282Sdim // The BlockMarker has no security purpose, but is specifically meant for 276351282Sdim // the chunk iteration function that can be used in debugging situations. 277351282Sdim // It is the only situation where we have to locate the start of a chunk 278351282Sdim // based on its block address. 279351282Sdim reinterpret_cast<u32 *>(Block)[0] = BlockMarker; 280351282Sdim reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset); 281360784Sdim Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask; 282351282Sdim } 283360784Sdim Header.ClassId = ClassId & Chunk::ClassIdMask; 284351282Sdim Header.State = Chunk::State::Allocated; 285351282Sdim Header.Origin = Origin & Chunk::OriginMask; 286360784Sdim Header.SizeOrUnusedBytes = (ClassId ? Size : BlockEnd - (UserPtr + Size)) & 287360784Sdim Chunk::SizeOrUnusedBytesMask; 288351282Sdim void *Ptr = reinterpret_cast<void *>(UserPtr); 289351282Sdim Chunk::storeHeader(Cookie, Ptr, &Header); 290351282Sdim 291351282Sdim if (&__scudo_allocate_hook) 292351282Sdim __scudo_allocate_hook(Ptr, Size); 293351282Sdim 294351282Sdim return Ptr; 295351282Sdim } 296351282Sdim 297351282Sdim NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0, 298351282Sdim UNUSED uptr Alignment = MinAlignment) { 299351282Sdim // For a deallocation, we only ensure minimal initialization, meaning thread 300351282Sdim // local data will be left uninitialized for now (when using ELF TLS). The 301351282Sdim // fallback cache will be used instead. This is a workaround for a situation 302351282Sdim // where the only heap operation performed in a thread would be a free past 303351282Sdim // the TLS destructors, ending up in initialized thread specific data never 304351282Sdim // being destroyed properly. Any other heap operation will do a full init. 305351282Sdim initThreadMaybe(/*MinimalInit=*/true); 306351282Sdim 307360784Sdim#ifdef GWP_ASAN_HOOKS 308360784Sdim if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { 309360784Sdim GuardedAlloc.deallocate(Ptr); 310360784Sdim return; 311360784Sdim } 312360784Sdim#endif // GWP_ASAN_HOOKS 313360784Sdim 314351282Sdim if (&__scudo_deallocate_hook) 315351282Sdim __scudo_deallocate_hook(Ptr); 316351282Sdim 317351282Sdim if (UNLIKELY(!Ptr)) 318351282Sdim return; 319351282Sdim if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment))) 320351282Sdim reportMisalignedPointer(AllocatorAction::Deallocating, Ptr); 321351282Sdim 322351282Sdim Chunk::UnpackedHeader Header; 323351282Sdim Chunk::loadHeader(Cookie, Ptr, &Header); 324351282Sdim 325351282Sdim if (UNLIKELY(Header.State != Chunk::State::Allocated)) 326351282Sdim reportInvalidChunkState(AllocatorAction::Deallocating, Ptr); 327351282Sdim if (Options.DeallocTypeMismatch) { 328351282Sdim if (Header.Origin != Origin) { 329351282Sdim // With the exception of memalign'd chunks, that can be still be free'd. 330351282Sdim if (UNLIKELY(Header.Origin != Chunk::Origin::Memalign || 331351282Sdim Origin != Chunk::Origin::Malloc)) 332351282Sdim reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr, 333351282Sdim Header.Origin, Origin); 334351282Sdim } 335351282Sdim } 336351282Sdim 337351282Sdim const uptr Size = getSize(Ptr, &Header); 338351282Sdim if (DeleteSize && Options.DeleteSizeMismatch) { 339351282Sdim if (UNLIKELY(DeleteSize != Size)) 340351282Sdim reportDeleteSizeMismatch(Ptr, DeleteSize, Size); 341351282Sdim } 342351282Sdim 343351282Sdim quarantineOrDeallocateChunk(Ptr, &Header, Size); 344351282Sdim } 345351282Sdim 346351282Sdim void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) { 347351282Sdim initThreadMaybe(); 348351282Sdim 349351282Sdim // The following cases are handled by the C wrappers. 350351282Sdim DCHECK_NE(OldPtr, nullptr); 351351282Sdim DCHECK_NE(NewSize, 0); 352351282Sdim 353360784Sdim#ifdef GWP_ASAN_HOOKS 354360784Sdim if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) { 355360784Sdim uptr OldSize = GuardedAlloc.getSize(OldPtr); 356360784Sdim void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment); 357360784Sdim if (NewPtr) 358360784Sdim memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize); 359360784Sdim GuardedAlloc.deallocate(OldPtr); 360360784Sdim return NewPtr; 361360784Sdim } 362360784Sdim#endif // GWP_ASAN_HOOKS 363360784Sdim 364351282Sdim if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment))) 365351282Sdim reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr); 366351282Sdim 367351282Sdim Chunk::UnpackedHeader OldHeader; 368351282Sdim Chunk::loadHeader(Cookie, OldPtr, &OldHeader); 369351282Sdim 370351282Sdim if (UNLIKELY(OldHeader.State != Chunk::State::Allocated)) 371351282Sdim reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr); 372351282Sdim 373351282Sdim // Pointer has to be allocated with a malloc-type function. Some 374351282Sdim // applications think that it is OK to realloc a memalign'ed pointer, which 375351282Sdim // will trigger this check. It really isn't. 376351282Sdim if (Options.DeallocTypeMismatch) { 377351282Sdim if (UNLIKELY(OldHeader.Origin != Chunk::Origin::Malloc)) 378351282Sdim reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr, 379351282Sdim OldHeader.Origin, Chunk::Origin::Malloc); 380351282Sdim } 381351282Sdim 382360784Sdim void *BlockBegin = getBlockBegin(OldPtr, &OldHeader); 383360784Sdim uptr BlockEnd; 384360784Sdim uptr OldSize; 385360784Sdim const uptr ClassId = OldHeader.ClassId; 386360784Sdim if (LIKELY(ClassId)) { 387360784Sdim BlockEnd = reinterpret_cast<uptr>(BlockBegin) + 388360784Sdim SizeClassMap::getSizeByClassId(ClassId); 389360784Sdim OldSize = OldHeader.SizeOrUnusedBytes; 390360784Sdim } else { 391360784Sdim BlockEnd = SecondaryT::getBlockEnd(BlockBegin); 392360784Sdim OldSize = BlockEnd - 393360784Sdim (reinterpret_cast<uptr>(OldPtr) + OldHeader.SizeOrUnusedBytes); 394360784Sdim } 395360784Sdim // If the new chunk still fits in the previously allocated block (with a 396360784Sdim // reasonable delta), we just keep the old block, and update the chunk 397360784Sdim // header to reflect the size change. 398360784Sdim if (reinterpret_cast<uptr>(OldPtr) + NewSize <= BlockEnd) { 399360784Sdim const uptr Delta = 400360784Sdim OldSize < NewSize ? NewSize - OldSize : OldSize - NewSize; 401360784Sdim if (Delta <= SizeClassMap::MaxSize / 2) { 402351282Sdim Chunk::UnpackedHeader NewHeader = OldHeader; 403351282Sdim NewHeader.SizeOrUnusedBytes = 404360784Sdim (ClassId ? NewSize 405360784Sdim : BlockEnd - (reinterpret_cast<uptr>(OldPtr) + NewSize)) & 406351282Sdim Chunk::SizeOrUnusedBytesMask; 407351282Sdim Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); 408351282Sdim return OldPtr; 409351282Sdim } 410351282Sdim } 411351282Sdim 412351282Sdim // Otherwise we allocate a new one, and deallocate the old one. Some 413351282Sdim // allocators will allocate an even larger chunk (by a fixed factor) to 414351282Sdim // allow for potential further in-place realloc. The gains of such a trick 415351282Sdim // are currently unclear. 416351282Sdim void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment); 417351282Sdim if (NewPtr) { 418360784Sdim const uptr OldSize = getSize(OldPtr, &OldHeader); 419351282Sdim memcpy(NewPtr, OldPtr, Min(NewSize, OldSize)); 420351282Sdim quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize); 421351282Sdim } 422351282Sdim return NewPtr; 423351282Sdim } 424351282Sdim 425360784Sdim // TODO(kostyak): disable() is currently best-effort. There are some small 426360784Sdim // windows of time when an allocation could still succeed after 427360784Sdim // this function finishes. We will revisit that later. 428351282Sdim void disable() { 429351282Sdim initThreadMaybe(); 430360784Sdim TSDRegistry.disable(); 431360784Sdim Stats.disable(); 432360784Sdim Quarantine.disable(); 433351282Sdim Primary.disable(); 434351282Sdim Secondary.disable(); 435351282Sdim } 436351282Sdim 437351282Sdim void enable() { 438351282Sdim initThreadMaybe(); 439351282Sdim Secondary.enable(); 440351282Sdim Primary.enable(); 441360784Sdim Quarantine.enable(); 442360784Sdim Stats.enable(); 443360784Sdim TSDRegistry.enable(); 444351282Sdim } 445351282Sdim 446360784Sdim // The function returns the amount of bytes required to store the statistics, 447360784Sdim // which might be larger than the amount of bytes provided. Note that the 448360784Sdim // statistics buffer is not necessarily constant between calls to this 449360784Sdim // function. This can be called with a null buffer or zero size for buffer 450360784Sdim // sizing purposes. 451360784Sdim uptr getStats(char *Buffer, uptr Size) { 452360784Sdim ScopedString Str(1024); 453360784Sdim disable(); 454360784Sdim const uptr Length = getStats(&Str) + 1; 455360784Sdim enable(); 456360784Sdim if (Length < Size) 457360784Sdim Size = Length; 458360784Sdim if (Buffer && Size) { 459360784Sdim memcpy(Buffer, Str.data(), Size); 460360784Sdim Buffer[Size - 1] = '\0'; 461360784Sdim } 462360784Sdim return Length; 463360784Sdim } 464360784Sdim 465351282Sdim void printStats() { 466360784Sdim ScopedString Str(1024); 467351282Sdim disable(); 468360784Sdim getStats(&Str); 469351282Sdim enable(); 470360784Sdim Str.output(); 471351282Sdim } 472351282Sdim 473360784Sdim void releaseToOS() { 474360784Sdim initThreadMaybe(); 475360784Sdim Primary.releaseToOS(); 476360784Sdim } 477351282Sdim 478351282Sdim // Iterate over all chunks and call a callback for all busy chunks located 479351282Sdim // within the provided memory range. Said callback must not use this allocator 480351282Sdim // or a deadlock can ensue. This fits Android's malloc_iterate() needs. 481351282Sdim void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback, 482351282Sdim void *Arg) { 483351282Sdim initThreadMaybe(); 484351282Sdim const uptr From = Base; 485351282Sdim const uptr To = Base + Size; 486351282Sdim auto Lambda = [this, From, To, Callback, Arg](uptr Block) { 487360784Sdim if (Block < From || Block >= To) 488351282Sdim return; 489360784Sdim uptr Chunk; 490360784Sdim Chunk::UnpackedHeader Header; 491360784Sdim if (getChunkFromBlock(Block, &Chunk, &Header) && 492360784Sdim Header.State == Chunk::State::Allocated) 493360784Sdim Callback(Chunk, getSize(reinterpret_cast<void *>(Chunk), &Header), Arg); 494351282Sdim }; 495351282Sdim Primary.iterateOverBlocks(Lambda); 496351282Sdim Secondary.iterateOverBlocks(Lambda); 497351282Sdim } 498351282Sdim 499351282Sdim bool canReturnNull() { 500351282Sdim initThreadMaybe(); 501351282Sdim return Options.MayReturnNull; 502351282Sdim } 503351282Sdim 504351282Sdim // TODO(kostyak): implement this as a "backend" to mallopt. 505351282Sdim bool setOption(UNUSED uptr Option, UNUSED uptr Value) { return false; } 506351282Sdim 507351282Sdim // Return the usable size for a given chunk. Technically we lie, as we just 508351282Sdim // report the actual size of a chunk. This is done to counteract code actively 509351282Sdim // writing past the end of a chunk (like sqlite3) when the usable size allows 510351282Sdim // for it, which then forces realloc to copy the usable size of a chunk as 511351282Sdim // opposed to its actual size. 512351282Sdim uptr getUsableSize(const void *Ptr) { 513351282Sdim initThreadMaybe(); 514351282Sdim if (UNLIKELY(!Ptr)) 515351282Sdim return 0; 516360784Sdim 517360784Sdim#ifdef GWP_ASAN_HOOKS 518360784Sdim if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) 519360784Sdim return GuardedAlloc.getSize(Ptr); 520360784Sdim#endif // GWP_ASAN_HOOKS 521360784Sdim 522351282Sdim Chunk::UnpackedHeader Header; 523351282Sdim Chunk::loadHeader(Cookie, Ptr, &Header); 524351282Sdim // Getting the usable size of a chunk only makes sense if it's allocated. 525351282Sdim if (UNLIKELY(Header.State != Chunk::State::Allocated)) 526351282Sdim reportInvalidChunkState(AllocatorAction::Sizing, const_cast<void *>(Ptr)); 527351282Sdim return getSize(Ptr, &Header); 528351282Sdim } 529351282Sdim 530351282Sdim void getStats(StatCounters S) { 531351282Sdim initThreadMaybe(); 532351282Sdim Stats.get(S); 533351282Sdim } 534351282Sdim 535360784Sdim // Returns true if the pointer provided was allocated by the current 536360784Sdim // allocator instance, which is compliant with tcmalloc's ownership concept. 537360784Sdim // A corrupted chunk will not be reported as owned, which is WAI. 538360784Sdim bool isOwned(const void *Ptr) { 539360784Sdim initThreadMaybe(); 540360784Sdim#ifdef GWP_ASAN_HOOKS 541360784Sdim if (GuardedAlloc.pointerIsMine(Ptr)) 542360784Sdim return true; 543360784Sdim#endif // GWP_ASAN_HOOKS 544360784Sdim if (!Ptr || !isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)) 545360784Sdim return false; 546360784Sdim Chunk::UnpackedHeader Header; 547360784Sdim return Chunk::isValid(Cookie, Ptr, &Header) && 548360784Sdim Header.State == Chunk::State::Allocated; 549360784Sdim } 550360784Sdim 551351282Sdimprivate: 552360784Sdim using SecondaryT = typename Params::Secondary; 553351282Sdim typedef typename PrimaryT::SizeClassMap SizeClassMap; 554351282Sdim 555351282Sdim static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG; 556351282Sdim static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable. 557351282Sdim static const uptr MinAlignment = 1UL << MinAlignmentLog; 558351282Sdim static const uptr MaxAlignment = 1UL << MaxAlignmentLog; 559351282Sdim static const uptr MaxAllowedMallocSize = 560351282Sdim FIRST_32_SECOND_64(1UL << 31, 1ULL << 40); 561351282Sdim 562360784Sdim static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), 563360784Sdim "Minimal alignment must at least cover a chunk header."); 564360784Sdim 565351282Sdim static const u32 BlockMarker = 0x44554353U; 566351282Sdim 567351282Sdim GlobalStats Stats; 568351282Sdim TSDRegistryT TSDRegistry; 569351282Sdim PrimaryT Primary; 570351282Sdim SecondaryT Secondary; 571351282Sdim QuarantineT Quarantine; 572351282Sdim 573351282Sdim u32 Cookie; 574351282Sdim 575351282Sdim struct { 576351282Sdim u8 MayReturnNull : 1; // may_return_null 577351282Sdim u8 ZeroContents : 1; // zero_contents 578351282Sdim u8 DeallocTypeMismatch : 1; // dealloc_type_mismatch 579351282Sdim u8 DeleteSizeMismatch : 1; // delete_size_mismatch 580351282Sdim u32 QuarantineMaxChunkSize; // quarantine_max_chunk_size 581351282Sdim } Options; 582351282Sdim 583351282Sdim // The following might get optimized out by the compiler. 584351282Sdim NOINLINE void performSanityChecks() { 585351282Sdim // Verify that the header offset field can hold the maximum offset. In the 586351282Sdim // case of the Secondary allocator, it takes care of alignment and the 587351282Sdim // offset will always be small. In the case of the Primary, the worst case 588351282Sdim // scenario happens in the last size class, when the backend allocation 589351282Sdim // would already be aligned on the requested alignment, which would happen 590351282Sdim // to be the maximum alignment that would fit in that size class. As a 591351282Sdim // result, the maximum offset will be at most the maximum alignment for the 592351282Sdim // last size class minus the header size, in multiples of MinAlignment. 593351282Sdim Chunk::UnpackedHeader Header = {}; 594351282Sdim const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex( 595351282Sdim SizeClassMap::MaxSize - MinAlignment); 596351282Sdim const uptr MaxOffset = 597351282Sdim (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog; 598351282Sdim Header.Offset = MaxOffset & Chunk::OffsetMask; 599351282Sdim if (UNLIKELY(Header.Offset != MaxOffset)) 600351282Sdim reportSanityCheckError("offset"); 601351282Sdim 602351282Sdim // Verify that we can fit the maximum size or amount of unused bytes in the 603351282Sdim // header. Given that the Secondary fits the allocation to a page, the worst 604351282Sdim // case scenario happens in the Primary. It will depend on the second to 605351282Sdim // last and last class sizes, as well as the dynamic base for the Primary. 606351282Sdim // The following is an over-approximation that works for our needs. 607351282Sdim const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1; 608360784Sdim Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes; 609351282Sdim if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes)) 610351282Sdim reportSanityCheckError("size (or unused bytes)"); 611351282Sdim 612351282Sdim const uptr LargestClassId = SizeClassMap::LargestClassId; 613351282Sdim Header.ClassId = LargestClassId; 614351282Sdim if (UNLIKELY(Header.ClassId != LargestClassId)) 615351282Sdim reportSanityCheckError("class ID"); 616351282Sdim } 617351282Sdim 618360784Sdim static inline void *getBlockBegin(const void *Ptr, 619351282Sdim Chunk::UnpackedHeader *Header) { 620360784Sdim return reinterpret_cast<void *>( 621360784Sdim reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() - 622360784Sdim (static_cast<uptr>(Header->Offset) << MinAlignmentLog)); 623351282Sdim } 624351282Sdim 625351282Sdim // Return the size of a chunk as requested during its allocation. 626360784Sdim inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) { 627351282Sdim const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; 628360784Sdim if (LIKELY(Header->ClassId)) 629351282Sdim return SizeOrUnusedBytes; 630351282Sdim return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) - 631351282Sdim reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes; 632351282Sdim } 633351282Sdim 634351282Sdim ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { 635351282Sdim TSDRegistry.initThreadMaybe(this, MinimalInit); 636351282Sdim } 637351282Sdim 638351282Sdim void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header, 639351282Sdim uptr Size) { 640351282Sdim Chunk::UnpackedHeader NewHeader = *Header; 641351282Sdim // If the quarantine is disabled, the actual size of a chunk is 0 or larger 642351282Sdim // than the maximum allowed, we return a chunk directly to the backend. 643360784Sdim // Logical Or can be short-circuited, which introduces unnecessary 644360784Sdim // conditional jumps, so use bitwise Or and let the compiler be clever. 645360784Sdim const bool BypassQuarantine = !Quarantine.getCacheSize() | !Size | 646351282Sdim (Size > Options.QuarantineMaxChunkSize); 647351282Sdim if (BypassQuarantine) { 648351282Sdim NewHeader.State = Chunk::State::Available; 649351282Sdim Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); 650351282Sdim void *BlockBegin = getBlockBegin(Ptr, &NewHeader); 651351282Sdim const uptr ClassId = NewHeader.ClassId; 652360784Sdim if (LIKELY(ClassId)) { 653351282Sdim bool UnlockRequired; 654351282Sdim auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); 655351282Sdim TSD->Cache.deallocate(ClassId, BlockBegin); 656351282Sdim if (UnlockRequired) 657351282Sdim TSD->unlock(); 658351282Sdim } else { 659351282Sdim Secondary.deallocate(BlockBegin); 660351282Sdim } 661351282Sdim } else { 662351282Sdim NewHeader.State = Chunk::State::Quarantined; 663351282Sdim Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); 664351282Sdim bool UnlockRequired; 665351282Sdim auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); 666351282Sdim Quarantine.put(&TSD->QuarantineCache, 667351282Sdim QuarantineCallback(*this, TSD->Cache), Ptr, Size); 668351282Sdim if (UnlockRequired) 669351282Sdim TSD->unlock(); 670351282Sdim } 671351282Sdim } 672351282Sdim 673360784Sdim bool getChunkFromBlock(uptr Block, uptr *Chunk, 674360784Sdim Chunk::UnpackedHeader *Header) { 675351282Sdim u32 Offset = 0; 676351282Sdim if (reinterpret_cast<u32 *>(Block)[0] == BlockMarker) 677351282Sdim Offset = reinterpret_cast<u32 *>(Block)[1]; 678360784Sdim *Chunk = Block + Offset + Chunk::getHeaderSize(); 679360784Sdim return Chunk::isValid(Cookie, reinterpret_cast<void *>(*Chunk), Header); 680351282Sdim } 681360784Sdim 682360784Sdim uptr getStats(ScopedString *Str) { 683360784Sdim Primary.getStats(Str); 684360784Sdim Secondary.getStats(Str); 685360784Sdim Quarantine.getStats(Str); 686360784Sdim return Str->length(); 687360784Sdim } 688351282Sdim}; 689351282Sdim 690351282Sdim} // namespace scudo 691351282Sdim 692351282Sdim#endif // SCUDO_COMBINED_H_ 693