168349Sobrien/* 268349Sobrien Title: memmgr.cpp Memory segment manager 3284193Sdelphij 468349Sobrien Copyright (c) 2006-7, 2011-12, 2016-18 David C. J. Matthews 5186691Sobrien 6186691Sobrien This library is free software; you can redistribute it and/or 7234449Sobrien modify it under the terms of the GNU Lesser General Public 8268515Sdelphij License version 2.1 as published by the Free Software Foundation. 9234449Sobrien 10234449Sobrien This library is distributed in the hope that it will be useful, 1168349Sobrien but WITHOUT ANY WARRANTY; without even the implied warranty of 1268349Sobrien MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13234449Sobrien Lesser General Public License for more details. 14234449Sobrien 15234449Sobrien You should have received a copy of the GNU Lesser General Public 16139368Sobrien License along with this library; if not, write to the Free Software 17175296Sobrien Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 18175296Sobrien 19234449Sobrien*/ 20159764Sobrien#ifdef HAVE_CONFIG_H 21159764Sobrien#include "config.h" 22234449Sobrien#elif defined(_WIN32) 23159764Sobrien#include "winconfig.h" 24159764Sobrien#else 25175296Sobrien#error "No configuration file" 26175296Sobrien#endif 27234449Sobrien 28234449Sobrien#ifdef HAVE_ASSERT_H 29234449Sobrien#include <assert.h> 30234449Sobrien#define ASSERT(x) assert(x) 31234449Sobrien#else 32175296Sobrien#define ASSERT(x) 33139368Sobrien#endif 34139368Sobrien 35175296Sobrien#include <stdio.h> 36234449Sobrien 37234449Sobrien#include <new> 38234449Sobrien 39234449Sobrien#include "globals.h" 40234449Sobrien#include "memmgr.h" 41234449Sobrien#include "osmem.h" 42234449Sobrien#include "scanaddrs.h" 43234449Sobrien#include "bitmap.h" 44234449Sobrien#include "mpoly.h" 45234449Sobrien#include "diagnostics.h" 46234449Sobrien#include "statistics.h" 47234449Sobrien#include "processes.h" 48234449Sobrien#include "machine_dep.h" 49234449Sobrien 50234449Sobrien 51234449Sobrien#ifdef POLYML32IN64 52234449Sobrien// This contains the address of the base of the heap. 53234449SobrienPolyWord *globalHeapBase, *globalCodeBase; 54234449Sobrien#endif 55234449Sobrien 56175296Sobrien// heap resizing policy option requested on command line 57234449Sobrienunsigned heapsizingOption = 0; 58234449Sobrien 59234449Sobrien// If we are building for the interpreted version we don't need or want the 60234449Sobrien// code to be executable. 61234449Sobrienstatic const enum OSMem::_MemUsage executableCodeWhereNecessary = 62234449Sobrien#ifdef CODEISNOTEXECUTABLE 63234449Sobrien OSMem::UsageData; 64234449Sobrien#else 65234449Sobrien OSMem::UsageExecutableCode; 66234449Sobrien#endif 67234449Sobrien 68234449SobrienMemSpace::MemSpace(OSMem *alloc): SpaceTree(true) 69175296Sobrien{ 70234449Sobrien spaceType = ST_PERMANENT; 71234449Sobrien isMutable = false; 72175296Sobrien bottom = 0; 73175296Sobrien top = 0; 74234449Sobrien isCode = false; 75234449Sobrien allocator = alloc; 76234449Sobrien shadowSpace = 0; 77234449Sobrien} 78234449Sobrien 79234449SobrienMemSpace::~MemSpace() 80234449Sobrien{ 81139368Sobrien if (allocator != 0 && bottom != 0) 82139368Sobrien { 83139368Sobrien if (isCode) 84175296Sobrien allocator->FreeCodeArea(bottom, shadowSpace, (char*)top - (char*)bottom); 85175296Sobrien else allocator->FreeDataArea(bottom, (char*)top - (char*)bottom); 86175296Sobrien } 87175296Sobrien} 88175296Sobrien 89175296SobrienMarkableSpace::MarkableSpace(OSMem *alloc): MemSpace(alloc), spaceLock("Local space") 90175296Sobrien{ 91175296Sobrien} 92234449Sobrien 93234449SobrienLocalMemSpace::LocalMemSpace(OSMem *alloc): MarkableSpace(alloc) 94175296Sobrien{ 95175296Sobrien spaceType = ST_LOCAL; 96175296Sobrien upperAllocPtr = lowerAllocPtr = 0; 97175296Sobrien for (unsigned i = 0; i < NSTARTS; i++) 98175296Sobrien start[i] = 0; 99175296Sobrien start_index = 0; 100234449Sobrien i_marked = m_marked = updated = 0; 101234449Sobrien allocationSpace = false; 102175296Sobrien} 103175296Sobrien 104175296Sobrienbool LocalMemSpace::InitSpace(PolyWord *heapSpace, uintptr_t size, bool mut) 105175296Sobrien{ 106175296Sobrien isMutable = mut; 107175296Sobrien bottom = heapSpace; 108175296Sobrien top = bottom + size; 109175296Sobrien // Initialise all the fields. The partial GC in particular relies on this. 110175296Sobrien upperAllocPtr = partialGCTop = fullGCRescanStart = fullGCLowerLimit = lowestWeak = top; 111175296Sobrien lowerAllocPtr = partialGCScan = partialGCRootBase = partialGCRootTop = 112234449Sobrien fullGCRescanEnd = highestWeak = bottom; 113234449Sobrien#ifdef POLYML32IN64 114175296Sobrien // The address must be on an odd-word boundary so that after the length 115175296Sobrien // word is put in the actual cell address is on an even-word boundary. 116234449Sobrien lowerAllocPtr[0] = PolyWord::FromUnsigned(0); 117234449Sobrien lowerAllocPtr = bottom + 1; 118234449Sobrien#endif 119234449Sobrien spaceOwner = 0; 120234449Sobrien 121234449Sobrien allocationSpace = false; 122234449Sobrien 123175296Sobrien // Bitmap for the space. 124159764Sobrien return bitmap.Create(size); 125159764Sobrien} 126159764Sobrien 127139368SobrienMemMgr::MemMgr(): allocLock("Memmgr alloc"), codeBitmapLock("Code bitmap") 128159764Sobrien{ 129234449Sobrien nextIndex = 0; 130234449Sobrien reservedSpace = 0; 131234449Sobrien nextAllocator = 0; 132234449Sobrien defaultSpaceSize = 0; 133139368Sobrien spaceBeforeMinorGC = 0; 134175296Sobrien spaceForHeap = 0; 135234449Sobrien currentAllocSpace = currentHeapSize = 0; 136159764Sobrien defaultSpaceSize = 1024 * 1024 / sizeof(PolyWord); // 1Mbyte segments. 137268515Sdelphij spaceTree = new SpaceTreeTree; 138268515Sdelphij} 139268515Sdelphij 140268515SdelphijMemMgr::~MemMgr() 141268515Sdelphij{ 142268515Sdelphij delete(spaceTree); // Have to do this before we delete the spaces. 143268515Sdelphij for (std::vector<PermanentMemSpace *>::iterator i = pSpaces.begin(); i < pSpaces.end(); i++) 144268515Sdelphij delete(*i); 145268515Sdelphij for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) 146268515Sdelphij delete(*i); 147268515Sdelphij for (std::vector<PermanentMemSpace *>::iterator i = eSpaces.begin(); i < eSpaces.end(); i++) 148268515Sdelphij delete(*i); 149268515Sdelphij for (std::vector<StackSpace *>::iterator i = sSpaces.begin(); i < sSpaces.end(); i++) 150268515Sdelphij delete(*i); 151268515Sdelphij for (std::vector<CodeSpace *>::iterator i = cSpaces.begin(); i < cSpaces.end(); i++) 152268515Sdelphij delete(*i); 153268515Sdelphij} 154268515Sdelphij 155268515Sdelphijbool MemMgr::Initialise() 156268515Sdelphij{ 157268515Sdelphij#ifdef POLYML32IN64 158268515Sdelphij // Allocate a single 16G area but with no access. 159268515Sdelphij void *heapBase; 160268515Sdelphij if (!osHeapAlloc.Initialise(OSMem::UsageData, (size_t)16 * 1024 * 1024 * 1024, &heapBase)) 161268515Sdelphij return false; 162175296Sobrien globalHeapBase = (PolyWord*)heapBase; 163234449Sobrien 164234449Sobrien // Allocate a 4 gbyte area for the stacks. 165234449Sobrien // It's important that the stack and code areas have addresses with 166234449Sobrien // non-zero top 32-bits. 167234449Sobrien if (!osStackAlloc.Initialise(OSMem::UsageStack, (size_t)4 * 1024 * 1024 * 1024)) 168234449Sobrien return false; 169234449Sobrien 170175296Sobrien // Allocate a 2G area for the code. 171234449Sobrien void *codeBase; 172234449Sobrien if (!osCodeAlloc.Initialise(executableCodeWhereNecessary, 173234449Sobrien (size_t)2 * 1024 * 1024 * 1024, &codeBase)) 174234449Sobrien return false; 175234449Sobrien globalCodeBase = (PolyWord*)codeBase; 176234449Sobrien return true; 177133359Sobrien#else 178234449Sobrien return osHeapAlloc.Initialise(OSMem::UsageData) && osStackAlloc.Initialise(OSMem::UsageStack) && 179234449Sobrien osCodeAlloc.Initialise(executableCodeWhereNecessary); 180234449Sobrien#endif 181234449Sobrien} 182234449Sobrien 183234449Sobrien// Create and initialise a new local space and add it to the table. 184133359SobrienLocalMemSpace* MemMgr::NewLocalSpace(uintptr_t size, bool mut) 185175296Sobrien{ 186234449Sobrien try { 187234449Sobrien LocalMemSpace *space = new LocalMemSpace(&osHeapAlloc); 188234449Sobrien // Before trying to allocate the heap temporarily allocate the 189234449Sobrien // reserved space. This ensures that this much space will always 190234449Sobrien // be available for C stacks and the C++ heap. 191133359Sobrien void *reservation = 0; 192175296Sobrien size_t rSpace = reservedSpace*sizeof(PolyWord); 193234449Sobrien 194175296Sobrien if (reservedSpace != 0) { 195268515Sdelphij reservation = osHeapAlloc.AllocateDataArea(rSpace); 196268515Sdelphij if (reservation == NULL) { 197234449Sobrien // Insufficient space for the reservation. Can't allocate this local space. 198234449Sobrien if (debugOptions & DEBUG_MEMMGR) 199234449Sobrien Log("MMGR: New local %smutable space: insufficient reservation space\n", mut ? "": "im"); 200234449Sobrien delete space; 201268515Sdelphij return 0; 202268515Sdelphij } 203268515Sdelphij } 204268515Sdelphij 205268515Sdelphij // Allocate the heap itself. 206268515Sdelphij size_t iSpace = size * sizeof(PolyWord); 207268515Sdelphij PolyWord* heapSpace = (PolyWord*)osHeapAlloc.AllocateDataArea(iSpace); 208268515Sdelphij // The size may have been rounded up to a block boundary. 209268515Sdelphij size = iSpace / sizeof(PolyWord); 210234449Sobrien bool success = heapSpace != 0 && space->InitSpace(heapSpace, size, mut) && AddLocalSpace(space); 211234449Sobrien 212175296Sobrien if (reservation != 0) osHeapAlloc.FreeDataArea(reservation, rSpace); 213234449Sobrien if (success) 214175296Sobrien { 215234449Sobrien if (debugOptions & DEBUG_MEMMGR) 216175296Sobrien Log("MMGR: New local %smutable space %p, size=%luk words, bottom=%p, top=%p\n", mut ? "": "im", 217175296Sobrien space, space->spaceSize()/1024, space->bottom, space->top); 218234449Sobrien currentHeapSize += space->spaceSize(); 219234449Sobrien globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord)); 220133359Sobrien return space; 221133359Sobrien } 222133359Sobrien 223133359Sobrien // If something went wrong. 224234449Sobrien delete space; 225234449Sobrien if (debugOptions & DEBUG_MEMMGR) 226133359Sobrien Log("MMGR: New local %smutable space: insufficient space\n", mut ? "": "im"); 227175296Sobrien return 0; 228234449Sobrien } 229234449Sobrien catch (std::bad_alloc&) { 230234449Sobrien if (debugOptions & DEBUG_MEMMGR) 231234449Sobrien Log("MMGR: New local %smutable space: \"new\" failed\n", mut ? "": "im"); 232234449Sobrien return 0; 233234449Sobrien } 234234449Sobrien} 235234449Sobrien 236234449Sobrien// Create a local space for initial allocation. 237175296SobrienLocalMemSpace *MemMgr::CreateAllocationSpace(uintptr_t size) 238175296Sobrien{ 239234449Sobrien LocalMemSpace *result = NewLocalSpace(size, true); 240175296Sobrien if (result) 241234449Sobrien { 242234449Sobrien result->allocationSpace = true; 243234449Sobrien currentAllocSpace += result->spaceSize(); 244234449Sobrien globalStats.incSize(PSS_ALLOCATION, result->spaceSize()*sizeof(PolyWord)); 245175296Sobrien globalStats.incSize(PSS_ALLOCATION_FREE, result->freeSpace()*sizeof(PolyWord)); 246175296Sobrien } 247175296Sobrien return result; 248234449Sobrien} 249268515Sdelphij 250268515Sdelphij// If an allocation space has a lot of data left in it after a GC, particularly 251268515Sdelphij// a single large object we should turn it into a local area. 252268515Sdelphijvoid MemMgr::ConvertAllocationSpaceToLocal(LocalMemSpace *space) 253268515Sdelphij{ 254268515Sdelphij ASSERT(space->allocationSpace); 255268515Sdelphij space->allocationSpace = false; 256268515Sdelphij // Currently it is left as a mutable area but if the contents are all 257268515Sdelphij // immutable e.g. a large vector it could be better to turn it into an 258268515Sdelphij // immutable area. 259268515Sdelphij currentAllocSpace -= space->spaceSize(); 260268515Sdelphij} 261268515Sdelphij 262268515Sdelphij// Add a local memory space to the table. 263268515Sdelphijbool MemMgr::AddLocalSpace(LocalMemSpace *space) 264268515Sdelphij{ 265268515Sdelphij // Add to the table. 266268515Sdelphij // Update the B-tree. 267268515Sdelphij try { 268175296Sobrien AddTree(space); 269175296Sobrien // The entries in the local table are ordered so that the copy phase of the full 270234449Sobrien // GC simply has to copy to an entry earlier in the table. Immutable spaces come 271234449Sobrien // first, followed by mutable spaces and finally allocation spaces. 272234449Sobrien if (space->allocationSpace) 273234449Sobrien lSpaces.push_back(space); // Just add at the end 274234449Sobrien else if (space->isMutable) 275234449Sobrien { 276234449Sobrien // Add before the allocation spaces 277234449Sobrien std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); 278234449Sobrien while (i != lSpaces.end() && ! (*i)->allocationSpace) i++; 279234449Sobrien lSpaces.insert(i, space); 280234449Sobrien } 281234449Sobrien else 282234449Sobrien { 283234449Sobrien // Immutable space: Add before the mutable spaces 284175296Sobrien std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); 285175296Sobrien while (i != lSpaces.end() && ! (*i)->isMutable) i++; 286175296Sobrien lSpaces.insert(i, space); 287234449Sobrien } 288234449Sobrien } 289234449Sobrien catch (std::bad_alloc&) { 290234449Sobrien RemoveTree(space); 291234449Sobrien return false; 292175296Sobrien } 293234449Sobrien return true; 294234449Sobrien} 295234449Sobrien 296234449Sobrien// Create an entry for a permanent space. 297234449SobrienPermanentMemSpace* MemMgr::NewPermanentSpace(PolyWord *base, uintptr_t words, 298234449Sobrien unsigned flags, unsigned index, unsigned hierarchy /*= 0*/) 299234449Sobrien{ 300234449Sobrien try { 301234449Sobrien PermanentMemSpace *space = new PermanentMemSpace(0/* Not freed */); 302234449Sobrien space->bottom = base; 303234449Sobrien space->topPointer = space->top = space->bottom + words; 304175296Sobrien space->spaceType = ST_PERMANENT; 305234449Sobrien space->isMutable = flags & MTF_WRITEABLE ? true : false; 306234449Sobrien space->noOverwrite = flags & MTF_NO_OVERWRITE ? true : false; 307234449Sobrien space->byteOnly = flags & MTF_BYTES ? true : false; 308234449Sobrien space->isCode = flags & MTF_EXECUTABLE ? true : false; 309234449Sobrien space->index = index; 310234449Sobrien space->hierarchy = hierarchy; 311234449Sobrien if (index >= nextIndex) nextIndex = index+1; 312175296Sobrien 313234449Sobrien // Extend the permanent memory table and add this space to it. 314234449Sobrien try { 315234449Sobrien AddTree(space); 316234449Sobrien pSpaces.push_back(space); 317234449Sobrien } 318234449Sobrien catch (std::exception&) { 319234449Sobrien RemoveTree(space); 320234449Sobrien delete space; 321234449Sobrien return 0; 322175296Sobrien } 323234449Sobrien return space; 324234449Sobrien } 325234449Sobrien catch (std::bad_alloc&) { 326234449Sobrien return 0; 327234449Sobrien } 328175296Sobrien} 329234449Sobrien 330234449SobrienPermanentMemSpace *MemMgr::AllocateNewPermanentSpace(uintptr_t byteSize, unsigned flags, unsigned index, unsigned hierarchy) 331234449Sobrien{ 332234449Sobrien try { 333234449Sobrien OSMem *alloc = flags & MTF_EXECUTABLE ? &osCodeAlloc : &osHeapAlloc; 334234449Sobrien PermanentMemSpace *space = new PermanentMemSpace(alloc); 335234449Sobrien size_t actualSize = byteSize; 336234449Sobrien PolyWord* base; 337234449Sobrien void* newShadow=0; 338234449Sobrien if (flags & MTF_EXECUTABLE) 339234449Sobrien base = (PolyWord*)alloc->AllocateCodeArea(actualSize, newShadow); 340234449Sobrien else base = (PolyWord*)alloc->AllocateDataArea(actualSize); 341234449Sobrien if (base == 0) 342234449Sobrien { 343234449Sobrien delete(space); 344234449Sobrien return 0; 345234449Sobrien } 346234449Sobrien space->bottom = base; 347234449Sobrien space->shadowSpace = (PolyWord*)newShadow; 348234449Sobrien space->topPointer = space->top = space->bottom + actualSize/sizeof(PolyWord); 349234449Sobrien space->spaceType = ST_PERMANENT; 350234449Sobrien space->isMutable = flags & MTF_WRITEABLE ? true : false; 351234449Sobrien space->noOverwrite = flags & MTF_NO_OVERWRITE ? true : false; 352234449Sobrien space->byteOnly = flags & MTF_BYTES ? true : false; 353234449Sobrien space->isCode = flags & MTF_EXECUTABLE ? true : false; 354234449Sobrien space->index = index; 355234449Sobrien space->hierarchy = hierarchy; 356234449Sobrien if (index >= nextIndex) nextIndex = index + 1; 357234449Sobrien 358234449Sobrien // Extend the permanent memory table and add this space to it. 359234449Sobrien try { 360234449Sobrien AddTree(space); 361234449Sobrien pSpaces.push_back(space); 362234449Sobrien } 363234449Sobrien catch (std::exception&) { 364234449Sobrien RemoveTree(space); 365234449Sobrien delete space; 366234449Sobrien return 0; 367133359Sobrien } 368175296Sobrien return space; 369234449Sobrien } 370268515Sdelphij catch (std::bad_alloc&) { 371268515Sdelphij return 0; 372268515Sdelphij } 373268515Sdelphij} 374268515Sdelphij 375268515Sdelphijbool MemMgr::CompletePermanentSpaceAllocation(PermanentMemSpace *space) 376268515Sdelphij{ 377268515Sdelphij // Remove write access unless it is mutable. 378234449Sobrien // Don't remove write access unless this is top-level. Share-data assumes only hierarchy 0 is write-protected. 379234449Sobrien if (!space->isMutable && space->hierarchy == 0) 380234449Sobrien { 381234449Sobrien if (space->isCode) 382234449Sobrien osCodeAlloc.DisableWriteForCode(space->bottom, space->shadowSpace, (char*)space->top - (char*)space->bottom); 383234449Sobrien else osHeapAlloc.EnableWrite(false, space->bottom, (char*)space->top - (char*)space->bottom); 384234449Sobrien } 385234449Sobrien return true; 386234449Sobrien} 387234449Sobrien 388234449Sobrien// Delete a local space and remove it from the table. 389234449Sobrienvoid MemMgr::DeleteLocalSpace(std::vector<LocalMemSpace*>::iterator &iter) 390234449Sobrien{ 391234449Sobrien LocalMemSpace *sp = *iter; 392234449Sobrien if (debugOptions & DEBUG_MEMMGR) 393234449Sobrien Log("MMGR: Deleted local %s space %p at %p size %zu\n", sp->spaceTypeString(), sp, sp->bottom, sp->spaceSize()); 394234449Sobrien currentHeapSize -= sp->spaceSize(); 395175296Sobrien globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord)); 396234449Sobrien if (sp->allocationSpace) currentAllocSpace -= sp->spaceSize(); 397234449Sobrien RemoveTree(sp); 398234449Sobrien delete(sp); 399234449Sobrien iter = lSpaces.erase(iter); 400234449Sobrien} 401234449Sobrien 402234449Sobrien// Remove local areas that are now empty after a GC. 403234449Sobrien// It isn't clear if we always want to do this. 404234449Sobrienvoid MemMgr::RemoveEmptyLocals() 405234449Sobrien{ 406234449Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); ) 407234449Sobrien { 408234449Sobrien LocalMemSpace *space = *i; 409234449Sobrien if (space->isEmpty()) 410234449Sobrien DeleteLocalSpace(i); 411234449Sobrien else i++; 412175296Sobrien } 413175296Sobrien} 414234449Sobrien 415234449Sobrien// Create and initialise a new export space and add it to the table. 416234449SobrienPermanentMemSpace* MemMgr::NewExportSpace(uintptr_t size, bool mut, bool noOv, bool code) 417234449Sobrien{ 418234449Sobrien try { 419234449Sobrien OSMem *alloc = code ? &osCodeAlloc : &osHeapAlloc; 420234449Sobrien PermanentMemSpace *space = new PermanentMemSpace(alloc); 421234449Sobrien space->spaceType = ST_EXPORT; 422234449Sobrien space->isMutable = mut; 423234449Sobrien space->noOverwrite = noOv; 424234449Sobrien space->isCode = code; 425234449Sobrien space->index = nextIndex++; 426234449Sobrien // Allocate the memory itself. 427234449Sobrien size_t iSpace = size*sizeof(PolyWord); 428234449Sobrien if (code) 429175296Sobrien { 430234449Sobrien void* shadow; 431234449Sobrien space->bottom = (PolyWord*)alloc->AllocateCodeArea(iSpace, shadow); 432234449Sobrien if (space->bottom != 0) 433175296Sobrien space->shadowSpace = (PolyWord*)shadow; 434234449Sobrien } 435175296Sobrien else space->bottom = (PolyWord*)alloc->AllocateDataArea(iSpace); 436175296Sobrien 437234449Sobrien if (space->bottom == 0) 438234449Sobrien { 439175296Sobrien delete space; 440234449Sobrien if (debugOptions & DEBUG_MEMMGR) 441175296Sobrien Log("MMGR: New export %smutable space: insufficient space\n", mut ? "" : "im"); 442175296Sobrien return 0; 443234449Sobrien } 444234449Sobrien 445175296Sobrien // The size may have been rounded up to a block boundary. 446234449Sobrien size = iSpace/sizeof(PolyWord); 447175296Sobrien space->top = space->bottom + size; 448175296Sobrien space->topPointer = space->bottom; 449234449Sobrien#ifdef POLYML32IN64 450234449Sobrien // The address must be on an odd-word boundary so that after the length 451234449Sobrien // word is put in the actual cell address is on an even-word boundary. 452234449Sobrien space->writeAble(space->topPointer)[0] = PolyWord::FromUnsigned(0); 453234449Sobrien space->topPointer = space->bottom + 1; 454234449Sobrien#endif 455234449Sobrien 456234449Sobrien if (debugOptions & DEBUG_MEMMGR) 457234449Sobrien Log("MMGR: New export %smutable %s%sspace %p, size=%luk words, bottom=%p, top=%p\n", mut ? "" : "im", 458234449Sobrien noOv ? "no-overwrite " : "", code ? "code " : "", space, 459234449Sobrien space->spaceSize() / 1024, space->bottom, space->top); 460234449Sobrien 461234449Sobrien // Add to the table. 462234449Sobrien try { 463234449Sobrien AddTree(space); 464234449Sobrien eSpaces.push_back(space); 465234449Sobrien } 466234449Sobrien catch (std::exception&) { 467175296Sobrien RemoveTree(space); 468234449Sobrien delete space; 469234449Sobrien if (debugOptions & DEBUG_MEMMGR) 470234449Sobrien Log("MMGR: New export %smutable space: Adding to tree failed\n", mut ? "" : "im"); 471234449Sobrien return 0; 472234449Sobrien } 473234449Sobrien return space; 474175296Sobrien } 475175296Sobrien catch (std::bad_alloc&) { 476234449Sobrien if (debugOptions & DEBUG_MEMMGR) 477234449Sobrien Log("MMGR: New export %smutable space: \"new\" failed\n", mut ? "" : "im"); 478234449Sobrien return 0; 479234449Sobrien } 480234449Sobrien} 481175296Sobrien 482175296Sobrienvoid MemMgr::DeleteExportSpaces(void) 483175296Sobrien{ 484175296Sobrien for (std::vector<PermanentMemSpace *>::iterator i = eSpaces.begin(); i < eSpaces.end(); i++) 485133359Sobrien { 486175296Sobrien PermanentMemSpace *space = *i; 487175296Sobrien RemoveTree(space); 488175296Sobrien delete(space); 489175296Sobrien } 490133359Sobrien eSpaces.clear(); 491175296Sobrien} 492175296Sobrien 493133359Sobrien// If we have saved the state rather than exported a function we turn the exported 494175296Sobrien// spaces into permanent ones, removing existing permanent spaces at the same or 495133359Sobrien// lower level. 496175296Sobrienbool MemMgr::PromoteExportSpaces(unsigned hierarchy) 497234449Sobrien{ 498133359Sobrien // Save permanent spaces at a lower hierarchy. Others are converted into 499268515Sdelphij // local spaces. Most or all items will have been copied from these spaces 500268515Sdelphij // into an export space but there could be items reachable only from the stack. 501268515Sdelphij std::vector<PermanentMemSpace*>::iterator i = pSpaces.begin(); 502268515Sdelphij while (i != pSpaces.end()) 503133359Sobrien { 504133359Sobrien PermanentMemSpace *pSpace = *i; 505175296Sobrien if (pSpace->hierarchy < hierarchy) 506175296Sobrien i++; 507133359Sobrien else 508133359Sobrien { 509133359Sobrien try { 510133359Sobrien // Turn this into a local space or a code space 511175296Sobrien // Remove this from the tree - AddLocalSpace will make an entry for the local version. 512234449Sobrien RemoveTree(pSpace); 513175296Sobrien 514234449Sobrien if (pSpace->isCode) 515175296Sobrien { 516234449Sobrien // Enable write access. Permanent spaces are read-only. 517234449Sobrien // osCodeAlloc.SetPermissions(pSpace->bottom, (char*)pSpace->top - (char*)pSpace->bottom, 518234449Sobrien // PERMISSION_READ | PERMISSION_WRITE | PERMISSION_EXEC); 519175296Sobrien CodeSpace *space = new CodeSpace(pSpace->bottom, pSpace->shadowSpace, pSpace->spaceSize(), &osCodeAlloc); 520175296Sobrien if (! space->headerMap.Create(space->spaceSize())) 521175296Sobrien { 522133359Sobrien if (debugOptions & DEBUG_MEMMGR) 523133359Sobrien Log("MMGR: Unable to create header map for state space %p\n", pSpace); 524133359Sobrien return false; 525175296Sobrien } 526175296Sobrien if (!AddCodeSpace(space)) 527175296Sobrien { 528175296Sobrien if (debugOptions & DEBUG_MEMMGR) 529234449Sobrien Log("MMGR: Unable to convert saved state space %p into code space\n", pSpace); 530175296Sobrien return false; 531234449Sobrien } 532234449Sobrien if (debugOptions & DEBUG_MEMMGR) 533234449Sobrien Log("MMGR: Converted saved state space %p into code space %p\n", pSpace, space); 534234449Sobrien // Set the bits in the header map. 535234449Sobrien for (PolyWord *ptr = space->bottom; ptr < space->top; ) 536234449Sobrien { 537268515Sdelphij PolyObject *obj = (PolyObject*)(ptr+1); 538234449Sobrien // We may have forwarded this if this has been 539268515Sdelphij // copied to the exported area. Restore the original length word. 540234449Sobrien if (obj->ContainsForwardingPtr()) 541234449Sobrien { 542234449Sobrien#ifdef POLYML32IN64 543268515Sdelphij PolyObject *forwardedTo = obj; 544234449Sobrien // This is relative to globalCodeBase not globalHeapBase 545133359Sobrien while (forwardedTo->ContainsForwardingPtr()) 546268515Sdelphij forwardedTo = (PolyObject*)(globalCodeBase + ((forwardedTo->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); 547133359Sobrien#else 548175296Sobrien PolyObject *forwardedTo = obj->FollowForwardingChain(); 549175296Sobrien#endif 550133359Sobrien obj->SetLengthWord(forwardedTo->LengthWord()); 551159764Sobrien } 552234449Sobrien // Set the "start" bit if this is allocated. It will be a byte seg if not. 553159764Sobrien if (obj->IsCodeObject()) 554159764Sobrien space->headerMap.SetBit(ptr-space->bottom); 555159764Sobrien ASSERT(!obj->IsClosureObject()); 556159764Sobrien ptr += obj->Length() + 1; 557159764Sobrien } 558268515Sdelphij } 559268515Sdelphij else 560133359Sobrien { 561133359Sobrien // Enable write access. Permanent spaces are read-only. 562159764Sobrien// osHeapAlloc.SetPermissions(pSpace->bottom, (char*)pSpace->top - (char*)pSpace->bottom, 563133359Sobrien// PERMISSION_READ | PERMISSION_WRITE); 564133359Sobrien LocalMemSpace *space = new LocalMemSpace(&osHeapAlloc); 565159764Sobrien space->top = pSpace->top; 566133359Sobrien // Space is allocated in local areas from the top down. This area is full and 567159764Sobrien // all data is in the old generation. The area can be recovered by a full GC. 568159764Sobrien space->bottom = space->upperAllocPtr = space->lowerAllocPtr = 569159764Sobrien space->fullGCLowerLimit = pSpace->bottom; 570234449Sobrien space->isMutable = pSpace->isMutable; 571234449Sobrien space->isCode = false; 572159764Sobrien if (! space->bitmap.Create(space->top-space->bottom) || ! AddLocalSpace(space)) 573133359Sobrien { 574234449Sobrien if (debugOptions & DEBUG_MEMMGR) 575133359Sobrien Log("MMGR: Unable to convert saved state space %p into local space\n", pSpace); 576133359Sobrien return false; 577133359Sobrien } 578133359Sobrien 579133359Sobrien if (debugOptions & DEBUG_MEMMGR) 580133359Sobrien Log("MMGR: Converted saved state space %p into local %smutable space %p\n", 58168349Sobrien pSpace, pSpace->isMutable ? "im": "", space); 582175296Sobrien currentHeapSize += space->spaceSize(); 583159764Sobrien globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord)); 584175296Sobrien } 585133359Sobrien i = pSpaces.erase(i); 586133359Sobrien } 587133359Sobrien catch (std::bad_alloc&) { 588133359Sobrien return false; 58968349Sobrien } 590133359Sobrien } 591186691Sobrien } 592186691Sobrien // Save newly exported spaces. 593284193Sdelphij for(std::vector<PermanentMemSpace *>::iterator j = eSpaces.begin(); j < eSpaces.end(); j++) 594284193Sdelphij { 595186691Sobrien PermanentMemSpace *space = *j; 596234449Sobrien space->hierarchy = hierarchy; // Set the hierarchy of the new spaces. 597133359Sobrien space->spaceType = ST_PERMANENT; 598133359Sobrien // Put a dummy object to fill up the unused space. 599133359Sobrien if (space->topPointer != space->top) 600133359Sobrien FillUnusedSpace(space->writeAble(space->topPointer), space->top - space->topPointer); 601175296Sobrien // Put in a dummy object to fill the rest of the space. 602133359Sobrien pSpaces.push_back(space); 603133359Sobrien } 604175296Sobrien eSpaces.clear(); 605133359Sobrien 606133359Sobrien return true; 607175296Sobrien} 608133359Sobrien 609133359Sobrien 610133359Sobrien// Before we import a hierarchical saved state we need to turn any previously imported 611175296Sobrien// spaces into local spaces. 612133359Sobrienbool MemMgr::DemoteImportSpaces() 613133359Sobrien{ 614133359Sobrien return PromoteExportSpaces(1); // Only truly permanent spaces are retained. 615175296Sobrien} 616175296Sobrien 617133359Sobrien// Return the space for a given index 618133359SobrienPermanentMemSpace *MemMgr::SpaceForIndex(unsigned index) 619133359Sobrien{ 620133359Sobrien for (std::vector<PermanentMemSpace*>::iterator i = pSpaces.begin(); i < pSpaces.end(); i++) 621175296Sobrien { 622133359Sobrien PermanentMemSpace *space = *i; 623133359Sobrien if (space->index == index) 624175296Sobrien return space; 625133359Sobrien } 626133359Sobrien return NULL; 627175296Sobrien} 628175296Sobrien 629175296Sobrien// In several places we assume that segments are filled with valid 630175296Sobrien// objects. This fills unused memory with one or more "byte" objects. 631133359Sobrienvoid MemMgr::FillUnusedSpace(PolyWord *base, uintptr_t words) 632133359Sobrien{ 633133359Sobrien PolyWord *pDummy = base+1; 634268515Sdelphij while (words > 0) 635234449Sobrien { 636234449Sobrien#ifdef POLYML32IN64 637234449Sobrien // Make sure that any dummy object we insert is properly aligned. 638234449Sobrien if (((uintptr_t)pDummy) & 4) 639234449Sobrien { 640234449Sobrien *pDummy++ = PolyWord::FromUnsigned(0); 641268515Sdelphij words--; 642268515Sdelphij continue; 643234449Sobrien } 644234449Sobrien#endif 645234449Sobrien POLYUNSIGNED oSize; 646234449Sobrien // If the space is larger than the maximum object size 647234449Sobrien // we will need several objects. 648268515Sdelphij if (words > MAX_OBJECT_SIZE) oSize = MAX_OBJECT_SIZE; 649234449Sobrien else oSize = (POLYUNSIGNED)(words-1); 650268515Sdelphij // Make this a byte object so it's always skipped. 651234449Sobrien ((PolyObject*)pDummy)->SetLengthWord(oSize, F_BYTE_OBJ); 652268515Sdelphij words -= oSize+1; 653234449Sobrien pDummy += oSize+1; 654234449Sobrien } 655234449Sobrien} 656234449Sobrien 657234449Sobrien// Allocate an area of the heap of at least minWords and at most maxWords. 658234449Sobrien// This is used both when allocating single objects (when minWords and maxWords 659234449Sobrien// are the same) and when allocating heap segments. If there is insufficient 660234449Sobrien// space to satisfy the minimum it will return 0. 661175296SobrienPolyWord *MemMgr::AllocHeapSpace(uintptr_t minWords, uintptr_t &maxWords, bool doAllocation) 662234449Sobrien{ 663175296Sobrien PLocker locker(&allocLock); 664175296Sobrien // We try to distribute the allocations between the memory spaces 665234449Sobrien // so that at the next GC we don't have all the most recent cells in 666234449Sobrien // one space. The most recent cells will be more likely to survive a 667234449Sobrien // GC so distibuting them improves the load balance for a multi-thread GC. 668268515Sdelphij nextAllocator++; 669234449Sobrien if (nextAllocator > gMem.lSpaces.size()) nextAllocator = 0; 670234449Sobrien 671234449Sobrien unsigned j = nextAllocator; 672234449Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) 673234449Sobrien { 674234449Sobrien if (j >= gMem.lSpaces.size()) j = 0; 675234449Sobrien LocalMemSpace *space = gMem.lSpaces[j++]; 676234449Sobrien if (space->allocationSpace) 677234449Sobrien { 678234449Sobrien uintptr_t available = space->freeSpace(); 679234449Sobrien if (available > 0 && available >= minWords) 680234449Sobrien { 681234449Sobrien // Reduce the maximum value if we had less than that. 682234449Sobrien if (available < maxWords) maxWords = available; 683234449Sobrien#ifdef POLYML32IN64 684234449Sobrien // If necessary round down to an even boundary 685234449Sobrien if (maxWords & 1) 686234449Sobrien { 687234449Sobrien maxWords--; 688234449Sobrien space->lowerAllocPtr[maxWords] = PolyWord::FromUnsigned(0); 689234449Sobrien } 690234449Sobrien#endif 691234449Sobrien PolyWord *result = space->lowerAllocPtr; // Return the address. 692234449Sobrien if (doAllocation) 693234449Sobrien space->lowerAllocPtr += maxWords; // Allocate it. 694234449Sobrien#ifdef POLYML32IN64 695234449Sobrien ASSERT((uintptr_t)result & 4); // Must be odd-word aligned 696234449Sobrien#endif 697234449Sobrien return result; 698234449Sobrien } 699234449Sobrien } 700268515Sdelphij } 701268515Sdelphij // There isn't space in the existing areas - can we create a new area? 702234449Sobrien // The reason we don't have enough space could simply be that we want to 703234449Sobrien // allocate an object larger than the default space size. Try deleting 704234449Sobrien // some other spaces to bring currentAllocSpace below spaceBeforeMinorGC - minWords. 705234449Sobrien if (minWords > defaultSpaceSize && minWords < spaceBeforeMinorGC) 706234449Sobrien RemoveExcessAllocation(spaceBeforeMinorGC - minWords); 707234449Sobrien 708234449Sobrien if (currentAllocSpace/* + minWords */ < spaceBeforeMinorGC) 709234449Sobrien { 710234449Sobrien // i.e. the current allocation space is less than the space allowed for the minor GC 711234449Sobrien // but it may be that allocating this object will take us over the limit. We allow 712234449Sobrien // that to happen so that we can successfully allocate very large objects even if 713234449Sobrien // we have a new GC very shortly. 714234449Sobrien uintptr_t spaceSize = defaultSpaceSize; 715234449Sobrien#ifdef POLYML32IN64 716234449Sobrien // When we create the allocation space we take one word so that the first 717234449Sobrien // length word is on an odd-word boundary. We need to allow for that otherwise 718234449Sobrien // we may have available < minWords. 719234449Sobrien if (minWords >= spaceSize) spaceSize = minWords+1; // If we really want a large space. 720234449Sobrien#else 721234449Sobrien if (minWords > spaceSize) spaceSize = minWords; // If we really want a large space. 722234449Sobrien#endif 723234449Sobrien LocalMemSpace *space = CreateAllocationSpace(spaceSize); 724234449Sobrien if (space == 0) return 0; // Can't allocate it 725234449Sobrien // Allocate our space in this new area. 726234449Sobrien uintptr_t available = space->freeSpace(); 727234449Sobrien ASSERT(available >= minWords); 728234449Sobrien if (available < maxWords) 729234449Sobrien { 730234449Sobrien maxWords = available; 731234449Sobrien#ifdef POLYML32IN64 732234449Sobrien // If necessary round down to an even boundary 733234449Sobrien if (maxWords & 1) 734234449Sobrien { 735234449Sobrien maxWords--; 736234449Sobrien space->lowerAllocPtr[maxWords] = PolyWord::FromUnsigned(0); 737234449Sobrien } 738234449Sobrien#endif 739234449Sobrien } 740234449Sobrien PolyWord *result = space->lowerAllocPtr; // Return the address. 741234449Sobrien if (doAllocation) 742234449Sobrien space->lowerAllocPtr += maxWords; // Allocate it. 743234449Sobrien#ifdef POLYML32IN64 744234449Sobrien ASSERT((uintptr_t)result & 4); // Must be odd-word aligned 745234449Sobrien#endif 746234449Sobrien return result; 747234449Sobrien } 748234449Sobrien return 0; // There isn't space even for the minimum. 749234449Sobrien} 750234449Sobrien 751234449SobrienCodeSpace::CodeSpace(PolyWord *start, PolyWord *shadow, uintptr_t spaceSize, OSMem *alloc): MarkableSpace(alloc) 752234449Sobrien{ 753234449Sobrien bottom = start; 754234449Sobrien shadowSpace = shadow; 755234449Sobrien top = start+spaceSize; 756234449Sobrien isMutable = true; // Make it mutable just in case. This will cause it to be scanned. 757234449Sobrien isCode = true; 758234449Sobrien spaceType = ST_CODE; 759234449Sobrien#ifdef POLYML32IN64 760234449Sobrien // Dummy word so that the cell itself, after the length word, is on an 8-byte boundary. 761234449Sobrien writeAble(start)[0] = PolyWord::FromUnsigned(0); 762234449Sobrien largestFree = spaceSize - 2; 763159764Sobrien firstFree = start+1; 764234449Sobrien#else 765234449Sobrien largestFree = spaceSize - 1; 766234449Sobrien firstFree = start; 767234449Sobrien#endif 768234449Sobrien} 769234449Sobrien 770234449SobrienCodeSpace *MemMgr::NewCodeSpace(uintptr_t size) 771234449Sobrien{ 772234449Sobrien // Allocate a new area and add it at the end of the table. 773268515Sdelphij CodeSpace *allocSpace = 0; 774234449Sobrien // Allocate a new mutable, code space. N.B. This may round up "actualSize". 775234449Sobrien size_t actualSize = size * sizeof(PolyWord); 776268515Sdelphij void* shadow; 777234449Sobrien PolyWord *mem = 778234449Sobrien (PolyWord*)osCodeAlloc.AllocateCodeArea(actualSize, shadow); 779234449Sobrien if (mem != 0) 780234449Sobrien { 781175296Sobrien try { 782175296Sobrien allocSpace = new CodeSpace(mem, (PolyWord*)shadow, actualSize / sizeof(PolyWord), &osCodeAlloc); 783175296Sobrien allocSpace->shadowSpace = (PolyWord*)shadow; 784175296Sobrien if (!allocSpace->headerMap.Create(allocSpace->spaceSize())) 785175296Sobrien { 786175296Sobrien delete allocSpace; 787175296Sobrien allocSpace = 0; 788175296Sobrien } 789234449Sobrien else if (!AddCodeSpace(allocSpace)) 790133359Sobrien { 791175296Sobrien delete allocSpace; 79268349Sobrien allocSpace = 0; 793133359Sobrien } 794133359Sobrien else if (debugOptions & DEBUG_MEMMGR) 795234449Sobrien Log("MMGR: New code space %p allocated at %p size %lu\n", allocSpace, allocSpace->bottom, allocSpace->spaceSize()); 796234449Sobrien // Put in a byte cell to mark the area as unallocated. 79768349Sobrien FillUnusedSpace(allocSpace->writeAble(allocSpace->firstFree), allocSpace->top- allocSpace->firstFree); 79868349Sobrien } 799133359Sobrien catch (std::bad_alloc&) 80068349Sobrien { 80168349Sobrien } 80268349Sobrien if (allocSpace == 0) 80368349Sobrien { 80468349Sobrien osCodeAlloc.FreeCodeArea(mem, shadow, actualSize); 80568349Sobrien mem = 0; 80668349Sobrien } 80768349Sobrien } 80868349Sobrien return allocSpace; 80968349Sobrien} 81068349Sobrien 81168349Sobrien// Allocate memory for a piece of code. This needs to be both mutable and executable, 81268349Sobrien// at least for native code. The interpreted version need not (should not?) make the 813133359Sobrien// area executable. It will not be executed until the mutable bit has been cleared. 814133359Sobrien// Once code is allocated it is not GCed or moved. 815133359Sobrien// initCell is a byte cell that is copied into the new code area. 816133359SobrienPolyObject* MemMgr::AllocCodeSpace(POLYUNSIGNED requiredSize) 817133359Sobrien{ 818133359Sobrien PLocker locker(&codeSpaceLock); 819175296Sobrien // Search the code spaces until we find a free area big enough. 82068349Sobrien size_t i = 0; 82168349Sobrien while (true) 82268349Sobrien { 823175296Sobrien if (i != cSpaces.size()) 824175296Sobrien { 82568349Sobrien CodeSpace *space = cSpaces[i]; 82668349Sobrien if (space->largestFree >= requiredSize) 82768349Sobrien { 82868349Sobrien POLYUNSIGNED actualLargest = 0; 82968349Sobrien while (space->firstFree < space->top) 830186691Sobrien { 831175296Sobrien PolyObject *obj = (PolyObject*)(space->firstFree+1); 832175296Sobrien // Skip over allocated areas or free areas that are too small. 833175296Sobrien if (obj->IsCodeObject() || obj->Length() < 8) 834175296Sobrien space->firstFree += obj->Length()+1; 835175296Sobrien else break; 836175296Sobrien } 837175296Sobrien PolyWord *pt = space->firstFree; 838175296Sobrien while (pt < space->top) 83968349Sobrien { 84068349Sobrien PolyObject *obj = (PolyObject*)(pt+1); 841175296Sobrien POLYUNSIGNED length = obj->Length(); 84268349Sobrien if (obj->IsByteObject()) 84368349Sobrien { 84468349Sobrien if (length >= requiredSize) 84568349Sobrien { 846175296Sobrien // Free and large enough 84768349Sobrien PolyWord *next = pt+requiredSize+1; 84868349Sobrien POLYUNSIGNED spare = length - requiredSize; 84968349Sobrien#ifdef POLYML32IN64 85068349Sobrien // Maintain alignment. 851175296Sobrien if (((requiredSize + 1) & 1) && spare != 0) 852234449Sobrien { 853234449Sobrien space->writeAble(next++)[0] = PolyWord::FromUnsigned(0); 854234449Sobrien spare--; 855175296Sobrien } 85668349Sobrien#endif 85768349Sobrien if (spare != 0) 85868349Sobrien FillUnusedSpace(space->writeAble(next), spare); 859175296Sobrien space->isMutable = true; // Set this - it ensures the area is scanned on GC. 860175296Sobrien space->headerMap.SetBit(pt-space->bottom); // Set the "header" bit 861175296Sobrien // Set the length word of the code area and copy the byte cell in. 86268349Sobrien // The code bit must be set before the lock is released to ensure 86368349Sobrien // another thread doesn't reuse this. 86468349Sobrien space->writeAble(obj)->SetLengthWord(requiredSize, F_CODE_OBJ|F_MUTABLE_BIT); 86568349Sobrien return obj; 866133359Sobrien } 86768349Sobrien else if (length >= actualLargest) actualLargest = length+1; 86868349Sobrien } 869133359Sobrien pt += length+1; 87068349Sobrien } 871133359Sobrien // Reached the end without finding what we wanted. Update the largest size. 87268349Sobrien space->largestFree = actualLargest; 87368349Sobrien } 87468349Sobrien i++; // Next area 87568349Sobrien } 87668349Sobrien else 87768349Sobrien { 878133359Sobrien // Allocate a new area and add it at the end of the table. 87968349Sobrien uintptr_t spaceSize = requiredSize + 1; 880133359Sobrien#ifdef POLYML32IN64 881133359Sobrien // We need to allow for the extra alignment word otherwise we 882133359Sobrien // may allocate less than we need. 883175296Sobrien spaceSize += 1; 88468349Sobrien#endif 885175296Sobrien CodeSpace *allocSpace = NewCodeSpace(spaceSize); 886133359Sobrien if (allocSpace == 0) 88768349Sobrien return 0; // Try a GC. 888175296Sobrien globalStats.incSize(PSS_CODE_SPACE, allocSpace->spaceSize() * sizeof(PolyWord)); 889175296Sobrien } 890175296Sobrien } 891175296Sobrien} 892175296Sobrien 893175296Sobrien// Remove code areas that are completely empty. This is probably better than waiting to reuse them. 894175296Sobrien// It's particularly important if we reload a saved state because the code areas for old saved states 89568349Sobrien// are made into local code areas just in case they are currently in use or reachable. 896234449Sobrienvoid MemMgr::RemoveEmptyCodeAreas() 89768349Sobrien{ 898234449Sobrien for (std::vector<CodeSpace *>::iterator i = cSpaces.begin(); i != cSpaces.end(); ) 899234449Sobrien { 900234449Sobrien CodeSpace *space = *i; 901234449Sobrien PolyObject *start = (PolyObject *)(space->bottom+1); 902234449Sobrien if (start->IsByteObject() && start->Length() == space->spaceSize()-1) 903234449Sobrien { 904234449Sobrien if (debugOptions & DEBUG_MEMMGR) 905234449Sobrien Log("MMGR: Deleted code space %p at %p size %zu\n", space, space->bottom, space->spaceSize()); 906234449Sobrien globalStats.decSize(PSS_CODE_SPACE, space->spaceSize() * sizeof(PolyWord)); 907234449Sobrien // We have an empty cell that fills the whole space. 908234449Sobrien RemoveTree(space); 909234449Sobrien delete(space); 91068349Sobrien i = cSpaces.erase(i); 911175296Sobrien } 912175296Sobrien else i++; 913175296Sobrien } 914175296Sobrien} 915175296Sobrien 916175296Sobrien// Add a code space to the tables. Used both for newly compiled code and also demoted saved spaces. 917175296Sobrienbool MemMgr::AddCodeSpace(CodeSpace *space) 918175296Sobrien{ 919175296Sobrien try { 920175296Sobrien AddTree(space); 92168349Sobrien cSpaces.push_back(space); 922234449Sobrien } 92368349Sobrien catch (std::exception&) { 924234449Sobrien RemoveTree(space); 925234449Sobrien return false; 926234449Sobrien } 927234449Sobrien return true; 928234449Sobrien} 929234449Sobrien 930234449Sobrien// Check that we have sufficient space for an allocation to succeed. 931234449Sobrien// Called from the GC to ensure that we will not get into an infinite 932234449Sobrien// loop trying to allocate, failing and garbage-collecting again. 933234449Sobrienbool MemMgr::CheckForAllocation(uintptr_t words) 934234449Sobrien{ 935234449Sobrien uintptr_t allocated = 0; 93668349Sobrien return AllocHeapSpace(words, allocated, false) != 0; 93768349Sobrien} 93868349Sobrien 93968349Sobrien// Adjust the allocation area by removing free areas so that the total 94068349Sobrien// size of the allocation area is less than the required value. This 94168349Sobrien// is used after the quick GC and also if we need to allocate a large 94268349Sobrien// object. 94368349Sobrienvoid MemMgr::RemoveExcessAllocation(uintptr_t words) 944133359Sobrien{ 94568349Sobrien // First remove any non-standard allocation areas. 94668349Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end();) 94768349Sobrien { 94868349Sobrien LocalMemSpace *space = *i; 94968349Sobrien if (space->allocationSpace && space->isEmpty() && 950133359Sobrien space->spaceSize() != defaultSpaceSize) 951133359Sobrien DeleteLocalSpace(i); 952133359Sobrien else i++; 953133359Sobrien } 954133359Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); currentAllocSpace > words && i < lSpaces.end(); ) 955133359Sobrien { 95668349Sobrien LocalMemSpace *space = *i; 95768349Sobrien if (space->allocationSpace && space->isEmpty()) 958133359Sobrien DeleteLocalSpace(i); 95968349Sobrien else i++; 960133359Sobrien } 96168349Sobrien} 962175296Sobrien 963175296Sobrien// Return number of words free in all allocation spaces. 964175296Sobrienuintptr_t MemMgr::GetFreeAllocSpace() 965175296Sobrien{ 966175296Sobrien uintptr_t freeSpace = 0; 967175296Sobrien PLocker lock(&allocLock); 96868349Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) 96968349Sobrien { 97068349Sobrien LocalMemSpace *space = *i; 97168349Sobrien if (space->allocationSpace) 97268349Sobrien freeSpace += space->freeSpace(); 973133359Sobrien } 97468349Sobrien return freeSpace; 97568349Sobrien} 97668349Sobrien 97768349SobrienStackSpace *MemMgr::NewStackSpace(uintptr_t size) 978133359Sobrien{ 97968349Sobrien PLocker lock(&stackSpaceLock); 98068349Sobrien 98168349Sobrien try { 98268349Sobrien StackSpace *space = new StackSpace(&osStackAlloc); 983133359Sobrien size_t iSpace = size*sizeof(PolyWord); 98468349Sobrien space->bottom = (PolyWord*)osStackAlloc.AllocateDataArea(iSpace); 98568349Sobrien if (space->bottom == 0) 98668349Sobrien { 98768349Sobrien if (debugOptions & DEBUG_MEMMGR) 98868349Sobrien Log("MMGR: New stack space: insufficient space\n"); 98968349Sobrien delete space; 990133359Sobrien return 0; 99168349Sobrien } 992175296Sobrien 993175296Sobrien // The size may have been rounded up to a block boundary. 994175296Sobrien size = iSpace/sizeof(PolyWord); 995175296Sobrien space->top = space->bottom + size; 996175296Sobrien space->spaceType = ST_STACK; 99768349Sobrien space->isMutable = true; 998175296Sobrien 99968349Sobrien // Add the stack space to the tree. This ensures that operations such as 100068349Sobrien // LocalSpaceForAddress will work for addresses within the stack. We can 1001175296Sobrien // get them in the RTS with functions such as quot_rem and exception stack. 1002133359Sobrien // It's not clear whether they really appear in the GC. 100368349Sobrien try { 100468349Sobrien AddTree(space); 100568349Sobrien sSpaces.push_back(space); 100668349Sobrien } 1007133359Sobrien catch (std::exception&) { 100868349Sobrien RemoveTree(space); 100968349Sobrien delete space; 101068349Sobrien return 0; 101168349Sobrien } 101268349Sobrien if (debugOptions & DEBUG_MEMMGR) 101368349Sobrien Log("MMGR: New stack space %p allocated at %p size %lu\n", space, space->bottom, space->spaceSize()); 1014133359Sobrien globalStats.incSize(PSS_STACK_SPACE, space->spaceSize() * sizeof(PolyWord)); 101568349Sobrien return space; 101668349Sobrien } 101768349Sobrien catch (std::bad_alloc&) { 101868349Sobrien if (debugOptions & DEBUG_MEMMGR) 101968349Sobrien Log("MMGR: New stack space: \"new\" failed\n"); 102068349Sobrien return 0; 102168349Sobrien } 102268349Sobrien} 102368349Sobrien 102468349Sobrien// If checkmem is given write protect the immutable areas except during a GC. 102568349Sobrienvoid MemMgr::ProtectImmutable(bool on) 102668349Sobrien{ 102768349Sobrien if (debugOptions & DEBUG_CHECK_OBJECTS) 1028133359Sobrien { 102968349Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) 103068349Sobrien { 103168349Sobrien LocalMemSpace *space = *i; 103268349Sobrien if (!space->isMutable) 1033133359Sobrien { 103468349Sobrien if (!space->isCode) 103568349Sobrien osHeapAlloc.EnableWrite(!on, space->bottom, (char*)space->top - (char*)space->bottom); 103668349Sobrien } 103768349Sobrien } 103868349Sobrien } 103968349Sobrien} 1040133359Sobrien 104168349Sobrienbool MemMgr::GrowOrShrinkStack(TaskData *taskData, uintptr_t newSize) 104268349Sobrien{ 104368349Sobrien StackSpace *space = taskData->stack; 104468349Sobrien size_t iSpace = newSize*sizeof(PolyWord); 104568349Sobrien 104668349Sobrien PolyWord *newSpace = (PolyWord*)osStackAlloc.AllocateDataArea(iSpace); 1047133359Sobrien if (newSpace == 0) 104868349Sobrien { 104968349Sobrien if (debugOptions & DEBUG_MEMMGR) 105068349Sobrien Log("MMGR: Unable to change size of stack %p from %lu to %lu: insufficient space\n", 105168349Sobrien space, space->spaceSize(), newSize); 105268349Sobrien return false; 105368349Sobrien } 105468349Sobrien // The size may have been rounded up to a block boundary. 105568349Sobrien newSize = iSpace/sizeof(PolyWord); 105668349Sobrien try { 105768349Sobrien AddTree(space, newSpace, newSpace+newSize); 105868349Sobrien } 105968349Sobrien catch (std::bad_alloc&) { 106068349Sobrien RemoveTree(space, newSpace, newSpace+newSize); 106168349Sobrien delete space; 106268349Sobrien return 0; 106368349Sobrien } 1064133359Sobrien taskData->CopyStackFrame(space->stack(), space->spaceSize(), (StackObject*)newSpace, newSize); 106568349Sobrien if (debugOptions & DEBUG_MEMMGR) 1066175296Sobrien Log("MMGR: Size of stack %p changed from %lu to %lu at %p\n", space, space->spaceSize(), newSize, newSpace); 1067175296Sobrien globalStats.incSize(PSS_STACK_SPACE, (newSize - space->spaceSize()) * sizeof(PolyWord)); 1068175296Sobrien RemoveTree(space); // Remove it BEFORE freeing the space - another thread may allocate it 1069175296Sobrien PolyWord *oldBottom = space->bottom; 1070175296Sobrien size_t oldSize = (char*)space->top - (char*)space->bottom; 1071175296Sobrien space->bottom = newSpace; // Switch this before freeing - We could get a profile trap during the free 1072175296Sobrien space->top = newSpace+newSize; 1073175296Sobrien osStackAlloc.FreeDataArea(oldBottom, oldSize); 1074175296Sobrien return true; 1075175296Sobrien} 107668349Sobrien 107768349Sobrien 107868349Sobrien// Delete a stack when a thread has finished. 107968349Sobrien// This can be called by an ML thread so needs an interlock. 108068349Sobrienbool MemMgr::DeleteStackSpace(StackSpace *space) 108168349Sobrien{ 108268349Sobrien PLocker lock(&stackSpaceLock); 108368349Sobrien 1084133359Sobrien for (std::vector<StackSpace *>::iterator i = sSpaces.begin(); i < sSpaces.end(); i++) 108568349Sobrien { 108668349Sobrien if (*i == space) 108768349Sobrien { 108868349Sobrien globalStats.decSize(PSS_STACK_SPACE, space->spaceSize() * sizeof(PolyWord)); 108968349Sobrien RemoveTree(space); 109068349Sobrien delete space; 109168349Sobrien sSpaces.erase(i); 109268349Sobrien if (debugOptions & DEBUG_MEMMGR) 109368349Sobrien Log("MMGR: Deleted stack space %p at %p size %zu\n", space, space->bottom, space->spaceSize()); 109468349Sobrien return true; 1095133359Sobrien } 109668349Sobrien } 109768349Sobrien ASSERT(false); // It should always be in the table. 109868349Sobrien return false; 109968349Sobrien} 1100133359Sobrien 110168349SobrienSpaceTreeTree::SpaceTreeTree(): SpaceTree(false) 110268349Sobrien{ 110368349Sobrien for (unsigned i = 0; i < 256; i++) 110468349Sobrien tree[i] = 0; 1105133359Sobrien} 110668349Sobrien 110768349SobrienSpaceTreeTree::~SpaceTreeTree() 110868349Sobrien{ 110968349Sobrien for (unsigned i = 0; i < 256; i++) 111068349Sobrien { 111168349Sobrien if (tree[i] && ! tree[i]->isSpace) 1112133359Sobrien delete(tree[i]); 111368349Sobrien } 111468349Sobrien} 1115133359Sobrien 111668349Sobrien// Add and remove entries in the space tree. 1117133359Sobrien 111868349Sobrienvoid MemMgr::AddTree(MemSpace *space, PolyWord *startS, PolyWord *endS) 111968349Sobrien{ 112068349Sobrien // It isn't clear we need to lock here but it's probably sensible. 112168349Sobrien PLocker lock(&spaceTreeLock); 1122133359Sobrien AddTreeRange(&spaceTree, space, (uintptr_t)startS, (uintptr_t)endS); 1123133359Sobrien} 112468349Sobrien 112568349Sobrienvoid MemMgr::RemoveTree(MemSpace *space, PolyWord *startS, PolyWord *endS) 1126234449Sobrien{ 112768349Sobrien PLocker lock(&spaceTreeLock); 1128234449Sobrien RemoveTreeRange(&spaceTree, space, (uintptr_t)startS, (uintptr_t)endS); 1129234449Sobrien} 1130234449Sobrien 1131234449Sobrien 1132234449Sobrienvoid MemMgr::AddTreeRange(SpaceTree **tt, MemSpace *space, uintptr_t startS, uintptr_t endS) 1133234449Sobrien{ 1134234449Sobrien if (*tt == 0) 1135234449Sobrien *tt = new SpaceTreeTree; 1136234449Sobrien ASSERT(! (*tt)->isSpace); 1137234449Sobrien SpaceTreeTree *t = (SpaceTreeTree*)*tt; 1138234449Sobrien 1139234449Sobrien const unsigned shift = (sizeof(void*)-1) * 8; // Takes the high-order byte 114068349Sobrien uintptr_t r = startS >> shift; 114168349Sobrien ASSERT(r < 256); 1142234449Sobrien const uintptr_t s = endS == 0 ? 256 : endS >> shift; 114368349Sobrien ASSERT(s >= r && s <= 256); 1144234449Sobrien 1145234449Sobrien if (r == s) // Wholly within this entry 1146234449Sobrien AddTreeRange(&(t->tree[r]), space, startS << 8, endS << 8); 1147234449Sobrien else 1148234449Sobrien { 1149234449Sobrien // Deal with any remainder at the start. 1150234449Sobrien if ((r << shift) != startS) 1151234449Sobrien { 1152234449Sobrien AddTreeRange(&(t->tree[r]), space, startS << 8, 0 /*End of range*/); 1153234449Sobrien r++; 1154234449Sobrien } 1155234449Sobrien // Whole entries. 115668349Sobrien while (r < s) 115768349Sobrien { 115868349Sobrien ASSERT(t->tree[r] == 0); 115968349Sobrien t->tree[r] = space; 116068349Sobrien r++; 116168349Sobrien } 116268349Sobrien // Remainder at the end. 116368349Sobrien if ((s << shift) != endS) 116468349Sobrien AddTreeRange(&(t->tree[r]), space, 0, endS << 8); 116568349Sobrien } 1166133359Sobrien} 116768349Sobrien 116868349Sobrien// Remove an entry from the tree for a range. Strictly speaking we don't need the 116968349Sobrien// space argument here but it's useful as a check. 117068349Sobrien// This may be called to remove a partially installed structure if we have 117168349Sobrien// run out of space in AddTreeRange. 117268349Sobrienvoid MemMgr::RemoveTreeRange(SpaceTree **tt, MemSpace *space, uintptr_t startS, uintptr_t endS) 1173133359Sobrien{ 117468349Sobrien SpaceTreeTree *t = (SpaceTreeTree*)*tt; 1175234449Sobrien if (t == 0) 1176234449Sobrien return; // This can only occur if we're recovering. 117768349Sobrien ASSERT(! t->isSpace); 117868349Sobrien const unsigned shift = (sizeof(void*)-1) * 8; 1179133359Sobrien uintptr_t r = startS >> shift; 1180133359Sobrien const uintptr_t s = endS == 0 ? 256 : endS >> shift; 1181133359Sobrien 1182234449Sobrien if (r == s) 1183234449Sobrien RemoveTreeRange(&(t->tree[r]), space, startS << 8, endS << 8); 1184234449Sobrien else 1185234449Sobrien { 1186175296Sobrien // Deal with any remainder at the start. 1187133359Sobrien if ((r << shift) != startS) 1188133359Sobrien { 118968349Sobrien RemoveTreeRange(&(t->tree[r]), space, startS << 8, 0); 1190133359Sobrien r++; 1191234449Sobrien } 1192133359Sobrien // Whole entries. 1193234449Sobrien while (r < s) 1194234449Sobrien { 119568349Sobrien ASSERT(t->tree[r] == space || t->tree[r] == 0 /* Recovery only */); 119668349Sobrien t->tree[r] = 0; 119768349Sobrien r++; 119868349Sobrien } 119968349Sobrien // Remainder at the end. 120068349Sobrien if ((s << shift) != endS) 1201133359Sobrien RemoveTreeRange(&(t->tree[r]), space, 0, endS << 8); 1202234449Sobrien } 120368349Sobrien // See if the whole vector is now empty. 120468349Sobrien for (unsigned j = 0; j < 256; j++) 1205234449Sobrien { 1206234449Sobrien if (t->tree[j]) 1207234449Sobrien return; // It's not empty - we're done. 1208234449Sobrien } 1209234449Sobrien delete(t); 1210234449Sobrien *tt = 0; 1211234449Sobrien} 1212234449Sobrien 1213234449Sobrienuintptr_t MemMgr::AllocatedInAlloc() 1214175296Sobrien{ 1215175296Sobrien uintptr_t inAlloc = 0; 1216175296Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) 1217175296Sobrien { 1218133359Sobrien LocalMemSpace *sp = *i; 1219175296Sobrien if (sp->allocationSpace) inAlloc += sp->allocatedSpace(); 1220234449Sobrien } 1221133359Sobrien return inAlloc; 1222234449Sobrien} 1223234449Sobrien 1224234449Sobrien// Report heap sizes and occupancy before and after GC 1225234449Sobrienvoid MemMgr::ReportHeapSizes(const char *phase) 1226234449Sobrien{ 1227234449Sobrien uintptr_t alloc = 0, nonAlloc = 0, inAlloc = 0, inNonAlloc = 0; 1228175296Sobrien for (std::vector<LocalMemSpace*>::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) 1229175296Sobrien { 1230133359Sobrien LocalMemSpace *sp = *i; 1231234449Sobrien if (sp->allocationSpace) 1232133359Sobrien { 123368349Sobrien alloc += sp->spaceSize(); 1234133359Sobrien inAlloc += sp->allocatedSpace(); 1235133359Sobrien } 1236133359Sobrien else 1237133359Sobrien { 1238133359Sobrien nonAlloc += sp->spaceSize(); 1239133359Sobrien inNonAlloc += sp->allocatedSpace(); 124068349Sobrien } 1241133359Sobrien } 1242133359Sobrien Log("Heap: %s Major heap used ", phase); 1243133359Sobrien LogSize(inNonAlloc); Log(" of "); 1244133359Sobrien LogSize(nonAlloc); 1245133359Sobrien Log(" (%1.0f%%). Alloc space used ", (float)inNonAlloc / (float)nonAlloc * 100.0F); 1246133359Sobrien LogSize(inAlloc); Log(" of "); 1247133359Sobrien LogSize(alloc); 1248133359Sobrien Log(" (%1.0f%%). Total space ", (float)inAlloc / (float)alloc * 100.0F); 124968349Sobrien LogSize(spaceForHeap); 1250133359Sobrien Log(" %1.0f%% full.\n", (float)(inAlloc + inNonAlloc) / (float)spaceForHeap * 100.0F); 1251133359Sobrien Log("Heap: Local spaces %" PRI_SIZET ", permanent spaces %" PRI_SIZET ", code spaces %" PRI_SIZET ", stack spaces %" PRI_SIZET "\n", 125268349Sobrien lSpaces.size(), pSpaces.size(), cSpaces.size(), sSpaces.size()); 1253133359Sobrien uintptr_t cTotal = 0, cOccupied = 0; 1254133359Sobrien for (std::vector<CodeSpace*>::iterator c = cSpaces.begin(); c != cSpaces.end(); c++) 1255133359Sobrien { 1256175296Sobrien cTotal += (*c)->spaceSize(); 1257175296Sobrien PolyWord *pt = (*c)->bottom; 1258175296Sobrien while (pt < (*c)->top) 1259234449Sobrien { 1260175296Sobrien pt++; 1261234449Sobrien PolyObject *obj = (PolyObject*)pt; 1262175296Sobrien if (obj->ContainsForwardingPtr()) 1263175296Sobrien { 126468349Sobrien#ifdef POLYML32IN64 126568349Sobrien // This is relative to globalCodeBase not globalHeapBase 126668349Sobrien while (obj->ContainsForwardingPtr()) 1267175296Sobrien obj = (PolyObject*)(globalCodeBase + ((obj->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); 1268234449Sobrien#else 1269234449Sobrien obj = obj->FollowForwardingChain(); 1270234449Sobrien#endif 1271234449Sobrien pt += obj->Length(); 1272234449Sobrien } 1273234449Sobrien else 1274175296Sobrien { 1275175296Sobrien if (obj->IsCodeObject()) 1276175296Sobrien cOccupied += obj->Length() + 1; 1277175296Sobrien pt += obj->Length(); 1278175296Sobrien } 1279175296Sobrien } 1280175296Sobrien } 1281175296Sobrien Log("Heap: Code area: total "); LogSize(cTotal); Log(" occupied: "); LogSize(cOccupied); Log("\n"); 1282175296Sobrien uintptr_t stackSpace = 0; 1283175296Sobrien for (std::vector<StackSpace*>::iterator s = sSpaces.begin(); s != sSpaces.end(); s++) 1284175296Sobrien { 1285175296Sobrien stackSpace += (*s)->spaceSize(); 1286175296Sobrien } 1287175296Sobrien Log("Heap: Stack area: total "); LogSize(stackSpace); Log("\n"); 1288175296Sobrien} 1289175296Sobrien 1290175296Sobrien// Profiling - Find a code object or return zero if not found. 129168349Sobrien// This can be called on a "user" thread. 1292175296SobrienPolyObject *MemMgr::FindCodeObject(const byte *addr) 129368349Sobrien{ 129468349Sobrien MemSpace *space = SpaceForAddress(addr); 129568349Sobrien if (space == 0) return 0; 129668349Sobrien Bitmap *profMap = 0; 129768349Sobrien if (! space->isCode) return 0; 1298175296Sobrien if (space->spaceType == ST_CODE) 1299175296Sobrien { 1300234449Sobrien CodeSpace *cSpace = (CodeSpace*)space; 1301175296Sobrien profMap = &cSpace->headerMap; 1302175296Sobrien } 1303175296Sobrien else if (space->spaceType == ST_PERMANENT) 1304234449Sobrien { 1305175296Sobrien PermanentMemSpace *pSpace = (PermanentMemSpace*)space; 1306175296Sobrien profMap = &pSpace->profileCode; 1307175296Sobrien } 1308175296Sobrien else return 0; // Must be in code or permanent code. 130968349Sobrien 1310175296Sobrien // For the permanent areas the header maps are created and initialised on demand. 1311175296Sobrien if (! profMap->Created()) 1312175296Sobrien { 1313175296Sobrien PLocker lock(&codeBitmapLock); 1314175296Sobrien if (! profMap->Created()) // Second check now we've got the lock. 1315175296Sobrien { 1316175296Sobrien // Create the bitmap. If it failed just say "not in this area" 1317175296Sobrien if (! profMap->Create(space->spaceSize())) 1318175296Sobrien return 0; 1319175296Sobrien // Set the first bit before releasing the lock. 1320175296Sobrien profMap->SetBit(0); 1321175296Sobrien } 132268349Sobrien } 1323133359Sobrien 1324133359Sobrien // A bit is set if it is a length word. 1325133359Sobrien while ((uintptr_t)addr & (sizeof(POLYUNSIGNED)-1)) addr--; // Make it word aligned 1326133359Sobrien PolyWord *wordAddr = (PolyWord*)addr; 1327133359Sobrien // Work back to find the first set bit before this. 1328133359Sobrien // Normally we will find one but if we're looking up a value that 1329133359Sobrien // is actually an integer it might be in a piece of code that is now free. 1330284193Sdelphij uintptr_t bitOffset = profMap->FindLastSet(wordAddr - space->bottom); 1331133359Sobrien if (space->spaceType == ST_CODE) 1332133359Sobrien { 1333133359Sobrien PolyWord *ptr = space->bottom+bitOffset; 1334133359Sobrien if (ptr >= space->top) return 0; 1335133359Sobrien // This will find the last non-free code cell or the first cell. 1336133359Sobrien // Return zero if the value was not actually in the cell or it wasn't code. 1337133359Sobrien PolyObject *obj = (PolyObject*)(ptr+1); 1338133359Sobrien#ifdef POLYML32IN64 1339133359Sobrien PolyObject *lastObj = obj; 1340133359Sobrien // This is relative to globalCodeBase not globalHeapBase. 1341133359Sobrien while (lastObj->ContainsForwardingPtr()) 1342133359Sobrien lastObj = (PolyObject*)(globalCodeBase + ((lastObj->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); 1343133359Sobrien#else 1344234449Sobrien PolyObject *lastObj = obj->FollowForwardingChain(); 1345133359Sobrien#endif 1346133359Sobrien // We normally replace forwarding pointers but when scanning to update 1347133359Sobrien // addresses after a saved state we may not have yet done that. 1348133359Sobrien if (wordAddr > ptr && wordAddr < ptr + 1 + lastObj->Length() && lastObj->IsCodeObject()) 1349133359Sobrien return obj; 1350133359Sobrien else return 0; 1351133359Sobrien } 1352234449Sobrien // Permanent area - the bits are set on demand. 1353133359Sobrien // Now work forward, setting any bits if necessary. We don't need a lock 1354234449Sobrien // because this is monotonic. 1355133359Sobrien for (;;) 1356133359Sobrien { 1357133359Sobrien PolyWord *ptr = space->bottom+bitOffset; 1358133359Sobrien if (ptr >= space->top) return 0; 1359133359Sobrien PolyObject *obj = (PolyObject*)(ptr+1); 1360133359Sobrien ASSERT(obj->ContainsNormalLengthWord()); 1361133359Sobrien if (wordAddr > ptr && wordAddr < ptr + obj->Length()) 1362133359Sobrien return obj; 1363133359Sobrien bitOffset += obj->Length()+1; 1364234449Sobrien profMap->SetBit(bitOffset); 1365234449Sobrien } 1366234449Sobrien return 0; 1367234449Sobrien} 1368234449Sobrien 1369234449Sobrien// Remove profiling bitmaps from permanent areas to free up memory. 1370234449Sobrienvoid MemMgr::RemoveProfilingBitmaps() 1371234449Sobrien{ 1372234449Sobrien for (std::vector<PermanentMemSpace*>::iterator i = pSpaces.begin(); i < pSpaces.end(); i++) 1373234449Sobrien (*i)->profileCode.Destroy(); 1374234449Sobrien} 1375234449Sobrien 1376234449Sobrien#ifdef POLYML32IN64DEBUG 1377234449SobrienPOLYOBJECTPTR PolyWord::AddressToObjectPtr(void *address) 1378234449Sobrien{ 1379234449Sobrien ASSERT(address >= globalHeapBase); 1380234449Sobrien uintptr_t offset = (PolyWord*)address - globalHeapBase; 1381234449Sobrien ASSERT(offset <= 0x7fffffff); // Currently limited to 8Gbytes 1382234449Sobrien ASSERT((offset & 1) == 0); 1383133359Sobrien return (POLYOBJECTPTR)offset; 1384133359Sobrien} 1385133359Sobrien#endif 1386133359Sobrien 1387133359SobrienMemMgr gMem; // The one and only memory manager object 1388133359Sobrien 1389133359Sobrien