1/** 2 * Contains the garbage collector implementation. 3 * 4 * Copyright: D Language Foundation 2001 - 2021. 5 * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 6 * Authors: Walter Bright, David Friedman, Sean Kelly 7 */ 8module core.internal.gc.impl.conservative.gc; 9 10// D Programming Language Garbage Collector implementation 11 12/************** Debugging ***************************/ 13 14//debug = PRINTF; // turn on printf's 15//debug = PARALLEL_PRINTF; // turn on printf's 16//debug = COLLECT_PRINTF; // turn on printf's 17//debug = MARK_PRINTF; // turn on printf's 18//debug = PRINTF_TO_FILE; // redirect printf's ouptut to file "gcx.log" 19//debug = LOGGING; // log allocations / frees 20//debug = MEMSTOMP; // stomp on memory 21//debug = SENTINEL; // add underrun/overrrun protection 22 // NOTE: this needs to be enabled globally in the makefiles 23 // (-debug=SENTINEL) to pass druntime's unittests. 24//debug = PTRCHECK; // more pointer checking 25//debug = PTRCHECK2; // thorough but slow pointer checking 26//debug = INVARIANT; // enable invariants 27//debug = PROFILE_API; // profile API calls for config.profile > 1 28//debug = GC_RECURSIVE_LOCK; // check for recursive locking on the same thread 29 30/***************************************************/ 31version = COLLECT_PARALLEL; // parallel scanning 32version (Posix) 33 version = COLLECT_FORK; 34 35import core.internal.gc.bits; 36import core.internal.gc.os; 37import core.gc.config; 38import core.gc.gcinterface; 39 40import core.internal.container.treap; 41import core.internal.spinlock; 42import core.internal.gc.pooltable; 43 44import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; 45import core.stdc.string : memcpy, memset, memmove; 46import core.bitop; 47import core.thread; 48static import core.memory; 49 50version (GNU) import gcc.builtins; 51 52debug (PRINTF_TO_FILE) import core.stdc.stdio : sprintf, fprintf, fopen, fflush, FILE; 53else import core.stdc.stdio : sprintf, printf; // needed to output profiling results 54 55import core.time; 56alias currTime = MonoTime.currTime; 57 58// Track total time spent preparing for GC, 59// marking, sweeping and recovering pages. 60__gshared Duration prepTime; 61__gshared Duration markTime; 62__gshared Duration sweepTime; 63__gshared Duration pauseTime; 64__gshared Duration maxPauseTime; 65__gshared Duration maxCollectionTime; 66__gshared size_t numCollections; 67__gshared size_t maxPoolMemory; 68 69__gshared long numMallocs; 70__gshared long numFrees; 71__gshared long numReallocs; 72__gshared long numExtends; 73__gshared long numOthers; 74__gshared long mallocTime; // using ticks instead of MonoTime for better performance 75__gshared long freeTime; 76__gshared long reallocTime; 77__gshared long extendTime; 78__gshared long otherTime; 79__gshared long lockTime; 80 81ulong bytesAllocated; // thread local counter 82 83private 84{ 85 extern (C) 86 { 87 // to allow compilation of this module without access to the rt package, 88 // make these functions available from rt.lifetime 89 void rt_finalizeFromGC(void* p, size_t size, uint attr) nothrow; 90 int rt_hasFinalizerInSegment(void* p, size_t size, uint attr, const scope void[] segment) nothrow; 91 92 // Declared as an extern instead of importing core.exception 93 // to avoid inlining - see issue 13725. 94 void onInvalidMemoryOperationError(void* pretend_sideffect = null) @trusted pure nothrow @nogc; 95 void onOutOfMemoryErrorNoGC() @trusted nothrow @nogc; 96 97 version (COLLECT_FORK) 98 version (OSX) 99 pid_t __fork() nothrow; 100 } 101 102 enum 103 { 104 OPFAIL = ~cast(size_t)0 105 } 106} 107 108alias GC gc_t; 109 110/* ============================ GC =============================== */ 111 112// register GC in C constructor (_STI_) 113extern(C) pragma(crt_constructor) void _d_register_conservative_gc() 114{ 115 import core.gc.registry; 116 registerGCFactory("conservative", &initialize); 117} 118 119extern(C) pragma(crt_constructor) void _d_register_precise_gc() 120{ 121 import core.gc.registry; 122 registerGCFactory("precise", &initialize_precise); 123} 124 125private GC initialize() 126{ 127 import core.lifetime : emplace; 128 129 auto gc = cast(ConservativeGC) cstdlib.malloc(__traits(classInstanceSize, ConservativeGC)); 130 if (!gc) 131 onOutOfMemoryErrorNoGC(); 132 133 return emplace(gc); 134} 135 136private GC initialize_precise() 137{ 138 ConservativeGC.isPrecise = true; 139 return initialize(); 140} 141 142class ConservativeGC : GC 143{ 144 // For passing to debug code (not thread safe) 145 __gshared size_t line; 146 __gshared char* file; 147 148 Gcx *gcx; // implementation 149 150 static gcLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy); 151 static bool _inFinalizer; 152 __gshared bool isPrecise = false; 153 154 /* 155 * Lock the GC. 156 * 157 * Throws: InvalidMemoryOperationError on recursive locking during finalization. 158 */ 159 static void lockNR() @safe @nogc nothrow 160 { 161 if (_inFinalizer) 162 onInvalidMemoryOperationError(); 163 gcLock.lock(); 164 } 165 166 /* 167 * Initialize the GC based on command line configuration. 168 * 169 * Throws: 170 * OutOfMemoryError if failed to initialize GC due to not enough memory. 171 */ 172 this() 173 { 174 //config is assumed to have already been initialized 175 176 gcx = cast(Gcx*)cstdlib.calloc(1, Gcx.sizeof); 177 if (!gcx) 178 onOutOfMemoryErrorNoGC(); 179 gcx.initialize(); 180 181 if (config.initReserve) 182 gcx.reserve(config.initReserve); 183 if (config.disable) 184 gcx.disabled++; 185 } 186 187 188 ~this() 189 { 190 version (linux) 191 { 192 //debug(PRINTF) printf("Thread %x ", pthread_self()); 193 //debug(PRINTF) printf("GC.Dtor()\n"); 194 } 195 196 if (gcx) 197 { 198 gcx.Dtor(); 199 cstdlib.free(gcx); 200 gcx = null; 201 } 202 // TODO: cannot free as memory is overwritten and 203 // the monitor is still read in rt_finalize (called by destroy) 204 // cstdlib.free(cast(void*) this); 205 } 206 207 208 /** 209 * Enables the GC if disable() was previously called. Must be called 210 * for each time disable was called in order to enable the GC again. 211 */ 212 void enable() 213 { 214 static void go(Gcx* gcx) nothrow 215 { 216 assert(gcx.disabled > 0); 217 gcx.disabled--; 218 } 219 runLocked!(go, otherTime, numOthers)(gcx); 220 } 221 222 223 /** 224 * Disable the GC. The GC may still run if it deems necessary. 225 */ 226 void disable() 227 { 228 static void go(Gcx* gcx) nothrow 229 { 230 gcx.disabled++; 231 } 232 runLocked!(go, otherTime, numOthers)(gcx); 233 } 234 235 debug (GC_RECURSIVE_LOCK) static bool lockedOnThisThread; 236 237 /** 238 * Run a function inside a lock/unlock set. 239 * 240 * Params: 241 * func = The function to run. 242 * args = The function arguments. 243 */ 244 auto runLocked(alias func, Args...)(auto ref Args args) 245 { 246 debug(PROFILE_API) immutable tm = (config.profile > 1 ? currTime.ticks : 0); 247 debug(GC_RECURSIVE_LOCK) 248 { 249 if (lockedOnThisThread) 250 onInvalidMemoryOperationError(); 251 lockedOnThisThread = true; 252 } 253 lockNR(); 254 scope (failure) gcLock.unlock(); 255 debug(PROFILE_API) immutable tm2 = (config.profile > 1 ? currTime.ticks : 0); 256 257 static if (is(typeof(func(args)) == void)) 258 func(args); 259 else 260 auto res = func(args); 261 262 debug(PROFILE_API) if (config.profile > 1) 263 lockTime += tm2 - tm; 264 gcLock.unlock(); 265 debug(GC_RECURSIVE_LOCK) 266 { 267 if (!lockedOnThisThread) 268 onInvalidMemoryOperationError(); 269 lockedOnThisThread = false; 270 } 271 272 static if (!is(typeof(func(args)) == void)) 273 return res; 274 } 275 276 /** 277 * Run a function in an lock/unlock set that keeps track of 278 * how much time was spend inside this function (in ticks) 279 * and how many times this fuction was called. 280 * 281 * Params: 282 * func = The function to run. 283 * time = The variable keeping track of the time (in ticks). 284 * count = The variable keeping track of how many times this fuction was called. 285 * args = The function arguments. 286 */ 287 auto runLocked(alias func, alias time, alias count, Args...)(auto ref Args args) 288 { 289 debug(PROFILE_API) immutable tm = (config.profile > 1 ? currTime.ticks : 0); 290 debug(GC_RECURSIVE_LOCK) 291 { 292 if (lockedOnThisThread) 293 onInvalidMemoryOperationError(); 294 lockedOnThisThread = true; 295 } 296 lockNR(); 297 scope (failure) gcLock.unlock(); 298 debug(PROFILE_API) immutable tm2 = (config.profile > 1 ? currTime.ticks : 0); 299 300 static if (is(typeof(func(args)) == void)) 301 func(args); 302 else 303 auto res = func(args); 304 305 debug(PROFILE_API) if (config.profile > 1) 306 { 307 count++; 308 immutable now = currTime.ticks; 309 lockTime += tm2 - tm; 310 time += now - tm2; 311 } 312 gcLock.unlock(); 313 debug(GC_RECURSIVE_LOCK) 314 { 315 if (!lockedOnThisThread) 316 onInvalidMemoryOperationError(); 317 lockedOnThisThread = false; 318 } 319 320 static if (!is(typeof(func(args)) == void)) 321 return res; 322 } 323 324 325 /** 326 * Returns a bit field representing all block attributes set for the memory 327 * referenced by p. 328 * 329 * Params: 330 * p = A pointer to the base of a valid memory block or to null. 331 * 332 * Returns: 333 * A bit field containing any bits set for the memory block referenced by 334 * p or zero on error. 335 */ 336 uint getAttr(void* p) nothrow 337 { 338 if (!p) 339 { 340 return 0; 341 } 342 343 static uint go(Gcx* gcx, void* p) nothrow 344 { 345 Pool* pool = gcx.findPool(p); 346 uint oldb = 0; 347 348 if (pool) 349 { 350 p = sentinel_sub(p); 351 if (p != pool.findBase(p)) 352 return 0; 353 auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; 354 355 oldb = pool.getBits(biti); 356 } 357 return oldb; 358 } 359 360 return runLocked!(go, otherTime, numOthers)(gcx, p); 361 } 362 363 /** 364 * Sets the specified bits for the memory references by p. 365 * 366 * If p was not allocated by the GC, points inside a block, or is null, no 367 * action will be taken. 368 * 369 * Params: 370 * p = A pointer to the base of a valid memory block or to null. 371 * mask = A bit field containing any bits to set for this memory block. 372 * 373 * Returns: 374 * The result of a call to getAttr after the specified bits have been 375 * set. 376 */ 377 uint setAttr(void* p, uint mask) nothrow 378 { 379 if (!p) 380 { 381 return 0; 382 } 383 384 static uint go(Gcx* gcx, void* p, uint mask) nothrow 385 { 386 Pool* pool = gcx.findPool(p); 387 uint oldb = 0; 388 389 if (pool) 390 { 391 p = sentinel_sub(p); 392 if (p != pool.findBase(p)) 393 return 0; 394 auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; 395 396 oldb = pool.getBits(biti); 397 pool.setBits(biti, mask); 398 } 399 return oldb; 400 } 401 402 return runLocked!(go, otherTime, numOthers)(gcx, p, mask); 403 } 404 405 406 /** 407 * Clears the specified bits for the memory references by p. 408 * 409 * If p was not allocated by the GC, points inside a block, or is null, no 410 * action will be taken. 411 * 412 * Params: 413 * p = A pointer to the base of a valid memory block or to null. 414 * mask = A bit field containing any bits to clear for this memory block. 415 * 416 * Returns: 417 * The result of a call to getAttr after the specified bits have been 418 * cleared 419 */ 420 uint clrAttr(void* p, uint mask) nothrow 421 { 422 if (!p) 423 { 424 return 0; 425 } 426 427 static uint go(Gcx* gcx, void* p, uint mask) nothrow 428 { 429 Pool* pool = gcx.findPool(p); 430 uint oldb = 0; 431 432 if (pool) 433 { 434 p = sentinel_sub(p); 435 if (p != pool.findBase(p)) 436 return 0; 437 auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; 438 439 oldb = pool.getBits(biti); 440 pool.clrBits(biti, mask); 441 } 442 return oldb; 443 } 444 445 return runLocked!(go, otherTime, numOthers)(gcx, p, mask); 446 } 447 448 /** 449 * Requests an aligned block of managed memory from the garbage collector. 450 * 451 * Params: 452 * size = The desired allocation size in bytes. 453 * bits = A bitmask of the attributes to set on this block. 454 * ti = TypeInfo to describe the memory. 455 * 456 * Returns: 457 * A reference to the allocated memory or null if no memory was requested. 458 * 459 * Throws: 460 * OutOfMemoryError on allocation failure 461 */ 462 void *malloc(size_t size, uint bits = 0, const TypeInfo ti = null) nothrow 463 { 464 if (!size) 465 { 466 return null; 467 } 468 469 size_t localAllocSize = void; 470 471 auto p = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, localAllocSize, ti); 472 473 if (!(bits & BlkAttr.NO_SCAN)) 474 { 475 memset(p + size, 0, localAllocSize - size); 476 } 477 478 return p; 479 } 480 481 482 // 483 // Implementation for malloc and calloc. 484 // 485 private void *mallocNoSync(size_t size, uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow 486 { 487 assert(size != 0); 488 489 debug(PRINTF) 490 printf("GC::malloc(gcx = %p, size = %d bits = %x, ti = %s)\n", gcx, size, bits, debugTypeName(ti).ptr); 491 492 assert(gcx); 493 //debug(PRINTF) printf("gcx.self = %x, pthread_self() = %x\n", gcx.self, pthread_self()); 494 495 auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits, ti); 496 if (!p) 497 onOutOfMemoryErrorNoGC(); 498 499 debug (SENTINEL) 500 { 501 p = sentinel_add(p); 502 sentinel_init(p, size); 503 alloc_size = size; 504 } 505 gcx.leakDetector.log_malloc(p, size); 506 bytesAllocated += alloc_size; 507 508 debug(PRINTF) printf(" => p = %p\n", p); 509 return p; 510 } 511 512 BlkInfo qalloc( size_t size, uint bits, const scope TypeInfo ti) nothrow 513 { 514 515 if (!size) 516 { 517 return BlkInfo.init; 518 } 519 520 BlkInfo retval; 521 522 retval.base = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, retval.size, ti); 523 524 if (!(bits & BlkAttr.NO_SCAN)) 525 { 526 memset(retval.base + size, 0, retval.size - size); 527 } 528 529 retval.attr = bits; 530 return retval; 531 } 532 533 534 /** 535 * Requests an aligned block of managed memory from the garbage collector, 536 * which is initialized with all bits set to zero. 537 * 538 * Params: 539 * size = The desired allocation size in bytes. 540 * bits = A bitmask of the attributes to set on this block. 541 * ti = TypeInfo to describe the memory. 542 * 543 * Returns: 544 * A reference to the allocated memory or null if no memory was requested. 545 * 546 * Throws: 547 * OutOfMemoryError on allocation failure. 548 */ 549 void *calloc(size_t size, uint bits = 0, const TypeInfo ti = null) nothrow 550 { 551 if (!size) 552 { 553 return null; 554 } 555 556 size_t localAllocSize = void; 557 558 auto p = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, localAllocSize, ti); 559 560 memset(p, 0, size); 561 if (!(bits & BlkAttr.NO_SCAN)) 562 { 563 memset(p + size, 0, localAllocSize - size); 564 } 565 566 return p; 567 } 568 569 /** 570 * Request that the GC reallocate a block of memory, attempting to adjust 571 * the size in place if possible. If size is 0, the memory will be freed. 572 * 573 * If p was not allocated by the GC, points inside a block, or is null, no 574 * action will be taken. 575 * 576 * Params: 577 * p = A pointer to the root of a valid memory block or to null. 578 * size = The desired allocation size in bytes. 579 * bits = A bitmask of the attributes to set on this block. 580 * ti = TypeInfo to describe the memory. 581 * 582 * Returns: 583 * A reference to the allocated memory on success or null if size is 584 * zero. 585 * 586 * Throws: 587 * OutOfMemoryError on allocation failure. 588 */ 589 void *realloc(void *p, size_t size, uint bits = 0, const TypeInfo ti = null) nothrow 590 { 591 size_t localAllocSize = void; 592 auto oldp = p; 593 594 p = runLocked!(reallocNoSync, mallocTime, numMallocs)(p, size, bits, localAllocSize, ti); 595 596 if (p && !(bits & BlkAttr.NO_SCAN)) 597 { 598 memset(p + size, 0, localAllocSize - size); 599 } 600 601 return p; 602 } 603 604 605 // 606 // The implementation of realloc. 607 // 608 // bits will be set to the resulting bits of the new block 609 // 610 private void *reallocNoSync(void *p, size_t size, ref uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow 611 { 612 if (!size) 613 { 614 if (p) 615 freeNoSync(p); 616 alloc_size = 0; 617 return null; 618 } 619 if (!p) 620 return mallocNoSync(size, bits, alloc_size, ti); 621 622 debug(PRINTF) printf("GC::realloc(p = %p, size = %llu)\n", p, cast(ulong)size); 623 624 Pool *pool = gcx.findPool(p); 625 if (!pool) 626 return null; 627 628 size_t psize; 629 size_t biti; 630 631 debug(SENTINEL) 632 { 633 void* q = p; 634 p = sentinel_sub(p); 635 bool alwaysMalloc = true; 636 } 637 else 638 { 639 alias q = p; 640 enum alwaysMalloc = false; 641 } 642 643 void* doMalloc() 644 { 645 if (!bits) 646 bits = pool.getBits(biti); 647 648 void* p2 = mallocNoSync(size, bits, alloc_size, ti); 649 debug (SENTINEL) 650 psize = sentinel_size(q, psize); 651 if (psize < size) 652 size = psize; 653 //debug(PRINTF) printf("\tcopying %d bytes\n",size); 654 memcpy(p2, q, size); 655 freeNoSync(q); 656 return p2; 657 } 658 659 if (pool.isLargeObject) 660 { 661 auto lpool = cast(LargeObjectPool*) pool; 662 auto psz = lpool.getPages(p); // get allocated size 663 if (psz == 0) 664 return null; // interior pointer 665 psize = psz * PAGESIZE; 666 667 alias pagenum = biti; // happens to be the same, but rename for clarity 668 pagenum = lpool.pagenumOf(p); 669 670 if (size <= PAGESIZE / 2 || alwaysMalloc) 671 return doMalloc(); // switching from large object pool to small object pool 672 673 auto newsz = lpool.numPages(size); 674 if (newsz == psz) 675 { 676 // nothing to do 677 } 678 else if (newsz < psz) 679 { 680 // Shrink in place 681 debug (MEMSTOMP) memset(p + size, 0xF2, psize - size); 682 lpool.freePages(pagenum + newsz, psz - newsz); 683 lpool.mergeFreePageOffsets!(false, true)(pagenum + newsz, psz - newsz); 684 lpool.bPageOffsets[pagenum] = cast(uint) newsz; 685 } 686 else if (pagenum + newsz <= pool.npages) 687 { 688 // Attempt to expand in place (TODO: merge with extend) 689 if (lpool.pagetable[pagenum + psz] != Bins.B_FREE) 690 return doMalloc(); 691 692 auto newPages = newsz - psz; 693 auto freesz = lpool.bPageOffsets[pagenum + psz]; 694 if (freesz < newPages) 695 return doMalloc(); // free range too small 696 697 debug (MEMSTOMP) memset(p + psize, 0xF0, size - psize); 698 debug (PRINTF) printFreeInfo(pool); 699 memset(&lpool.pagetable[pagenum + psz], Bins.B_PAGEPLUS, newPages); 700 lpool.bPageOffsets[pagenum] = cast(uint) newsz; 701 for (auto offset = psz; offset < newsz; offset++) 702 lpool.bPageOffsets[pagenum + offset] = cast(uint) offset; 703 if (freesz > newPages) 704 lpool.setFreePageOffsets(pagenum + newsz, freesz - newPages); 705 gcx.usedLargePages += newPages; 706 lpool.freepages -= newPages; 707 debug (PRINTF) printFreeInfo(pool); 708 } 709 else 710 return doMalloc(); // does not fit into current pool 711 712 alloc_size = newsz * PAGESIZE; 713 } 714 else 715 { 716 psize = (cast(SmallObjectPool*) pool).getSize(p); // get allocated bin size 717 if (psize == 0) 718 return null; // interior pointer 719 biti = cast(size_t)(p - pool.baseAddr) >> Pool.ShiftBy.Small; 720 if (pool.freebits.test (biti)) 721 return null; 722 723 // allocate if new size is bigger or less than half 724 if (psize < size || psize > size * 2 || alwaysMalloc) 725 return doMalloc(); 726 727 alloc_size = psize; 728 if (isPrecise) 729 pool.setPointerBitmapSmall(p, size, psize, bits, ti); 730 } 731 732 if (bits) 733 { 734 pool.clrBits(biti, ~BlkAttr.NONE); 735 pool.setBits(biti, bits); 736 } 737 return p; 738 } 739 740 741 size_t extend(void* p, size_t minsize, size_t maxsize, const TypeInfo ti) nothrow 742 { 743 return runLocked!(extendNoSync, extendTime, numExtends)(p, minsize, maxsize, ti); 744 } 745 746 747 // 748 // Implementation of extend. 749 // 750 private size_t extendNoSync(void* p, size_t minsize, size_t maxsize, const TypeInfo ti = null) nothrow 751 in 752 { 753 assert(minsize <= maxsize); 754 } 755 do 756 { 757 debug(PRINTF) printf("GC::extend(p = %p, minsize = %zu, maxsize = %zu)\n", p, minsize, maxsize); 758 debug (SENTINEL) 759 { 760 return 0; 761 } 762 else 763 { 764 auto pool = gcx.findPool(p); 765 if (!pool || !pool.isLargeObject) 766 return 0; 767 768 auto lpool = cast(LargeObjectPool*) pool; 769 size_t pagenum = lpool.pagenumOf(p); 770 if (lpool.pagetable[pagenum] != Bins.B_PAGE) 771 return 0; 772 773 size_t psz = lpool.bPageOffsets[pagenum]; 774 assert(psz > 0); 775 776 auto minsz = lpool.numPages(minsize); 777 auto maxsz = lpool.numPages(maxsize); 778 779 if (pagenum + psz >= lpool.npages) 780 return 0; 781 if (lpool.pagetable[pagenum + psz] != Bins.B_FREE) 782 return 0; 783 784 size_t freesz = lpool.bPageOffsets[pagenum + psz]; 785 if (freesz < minsz) 786 return 0; 787 size_t sz = freesz > maxsz ? maxsz : freesz; 788 debug (MEMSTOMP) memset(pool.baseAddr + (pagenum + psz) * PAGESIZE, 0xF0, sz * PAGESIZE); 789 memset(lpool.pagetable + pagenum + psz, Bins.B_PAGEPLUS, sz); 790 lpool.bPageOffsets[pagenum] = cast(uint) (psz + sz); 791 for (auto offset = psz; offset < psz + sz; offset++) 792 lpool.bPageOffsets[pagenum + offset] = cast(uint) offset; 793 if (freesz > sz) 794 lpool.setFreePageOffsets(pagenum + psz + sz, freesz - sz); 795 lpool.freepages -= sz; 796 gcx.usedLargePages += sz; 797 return (psz + sz) * PAGESIZE; 798 } 799 } 800 801 802 /** 803 * Requests that at least size bytes of memory be obtained from the operating 804 * system and marked as free. 805 * 806 * Params: 807 * size = The desired size in bytes. 808 * 809 * Returns: 810 * The actual number of bytes reserved or zero on error. 811 */ 812 size_t reserve(size_t size) nothrow 813 { 814 if (!size) 815 { 816 return 0; 817 } 818 819 return runLocked!(reserveNoSync, otherTime, numOthers)(size); 820 } 821 822 823 // 824 // Implementation of reserve 825 // 826 private size_t reserveNoSync(size_t size) nothrow 827 { 828 assert(size != 0); 829 assert(gcx); 830 831 return gcx.reserve(size); 832 } 833 834 835 /** 836 * Deallocates the memory referenced by p. 837 * 838 * If p was not allocated by the GC, points inside a block, is null, or 839 * if free is called from a finalizer, no action will be taken. 840 * 841 * Params: 842 * p = A pointer to the root of a valid memory block or to null. 843 */ 844 void free(void *p) nothrow 845 { 846 if (!p || _inFinalizer) 847 { 848 return; 849 } 850 851 return runLocked!(freeNoSync, freeTime, numFrees)(p); 852 } 853 854 855 // 856 // Implementation of free. 857 // 858 private void freeNoSync(void *p) nothrow @nogc 859 { 860 debug(PRINTF) printf("Freeing %p\n", cast(size_t) p); 861 assert (p); 862 863 Pool* pool; 864 size_t pagenum; 865 Bins bin; 866 size_t biti; 867 868 // Find which page it is in 869 pool = gcx.findPool(p); 870 if (!pool) // if not one of ours 871 return; // ignore 872 873 pagenum = pool.pagenumOf(p); 874 875 debug(PRINTF) printf("pool base = %p, PAGENUM = %d of %d, bin = %d\n", pool.baseAddr, pagenum, pool.npages, pool.pagetable[pagenum]); 876 debug(PRINTF) if (pool.isLargeObject) printf("Block size = %d\n", pool.bPageOffsets[pagenum]); 877 878 bin = pool.pagetable[pagenum]; 879 880 // Verify that the pointer is at the beginning of a block, 881 // no action should be taken if p is an interior pointer 882 if (bin > Bins.B_PAGE) // B_PAGEPLUS or B_FREE 883 return; 884 size_t off = (sentinel_sub(p) - pool.baseAddr); 885 size_t base = baseOffset(off, bin); 886 if (off != base) 887 return; 888 889 sentinel_Invariant(p); 890 auto q = p; 891 p = sentinel_sub(p); 892 size_t ssize; 893 894 if (pool.isLargeObject) // if large alloc 895 { 896 biti = cast(size_t)(p - pool.baseAddr) >> pool.ShiftBy.Large; 897 assert(bin == Bins.B_PAGE); 898 auto lpool = cast(LargeObjectPool*) pool; 899 900 // Free pages 901 size_t npages = lpool.bPageOffsets[pagenum]; 902 auto size = npages * PAGESIZE; 903 ssize = sentinel_size(q, size); 904 debug (MEMSTOMP) memset(p, 0xF2, size); 905 lpool.freePages(pagenum, npages); 906 lpool.mergeFreePageOffsets!(true, true)(pagenum, npages); 907 } 908 else 909 { 910 biti = cast(size_t)(p - pool.baseAddr) >> pool.ShiftBy.Small; 911 if (pool.freebits.test (biti)) 912 return; 913 // Add to free list 914 List *list = cast(List*)p; 915 916 auto size = binsize[bin]; 917 ssize = sentinel_size(q, size); 918 debug (MEMSTOMP) memset(p, 0xF2, size); 919 920 // in case the page hasn't been recovered yet, don't add the object to the free list 921 if (!gcx.recoverPool[bin] || pool.binPageChain[pagenum] == Pool.PageRecovered) 922 { 923 list.next = gcx.bucket[bin]; 924 list.pool = pool; 925 gcx.bucket[bin] = list; 926 } 927 pool.freebits.set(biti); 928 } 929 pool.clrBits(biti, ~BlkAttr.NONE); 930 931 gcx.leakDetector.log_free(sentinel_add(p), ssize); 932 } 933 934 935 /** 936 * Determine the base address of the block containing p. If p is not a gc 937 * allocated pointer, return null. 938 * 939 * Params: 940 * p = A pointer to the root or the interior of a valid memory block or to 941 * null. 942 * 943 * Returns: 944 * The base address of the memory block referenced by p or null on error. 945 */ 946 void* addrOf(void *p) nothrow 947 { 948 if (!p) 949 { 950 return null; 951 } 952 953 return runLocked!(addrOfNoSync, otherTime, numOthers)(p); 954 } 955 956 957 // 958 // Implementation of addrOf. 959 // 960 void* addrOfNoSync(void *p) nothrow @nogc 961 { 962 if (!p) 963 { 964 return null; 965 } 966 967 auto q = gcx.findBase(p); 968 if (q) 969 q = sentinel_add(q); 970 return q; 971 } 972 973 974 /** 975 * Determine the allocated size of pointer p. If p is an interior pointer 976 * or not a gc allocated pointer, return 0. 977 * 978 * Params: 979 * p = A pointer to the root of a valid memory block or to null. 980 * 981 * Returns: 982 * The size in bytes of the memory block referenced by p or zero on error. 983 */ 984 size_t sizeOf(void *p) nothrow 985 { 986 if (!p) 987 { 988 return 0; 989 } 990 991 return runLocked!(sizeOfNoSync, otherTime, numOthers)(p); 992 } 993 994 995 // 996 // Implementation of sizeOf. 997 // 998 private size_t sizeOfNoSync(void *p) nothrow @nogc 999 { 1000 assert (p); 1001 1002 debug (SENTINEL) 1003 { 1004 p = sentinel_sub(p); 1005 size_t size = gcx.findSize(p); 1006 return size ? size - SENTINEL_EXTRA : 0; 1007 } 1008 else 1009 { 1010 size_t size = gcx.findSize(p); 1011 return size; 1012 } 1013 } 1014 1015 1016 /** 1017 * Determine the base address of the block containing p. If p is not a gc 1018 * allocated pointer, return null. 1019 * 1020 * Params: 1021 * p = A pointer to the root or the interior of a valid memory block or to 1022 * null. 1023 * 1024 * Returns: 1025 * Information regarding the memory block referenced by p or BlkInfo.init 1026 * on error. 1027 */ 1028 BlkInfo query(void *p) nothrow 1029 { 1030 if (!p) 1031 { 1032 BlkInfo i; 1033 return i; 1034 } 1035 1036 return runLocked!(queryNoSync, otherTime, numOthers)(p); 1037 } 1038 1039 // 1040 // Implementation of query 1041 // 1042 BlkInfo queryNoSync(void *p) nothrow 1043 { 1044 assert(p); 1045 1046 BlkInfo info = gcx.getInfo(p); 1047 debug(SENTINEL) 1048 { 1049 if (info.base) 1050 { 1051 info.base = sentinel_add(info.base); 1052 info.size = *sentinel_psize(info.base); 1053 } 1054 } 1055 return info; 1056 } 1057 1058 1059 /** 1060 * Performs certain checks on a pointer. These checks will cause asserts to 1061 * fail unless the following conditions are met: 1062 * 1) The poiinter belongs to this memory pool. 1063 * 2) The pointer points to the start of an allocated piece of memory. 1064 * 3) The pointer is not on a free list. 1065 * 1066 * Params: 1067 * p = The pointer to be checked. 1068 */ 1069 void check(void *p) nothrow 1070 { 1071 if (!p) 1072 { 1073 return; 1074 } 1075 1076 return runLocked!(checkNoSync, otherTime, numOthers)(p); 1077 } 1078 1079 1080 // 1081 // Implementation of check 1082 // 1083 private void checkNoSync(void *p) nothrow 1084 { 1085 assert(p); 1086 1087 sentinel_Invariant(p); 1088 debug (PTRCHECK) 1089 { 1090 Pool* pool; 1091 size_t pagenum; 1092 Bins bin; 1093 1094 p = sentinel_sub(p); 1095 pool = gcx.findPool(p); 1096 assert(pool); 1097 pagenum = pool.pagenumOf(p); 1098 bin = pool.pagetable[pagenum]; 1099 assert(bin <= Bins.B_PAGE); 1100 assert(p == cast(void*)baseOffset(cast(size_t)p, bin)); 1101 1102 debug (PTRCHECK2) 1103 { 1104 if (bin < Bins.B_PAGE) 1105 { 1106 // Check that p is not on a free list 1107 List *list; 1108 1109 for (list = gcx.bucket[bin]; list; list = list.next) 1110 { 1111 assert(cast(void*)list != p); 1112 } 1113 } 1114 } 1115 } 1116 } 1117 1118 1119 /** 1120 * Add p to list of roots. If p is null, no operation is performed. 1121 * 1122 * Params: 1123 * p = A pointer into a GC-managed memory block or null. 1124 */ 1125 void addRoot(void *p) nothrow @nogc 1126 { 1127 if (!p) 1128 { 1129 return; 1130 } 1131 1132 gcx.addRoot(p); 1133 } 1134 1135 1136 /** 1137 * Remove p from list of roots. If p is null or is not a value 1138 * previously passed to addRoot() then no operation is performed. 1139 * 1140 * Params: 1141 * p = A pointer into a GC-managed memory block or null. 1142 */ 1143 void removeRoot(void *p) nothrow @nogc 1144 { 1145 if (!p) 1146 { 1147 return; 1148 } 1149 1150 gcx.removeRoot(p); 1151 } 1152 1153 /** 1154 * Returns an iterator allowing roots to be traversed via a foreach loop. 1155 */ 1156 @property RootIterator rootIter() @nogc 1157 { 1158 return &gcx.rootsApply; 1159 } 1160 1161 1162 /** 1163 * Add range to scan for roots. If p is null or sz is 0, no operation is performed. 1164 * 1165 * Params: 1166 * p = A pointer to a valid memory address or to null. 1167 * sz = The size in bytes of the block to add. 1168 * ti = TypeInfo to describe the memory. 1169 */ 1170 void addRange(void *p, size_t sz, const TypeInfo ti = null) nothrow @nogc 1171 { 1172 if (!p || !sz) 1173 { 1174 return; 1175 } 1176 1177 gcx.addRange(p, p + sz, ti); 1178 } 1179 1180 1181 /** 1182 * Remove range from list of ranges. If p is null or does not represent 1183 * a value previously passed to addRange() then no operation is 1184 * performed. 1185 * 1186 * Params: 1187 * p = A pointer to a valid memory address or to null. 1188 */ 1189 void removeRange(void *p) nothrow @nogc 1190 { 1191 if (!p) 1192 { 1193 return; 1194 } 1195 1196 gcx.removeRange(p); 1197 } 1198 1199 1200 /** 1201 * Returns an iterator allowing ranges to be traversed via a foreach loop. 1202 */ 1203 @property RangeIterator rangeIter() @nogc 1204 { 1205 return &gcx.rangesApply; 1206 } 1207 1208 1209 /** 1210 * Run all finalizers in the code segment. 1211 * 1212 * Params: 1213 * segment = address range of a code segment 1214 */ 1215 void runFinalizers(const scope void[] segment) nothrow 1216 { 1217 static void go(Gcx* gcx, const scope void[] segment) nothrow 1218 { 1219 gcx.runFinalizers(segment); 1220 } 1221 return runLocked!(go, otherTime, numOthers)(gcx, segment); 1222 } 1223 1224 1225 bool inFinalizer() nothrow @nogc 1226 { 1227 return _inFinalizer; 1228 } 1229 1230 1231 void collect() nothrow 1232 { 1233 fullCollect(); 1234 } 1235 1236 1237 void collectNoStack() nothrow 1238 { 1239 fullCollectNoStack(); 1240 } 1241 1242 1243 /** 1244 * Begins a full collection, scanning all stack segments for roots. 1245 * 1246 * Returns: 1247 * The number of pages freed. 1248 */ 1249 size_t fullCollect() nothrow 1250 { 1251 debug(PRINTF) printf("GC.fullCollect()\n"); 1252 1253 // Since a finalizer could launch a new thread, we always need to lock 1254 // when collecting. 1255 static size_t go(Gcx* gcx) nothrow 1256 { 1257 return gcx.fullcollect(false, true); // standard stop the world 1258 } 1259 immutable result = runLocked!go(gcx); 1260 1261 version (none) 1262 { 1263 GCStats stats; 1264 1265 getStats(stats); 1266 debug(PRINTF) printf("heapSize = %zx, freeSize = %zx\n", 1267 stats.heapSize, stats.freeSize); 1268 } 1269 1270 gcx.leakDetector.log_collect(); 1271 return result; 1272 } 1273 1274 1275 /** 1276 * Begins a full collection while ignoring all stack segments for roots. 1277 */ 1278 void fullCollectNoStack() nothrow 1279 { 1280 // Since a finalizer could launch a new thread, we always need to lock 1281 // when collecting. 1282 static size_t go(Gcx* gcx) nothrow 1283 { 1284 return gcx.fullcollect(true, true, true); // standard stop the world 1285 } 1286 runLocked!go(gcx); 1287 } 1288 1289 1290 /** 1291 * Minimize free space usage. 1292 */ 1293 void minimize() nothrow 1294 { 1295 static void go(Gcx* gcx) nothrow 1296 { 1297 gcx.minimize(); 1298 } 1299 runLocked!(go, otherTime, numOthers)(gcx); 1300 } 1301 1302 1303 core.memory.GC.Stats stats() @safe nothrow @nogc 1304 { 1305 typeof(return) ret; 1306 1307 runLocked!(getStatsNoSync, otherTime, numOthers)(ret); 1308 1309 return ret; 1310 } 1311 1312 1313 core.memory.GC.ProfileStats profileStats() nothrow @trusted 1314 { 1315 typeof(return) ret; 1316 1317 ret.numCollections = numCollections; 1318 ret.totalCollectionTime = prepTime + markTime + sweepTime; 1319 ret.totalPauseTime = pauseTime; 1320 ret.maxCollectionTime = maxCollectionTime; 1321 ret.maxPauseTime = maxPauseTime; 1322 1323 return ret; 1324 } 1325 1326 1327 ulong allocatedInCurrentThread() nothrow 1328 { 1329 return bytesAllocated; 1330 } 1331 1332 1333 // 1334 // Implementation of getStats 1335 // 1336 private void getStatsNoSync(out core.memory.GC.Stats stats) @trusted nothrow @nogc 1337 { 1338 // This function is trusted for two reasons: `pool.pagetable` is a pointer, 1339 // which is being sliced in the below foreach, and so is `binPageChain`, 1340 // also sliced a bit later in this function. 1341 // However, both usages are safe as long as the assumption that `npools` 1342 // defines the limit for `pagetable`'s length holds true (see allocation). 1343 // The slicing happens at __LINE__ + 4 and __LINE__ + 24. 1344 // `@trusted` delegates are not used to prevent any performance issue. 1345 foreach (pool; gcx.pooltable[]) 1346 { 1347 foreach (bin; pool.pagetable[0 .. pool.npages]) 1348 { 1349 if (bin == Bins.B_FREE) 1350 stats.freeSize += PAGESIZE; 1351 else 1352 stats.usedSize += PAGESIZE; 1353 } 1354 } 1355 1356 size_t freeListSize; 1357 foreach (n; 0 .. Bins.B_PAGE) 1358 { 1359 immutable sz = binsize[n]; 1360 for (List *list = gcx.bucket[n]; list; list = list.next) 1361 freeListSize += sz; 1362 1363 foreach (pool; gcx.pooltable[]) 1364 { 1365 if (pool.isLargeObject) 1366 continue; 1367 for (uint pn = pool.recoverPageFirst[n]; pn < pool.npages; pn = pool.binPageChain[pn]) 1368 { 1369 const bitbase = pn * PAGESIZE / 16; 1370 const top = PAGESIZE - sz + 1; // ensure <size> bytes available even if unaligned 1371 for (size_t u = 0; u < top; u += sz) 1372 if (pool.freebits.test(bitbase + u / 16)) 1373 freeListSize += sz; 1374 } 1375 } 1376 } 1377 1378 stats.usedSize -= freeListSize; 1379 stats.freeSize += freeListSize; 1380 stats.allocatedInCurrentThread = bytesAllocated; 1381 } 1382} 1383 1384 1385/* ============================ Gcx =============================== */ 1386 1387enum 1388{ PAGESIZE = 4096, 1389} 1390 1391 1392enum Bins : ubyte 1393{ 1394 B_16, 1395 B_32, 1396 B_48, 1397 B_64, 1398 B_96, 1399 B_128, 1400 B_176, 1401 B_256, 1402 B_368, 1403 B_512, 1404 B_816, 1405 B_1024, 1406 B_1360, 1407 B_2048, 1408 B_NUMSMALL, 1409 1410 B_PAGE = B_NUMSMALL,// start of large alloc 1411 B_PAGEPLUS, // continuation of large alloc 1412 B_FREE, // free page 1413 B_MAX, 1414} 1415 1416struct List 1417{ 1418 List *next; 1419 Pool *pool; 1420} 1421 1422// non power of two sizes optimized for small remainder within page (<= 64 bytes) 1423immutable short[Bins.B_NUMSMALL + 1] binsize = [ 16, 32, 48, 64, 96, 128, 176, 256, 368, 512, 816, 1024, 1360, 2048, 4096 ]; 1424immutable short[PAGESIZE / 16][Bins.B_NUMSMALL + 1] binbase = calcBinBase(); 1425 1426short[PAGESIZE / 16][Bins.B_NUMSMALL + 1] calcBinBase() 1427{ 1428 short[PAGESIZE / 16][Bins.B_NUMSMALL + 1] bin; 1429 1430 foreach (i, size; binsize) 1431 { 1432 short end = (PAGESIZE / size) * size; 1433 short bsz = size / 16; 1434 foreach (off; 0..PAGESIZE/16) 1435 { 1436 // add the remainder to the last bin, so no check during scanning 1437 // is needed if a false pointer targets that area 1438 const base = (off - off % bsz) * 16; 1439 bin[i][off] = cast(short)(base < end ? base : end - size); 1440 } 1441 } 1442 return bin; 1443} 1444 1445size_t baseOffset(size_t offset, Bins bin) @nogc nothrow 1446{ 1447 assert(bin <= Bins.B_PAGE); 1448 return (offset & ~(PAGESIZE - 1)) + binbase[bin][(offset & (PAGESIZE - 1)) >> 4]; 1449} 1450 1451alias PageBits = GCBits.wordtype[PAGESIZE / 16 / GCBits.BITS_PER_WORD]; 1452static assert(PAGESIZE % (GCBits.BITS_PER_WORD * 16) == 0); 1453 1454// bitmask with bits set at base offsets of objects 1455immutable PageBits[Bins.B_NUMSMALL] baseOffsetBits = (){ 1456 PageBits[Bins.B_NUMSMALL] bits; 1457 foreach (bin; 0 .. Bins.B_NUMSMALL) 1458 { 1459 size_t size = binsize[bin]; 1460 const top = PAGESIZE - size + 1; // ensure <size> bytes available even if unaligned 1461 for (size_t u = 0; u < top; u += size) 1462 { 1463 size_t biti = u / 16; 1464 size_t off = biti / GCBits.BITS_PER_WORD; 1465 size_t mod = biti % GCBits.BITS_PER_WORD; 1466 bits[bin][off] |= GCBits.BITS_1 << mod; 1467 } 1468 } 1469 return bits; 1470}(); 1471 1472private void set(ref PageBits bits, size_t i) @nogc pure nothrow 1473{ 1474 assert(i < PageBits.sizeof * 8); 1475 bts(bits.ptr, i); 1476} 1477 1478/* ============================ Gcx =============================== */ 1479 1480struct Gcx 1481{ 1482 auto rootsLock = shared(AlignedSpinLock)(SpinLock.Contention.brief); 1483 auto rangesLock = shared(AlignedSpinLock)(SpinLock.Contention.brief); 1484 Treap!Root roots; 1485 Treap!Range ranges; 1486 bool minimizeAfterNextCollection = false; 1487 version (COLLECT_FORK) 1488 { 1489 private pid_t markProcPid = 0; 1490 bool shouldFork; 1491 } 1492 1493 debug(INVARIANT) bool initialized; 1494 debug(INVARIANT) bool inCollection; 1495 uint disabled; // turn off collections if >0 1496 1497 PoolTable!Pool pooltable; 1498 1499 List*[Bins.B_NUMSMALL] bucket; // free list for each small size 1500 1501 // run a collection when reaching those thresholds (number of used pages) 1502 float smallCollectThreshold, largeCollectThreshold; 1503 uint usedSmallPages, usedLargePages; 1504 // total number of mapped pages 1505 uint mappedPages; 1506 1507 debug (LOGGING) 1508 LeakDetector leakDetector; 1509 else 1510 alias leakDetector = LeakDetector; 1511 1512 SmallObjectPool*[Bins.B_NUMSMALL] recoverPool; 1513 version (Posix) __gshared Gcx* instance; 1514 1515 void initialize() 1516 { 1517 (cast(byte*)&this)[0 .. Gcx.sizeof] = 0; 1518 leakDetector.initialize(&this); 1519 roots.initialize(0x243F6A8885A308D3UL); 1520 ranges.initialize(0x13198A2E03707344UL); 1521 smallCollectThreshold = largeCollectThreshold = 0.0f; 1522 usedSmallPages = usedLargePages = 0; 1523 mappedPages = 0; 1524 //printf("gcx = %p, self = %x\n", &this, self); 1525 version (Posix) 1526 { 1527 import core.sys.posix.pthread : pthread_atfork; 1528 instance = &this; 1529 __gshared atforkHandlersInstalled = false; 1530 if (!atforkHandlersInstalled) 1531 { 1532 pthread_atfork( 1533 &_d_gcx_atfork_prepare, 1534 &_d_gcx_atfork_parent, 1535 &_d_gcx_atfork_child); 1536 atforkHandlersInstalled = true; 1537 } 1538 } 1539 debug(INVARIANT) initialized = true; 1540 version (COLLECT_FORK) 1541 shouldFork = config.fork; 1542 1543 } 1544 1545 void Dtor() 1546 { 1547 if (config.profile) 1548 { 1549 printf("\tNumber of collections: %llu\n", cast(ulong)numCollections); 1550 printf("\tTotal GC prep time: %lld milliseconds\n", 1551 prepTime.total!("msecs")); 1552 printf("\tTotal mark time: %lld milliseconds\n", 1553 markTime.total!("msecs")); 1554 printf("\tTotal sweep time: %lld milliseconds\n", 1555 sweepTime.total!("msecs")); 1556 long maxPause = maxPauseTime.total!("msecs"); 1557 printf("\tMax Pause Time: %lld milliseconds\n", maxPause); 1558 long gcTime = (sweepTime + markTime + prepTime).total!("msecs"); 1559 printf("\tGrand total GC time: %lld milliseconds\n", gcTime); 1560 long pauseTime = (markTime + prepTime).total!("msecs"); 1561 1562 char[30] apitxt = void; 1563 apitxt[0] = 0; 1564 debug(PROFILE_API) if (config.profile > 1) 1565 { 1566 static Duration toDuration(long dur) 1567 { 1568 return MonoTime(dur) - MonoTime(0); 1569 } 1570 1571 printf("\n"); 1572 printf("\tmalloc: %llu calls, %lld ms\n", cast(ulong)numMallocs, toDuration(mallocTime).total!"msecs"); 1573 printf("\trealloc: %llu calls, %lld ms\n", cast(ulong)numReallocs, toDuration(reallocTime).total!"msecs"); 1574 printf("\tfree: %llu calls, %lld ms\n", cast(ulong)numFrees, toDuration(freeTime).total!"msecs"); 1575 printf("\textend: %llu calls, %lld ms\n", cast(ulong)numExtends, toDuration(extendTime).total!"msecs"); 1576 printf("\tother: %llu calls, %lld ms\n", cast(ulong)numOthers, toDuration(otherTime).total!"msecs"); 1577 printf("\tlock time: %lld ms\n", toDuration(lockTime).total!"msecs"); 1578 1579 long apiTime = mallocTime + reallocTime + freeTime + extendTime + otherTime + lockTime; 1580 printf("\tGC API: %lld ms\n", toDuration(apiTime).total!"msecs"); 1581 sprintf(apitxt.ptr, " API%5ld ms", toDuration(apiTime).total!"msecs"); 1582 } 1583 1584 printf("GC summary:%5lld MB,%5lld GC%5lld ms, Pauses%5lld ms <%5lld ms%s\n", 1585 cast(long) maxPoolMemory >> 20, cast(ulong)numCollections, gcTime, 1586 pauseTime, maxPause, apitxt.ptr); 1587 } 1588 1589 version (Posix) 1590 instance = null; 1591 version (COLLECT_PARALLEL) 1592 stopScanThreads(); 1593 1594 debug(INVARIANT) initialized = false; 1595 1596 foreach (Pool* pool; this.pooltable[]) 1597 { 1598 mappedPages -= pool.npages; 1599 pool.Dtor(); 1600 cstdlib.free(pool); 1601 } 1602 assert(!mappedPages); 1603 pooltable.Dtor(); 1604 1605 roots.removeAll(); 1606 ranges.removeAll(); 1607 toscanConservative.reset(); 1608 toscanPrecise.reset(); 1609 } 1610 1611 1612 void Invariant() const { } 1613 1614 debug(INVARIANT) 1615 invariant() 1616 { 1617 if (initialized) 1618 { 1619 //printf("Gcx.invariant(): this = %p\n", &this); 1620 pooltable.Invariant(); 1621 for (size_t p = 0; p < pooltable.length; p++) 1622 if (pooltable.pools[p].isLargeObject) 1623 (cast(LargeObjectPool*)(pooltable.pools[p])).Invariant(); 1624 else 1625 (cast(SmallObjectPool*)(pooltable.pools[p])).Invariant(); 1626 1627 if (!inCollection) 1628 (cast()rangesLock).lock(); 1629 foreach (range; ranges) 1630 { 1631 assert(range.pbot); 1632 assert(range.ptop); 1633 assert(range.pbot <= range.ptop); 1634 } 1635 if (!inCollection) 1636 (cast()rangesLock).unlock(); 1637 1638 for (size_t i = 0; i < Bins.B_NUMSMALL; i++) 1639 { 1640 size_t j = 0; 1641 List* prev, pprev, ppprev; // keep a short history to inspect in the debugger 1642 for (auto list = cast(List*)bucket[i]; list; list = list.next) 1643 { 1644 auto pool = list.pool; 1645 auto biti = cast(size_t)(cast(void*)list - pool.baseAddr) >> Pool.ShiftBy.Small; 1646 assert(pool.freebits.test(biti)); 1647 ppprev = pprev; 1648 pprev = prev; 1649 prev = list; 1650 } 1651 } 1652 } 1653 } 1654 1655 @property bool collectInProgress() const nothrow 1656 { 1657 version (COLLECT_FORK) 1658 return markProcPid != 0; 1659 else 1660 return false; 1661 } 1662 1663 1664 /** 1665 * 1666 */ 1667 void addRoot(void *p) nothrow @nogc 1668 { 1669 rootsLock.lock(); 1670 scope (failure) rootsLock.unlock(); 1671 roots.insert(Root(p)); 1672 rootsLock.unlock(); 1673 } 1674 1675 1676 /** 1677 * 1678 */ 1679 void removeRoot(void *p) nothrow @nogc 1680 { 1681 rootsLock.lock(); 1682 scope (failure) rootsLock.unlock(); 1683 roots.remove(Root(p)); 1684 rootsLock.unlock(); 1685 } 1686 1687 1688 /** 1689 * 1690 */ 1691 int rootsApply(scope int delegate(ref Root) nothrow dg) nothrow 1692 { 1693 rootsLock.lock(); 1694 scope (failure) rootsLock.unlock(); 1695 auto ret = roots.opApply(dg); 1696 rootsLock.unlock(); 1697 return ret; 1698 } 1699 1700 1701 /** 1702 * 1703 */ 1704 void addRange(void *pbot, void *ptop, const TypeInfo ti) nothrow @nogc 1705 { 1706 //debug(PRINTF) printf("Thread %x ", pthread_self()); 1707 debug(PRINTF) printf("%p.Gcx::addRange(%p, %p)\n", &this, pbot, ptop); 1708 rangesLock.lock(); 1709 scope (failure) rangesLock.unlock(); 1710 ranges.insert(Range(pbot, ptop)); 1711 rangesLock.unlock(); 1712 } 1713 1714 1715 /** 1716 * 1717 */ 1718 void removeRange(void *pbot) nothrow @nogc 1719 { 1720 //debug(PRINTF) printf("Thread %x ", pthread_self()); 1721 debug(PRINTF) printf("Gcx.removeRange(%p)\n", pbot); 1722 rangesLock.lock(); 1723 scope (failure) rangesLock.unlock(); 1724 ranges.remove(Range(pbot, pbot)); // only pbot is used, see Range.opCmp 1725 rangesLock.unlock(); 1726 1727 // debug(PRINTF) printf("Wrong thread\n"); 1728 // This is a fatal error, but ignore it. 1729 // The problem is that we can get a Close() call on a thread 1730 // other than the one the range was allocated on. 1731 //assert(zero); 1732 } 1733 1734 /** 1735 * 1736 */ 1737 int rangesApply(scope int delegate(ref Range) nothrow dg) nothrow 1738 { 1739 rangesLock.lock(); 1740 scope (failure) rangesLock.unlock(); 1741 auto ret = ranges.opApply(dg); 1742 rangesLock.unlock(); 1743 return ret; 1744 } 1745 1746 1747 /** 1748 * 1749 */ 1750 void runFinalizers(const scope void[] segment) nothrow 1751 { 1752 ConservativeGC._inFinalizer = true; 1753 scope (failure) ConservativeGC._inFinalizer = false; 1754 1755 foreach (pool; this.pooltable[]) 1756 { 1757 if (!pool.finals.nbits) continue; 1758 1759 if (pool.isLargeObject) 1760 { 1761 auto lpool = cast(LargeObjectPool*) pool; 1762 lpool.runFinalizers(segment); 1763 } 1764 else 1765 { 1766 auto spool = cast(SmallObjectPool*) pool; 1767 spool.runFinalizers(segment); 1768 } 1769 } 1770 ConservativeGC._inFinalizer = false; 1771 } 1772 1773 Pool* findPool(void* p) pure nothrow @nogc 1774 { 1775 return pooltable.findPool(p); 1776 } 1777 1778 /** 1779 * Find base address of block containing pointer p. 1780 * Returns null if not a gc'd pointer 1781 */ 1782 void* findBase(void *p) nothrow @nogc 1783 { 1784 Pool *pool; 1785 1786 pool = findPool(p); 1787 if (pool) 1788 return pool.findBase(p); 1789 return null; 1790 } 1791 1792 1793 /** 1794 * Find size of pointer p. 1795 * Returns 0 if not a gc'd pointer 1796 */ 1797 size_t findSize(void *p) nothrow @nogc 1798 { 1799 Pool* pool = findPool(p); 1800 if (pool) 1801 return pool.slGetSize(p); 1802 return 0; 1803 } 1804 1805 /** 1806 * 1807 */ 1808 BlkInfo getInfo(void* p) nothrow 1809 { 1810 Pool* pool = findPool(p); 1811 if (pool) 1812 return pool.slGetInfo(p); 1813 return BlkInfo(); 1814 } 1815 1816 /** 1817 * Computes the bin table using CTFE. 1818 */ 1819 static Bins[2049] ctfeBins() nothrow 1820 { 1821 Bins[2049] ret; 1822 size_t p = 0; 1823 for (Bins b = Bins.B_16; b <= Bins.B_2048; b++) 1824 for ( ; p <= binsize[b]; p++) 1825 ret[p] = b; 1826 1827 return ret; 1828 } 1829 1830 static immutable Bins[2049] binTable = ctfeBins(); 1831 1832 /** 1833 * Allocate a new pool of at least size bytes. 1834 * Sort it into pooltable[]. 1835 * Mark all memory in the pool as B_FREE. 1836 * Return the actual number of bytes reserved or 0 on error. 1837 */ 1838 size_t reserve(size_t size) nothrow 1839 { 1840 size_t npages = (size + PAGESIZE - 1) / PAGESIZE; 1841 1842 // Assume reserve() is for small objects. 1843 Pool* pool = newPool(npages, false); 1844 1845 if (!pool) 1846 return 0; 1847 return pool.npages * PAGESIZE; 1848 } 1849 1850 /** 1851 * Update the thresholds for when to collect the next time 1852 */ 1853 void updateCollectThresholds() nothrow 1854 { 1855 static float max(float a, float b) nothrow 1856 { 1857 return a >= b ? a : b; 1858 } 1859 1860 // instantly increases, slowly decreases 1861 static float smoothDecay(float oldVal, float newVal) nothrow 1862 { 1863 // decay to 63.2% of newVal over 5 collections 1864 // http://en.wikipedia.org/wiki/Low-pass_filter#Simple_infinite_impulse_response_filter 1865 enum alpha = 1.0 / (5 + 1); 1866 immutable decay = (newVal - oldVal) * alpha + oldVal; 1867 return max(newVal, decay); 1868 } 1869 1870 immutable smTarget = usedSmallPages * config.heapSizeFactor; 1871 smallCollectThreshold = smoothDecay(smallCollectThreshold, smTarget); 1872 immutable lgTarget = usedLargePages * config.heapSizeFactor; 1873 largeCollectThreshold = smoothDecay(largeCollectThreshold, lgTarget); 1874 } 1875 1876 /** 1877 * Minimizes physical memory usage by returning free pools to the OS. 1878 */ 1879 void minimize() nothrow 1880 { 1881 debug(PRINTF) printf("Minimizing.\n"); 1882 1883 foreach (pool; pooltable.minimize()) 1884 { 1885 debug(PRINTF) printFreeInfo(pool); 1886 mappedPages -= pool.npages; 1887 pool.Dtor(); 1888 cstdlib.free(pool); 1889 } 1890 1891 debug(PRINTF) printf("Done minimizing.\n"); 1892 } 1893 1894 private @property bool lowMem() const nothrow 1895 { 1896 return isLowOnMem(cast(size_t)mappedPages * PAGESIZE); 1897 } 1898 1899 void* alloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow 1900 { 1901 return size <= PAGESIZE/2 ? smallAlloc(size, alloc_size, bits, ti) 1902 : bigAlloc(size, alloc_size, bits, ti); 1903 } 1904 1905 void* smallAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow 1906 { 1907 immutable bin = binTable[size]; 1908 alloc_size = binsize[bin]; 1909 1910 void* p = bucket[bin]; 1911 if (p) 1912 goto L_hasBin; 1913 1914 if (recoverPool[bin]) 1915 recoverNextPage(bin); 1916 1917 bool tryAlloc() nothrow 1918 { 1919 if (!bucket[bin]) 1920 { 1921 bucket[bin] = allocPage(bin); 1922 if (!bucket[bin]) 1923 return false; 1924 } 1925 p = bucket[bin]; 1926 return true; 1927 } 1928 1929 if (!tryAlloc()) 1930 { 1931 if (!lowMem && (disabled || usedSmallPages < smallCollectThreshold)) 1932 { 1933 // disabled or threshold not reached => allocate a new pool instead of collecting 1934 if (!newPool(1, false)) 1935 { 1936 // out of memory => try to free some memory 1937 fullcollect(false, true); // stop the world 1938 if (lowMem) 1939 minimize(); 1940 recoverNextPage(bin); 1941 } 1942 } 1943 else if (usedSmallPages > 0) 1944 { 1945 fullcollect(); 1946 if (lowMem) 1947 minimize(); 1948 recoverNextPage(bin); 1949 } 1950 // tryAlloc will succeed if a new pool was allocated above, if it fails allocate a new pool now 1951 if (!tryAlloc() && (!newPool(1, false) || !tryAlloc())) 1952 // out of luck or memory 1953 onOutOfMemoryErrorNoGC(); 1954 } 1955 assert(p !is null); 1956 L_hasBin: 1957 // Return next item from free list 1958 bucket[bin] = (cast(List*)p).next; 1959 auto pool = (cast(List*)p).pool; 1960 1961 auto biti = (p - pool.baseAddr) >> pool.shiftBy; 1962 assert(pool.freebits.test(biti)); 1963 if (collectInProgress) 1964 pool.mark.setLocked(biti); // be sure that the child is aware of the page being used 1965 pool.freebits.clear(biti); 1966 if (bits) 1967 pool.setBits(biti, bits); 1968 //debug(PRINTF) printf("\tmalloc => %p\n", p); 1969 debug (MEMSTOMP) memset(p, 0xF0, alloc_size); 1970 1971 if (ConservativeGC.isPrecise) 1972 { 1973 debug(SENTINEL) 1974 pool.setPointerBitmapSmall(sentinel_add(p), size - SENTINEL_EXTRA, size - SENTINEL_EXTRA, bits, ti); 1975 else 1976 pool.setPointerBitmapSmall(p, size, alloc_size, bits, ti); 1977 } 1978 return p; 1979 } 1980 1981 /** 1982 * Allocate a chunk of memory that is larger than a page. 1983 * Return null if out of memory. 1984 */ 1985 void* bigAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti = null) nothrow 1986 { 1987 debug(PRINTF) printf("In bigAlloc. Size: %d\n", size); 1988 1989 LargeObjectPool* pool; 1990 size_t pn; 1991 immutable npages = LargeObjectPool.numPages(size); 1992 if (npages == size_t.max) 1993 onOutOfMemoryErrorNoGC(); // size just below size_t.max requested 1994 1995 bool tryAlloc() nothrow 1996 { 1997 foreach (p; this.pooltable[]) 1998 { 1999 if (!p.isLargeObject || p.freepages < npages) 2000 continue; 2001 auto lpool = cast(LargeObjectPool*) p; 2002 if ((pn = lpool.allocPages(npages)) == OPFAIL) 2003 continue; 2004 pool = lpool; 2005 return true; 2006 } 2007 return false; 2008 } 2009 2010 bool tryAllocNewPool() nothrow 2011 { 2012 pool = cast(LargeObjectPool*) newPool(npages, true); 2013 if (!pool) return false; 2014 pn = pool.allocPages(npages); 2015 assert(pn != OPFAIL); 2016 return true; 2017 } 2018 2019 if (!tryAlloc()) 2020 { 2021 if (!lowMem && (disabled || usedLargePages < largeCollectThreshold)) 2022 { 2023 // disabled or threshold not reached => allocate a new pool instead of collecting 2024 if (!tryAllocNewPool()) 2025 { 2026 // disabled but out of memory => try to free some memory 2027 minimizeAfterNextCollection = true; 2028 fullcollect(false, true); 2029 } 2030 } 2031 else if (usedLargePages > 0) 2032 { 2033 minimizeAfterNextCollection = true; 2034 fullcollect(); 2035 } 2036 // If alloc didn't yet succeed retry now that we collected/minimized 2037 if (!pool && !tryAlloc() && !tryAllocNewPool()) 2038 // out of luck or memory 2039 return null; 2040 } 2041 assert(pool); 2042 2043 debug(PRINTF) printFreeInfo(&pool.base); 2044 if (collectInProgress) 2045 pool.mark.setLocked(pn); 2046 usedLargePages += npages; 2047 2048 debug(PRINTF) printFreeInfo(&pool.base); 2049 2050 auto p = pool.baseAddr + pn * PAGESIZE; 2051 debug(PRINTF) printf("Got large alloc: %p, pt = %d, np = %d\n", p, pool.pagetable[pn], npages); 2052 debug (MEMSTOMP) memset(p, 0xF1, size); 2053 alloc_size = npages * PAGESIZE; 2054 //debug(PRINTF) printf("\tp = %p\n", p); 2055 2056 if (bits) 2057 pool.setBits(pn, bits); 2058 2059 if (ConservativeGC.isPrecise) 2060 { 2061 // an array of classes is in fact an array of pointers 2062 immutable(void)* rtinfo; 2063 if (!ti) 2064 rtinfo = rtinfoHasPointers; 2065 else if ((bits & BlkAttr.APPENDABLE) && (typeid(ti) is typeid(TypeInfo_Class))) 2066 rtinfo = rtinfoHasPointers; 2067 else 2068 rtinfo = ti.rtInfo(); 2069 pool.rtinfo[pn] = cast(immutable(size_t)*)rtinfo; 2070 } 2071 2072 return p; 2073 } 2074 2075 2076 /** 2077 * Allocate a new pool with at least npages in it. 2078 * Sort it into pooltable[]. 2079 * Return null if failed. 2080 */ 2081 Pool *newPool(size_t npages, bool isLargeObject) nothrow 2082 { 2083 //debug(PRINTF) printf("************Gcx::newPool(npages = %d)****************\n", npages); 2084 2085 // Minimum of POOLSIZE 2086 size_t minPages = config.minPoolSize / PAGESIZE; 2087 if (npages < minPages) 2088 npages = minPages; 2089 else if (npages > minPages) 2090 { // Give us 150% of requested size, so there's room to extend 2091 auto n = npages + (npages >> 1); 2092 if (n < size_t.max/PAGESIZE) 2093 npages = n; 2094 } 2095 2096 // Allocate successively larger pools up to 8 megs 2097 if (this.pooltable.length) 2098 { 2099 size_t n; 2100 2101 n = config.minPoolSize + config.incPoolSize * this.pooltable.length; 2102 if (n > config.maxPoolSize) 2103 n = config.maxPoolSize; // cap pool size 2104 n /= PAGESIZE; // convert bytes to pages 2105 if (npages < n) 2106 npages = n; 2107 } 2108 2109 //printf("npages = %d\n", npages); 2110 2111 auto pool = cast(Pool *)cstdlib.calloc(1, isLargeObject ? LargeObjectPool.sizeof : SmallObjectPool.sizeof); 2112 if (pool) 2113 { 2114 pool.initialize(npages, isLargeObject); 2115 if (collectInProgress) 2116 pool.mark.setAll(); 2117 if (!pool.baseAddr || !pooltable.insert(pool)) 2118 { 2119 pool.Dtor(); 2120 cstdlib.free(pool); 2121 return null; 2122 } 2123 } 2124 2125 mappedPages += npages; 2126 2127 if (config.profile) 2128 { 2129 if (cast(size_t)mappedPages * PAGESIZE > maxPoolMemory) 2130 maxPoolMemory = cast(size_t)mappedPages * PAGESIZE; 2131 } 2132 return pool; 2133 } 2134 2135 /** 2136 * Allocate a page of bin's. 2137 * Returns: 2138 * head of a single linked list of new entries 2139 */ 2140 List* allocPage(Bins bin) nothrow 2141 { 2142 //debug(PRINTF) printf("Gcx::allocPage(bin = %d)\n", bin); 2143 foreach (Pool* pool; this.pooltable[]) 2144 { 2145 if (pool.isLargeObject) 2146 continue; 2147 if (List* p = (cast(SmallObjectPool*)pool).allocPage(bin)) 2148 { 2149 ++usedSmallPages; 2150 return p; 2151 } 2152 } 2153 return null; 2154 } 2155 2156 static struct ScanRange(bool precise) 2157 { 2158 void* pbot; 2159 void* ptop; 2160 static if (precise) 2161 { 2162 void** pbase; // start of memory described by ptrbitmap 2163 size_t* ptrbmp; // bits from is_pointer or rtinfo 2164 size_t bmplength; // number of valid bits 2165 } 2166 } 2167 2168 static struct ToScanStack(RANGE) 2169 { 2170 nothrow: 2171 @disable this(this); 2172 auto stackLock = shared(AlignedSpinLock)(SpinLock.Contention.brief); 2173 2174 void reset() 2175 { 2176 _length = 0; 2177 if (_p) 2178 { 2179 os_mem_unmap(_p, _cap * RANGE.sizeof); 2180 _p = null; 2181 } 2182 _cap = 0; 2183 } 2184 void clear() 2185 { 2186 _length = 0; 2187 } 2188 2189 void push(RANGE rng) 2190 { 2191 if (_length == _cap) grow(); 2192 _p[_length++] = rng; 2193 } 2194 2195 RANGE pop() 2196 in { assert(!empty); } 2197 do 2198 { 2199 return _p[--_length]; 2200 } 2201 2202 bool popLocked(ref RANGE rng) 2203 { 2204 if (_length == 0) 2205 return false; 2206 2207 stackLock.lock(); 2208 scope(exit) stackLock.unlock(); 2209 if (_length == 0) 2210 return false; 2211 rng = _p[--_length]; 2212 return true; 2213 } 2214 2215 ref inout(RANGE) opIndex(size_t idx) inout 2216 in { assert(idx < _length); } 2217 do 2218 { 2219 return _p[idx]; 2220 } 2221 2222 @property size_t length() const { return _length; } 2223 @property bool empty() const { return !length; } 2224 2225 private: 2226 void grow() 2227 { 2228 pragma(inline, false); 2229 2230 enum initSize = 64 * 1024; // Windows VirtualAlloc granularity 2231 immutable ncap = _cap ? 2 * _cap : initSize / RANGE.sizeof; 2232 auto p = cast(RANGE*)os_mem_map(ncap * RANGE.sizeof); 2233 if (p is null) onOutOfMemoryErrorNoGC(); 2234 if (_p !is null) 2235 { 2236 p[0 .. _length] = _p[0 .. _length]; 2237 os_mem_unmap(_p, _cap * RANGE.sizeof); 2238 } 2239 _p = p; 2240 _cap = ncap; 2241 } 2242 2243 size_t _length; 2244 RANGE* _p; 2245 size_t _cap; 2246 } 2247 2248 ToScanStack!(ScanRange!false) toscanConservative; 2249 ToScanStack!(ScanRange!true) toscanPrecise; 2250 2251 template scanStack(bool precise) 2252 { 2253 static if (precise) 2254 alias scanStack = toscanPrecise; 2255 else 2256 alias scanStack = toscanConservative; 2257 } 2258 2259 /** 2260 * Search a range of memory values and mark any pointers into the GC pool. 2261 */ 2262 private void mark(bool precise, bool parallel, bool shared_mem)(ScanRange!precise rng) scope nothrow 2263 { 2264 alias toscan = scanStack!precise; 2265 2266 debug(MARK_PRINTF) 2267 printf("marking range: [%p..%p] (%#llx)\n", pbot, ptop, cast(long)(ptop - pbot)); 2268 2269 // limit the amount of ranges added to the toscan stack 2270 enum FANOUT_LIMIT = 32; 2271 size_t stackPos; 2272 ScanRange!precise[FANOUT_LIMIT] stack = void; 2273 2274 size_t pcache = 0; 2275 2276 // let dmd allocate a register for this.pools 2277 auto pools = pooltable.pools; 2278 const highpool = pooltable.length - 1; 2279 const minAddr = pooltable.minAddr; 2280 size_t memSize = pooltable.maxAddr - minAddr; 2281 Pool* pool = null; 2282 2283 // properties of allocation pointed to 2284 ScanRange!precise tgt = void; 2285 2286 for (;;) 2287 { 2288 auto p = *cast(void**)(rng.pbot); 2289 2290 debug(MARK_PRINTF) printf("\tmark %p: %p\n", rng.pbot, p); 2291 2292 if (cast(size_t)(p - minAddr) < memSize && 2293 (cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) != pcache) 2294 { 2295 static if (precise) if (rng.pbase) 2296 { 2297 size_t bitpos = cast(void**)rng.pbot - rng.pbase; 2298 while (bitpos >= rng.bmplength) 2299 { 2300 bitpos -= rng.bmplength; 2301 rng.pbase += rng.bmplength; 2302 } 2303 if (!core.bitop.bt(rng.ptrbmp, bitpos)) 2304 { 2305 debug(MARK_PRINTF) printf("\t\tskipping non-pointer\n"); 2306 goto LnextPtr; 2307 } 2308 } 2309 2310 if (!pool || p < pool.baseAddr || p >= pool.topAddr) 2311 { 2312 size_t low = 0; 2313 size_t high = highpool; 2314 while (true) 2315 { 2316 size_t mid = (low + high) >> 1; 2317 pool = pools[mid]; 2318 if (p < pool.baseAddr) 2319 high = mid - 1; 2320 else if (p >= pool.topAddr) 2321 low = mid + 1; 2322 else break; 2323 2324 if (low > high) 2325 goto LnextPtr; 2326 } 2327 } 2328 size_t offset = cast(size_t)(p - pool.baseAddr); 2329 size_t biti = void; 2330 size_t pn = offset / PAGESIZE; 2331 size_t bin = pool.pagetable[pn]; // not Bins to avoid multiple size extension instructions 2332 2333 debug(MARK_PRINTF) 2334 printf("\t\tfound pool %p, base=%p, pn = %lld, bin = %d\n", pool, pool.baseAddr, cast(long)pn, bin); 2335 2336 // Adjust bit to be at start of allocated memory block 2337 if (bin < Bins.B_PAGE) 2338 { 2339 // We don't care abou setting pointsToBase correctly 2340 // because it's ignored for small object pools anyhow. 2341 auto offsetBase = baseOffset(offset, cast(Bins)bin); 2342 biti = offsetBase >> Pool.ShiftBy.Small; 2343 //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); 2344 2345 if (!pool.mark.testAndSet!shared_mem(biti) && !pool.noscan.test(biti)) 2346 { 2347 tgt.pbot = pool.baseAddr + offsetBase; 2348 tgt.ptop = tgt.pbot + binsize[bin]; 2349 static if (precise) 2350 { 2351 tgt.pbase = cast(void**)pool.baseAddr; 2352 tgt.ptrbmp = pool.is_pointer.data; 2353 tgt.bmplength = size_t.max; // no repetition 2354 } 2355 goto LaddRange; 2356 } 2357 } 2358 else if (bin == Bins.B_PAGE) 2359 { 2360 biti = offset >> Pool.ShiftBy.Large; 2361 //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); 2362 2363 pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); 2364 tgt.pbot = cast(void*)pcache; 2365 2366 // For the NO_INTERIOR attribute. This tracks whether 2367 // the pointer is an interior pointer or points to the 2368 // base address of a block. 2369 if (tgt.pbot != sentinel_sub(p) && pool.nointerior.nbits && pool.nointerior.test(biti)) 2370 goto LnextPtr; 2371 2372 if (!pool.mark.testAndSet!shared_mem(biti) && !pool.noscan.test(biti)) 2373 { 2374 tgt.ptop = tgt.pbot + (cast(LargeObjectPool*)pool).getSize(pn); 2375 goto LaddLargeRange; 2376 } 2377 } 2378 else if (bin == Bins.B_PAGEPLUS) 2379 { 2380 pn -= pool.bPageOffsets[pn]; 2381 biti = pn * (PAGESIZE >> Pool.ShiftBy.Large); 2382 2383 pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); 2384 if (pool.nointerior.nbits && pool.nointerior.test(biti)) 2385 goto LnextPtr; 2386 2387 if (!pool.mark.testAndSet!shared_mem(biti) && !pool.noscan.test(biti)) 2388 { 2389 tgt.pbot = pool.baseAddr + (pn * PAGESIZE); 2390 tgt.ptop = tgt.pbot + (cast(LargeObjectPool*)pool).getSize(pn); 2391 LaddLargeRange: 2392 static if (precise) 2393 { 2394 auto rtinfo = pool.rtinfo[biti]; 2395 if (rtinfo is rtinfoNoPointers) 2396 goto LnextPtr; // only if inconsistent with noscan 2397 if (rtinfo is rtinfoHasPointers) 2398 { 2399 tgt.pbase = null; // conservative 2400 } 2401 else 2402 { 2403 tgt.ptrbmp = cast(size_t*)rtinfo; 2404 size_t element_size = *tgt.ptrbmp++; 2405 tgt.bmplength = (element_size + (void*).sizeof - 1) / (void*).sizeof; 2406 assert(tgt.bmplength); 2407 2408 debug(SENTINEL) 2409 tgt.pbot = sentinel_add(tgt.pbot); 2410 if (pool.appendable.test(biti)) 2411 { 2412 // take advantage of knowing array layout in rt.lifetime 2413 void* arrtop = tgt.pbot + 16 + *cast(size_t*)tgt.pbot; 2414 assert (arrtop > tgt.pbot && arrtop <= tgt.ptop); 2415 tgt.pbot += 16; 2416 tgt.ptop = arrtop; 2417 } 2418 else 2419 { 2420 tgt.ptop = tgt.pbot + element_size; 2421 } 2422 tgt.pbase = cast(void**)tgt.pbot; 2423 } 2424 } 2425 goto LaddRange; 2426 } 2427 } 2428 else 2429 { 2430 // Don't mark bits in B_FREE pages 2431 assert(bin == Bins.B_FREE); 2432 } 2433 } 2434 LnextPtr: 2435 rng.pbot += (void*).sizeof; 2436 if (rng.pbot < rng.ptop) 2437 continue; 2438 2439 LnextRange: 2440 if (stackPos) 2441 { 2442 // pop range from local stack and recurse 2443 rng = stack[--stackPos]; 2444 } 2445 else 2446 { 2447 static if (parallel) 2448 { 2449 if (!toscan.popLocked(rng)) 2450 break; // nothing more to do 2451 } 2452 else 2453 { 2454 if (toscan.empty) 2455 break; // nothing more to do 2456 2457 // pop range from global stack and recurse 2458 rng = toscan.pop(); 2459 } 2460 } 2461 // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); 2462 goto LcontRange; 2463 2464 LaddRange: 2465 rng.pbot += (void*).sizeof; 2466 if (rng.pbot < rng.ptop) 2467 { 2468 if (stackPos < stack.length) 2469 { 2470 stack[stackPos] = tgt; 2471 stackPos++; 2472 continue; 2473 } 2474 static if (parallel) 2475 { 2476 toscan.stackLock.lock(); 2477 scope(exit) toscan.stackLock.unlock(); 2478 } 2479 toscan.push(rng); 2480 // reverse order for depth-first-order traversal 2481 foreach_reverse (ref range; stack) 2482 toscan.push(range); 2483 stackPos = 0; 2484 } 2485 LendOfRange: 2486 // continue with last found range 2487 rng = tgt; 2488 2489 LcontRange: 2490 pcache = 0; 2491 } 2492 } 2493 2494 void markConservative(bool shared_mem)(void *pbot, void *ptop) scope nothrow 2495 { 2496 if (pbot < ptop) 2497 mark!(false, false, shared_mem)(ScanRange!false(pbot, ptop)); 2498 } 2499 2500 void markPrecise(bool shared_mem)(void *pbot, void *ptop) scope nothrow 2501 { 2502 if (pbot < ptop) 2503 mark!(true, false, shared_mem)(ScanRange!true(pbot, ptop, null)); 2504 } 2505 2506 version (COLLECT_PARALLEL) 2507 ToScanStack!(void*) toscanRoots; 2508 2509 version (COLLECT_PARALLEL) 2510 void collectRoots(void *pbot, void *ptop) scope nothrow 2511 { 2512 const minAddr = pooltable.minAddr; 2513 size_t memSize = pooltable.maxAddr - minAddr; 2514 2515 for (auto p = cast(void**)pbot; cast(void*)p < ptop; p++) 2516 { 2517 auto ptr = *p; 2518 if (cast(size_t)(ptr - minAddr) < memSize) 2519 toscanRoots.push(ptr); 2520 } 2521 } 2522 2523 // collection step 1: prepare freebits and mark bits 2524 void prepare() nothrow 2525 { 2526 debug(COLLECT_PRINTF) printf("preparing mark.\n"); 2527 2528 foreach (Pool* pool; this.pooltable[]) 2529 { 2530 if (pool.isLargeObject) 2531 pool.mark.zero(); 2532 else 2533 pool.mark.copy(&pool.freebits); 2534 } 2535 } 2536 2537 // collection step 2: mark roots and heap 2538 void markAll(alias markFn)(bool nostack) nothrow 2539 { 2540 if (!nostack) 2541 { 2542 debug(COLLECT_PRINTF) printf("\tscan stacks.\n"); 2543 // Scan stacks and registers for each paused thread 2544 thread_scanAll(&markFn); 2545 } 2546 2547 // Scan roots[] 2548 debug(COLLECT_PRINTF) printf("\tscan roots[]\n"); 2549 foreach (root; roots) 2550 { 2551 markFn(cast(void*)&root.proot, cast(void*)(&root.proot + 1)); 2552 } 2553 2554 // Scan ranges[] 2555 debug(COLLECT_PRINTF) printf("\tscan ranges[]\n"); 2556 //log++; 2557 foreach (range; ranges) 2558 { 2559 debug(COLLECT_PRINTF) printf("\t\t%p .. %p\n", range.pbot, range.ptop); 2560 markFn(range.pbot, range.ptop); 2561 } 2562 //log--; 2563 } 2564 2565 version (COLLECT_PARALLEL) 2566 void collectAllRoots(bool nostack) nothrow 2567 { 2568 if (!nostack) 2569 { 2570 debug(COLLECT_PRINTF) printf("\tcollect stacks.\n"); 2571 // Scan stacks and registers for each paused thread 2572 thread_scanAll(&collectRoots); 2573 } 2574 2575 // Scan roots[] 2576 debug(COLLECT_PRINTF) printf("\tcollect roots[]\n"); 2577 foreach (root; roots) 2578 { 2579 toscanRoots.push(root); 2580 } 2581 2582 // Scan ranges[] 2583 debug(COLLECT_PRINTF) printf("\tcollect ranges[]\n"); 2584 foreach (range; ranges) 2585 { 2586 debug(COLLECT_PRINTF) printf("\t\t%p .. %p\n", range.pbot, range.ptop); 2587 collectRoots(range.pbot, range.ptop); 2588 } 2589 } 2590 2591 // collection step 3: finalize unreferenced objects, recover full pages with no live objects 2592 size_t sweep() nothrow 2593 { 2594 // Free up everything not marked 2595 debug(COLLECT_PRINTF) printf("\tfree'ing\n"); 2596 size_t freedLargePages; 2597 size_t freedSmallPages; 2598 size_t freed; 2599 foreach (Pool* pool; this.pooltable[]) 2600 { 2601 size_t pn; 2602 2603 if (pool.isLargeObject) 2604 { 2605 auto lpool = cast(LargeObjectPool*)pool; 2606 size_t numFree = 0; 2607 size_t npages; 2608 for (pn = 0; pn < pool.npages; pn += npages) 2609 { 2610 npages = pool.bPageOffsets[pn]; 2611 Bins bin = cast(Bins)pool.pagetable[pn]; 2612 if (bin == Bins.B_FREE) 2613 { 2614 numFree += npages; 2615 continue; 2616 } 2617 assert(bin == Bins.B_PAGE); 2618 size_t biti = pn; 2619 2620 if (!pool.mark.test(biti)) 2621 { 2622 void *p = pool.baseAddr + pn * PAGESIZE; 2623 void* q = sentinel_add(p); 2624 sentinel_Invariant(q); 2625 2626 if (pool.finals.nbits && pool.finals.clear(biti)) 2627 { 2628 size_t size = npages * PAGESIZE - SENTINEL_EXTRA; 2629 uint attr = pool.getBits(biti); 2630 rt_finalizeFromGC(q, sentinel_size(q, size), attr); 2631 } 2632 2633 pool.clrBits(biti, ~BlkAttr.NONE ^ BlkAttr.FINALIZE); 2634 2635 debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p); 2636 leakDetector.log_free(q, sentinel_size(q, npages * PAGESIZE - SENTINEL_EXTRA)); 2637 pool.pagetable[pn..pn+npages] = Bins.B_FREE; 2638 if (pn < pool.searchStart) pool.searchStart = pn; 2639 freedLargePages += npages; 2640 pool.freepages += npages; 2641 numFree += npages; 2642 2643 debug (MEMSTOMP) memset(p, 0xF3, npages * PAGESIZE); 2644 // Don't need to update searchStart here because 2645 // pn is guaranteed to be greater than last time 2646 // we updated it. 2647 2648 pool.largestFree = pool.freepages; // invalidate 2649 } 2650 else 2651 { 2652 if (numFree > 0) 2653 { 2654 lpool.setFreePageOffsets(pn - numFree, numFree); 2655 numFree = 0; 2656 } 2657 } 2658 } 2659 if (numFree > 0) 2660 lpool.setFreePageOffsets(pn - numFree, numFree); 2661 } 2662 else 2663 { 2664 // reinit chain of pages to rebuild free list 2665 pool.recoverPageFirst[] = cast(uint)pool.npages; 2666 2667 for (pn = 0; pn < pool.npages; pn++) 2668 { 2669 Bins bin = cast(Bins)pool.pagetable[pn]; 2670 2671 if (bin < Bins.B_PAGE) 2672 { 2673 auto freebitsdata = pool.freebits.data + pn * PageBits.length; 2674 auto markdata = pool.mark.data + pn * PageBits.length; 2675 2676 // the entries to free are allocated objects (freebits == false) 2677 // that are not marked (mark == false) 2678 PageBits toFree; 2679 static foreach (w; 0 .. PageBits.length) 2680 toFree[w] = (~freebitsdata[w] & ~markdata[w]); 2681 2682 // the page is unchanged if there is nothing to free 2683 bool unchanged = true; 2684 static foreach (w; 0 .. PageBits.length) 2685 unchanged = unchanged && (toFree[w] == 0); 2686 if (unchanged) 2687 { 2688 bool hasDead = false; 2689 static foreach (w; 0 .. PageBits.length) 2690 hasDead = hasDead || (~freebitsdata[w] != baseOffsetBits[bin][w]); 2691 if (hasDead) 2692 { 2693 // add to recover chain 2694 pool.binPageChain[pn] = pool.recoverPageFirst[bin]; 2695 pool.recoverPageFirst[bin] = cast(uint)pn; 2696 } 2697 else 2698 { 2699 pool.binPageChain[pn] = Pool.PageRecovered; 2700 } 2701 continue; 2702 } 2703 2704 // the page can be recovered if all of the allocated objects (freebits == false) 2705 // are freed 2706 bool recoverPage = true; 2707 static foreach (w; 0 .. PageBits.length) 2708 recoverPage = recoverPage && (~freebitsdata[w] == toFree[w]); 2709 2710 // We need to loop through each object if any have a finalizer, 2711 // or, if any of the debug hooks are enabled. 2712 bool doLoop = false; 2713 debug (SENTINEL) 2714 doLoop = true; 2715 else version (assert) 2716 doLoop = true; 2717 else debug (COLLECT_PRINTF) // need output for each object 2718 doLoop = true; 2719 else debug (LOGGING) 2720 doLoop = true; 2721 else debug (MEMSTOMP) 2722 doLoop = true; 2723 else if (pool.finals.data) 2724 { 2725 // finalizers must be called on objects that are about to be freed 2726 auto finalsdata = pool.finals.data + pn * PageBits.length; 2727 static foreach (w; 0 .. PageBits.length) 2728 doLoop = doLoop || (toFree[w] & finalsdata[w]) != 0; 2729 } 2730 2731 if (doLoop) 2732 { 2733 immutable size = binsize[bin]; 2734 void *p = pool.baseAddr + pn * PAGESIZE; 2735 immutable base = pn * (PAGESIZE/16); 2736 immutable bitstride = size / 16; 2737 2738 // ensure that there are at least <size> bytes for every address 2739 // below ptop even if unaligned 2740 void *ptop = p + PAGESIZE - size + 1; 2741 for (size_t i; p < ptop; p += size, i += bitstride) 2742 { 2743 immutable biti = base + i; 2744 2745 if (!pool.mark.test(biti)) 2746 { 2747 void* q = sentinel_add(p); 2748 sentinel_Invariant(q); 2749 2750 if (pool.finals.nbits && pool.finals.test(biti)) 2751 rt_finalizeFromGC(q, sentinel_size(q, size), pool.getBits(biti)); 2752 2753 assert(core.bitop.bt(toFree.ptr, i)); 2754 2755 debug(COLLECT_PRINTF) printf("\tcollecting %p\n", p); 2756 leakDetector.log_free(q, sentinel_size(q, size)); 2757 2758 debug (MEMSTOMP) memset(p, 0xF3, size); 2759 } 2760 } 2761 } 2762 2763 if (recoverPage) 2764 { 2765 pool.freeAllPageBits(pn); 2766 2767 pool.pagetable[pn] = Bins.B_FREE; 2768 // add to free chain 2769 pool.binPageChain[pn] = cast(uint) pool.searchStart; 2770 pool.searchStart = pn; 2771 pool.freepages++; 2772 freedSmallPages++; 2773 } 2774 else 2775 { 2776 pool.freePageBits(pn, toFree); 2777 2778 // add to recover chain 2779 pool.binPageChain[pn] = pool.recoverPageFirst[bin]; 2780 pool.recoverPageFirst[bin] = cast(uint)pn; 2781 } 2782 } 2783 } 2784 } 2785 } 2786 2787 assert(freedLargePages <= usedLargePages); 2788 usedLargePages -= freedLargePages; 2789 debug(COLLECT_PRINTF) printf("\tfree'd %u bytes, %u pages from %u pools\n", 2790 freed, freedLargePages, this.pooltable.length); 2791 2792 assert(freedSmallPages <= usedSmallPages); 2793 usedSmallPages -= freedSmallPages; 2794 debug(COLLECT_PRINTF) printf("\trecovered small pages = %d\n", freedSmallPages); 2795 2796 return freedLargePages + freedSmallPages; 2797 } 2798 2799 bool recoverPage(SmallObjectPool* pool, size_t pn, Bins bin) nothrow 2800 { 2801 size_t size = binsize[bin]; 2802 size_t bitbase = pn * (PAGESIZE / 16); 2803 2804 auto freebitsdata = pool.freebits.data + pn * PageBits.length; 2805 2806 // the page had dead objects when collecting, these cannot have been resurrected 2807 bool hasDead = false; 2808 static foreach (w; 0 .. PageBits.length) 2809 hasDead = hasDead || (freebitsdata[w] != 0); 2810 assert(hasDead); 2811 2812 // prepend to buckets, but with forward addresses inside the page 2813 assert(bucket[bin] is null); 2814 List** bucketTail = &bucket[bin]; 2815 2816 void* p = pool.baseAddr + pn * PAGESIZE; 2817 const top = PAGESIZE - size + 1; // ensure <size> bytes available even if unaligned 2818 for (size_t u = 0; u < top; u += size) 2819 { 2820 if (!core.bitop.bt(freebitsdata, u / 16)) 2821 continue; 2822 auto elem = cast(List *)(p + u); 2823 elem.pool = &pool.base; 2824 *bucketTail = elem; 2825 bucketTail = &elem.next; 2826 } 2827 *bucketTail = null; 2828 assert(bucket[bin] !is null); 2829 return true; 2830 } 2831 2832 bool recoverNextPage(Bins bin) nothrow 2833 { 2834 SmallObjectPool* pool = recoverPool[bin]; 2835 while (pool) 2836 { 2837 auto pn = pool.recoverPageFirst[bin]; 2838 while (pn < pool.npages) 2839 { 2840 auto next = pool.binPageChain[pn]; 2841 pool.binPageChain[pn] = Pool.PageRecovered; 2842 pool.recoverPageFirst[bin] = next; 2843 if (recoverPage(pool, pn, bin)) 2844 return true; 2845 pn = next; 2846 } 2847 pool = setNextRecoverPool(bin, pool.ptIndex + 1); 2848 } 2849 return false; 2850 } 2851 2852 private SmallObjectPool* setNextRecoverPool(Bins bin, size_t poolIndex) nothrow 2853 { 2854 Pool* pool; 2855 while (poolIndex < this.pooltable.length && 2856 ((pool = this.pooltable[poolIndex]).isLargeObject || 2857 pool.recoverPageFirst[bin] >= pool.npages)) 2858 poolIndex++; 2859 2860 return recoverPool[bin] = poolIndex < this.pooltable.length ? cast(SmallObjectPool*)pool : null; 2861 } 2862 2863 version (COLLECT_FORK) 2864 void disableFork() nothrow 2865 { 2866 markProcPid = 0; 2867 shouldFork = false; 2868 } 2869 2870 version (COLLECT_FORK) 2871 ChildStatus collectFork(bool block) nothrow 2872 { 2873 typeof(return) rc = wait_pid(markProcPid, block); 2874 final switch (rc) 2875 { 2876 case ChildStatus.done: 2877 debug(COLLECT_PRINTF) printf("\t\tmark proc DONE (block=%d)\n", 2878 cast(int) block); 2879 markProcPid = 0; 2880 // process GC marks then sweep 2881 thread_suspendAll(); 2882 thread_processGCMarks(&isMarked); 2883 thread_resumeAll(); 2884 break; 2885 case ChildStatus.running: 2886 debug(COLLECT_PRINTF) printf("\t\tmark proc RUNNING\n"); 2887 if (!block) 2888 break; 2889 // Something went wrong, if block is true, wait() should never 2890 // return RUNNING. 2891 goto case ChildStatus.error; 2892 case ChildStatus.error: 2893 debug(COLLECT_PRINTF) printf("\t\tmark proc ERROR\n"); 2894 // Try to keep going without forking 2895 // and do the marking in this thread 2896 break; 2897 } 2898 return rc; 2899 } 2900 2901 version (COLLECT_FORK) 2902 ChildStatus markFork(bool nostack, bool block, bool doParallel) nothrow 2903 { 2904 // Forking is enabled, so we fork() and start a new concurrent mark phase 2905 // in the child. If the collection should not block, the parent process 2906 // tells the caller no memory could be recycled immediately (if this collection 2907 // was triggered by an allocation, the caller should allocate more memory 2908 // to fulfill the request). 2909 // If the collection should block, the parent will wait for the mark phase 2910 // to finish before returning control to the mutator, 2911 // but other threads are restarted and may run in parallel with the mark phase 2912 // (unless they allocate or use the GC themselves, in which case 2913 // the global GC lock will stop them). 2914 // fork now and sweep later 2915 int child_mark() scope 2916 { 2917 if (doParallel) 2918 markParallel(nostack); 2919 else if (ConservativeGC.isPrecise) 2920 markAll!(markPrecise!true)(nostack); 2921 else 2922 markAll!(markConservative!true)(nostack); 2923 return 0; 2924 } 2925 2926 import core.stdc.stdlib : _Exit; 2927 debug (PRINTF_TO_FILE) 2928 { 2929 fflush(null); // avoid duplicated FILE* output 2930 } 2931 version (OSX) 2932 { 2933 auto pid = __fork(); // avoids calling handlers (from libc source code) 2934 } 2935 else version (linux) 2936 { 2937 // clone() fits better as we don't want to do anything but scanning in the child process. 2938 // no fork-handlers are called, so we can avoid deadlocks due to malloc locks. Probably related: 2939 // https://sourceware.org/bugzilla/show_bug.cgi?id=4737 2940 import core.sys.linux.sched : clone; 2941 import core.sys.posix.signal : SIGCHLD; 2942 const flags = SIGCHLD; // exit signal 2943 scope int delegate() scope dg = &child_mark; 2944 extern(C) static int wrap_delegate(void* arg) 2945 { 2946 auto dg = cast(int delegate() scope*)arg; 2947 return (*dg)(); 2948 } 2949 ubyte[256] stackbuf; // enough stack space for clone() to place some info for the child without stomping the parent stack 2950 auto stack = stackbuf.ptr + (isStackGrowingDown ? stackbuf.length : 0); 2951 auto pid = clone(&wrap_delegate, stack, flags, &dg); 2952 } 2953 else 2954 { 2955 fork_needs_lock = false; 2956 auto pid = fork(); 2957 fork_needs_lock = true; 2958 } 2959 switch (pid) 2960 { 2961 case -1: // fork() failed, retry without forking 2962 return ChildStatus.error; 2963 case 0: // child process (not run with clone) 2964 child_mark(); 2965 _Exit(0); 2966 default: // the parent 2967 thread_resumeAll(); 2968 if (!block) 2969 { 2970 markProcPid = pid; 2971 return ChildStatus.running; 2972 } 2973 ChildStatus r = wait_pid(pid); // block until marking is done 2974 if (r == ChildStatus.error) 2975 { 2976 thread_suspendAll(); 2977 // there was an error 2978 // do the marking in this thread 2979 disableFork(); 2980 if (doParallel) 2981 markParallel(nostack); 2982 else if (ConservativeGC.isPrecise) 2983 markAll!(markPrecise!false)(nostack); 2984 else 2985 markAll!(markConservative!false)(nostack); 2986 } else { 2987 assert(r == ChildStatus.done); 2988 assert(r != ChildStatus.running); 2989 } 2990 } 2991 return ChildStatus.done; // waited for the child 2992 } 2993 2994 /** 2995 * Return number of full pages free'd. 2996 * The collection is done concurrently only if block and isFinal are false. 2997 */ 2998 size_t fullcollect(bool nostack = false, bool block = false, bool isFinal = false) nothrow 2999 { 3000 // It is possible that `fullcollect` will be called from a thread which 3001 // is not yet registered in runtime (because allocating `new Thread` is 3002 // part of `thread_attachThis` implementation). In that case it is 3003 // better not to try actually collecting anything 3004 3005 if (Thread.getThis() is null) 3006 return 0; 3007 3008 MonoTime start, stop, begin; 3009 begin = start = currTime; 3010 3011 debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n"); 3012 version (COLLECT_PARALLEL) 3013 { 3014 bool doParallel = config.parallel > 0 && !config.fork; 3015 if (doParallel && !scanThreadData) 3016 { 3017 if (isFinal) // avoid starting threads for parallel marking 3018 doParallel = false; 3019 else 3020 startScanThreads(); 3021 } 3022 } 3023 else 3024 enum doParallel = false; 3025 3026 //printf("\tpool address range = %p .. %p\n", minAddr, maxAddr); 3027 3028 version (COLLECT_FORK) 3029 alias doFork = shouldFork; 3030 else 3031 enum doFork = false; 3032 3033 if (doFork && collectInProgress) 3034 { 3035 version (COLLECT_FORK) 3036 { 3037 // If there is a mark process running, check if it already finished. 3038 // If that is the case, we move to the sweep phase. 3039 // If it's still running, either we block until the mark phase is 3040 // done (and then sweep to finish the collection), or in case of error 3041 // we redo the mark phase without forking. 3042 ChildStatus rc = collectFork(block); 3043 final switch (rc) 3044 { 3045 case ChildStatus.done: 3046 break; 3047 case ChildStatus.running: 3048 return 0; 3049 case ChildStatus.error: 3050 disableFork(); 3051 goto Lmark; 3052 } 3053 } 3054 } 3055 else 3056 { 3057Lmark: 3058 // lock roots and ranges around suspending threads b/c they're not reentrant safe 3059 rangesLock.lock(); 3060 rootsLock.lock(); 3061 debug(INVARIANT) inCollection = true; 3062 scope (exit) 3063 { 3064 debug(INVARIANT) inCollection = false; 3065 rangesLock.unlock(); 3066 rootsLock.unlock(); 3067 } 3068 thread_suspendAll(); 3069 3070 prepare(); 3071 3072 stop = currTime; 3073 prepTime += (stop - start); 3074 start = stop; 3075 3076 if (doFork && !isFinal && !block) // don't start a new fork during termination 3077 { 3078 version (COLLECT_FORK) 3079 { 3080 auto forkResult = markFork(nostack, block, doParallel); 3081 final switch (forkResult) 3082 { 3083 case ChildStatus.error: 3084 // fork() failed, retry without forking 3085 disableFork(); 3086 goto Lmark; 3087 case ChildStatus.running: 3088 // update profiling informations 3089 stop = currTime; 3090 markTime += (stop - start); 3091 Duration pause = stop - begin; 3092 if (pause > maxPauseTime) 3093 maxPauseTime = pause; 3094 pauseTime += pause; 3095 return 0; 3096 case ChildStatus.done: 3097 break; 3098 } 3099 // if we get here, forking failed and a standard STW collection got issued 3100 // threads were suspended again, restart them 3101 thread_suspendAll(); 3102 } 3103 } 3104 else if (doParallel) 3105 { 3106 version (COLLECT_PARALLEL) 3107 markParallel(nostack); 3108 } 3109 else 3110 { 3111 if (ConservativeGC.isPrecise) 3112 markAll!(markPrecise!false)(nostack); 3113 else 3114 markAll!(markConservative!false)(nostack); 3115 } 3116 3117 thread_processGCMarks(&isMarked); 3118 thread_resumeAll(); 3119 isFinal = false; 3120 } 3121 3122 // If we get here with the forking GC, the child process has finished the marking phase 3123 // or block == true and we are using standard stop the world collection. 3124 // It is time to sweep 3125 3126 stop = currTime; 3127 markTime += (stop - start); 3128 Duration pause = stop - begin; 3129 if (pause > maxPauseTime) 3130 maxPauseTime = pause; 3131 pauseTime += pause; 3132 start = stop; 3133 3134 ConservativeGC._inFinalizer = true; 3135 size_t freedPages = void; 3136 { 3137 scope (failure) ConservativeGC._inFinalizer = false; 3138 freedPages = sweep(); 3139 ConservativeGC._inFinalizer = false; 3140 } 3141 3142 // minimize() should be called only after a call to fullcollect 3143 // terminates with a sweep 3144 if (minimizeAfterNextCollection || lowMem) 3145 { 3146 minimizeAfterNextCollection = false; 3147 minimize(); 3148 } 3149 3150 // init bucket lists 3151 bucket[] = null; 3152 foreach (Bins bin; Bins.B_16 .. Bins.B_NUMSMALL) 3153 setNextRecoverPool(bin, 0); 3154 3155 stop = currTime; 3156 sweepTime += (stop - start); 3157 3158 Duration collectionTime = stop - begin; 3159 if (collectionTime > maxCollectionTime) 3160 maxCollectionTime = collectionTime; 3161 3162 ++numCollections; 3163 3164 updateCollectThresholds(); 3165 if (doFork && isFinal) 3166 return fullcollect(true, true, false); 3167 return freedPages; 3168 } 3169 3170 /** 3171 * Returns true if the addr lies within a marked block. 3172 * 3173 * Warning! This should only be called while the world is stopped inside 3174 * the fullcollect function after all live objects have been marked, but before sweeping. 3175 */ 3176 int isMarked(void *addr) scope nothrow 3177 { 3178 // first, we find the Pool this block is in, then check to see if the 3179 // mark bit is clear. 3180 auto pool = findPool(addr); 3181 if (pool) 3182 { 3183 auto offset = cast(size_t)(addr - pool.baseAddr); 3184 auto pn = offset / PAGESIZE; 3185 auto bins = cast(Bins)pool.pagetable[pn]; 3186 size_t biti = void; 3187 if (bins < Bins.B_PAGE) 3188 { 3189 biti = baseOffset(offset, bins) >> pool.ShiftBy.Small; 3190 // doesn't need to check freebits because no pointer must exist 3191 // to a block that was free before starting the collection 3192 } 3193 else if (bins == Bins.B_PAGE) 3194 { 3195 biti = pn * (PAGESIZE >> pool.ShiftBy.Large); 3196 } 3197 else if (bins == Bins.B_PAGEPLUS) 3198 { 3199 pn -= pool.bPageOffsets[pn]; 3200 biti = pn * (PAGESIZE >> pool.ShiftBy.Large); 3201 } 3202 else // bins == Bins.B_FREE 3203 { 3204 assert(bins == Bins.B_FREE); 3205 return IsMarked.no; 3206 } 3207 return pool.mark.test(biti) ? IsMarked.yes : IsMarked.no; 3208 } 3209 return IsMarked.unknown; 3210 } 3211 3212 version (Posix) 3213 { 3214 // A fork might happen while GC code is running in a different thread. 3215 // Because that would leave the GC in an inconsistent state, 3216 // make sure no GC code is running by acquiring the lock here, 3217 // before a fork. 3218 // This must not happen if fork is called from the GC with the lock already held 3219 3220 __gshared bool fork_needs_lock = true; // racing condition with cocurrent calls of fork? 3221 3222 3223 extern(C) static void _d_gcx_atfork_prepare() 3224 { 3225 if (instance && fork_needs_lock) 3226 ConservativeGC.lockNR(); 3227 } 3228 3229 extern(C) static void _d_gcx_atfork_parent() 3230 { 3231 if (instance && fork_needs_lock) 3232 ConservativeGC.gcLock.unlock(); 3233 } 3234 3235 extern(C) static void _d_gcx_atfork_child() 3236 { 3237 if (instance && fork_needs_lock) 3238 { 3239 ConservativeGC.gcLock.unlock(); 3240 3241 // make sure the threads and event handles are reinitialized in a fork 3242 version (COLLECT_PARALLEL) 3243 { 3244 if (Gcx.instance.scanThreadData) 3245 { 3246 cstdlib.free(Gcx.instance.scanThreadData); 3247 Gcx.instance.numScanThreads = 0; 3248 Gcx.instance.scanThreadData = null; 3249 Gcx.instance.busyThreads = 0; 3250 3251 memset(&Gcx.instance.evStart, 0, Gcx.instance.evStart.sizeof); 3252 memset(&Gcx.instance.evDone, 0, Gcx.instance.evDone.sizeof); 3253 } 3254 } 3255 } 3256 } 3257 } 3258 3259 /* ============================ Parallel scanning =============================== */ 3260 version (COLLECT_PARALLEL): 3261 3262 import core.atomic; 3263 import core.cpuid; 3264 import core.sync.event; 3265 3266 private: // disable invariants for background threads 3267 3268 static struct ScanThreadData 3269 { 3270 ThreadID tid; 3271 } 3272 uint numScanThreads; 3273 ScanThreadData* scanThreadData; 3274 3275 Event evStart; 3276 Event evDone; 3277 3278 shared uint busyThreads; 3279 shared uint stoppedThreads; 3280 bool stopGC; 3281 3282 void markParallel(bool nostack) nothrow 3283 { 3284 toscanRoots.clear(); 3285 collectAllRoots(nostack); 3286 if (toscanRoots.empty) 3287 return; 3288 3289 void** pbot = toscanRoots._p; 3290 void** ptop = toscanRoots._p + toscanRoots._length; 3291 3292 debug(PARALLEL_PRINTF) printf("markParallel\n"); 3293 3294 size_t pointersPerThread = toscanRoots._length / (numScanThreads + 1); 3295 if (pointersPerThread > 0) 3296 { 3297 void pushRanges(bool precise)() 3298 { 3299 alias toscan = scanStack!precise; 3300 toscan.stackLock.lock(); 3301 3302 for (int idx = 0; idx < numScanThreads; idx++) 3303 { 3304 toscan.push(ScanRange!precise(pbot, pbot + pointersPerThread)); 3305 pbot += pointersPerThread; 3306 } 3307 toscan.stackLock.unlock(); 3308 } 3309 if (ConservativeGC.isPrecise) 3310 pushRanges!true(); 3311 else 3312 pushRanges!false(); 3313 } 3314 assert(pbot < ptop); 3315 3316 busyThreads.atomicOp!"+="(1); // main thread is busy 3317 3318 evStart.set(); 3319 3320 debug(PARALLEL_PRINTF) printf("mark %lld roots\n", cast(ulong)(ptop - pbot)); 3321 3322 if (ConservativeGC.isPrecise) 3323 mark!(true, true, true)(ScanRange!true(pbot, ptop, null)); 3324 else 3325 mark!(false, true, true)(ScanRange!false(pbot, ptop)); 3326 3327 busyThreads.atomicOp!"-="(1); 3328 3329 debug(PARALLEL_PRINTF) printf("waitForScanDone\n"); 3330 pullFromScanStack(); 3331 debug(PARALLEL_PRINTF) printf("waitForScanDone done\n"); 3332 } 3333 3334 int maxParallelThreads() nothrow 3335 { 3336 auto threads = threadsPerCPU(); 3337 3338 if (threads == 0) 3339 { 3340 // If the GC is called by module ctors no explicit 3341 // import dependency on the GC is generated. So the 3342 // GC module is not correctly inserted into the module 3343 // initialization chain. As it relies on core.cpuid being 3344 // initialized, force this here. 3345 try 3346 { 3347 foreach (m; ModuleInfo) 3348 if (m.name == "core.cpuid") 3349 if (auto ctor = m.ctor()) 3350 { 3351 ctor(); 3352 threads = threadsPerCPU(); 3353 break; 3354 } 3355 } 3356 catch (Exception) 3357 { 3358 assert(false, "unexpected exception iterating ModuleInfo"); 3359 } 3360 } 3361 return threads; 3362 } 3363 3364 3365 void startScanThreads() nothrow 3366 { 3367 auto threads = maxParallelThreads(); 3368 debug(PARALLEL_PRINTF) printf("startScanThreads: %d threads per CPU\n", threads); 3369 if (threads <= 1) 3370 return; // either core.cpuid not initialized or single core 3371 3372 numScanThreads = threads >= config.parallel ? config.parallel : threads - 1; 3373 3374 scanThreadData = cast(ScanThreadData*) cstdlib.calloc(numScanThreads, ScanThreadData.sizeof); 3375 if (!scanThreadData) 3376 onOutOfMemoryErrorNoGC(); 3377 3378 evStart.initialize(false, false); 3379 evDone.initialize(false, false); 3380 3381 version (Posix) 3382 { 3383 import core.sys.posix.signal; 3384 // block all signals, scanBackground inherits this mask. 3385 // see https://issues.dlang.org/show_bug.cgi?id=20256 3386 sigset_t new_mask, old_mask; 3387 sigfillset(&new_mask); 3388 auto sigmask_rc = pthread_sigmask(SIG_BLOCK, &new_mask, &old_mask); 3389 assert(sigmask_rc == 0, "failed to set up GC scan thread sigmask"); 3390 } 3391 3392 for (int idx = 0; idx < numScanThreads; idx++) 3393 scanThreadData[idx].tid = createLowLevelThread(&scanBackground, 0x4000, &stopScanThreads); 3394 3395 version (Posix) 3396 { 3397 sigmask_rc = pthread_sigmask(SIG_SETMASK, &old_mask, null); 3398 assert(sigmask_rc == 0, "failed to set up GC scan thread sigmask"); 3399 } 3400 } 3401 3402 void stopScanThreads() nothrow 3403 { 3404 if (!numScanThreads) 3405 return; 3406 3407 debug(PARALLEL_PRINTF) printf("stopScanThreads\n"); 3408 int startedThreads = 0; 3409 for (int idx = 0; idx < numScanThreads; idx++) 3410 if (scanThreadData[idx].tid != scanThreadData[idx].tid.init) 3411 startedThreads++; 3412 3413 version (Windows) 3414 alias allThreadsDead = thread_DLLProcessDetaching; 3415 else 3416 enum allThreadsDead = false; 3417 stopGC = true; 3418 while (atomicLoad(stoppedThreads) < startedThreads && !allThreadsDead) 3419 { 3420 evStart.set(); 3421 evDone.wait(dur!"msecs"(1)); 3422 } 3423 3424 for (int idx = 0; idx < numScanThreads; idx++) 3425 { 3426 if (scanThreadData[idx].tid != scanThreadData[idx].tid.init) 3427 { 3428 joinLowLevelThread(scanThreadData[idx].tid); 3429 scanThreadData[idx].tid = scanThreadData[idx].tid.init; 3430 } 3431 } 3432 3433 evDone.terminate(); 3434 evStart.terminate(); 3435 3436 cstdlib.free(scanThreadData); 3437 // scanThreadData = null; // keep non-null to not start again after shutdown 3438 numScanThreads = 0; 3439 3440 debug(PARALLEL_PRINTF) printf("stopScanThreads done\n"); 3441 } 3442 3443 void scanBackground() nothrow 3444 { 3445 while (!stopGC) 3446 { 3447 evStart.wait(); 3448 pullFromScanStack(); 3449 evDone.set(); 3450 } 3451 stoppedThreads.atomicOp!"+="(1); 3452 } 3453 3454 void pullFromScanStack() nothrow 3455 { 3456 if (ConservativeGC.isPrecise) 3457 pullFromScanStackImpl!true(); 3458 else 3459 pullFromScanStackImpl!false(); 3460 } 3461 3462 void pullFromScanStackImpl(bool precise)() nothrow 3463 { 3464 if (atomicLoad(busyThreads) == 0) 3465 return; 3466 3467 debug(PARALLEL_PRINTF) 3468 pthread_t threadId = pthread_self(); 3469 debug(PARALLEL_PRINTF) printf("scanBackground thread %d start\n", threadId); 3470 3471 ScanRange!precise rng; 3472 alias toscan = scanStack!precise; 3473 3474 while (atomicLoad(busyThreads) > 0) 3475 { 3476 if (toscan.empty) 3477 { 3478 evDone.wait(dur!"msecs"(1)); 3479 continue; 3480 } 3481 3482 busyThreads.atomicOp!"+="(1); 3483 if (toscan.popLocked(rng)) 3484 { 3485 debug(PARALLEL_PRINTF) printf("scanBackground thread %d scanning range [%p,%lld] from stack\n", threadId, 3486 rng.pbot, cast(long) (rng.ptop - rng.pbot)); 3487 mark!(precise, true, true)(rng); 3488 } 3489 busyThreads.atomicOp!"-="(1); 3490 } 3491 debug(PARALLEL_PRINTF) printf("scanBackground thread %d done\n", threadId); 3492 } 3493} 3494 3495/* ============================ Pool =============================== */ 3496 3497struct Pool 3498{ 3499 void* baseAddr; 3500 void* topAddr; 3501 size_t ptIndex; // index in pool table 3502 GCBits mark; // entries already scanned, or should not be scanned 3503 GCBits freebits; // entries that are on the free list (all bits set but for allocated objects at their base offset) 3504 GCBits finals; // entries that need finalizer run on them 3505 GCBits structFinals;// struct entries that need a finalzier run on them 3506 GCBits noscan; // entries that should not be scanned 3507 GCBits appendable; // entries that are appendable 3508 GCBits nointerior; // interior pointers should be ignored. 3509 // Only implemented for large object pools. 3510 GCBits is_pointer; // precise GC only: per-word, not per-block like the rest of them (SmallObjectPool only) 3511 size_t npages; 3512 size_t freepages; // The number of pages not in use. 3513 Bins* pagetable; 3514 3515 bool isLargeObject; 3516 3517 enum ShiftBy 3518 { 3519 Small = 4, 3520 Large = 12 3521 } 3522 ShiftBy shiftBy; // shift count for the divisor used for determining bit indices. 3523 3524 // This tracks how far back we have to go to find the nearest B_PAGE at 3525 // a smaller address than a B_PAGEPLUS. To save space, we use a uint. 3526 // This limits individual allocations to 16 terabytes, assuming a 4k 3527 // pagesize. (LargeObjectPool only) 3528 // For B_PAGE and B_FREE, this specifies the number of pages in this block. 3529 // As an optimization, a contiguous range of free pages tracks this information 3530 // only for the first and the last page. 3531 uint* bPageOffsets; 3532 3533 // The small object pool uses the same array to keep a chain of 3534 // - pages with the same bin size that are still to be recovered 3535 // - free pages (searchStart is first free page) 3536 // other pages are marked by value PageRecovered 3537 alias binPageChain = bPageOffsets; 3538 3539 enum PageRecovered = uint.max; 3540 3541 // first of chain of pages to recover (SmallObjectPool only) 3542 uint[Bins.B_NUMSMALL] recoverPageFirst; 3543 3544 // precise GC: TypeInfo.rtInfo for allocation (LargeObjectPool only) 3545 immutable(size_t)** rtinfo; 3546 3547 // This variable tracks a conservative estimate of where the first free 3548 // page in this pool is, so that if a lot of pages towards the beginning 3549 // are occupied, we can bypass them in O(1). 3550 size_t searchStart; 3551 size_t largestFree; // upper limit for largest free chunk in large object pool 3552 3553 void initialize(size_t npages, bool isLargeObject) nothrow 3554 { 3555 assert(npages >= 256); 3556 3557 this.isLargeObject = isLargeObject; 3558 size_t poolsize; 3559 3560 shiftBy = isLargeObject ? ShiftBy.Large : ShiftBy.Small; 3561 3562 //debug(PRINTF) printf("Pool::Pool(%u)\n", npages); 3563 poolsize = npages * PAGESIZE; 3564 baseAddr = cast(byte *)os_mem_map(poolsize); 3565 3566 // Some of the code depends on page alignment of memory pools 3567 assert((cast(size_t)baseAddr & (PAGESIZE - 1)) == 0); 3568 3569 if (!baseAddr) 3570 { 3571 //debug(PRINTF) printf("GC fail: poolsize = x%zx, errno = %d\n", poolsize, errno); 3572 //debug(PRINTF) printf("message = '%s'\n", sys_errlist[errno]); 3573 3574 npages = 0; 3575 poolsize = 0; 3576 } 3577 //assert(baseAddr); 3578 topAddr = baseAddr + poolsize; 3579 auto nbits = cast(size_t)poolsize >> shiftBy; 3580 3581 version (COLLECT_FORK) 3582 mark.alloc(nbits, config.fork); 3583 else 3584 mark.alloc(nbits); 3585 if (ConservativeGC.isPrecise) 3586 { 3587 if (isLargeObject) 3588 { 3589 rtinfo = cast(immutable(size_t)**)cstdlib.malloc(npages * (size_t*).sizeof); 3590 if (!rtinfo) 3591 onOutOfMemoryErrorNoGC(); 3592 memset(rtinfo, 0, npages * (size_t*).sizeof); 3593 } 3594 else 3595 { 3596 is_pointer.alloc(cast(size_t)poolsize/(void*).sizeof); 3597 is_pointer.clrRange(0, is_pointer.nbits); 3598 } 3599 } 3600 3601 // pagetable already keeps track of what's free for the large object 3602 // pool. 3603 if (!isLargeObject) 3604 { 3605 freebits.alloc(nbits); 3606 freebits.setRange(0, nbits); 3607 } 3608 3609 noscan.alloc(nbits); 3610 appendable.alloc(nbits); 3611 3612 pagetable = cast(Bins*)cstdlib.malloc(npages * Bins.sizeof); 3613 if (!pagetable) 3614 onOutOfMemoryErrorNoGC(); 3615 3616 if (npages > 0) 3617 { 3618 bPageOffsets = cast(uint*)cstdlib.malloc(npages * uint.sizeof); 3619 if (!bPageOffsets) 3620 onOutOfMemoryErrorNoGC(); 3621 3622 if (isLargeObject) 3623 { 3624 bPageOffsets[0] = cast(uint)npages; 3625 bPageOffsets[npages-1] = cast(uint)npages; 3626 } 3627 else 3628 { 3629 // all pages free 3630 foreach (n; 0..npages) 3631 binPageChain[n] = cast(uint)(n + 1); 3632 recoverPageFirst[] = cast(uint)npages; 3633 } 3634 } 3635 3636 memset(pagetable, Bins.B_FREE, npages); 3637 3638 this.npages = npages; 3639 this.freepages = npages; 3640 this.searchStart = 0; 3641 this.largestFree = npages; 3642 } 3643 3644 3645 void Dtor() nothrow 3646 { 3647 if (baseAddr) 3648 { 3649 int result; 3650 3651 if (npages) 3652 { 3653 result = os_mem_unmap(baseAddr, npages * PAGESIZE); 3654 assert(result == 0); 3655 npages = 0; 3656 } 3657 3658 baseAddr = null; 3659 topAddr = null; 3660 } 3661 if (pagetable) 3662 { 3663 cstdlib.free(pagetable); 3664 pagetable = null; 3665 } 3666 3667 if (bPageOffsets) 3668 { 3669 cstdlib.free(bPageOffsets); 3670 bPageOffsets = null; 3671 } 3672 3673 mark.Dtor(config.fork); 3674 if (ConservativeGC.isPrecise) 3675 { 3676 if (isLargeObject) 3677 cstdlib.free(rtinfo); 3678 else 3679 is_pointer.Dtor(); 3680 } 3681 if (isLargeObject) 3682 { 3683 nointerior.Dtor(); 3684 } 3685 else 3686 { 3687 freebits.Dtor(); 3688 } 3689 finals.Dtor(); 3690 structFinals.Dtor(); 3691 noscan.Dtor(); 3692 appendable.Dtor(); 3693 } 3694 3695 /** 3696 * 3697 */ 3698 uint getBits(size_t biti) nothrow 3699 { 3700 uint bits; 3701 3702 if (finals.nbits && finals.test(biti)) 3703 bits |= BlkAttr.FINALIZE; 3704 if (structFinals.nbits && structFinals.test(biti)) 3705 bits |= BlkAttr.STRUCTFINAL; 3706 if (noscan.test(biti)) 3707 bits |= BlkAttr.NO_SCAN; 3708 if (nointerior.nbits && nointerior.test(biti)) 3709 bits |= BlkAttr.NO_INTERIOR; 3710 if (appendable.test(biti)) 3711 bits |= BlkAttr.APPENDABLE; 3712 return bits; 3713 } 3714 3715 /** 3716 * 3717 */ 3718 void clrBits(size_t biti, uint mask) nothrow @nogc 3719 { 3720 immutable dataIndex = biti >> GCBits.BITS_SHIFT; 3721 immutable bitOffset = biti & GCBits.BITS_MASK; 3722 immutable keep = ~(GCBits.BITS_1 << bitOffset); 3723 3724 if (mask & BlkAttr.FINALIZE && finals.nbits) 3725 finals.data[dataIndex] &= keep; 3726 3727 if (structFinals.nbits && (mask & BlkAttr.STRUCTFINAL)) 3728 structFinals.data[dataIndex] &= keep; 3729 3730 if (mask & BlkAttr.NO_SCAN) 3731 noscan.data[dataIndex] &= keep; 3732 if (mask & BlkAttr.APPENDABLE) 3733 appendable.data[dataIndex] &= keep; 3734 if (nointerior.nbits && (mask & BlkAttr.NO_INTERIOR)) 3735 nointerior.data[dataIndex] &= keep; 3736 } 3737 3738 /** 3739 * 3740 */ 3741 void setBits(size_t biti, uint mask) nothrow 3742 { 3743 // Calculate the mask and bit offset once and then use it to 3744 // set all of the bits we need to set. 3745 immutable dataIndex = biti >> GCBits.BITS_SHIFT; 3746 immutable bitOffset = biti & GCBits.BITS_MASK; 3747 immutable orWith = GCBits.BITS_1 << bitOffset; 3748 3749 if (mask & BlkAttr.STRUCTFINAL) 3750 { 3751 if (!structFinals.nbits) 3752 structFinals.alloc(mark.nbits); 3753 structFinals.data[dataIndex] |= orWith; 3754 } 3755 3756 if (mask & BlkAttr.FINALIZE) 3757 { 3758 if (!finals.nbits) 3759 finals.alloc(mark.nbits); 3760 finals.data[dataIndex] |= orWith; 3761 } 3762 3763 if (mask & BlkAttr.NO_SCAN) 3764 { 3765 noscan.data[dataIndex] |= orWith; 3766 } 3767// if (mask & BlkAttr.NO_MOVE) 3768// { 3769// if (!nomove.nbits) 3770// nomove.alloc(mark.nbits); 3771// nomove.data[dataIndex] |= orWith; 3772// } 3773 if (mask & BlkAttr.APPENDABLE) 3774 { 3775 appendable.data[dataIndex] |= orWith; 3776 } 3777 3778 if (isLargeObject && (mask & BlkAttr.NO_INTERIOR)) 3779 { 3780 if (!nointerior.nbits) 3781 nointerior.alloc(mark.nbits); 3782 nointerior.data[dataIndex] |= orWith; 3783 } 3784 } 3785 3786 void freePageBits(size_t pagenum, const scope ref PageBits toFree) nothrow 3787 { 3788 assert(!isLargeObject); 3789 assert(!nointerior.nbits); // only for large objects 3790 3791 import core.internal.traits : staticIota; 3792 immutable beg = pagenum * (PAGESIZE / 16 / GCBits.BITS_PER_WORD); 3793 foreach (i; staticIota!(0, PageBits.length)) 3794 { 3795 immutable w = toFree[i]; 3796 if (!w) continue; 3797 3798 immutable wi = beg + i; 3799 freebits.data[wi] |= w; 3800 noscan.data[wi] &= ~w; 3801 appendable.data[wi] &= ~w; 3802 } 3803 3804 if (finals.nbits) 3805 { 3806 foreach (i; staticIota!(0, PageBits.length)) 3807 if (toFree[i]) 3808 finals.data[beg + i] &= ~toFree[i]; 3809 } 3810 3811 if (structFinals.nbits) 3812 { 3813 foreach (i; staticIota!(0, PageBits.length)) 3814 if (toFree[i]) 3815 structFinals.data[beg + i] &= ~toFree[i]; 3816 } 3817 } 3818 3819 void freeAllPageBits(size_t pagenum) nothrow 3820 { 3821 assert(!isLargeObject); 3822 assert(!nointerior.nbits); // only for large objects 3823 3824 immutable beg = pagenum * PageBits.length; 3825 static foreach (i; 0 .. PageBits.length) 3826 {{ 3827 immutable w = beg + i; 3828 freebits.data[w] = ~0; 3829 noscan.data[w] = 0; 3830 appendable.data[w] = 0; 3831 if (finals.data) 3832 finals.data[w] = 0; 3833 if (structFinals.data) 3834 structFinals.data[w] = 0; 3835 }} 3836 } 3837 3838 /** 3839 * Given a pointer p in the p, return the pagenum. 3840 */ 3841 size_t pagenumOf(void *p) const nothrow @nogc 3842 in 3843 { 3844 assert(p >= baseAddr); 3845 assert(p < topAddr); 3846 } 3847 do 3848 { 3849 return cast(size_t)(p - baseAddr) / PAGESIZE; 3850 } 3851 3852 public 3853 @property bool isFree() const scope @safe pure nothrow @nogc 3854 { 3855 return npages == freepages; 3856 } 3857 3858 /** 3859 * Return number of pages necessary for an allocation of the given size 3860 * 3861 * returns size_t.max if more than uint.max pages are requested 3862 * (return type is still size_t to avoid truncation when being used 3863 * in calculations, e.g. npages * PAGESIZE) 3864 */ 3865 static size_t numPages(size_t size) nothrow @nogc 3866 { 3867 version (D_LP64) 3868 { 3869 if (size > PAGESIZE * cast(size_t)uint.max) 3870 return size_t.max; 3871 } 3872 else 3873 { 3874 if (size > size_t.max - PAGESIZE) 3875 return size_t.max; 3876 } 3877 return (size + PAGESIZE - 1) / PAGESIZE; 3878 } 3879 3880 void* findBase(void* p) nothrow @nogc 3881 { 3882 size_t offset = cast(size_t)(p - baseAddr); 3883 size_t pn = offset / PAGESIZE; 3884 Bins bin = pagetable[pn]; 3885 3886 // Adjust bit to be at start of allocated memory block 3887 if (bin < Bins.B_NUMSMALL) 3888 { 3889 auto baseOff = baseOffset(offset, bin); 3890 const biti = baseOff >> Pool.ShiftBy.Small; 3891 if (freebits.test (biti)) 3892 return null; 3893 return baseAddr + baseOff; 3894 } 3895 if (bin == Bins.B_PAGE) 3896 { 3897 return baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); 3898 } 3899 if (bin == Bins.B_PAGEPLUS) 3900 { 3901 size_t pageOffset = bPageOffsets[pn]; 3902 offset -= pageOffset * PAGESIZE; 3903 pn -= pageOffset; 3904 3905 return baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); 3906 } 3907 // we are in a B_FREE page 3908 assert(bin == Bins.B_FREE); 3909 return null; 3910 } 3911 3912 size_t slGetSize(void* p) nothrow @nogc 3913 { 3914 if (isLargeObject) 3915 return (cast(LargeObjectPool*)&this).getPages(p) * PAGESIZE; 3916 else 3917 return (cast(SmallObjectPool*)&this).getSize(p); 3918 } 3919 3920 BlkInfo slGetInfo(void* p) nothrow 3921 { 3922 if (isLargeObject) 3923 return (cast(LargeObjectPool*)&this).getInfo(p); 3924 else 3925 return (cast(SmallObjectPool*)&this).getInfo(p); 3926 } 3927 3928 3929 void Invariant() const {} 3930 3931 debug(INVARIANT) 3932 invariant() 3933 { 3934 if (baseAddr) 3935 { 3936 //if (baseAddr + npages * PAGESIZE != topAddr) 3937 //printf("baseAddr = %p, npages = %d, topAddr = %p\n", baseAddr, npages, topAddr); 3938 assert(baseAddr + npages * PAGESIZE == topAddr); 3939 } 3940 3941 if (pagetable !is null) 3942 { 3943 for (size_t i = 0; i < npages; i++) 3944 { 3945 Bins bin = pagetable[i]; 3946 assert(bin < Bins.B_MAX); 3947 } 3948 } 3949 } 3950 3951 void setPointerBitmapSmall(void* p, size_t s, size_t allocSize, uint attr, const TypeInfo ti) nothrow 3952 { 3953 if (!(attr & BlkAttr.NO_SCAN)) 3954 setPointerBitmap(p, s, allocSize, ti, attr); 3955 } 3956 3957 pragma(inline,false) 3958 void setPointerBitmap(void* p, size_t s, size_t allocSize, const TypeInfo ti, uint attr) nothrow 3959 { 3960 size_t offset = p - baseAddr; 3961 //debug(PRINTF) printGCBits(&pool.is_pointer); 3962 3963 debug(PRINTF) 3964 printf("Setting a pointer bitmap for %s at %p + %llu\n", debugTypeName(ti).ptr, p, cast(ulong)s); 3965 3966 if (ti) 3967 { 3968 if (attr & BlkAttr.APPENDABLE) 3969 { 3970 // an array of classes is in fact an array of pointers 3971 if (typeid(ti) is typeid(TypeInfo_Class)) 3972 goto L_conservative; 3973 s = allocSize; 3974 } 3975 3976 auto rtInfo = cast(const(size_t)*)ti.rtInfo(); 3977 3978 if (rtInfo is rtinfoNoPointers) 3979 { 3980 debug(PRINTF) printf("\tCompiler generated rtInfo: no pointers\n"); 3981 is_pointer.clrRange(offset/(void*).sizeof, s/(void*).sizeof); 3982 } 3983 else if (rtInfo is rtinfoHasPointers) 3984 { 3985 debug(PRINTF) printf("\tCompiler generated rtInfo: has pointers\n"); 3986 is_pointer.setRange(offset/(void*).sizeof, s/(void*).sizeof); 3987 } 3988 else 3989 { 3990 const(size_t)* bitmap = cast (size_t*) rtInfo; 3991 //first element of rtInfo is the size of the object the bitmap encodes 3992 size_t element_size = * bitmap; 3993 bitmap++; 3994 size_t tocopy; 3995 if (attr & BlkAttr.APPENDABLE) 3996 { 3997 tocopy = s/(void*).sizeof; 3998 is_pointer.copyRangeRepeating(offset/(void*).sizeof, tocopy, bitmap, element_size/(void*).sizeof); 3999 } 4000 else 4001 { 4002 tocopy = (s < element_size ? s : element_size)/(void*).sizeof; 4003 is_pointer.copyRange(offset/(void*).sizeof, tocopy, bitmap); 4004 } 4005 4006 debug(PRINTF) printf("\tSetting bitmap for new object (%s)\n\t\tat %p\t\tcopying from %p + %llu: ", 4007 debugTypeName(ti).ptr, p, bitmap, cast(ulong)element_size); 4008 debug(PRINTF) 4009 for (size_t i = 0; i < element_size/((void*).sizeof); i++) 4010 printf("%d", (bitmap[i/(8*size_t.sizeof)] >> (i%(8*size_t.sizeof))) & 1); 4011 debug(PRINTF) printf("\n"); 4012 4013 if (tocopy * (void*).sizeof < s) // better safe than sorry: if allocated more, assume pointers inside 4014 { 4015 debug(PRINTF) printf(" Appending %d pointer bits\n", s/(void*).sizeof - tocopy); 4016 is_pointer.setRange(offset/(void*).sizeof + tocopy, s/(void*).sizeof - tocopy); 4017 } 4018 } 4019 4020 if (s < allocSize) 4021 { 4022 offset = (offset + s + (void*).sizeof - 1) & ~((void*).sizeof - 1); 4023 is_pointer.clrRange(offset/(void*).sizeof, (allocSize - s)/(void*).sizeof); 4024 } 4025 } 4026 else 4027 { 4028 L_conservative: 4029 // limit pointers to actual size of allocation? might fail for arrays that append 4030 // without notifying the GC 4031 s = allocSize; 4032 4033 debug(PRINTF) printf("Allocating a block without TypeInfo\n"); 4034 is_pointer.setRange(offset/(void*).sizeof, s/(void*).sizeof); 4035 } 4036 //debug(PRINTF) printGCBits(&pool.is_pointer); 4037 } 4038} 4039 4040struct LargeObjectPool 4041{ 4042 Pool base; 4043 alias base this; 4044 4045 debug(INVARIANT) 4046 void Invariant() 4047 { 4048 //base.Invariant(); 4049 for (size_t n = 0; n < npages; ) 4050 { 4051 uint np = bPageOffsets[n]; 4052 assert(np > 0 && np <= npages - n); 4053 4054 if (pagetable[n] == Bins.B_PAGE) 4055 { 4056 for (uint p = 1; p < np; p++) 4057 { 4058 assert(pagetable[n + p] == Bins.B_PAGEPLUS); 4059 assert(bPageOffsets[n + p] == p); 4060 } 4061 } 4062 else if (pagetable[n] == Bins.B_FREE) 4063 { 4064 for (uint p = 1; p < np; p++) 4065 { 4066 assert(pagetable[n + p] == Bins.B_FREE); 4067 } 4068 assert(bPageOffsets[n + np - 1] == np); 4069 } 4070 else 4071 assert(false); 4072 n += np; 4073 } 4074 } 4075 4076 /** 4077 * Allocate n pages from Pool. 4078 * Returns OPFAIL on failure. 4079 */ 4080 size_t allocPages(size_t n) nothrow 4081 { 4082 if (largestFree < n || searchStart + n > npages) 4083 return OPFAIL; 4084 4085 //debug(PRINTF) printf("Pool::allocPages(n = %d)\n", n); 4086 size_t largest = 0; 4087 if (pagetable[searchStart] == Bins.B_PAGEPLUS) 4088 { 4089 searchStart -= bPageOffsets[searchStart]; // jump to B_PAGE 4090 searchStart += bPageOffsets[searchStart]; 4091 } 4092 while (searchStart < npages && pagetable[searchStart] == Bins.B_PAGE) 4093 searchStart += bPageOffsets[searchStart]; 4094 4095 for (size_t i = searchStart; i < npages; ) 4096 { 4097 assert(pagetable[i] == Bins.B_FREE); 4098 4099 auto p = bPageOffsets[i]; 4100 if (p > n) 4101 { 4102 setFreePageOffsets(i + n, p - n); 4103 goto L_found; 4104 } 4105 if (p == n) 4106 { 4107 L_found: 4108 pagetable[i] = Bins.B_PAGE; 4109 bPageOffsets[i] = cast(uint) n; 4110 if (n > 1) 4111 { 4112 memset(&pagetable[i + 1], Bins.B_PAGEPLUS, n - 1); 4113 for (auto offset = 1; offset < n; offset++) 4114 bPageOffsets[i + offset] = cast(uint) offset; 4115 } 4116 freepages -= n; 4117 return i; 4118 } 4119 if (p > largest) 4120 largest = p; 4121 4122 i += p; 4123 while (i < npages && pagetable[i] == Bins.B_PAGE) 4124 { 4125 // we have the size information, so we skip a whole bunch of pages. 4126 i += bPageOffsets[i]; 4127 } 4128 } 4129 4130 // not enough free pages found, remember largest free chunk 4131 largestFree = largest; 4132 return OPFAIL; 4133 } 4134 4135 /** 4136 * Free npages pages starting with pagenum. 4137 */ 4138 void freePages(size_t pagenum, size_t npages) nothrow @nogc 4139 { 4140 //memset(&pagetable[pagenum], B_FREE, npages); 4141 if (pagenum < searchStart) 4142 searchStart = pagenum; 4143 4144 for (size_t i = pagenum; i < npages + pagenum; i++) 4145 { 4146 assert(pagetable[i] < Bins.B_FREE); 4147 pagetable[i] = Bins.B_FREE; 4148 } 4149 freepages += npages; 4150 largestFree = freepages; // invalidate 4151 } 4152 4153 /** 4154 * Set the first and the last entry of a B_FREE block to the size 4155 */ 4156 void setFreePageOffsets(size_t page, size_t num) nothrow @nogc 4157 { 4158 assert(pagetable[page] == Bins.B_FREE); 4159 assert(pagetable[page + num - 1] == Bins.B_FREE); 4160 bPageOffsets[page] = cast(uint)num; 4161 if (num > 1) 4162 bPageOffsets[page + num - 1] = cast(uint)num; 4163 } 4164 4165 void mergeFreePageOffsets(bool bwd, bool fwd)(size_t page, size_t num) nothrow @nogc 4166 { 4167 static if (bwd) 4168 { 4169 if (page > 0 && pagetable[page - 1] == Bins.B_FREE) 4170 { 4171 auto sz = bPageOffsets[page - 1]; 4172 page -= sz; 4173 num += sz; 4174 } 4175 } 4176 static if (fwd) 4177 { 4178 if (page + num < npages && pagetable[page + num] == Bins.B_FREE) 4179 num += bPageOffsets[page + num]; 4180 } 4181 setFreePageOffsets(page, num); 4182 } 4183 4184 /** 4185 * Get pages of allocation at pointer p in pool. 4186 */ 4187 size_t getPages(void *p) const nothrow @nogc 4188 in 4189 { 4190 assert(p >= baseAddr); 4191 assert(p < topAddr); 4192 } 4193 do 4194 { 4195 if (cast(size_t)p & (PAGESIZE - 1)) // check for interior pointer 4196 return 0; 4197 size_t pagenum = pagenumOf(p); 4198 Bins bin = pagetable[pagenum]; 4199 if (bin != Bins.B_PAGE) 4200 return 0; 4201 return bPageOffsets[pagenum]; 4202 } 4203 4204 /** 4205 * Get size of allocation at page pn in pool. 4206 */ 4207 size_t getSize(size_t pn) const nothrow @nogc 4208 { 4209 assert(pagetable[pn] == Bins.B_PAGE); 4210 return cast(size_t) bPageOffsets[pn] * PAGESIZE; 4211 } 4212 4213 /** 4214 * 4215 */ 4216 BlkInfo getInfo(void* p) nothrow 4217 { 4218 BlkInfo info; 4219 4220 size_t offset = cast(size_t)(p - baseAddr); 4221 size_t pn = offset / PAGESIZE; 4222 Bins bin = pagetable[pn]; 4223 4224 if (bin == Bins.B_PAGEPLUS) 4225 pn -= bPageOffsets[pn]; 4226 else if (bin != Bins.B_PAGE) 4227 return info; // no info for free pages 4228 4229 info.base = baseAddr + pn * PAGESIZE; 4230 info.size = getSize(pn); 4231 info.attr = getBits(pn); 4232 return info; 4233 } 4234 4235 void runFinalizers(const scope void[] segment) nothrow 4236 { 4237 foreach (pn; 0 .. npages) 4238 { 4239 Bins bin = pagetable[pn]; 4240 if (bin > Bins.B_PAGE) 4241 continue; 4242 size_t biti = pn; 4243 4244 if (!finals.test(biti)) 4245 continue; 4246 4247 auto p = sentinel_add(baseAddr + pn * PAGESIZE); 4248 size_t size = sentinel_size(p, getSize(pn)); 4249 uint attr = getBits(biti); 4250 4251 if (!rt_hasFinalizerInSegment(p, size, attr, segment)) 4252 continue; 4253 4254 rt_finalizeFromGC(p, size, attr); 4255 4256 clrBits(biti, ~BlkAttr.NONE); 4257 4258 if (pn < searchStart) 4259 searchStart = pn; 4260 4261 debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p); 4262 //log_free(sentinel_add(p)); 4263 4264 size_t n = 1; 4265 for (; pn + n < npages; ++n) 4266 if (pagetable[pn + n] != Bins.B_PAGEPLUS) 4267 break; 4268 debug (MEMSTOMP) memset(baseAddr + pn * PAGESIZE, 0xF3, n * PAGESIZE); 4269 freePages(pn, n); 4270 mergeFreePageOffsets!(true, true)(pn, n); 4271 } 4272 } 4273} 4274 4275 4276struct SmallObjectPool 4277{ 4278 Pool base; 4279 alias base this; 4280 4281 debug(INVARIANT) 4282 void Invariant() 4283 { 4284 //base.Invariant(); 4285 uint cntRecover = 0; 4286 foreach (Bins bin; Bins.B_16 .. Bins.B_NUMSMALL) 4287 { 4288 for (auto pn = recoverPageFirst[bin]; pn < npages; pn = binPageChain[pn]) 4289 { 4290 assert(pagetable[pn] == bin); 4291 cntRecover++; 4292 } 4293 } 4294 uint cntFree = 0; 4295 for (auto pn = searchStart; pn < npages; pn = binPageChain[pn]) 4296 { 4297 assert(pagetable[pn] == Bins.B_FREE); 4298 cntFree++; 4299 } 4300 assert(cntFree == freepages); 4301 assert(cntFree + cntRecover <= npages); 4302 } 4303 4304 /** 4305 * Get size of pointer p in pool. 4306 */ 4307 size_t getSize(void *p) const nothrow @nogc 4308 in 4309 { 4310 assert(p >= baseAddr); 4311 assert(p < topAddr); 4312 } 4313 do 4314 { 4315 size_t pagenum = pagenumOf(p); 4316 Bins bin = pagetable[pagenum]; 4317 assert(bin < Bins.B_PAGE); 4318 if (p != cast(void*)baseOffset(cast(size_t)p, bin)) // check for interior pointer 4319 return 0; 4320 const biti = cast(size_t)(p - baseAddr) >> ShiftBy.Small; 4321 if (freebits.test (biti)) 4322 return 0; 4323 return binsize[bin]; 4324 } 4325 4326 BlkInfo getInfo(void* p) nothrow 4327 { 4328 BlkInfo info; 4329 size_t offset = cast(size_t)(p - baseAddr); 4330 size_t pn = offset / PAGESIZE; 4331 Bins bin = pagetable[pn]; 4332 4333 if (bin >= Bins.B_PAGE) 4334 return info; 4335 4336 auto base = cast(void*)baseOffset(cast(size_t)p, bin); 4337 const biti = cast(size_t)(base - baseAddr) >> ShiftBy.Small; 4338 if (freebits.test (biti)) 4339 return info; 4340 4341 info.base = base; 4342 info.size = binsize[bin]; 4343 offset = info.base - baseAddr; 4344 info.attr = getBits(biti); 4345 4346 return info; 4347 } 4348 4349 void runFinalizers(const scope void[] segment) nothrow 4350 { 4351 foreach (pn; 0 .. npages) 4352 { 4353 Bins bin = pagetable[pn]; 4354 if (bin >= Bins.B_PAGE) 4355 continue; 4356 4357 immutable size = binsize[bin]; 4358 auto p = baseAddr + pn * PAGESIZE; 4359 const ptop = p + PAGESIZE - size + 1; 4360 immutable base = pn * (PAGESIZE/16); 4361 immutable bitstride = size / 16; 4362 4363 bool freeBits; 4364 PageBits toFree; 4365 4366 for (size_t i; p < ptop; p += size, i += bitstride) 4367 { 4368 immutable biti = base + i; 4369 4370 if (!finals.test(biti)) 4371 continue; 4372 4373 auto q = sentinel_add(p); 4374 uint attr = getBits(biti); 4375 const ssize = sentinel_size(q, size); 4376 if (!rt_hasFinalizerInSegment(q, ssize, attr, segment)) 4377 continue; 4378 4379 rt_finalizeFromGC(q, ssize, attr); 4380 4381 freeBits = true; 4382 toFree.set(i); 4383 4384 debug(COLLECT_PRINTF) printf("\tcollecting %p\n", p); 4385 //log_free(sentinel_add(p)); 4386 4387 debug (MEMSTOMP) memset(p, 0xF3, size); 4388 } 4389 4390 if (freeBits) 4391 freePageBits(pn, toFree); 4392 } 4393 } 4394 4395 /** 4396 * Allocate a page of bin's. 4397 * Returns: 4398 * head of a single linked list of new entries 4399 */ 4400 List* allocPage(Bins bin) nothrow 4401 { 4402 if (searchStart >= npages) 4403 return null; 4404 4405 assert(pagetable[searchStart] == Bins.B_FREE); 4406 4407 L1: 4408 size_t pn = searchStart; 4409 searchStart = binPageChain[searchStart]; 4410 binPageChain[pn] = Pool.PageRecovered; 4411 pagetable[pn] = bin; 4412 freepages--; 4413 4414 // Convert page to free list 4415 size_t size = binsize[bin]; 4416 void* p = baseAddr + pn * PAGESIZE; 4417 auto first = cast(List*) p; 4418 4419 // ensure 2 <size> bytes blocks are available below ptop, one 4420 // being set in the loop, and one for the tail block 4421 void* ptop = p + PAGESIZE - 2 * size + 1; 4422 for (; p < ptop; p += size) 4423 { 4424 (cast(List *)p).next = cast(List *)(p + size); 4425 (cast(List *)p).pool = &base; 4426 } 4427 (cast(List *)p).next = null; 4428 (cast(List *)p).pool = &base; 4429 return first; 4430 } 4431} 4432 4433debug(SENTINEL) {} else // no additional capacity with SENTINEL 4434unittest // bugzilla 14467 4435{ 4436 int[] arr = new int[10]; 4437 assert(arr.capacity); 4438 arr = arr[$..$]; 4439 assert(arr.capacity); 4440} 4441 4442unittest // bugzilla 15353 4443{ 4444 import core.memory : GC; 4445 4446 static struct Foo 4447 { 4448 ~this() 4449 { 4450 GC.free(buf); // ignored in finalizer 4451 } 4452 4453 void* buf; 4454 } 4455 new Foo(GC.malloc(10)); 4456 GC.collect(); 4457} 4458 4459unittest // bugzilla 15822 4460{ 4461 import core.memory : GC; 4462 4463 __gshared ubyte[16] buf; 4464 static struct Foo 4465 { 4466 ~this() 4467 { 4468 GC.removeRange(ptr); 4469 GC.removeRoot(ptr); 4470 } 4471 4472 ubyte* ptr; 4473 } 4474 GC.addRoot(buf.ptr); 4475 GC.addRange(buf.ptr, buf.length); 4476 new Foo(buf.ptr); 4477 GC.collect(); 4478} 4479 4480unittest // bugzilla 1180 4481{ 4482 import core.exception; 4483 try 4484 { 4485 size_t x = size_t.max - 100; 4486 byte[] big_buf = new byte[x]; 4487 } 4488 catch (OutOfMemoryError) 4489 { 4490 } 4491} 4492 4493/* ============================ PRINTF =============================== */ 4494 4495debug(PRINTF_TO_FILE) 4496{ 4497 private __gshared MonoTime gcStartTick; 4498 private __gshared FILE* gcx_fh; 4499 private __gshared bool hadNewline = false; 4500 import core.internal.spinlock; 4501 static printLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy); 4502 4503 private int printf(ARGS...)(const char* fmt, ARGS args) nothrow 4504 { 4505 printLock.lock(); 4506 scope(exit) printLock.unlock(); 4507 4508 if (!gcx_fh) 4509 gcx_fh = fopen("gcx.log", "w"); 4510 if (!gcx_fh) 4511 return 0; 4512 4513 int len; 4514 if (MonoTime.ticksPerSecond == 0) 4515 { 4516 len = fprintf(gcx_fh, "before init: "); 4517 } 4518 else if (hadNewline) 4519 { 4520 if (gcStartTick == MonoTime.init) 4521 gcStartTick = MonoTime.currTime; 4522 immutable timeElapsed = MonoTime.currTime - gcStartTick; 4523 immutable secondsAsDouble = timeElapsed.total!"hnsecs" / cast(double)convert!("seconds", "hnsecs")(1); 4524 len = fprintf(gcx_fh, "%10.6f: ", secondsAsDouble); 4525 } 4526 len += fprintf(gcx_fh, fmt, args); 4527 fflush(gcx_fh); 4528 import core.stdc.string; 4529 hadNewline = fmt && fmt[0] && fmt[strlen(fmt) - 1] == '\n'; 4530 return len; 4531 } 4532} 4533 4534debug(PRINTF) void printFreeInfo(Pool* pool) nothrow 4535{ 4536 uint nReallyFree; 4537 foreach (i; 0..pool.npages) { 4538 if (pool.pagetable[i] >= Bins.B_FREE) nReallyFree++; 4539 } 4540 4541 printf("Pool %p: %d really free, %d supposedly free\n", pool, nReallyFree, pool.freepages); 4542} 4543 4544debug(PRINTF) 4545void printGCBits(GCBits* bits) 4546{ 4547 for (size_t i = 0; i < bits.nwords; i++) 4548 { 4549 if (i % 32 == 0) printf("\n\t"); 4550 printf("%x ", bits.data[i]); 4551 } 4552 printf("\n"); 4553} 4554 4555// we can assume the name is always from a literal, so it is zero terminated 4556debug(PRINTF) 4557string debugTypeName(const(TypeInfo) ti) nothrow 4558{ 4559 string name; 4560 if (ti is null) 4561 name = "null"; 4562 else if (auto ci = cast(TypeInfo_Class)ti) 4563 name = ci.name; 4564 else if (auto si = cast(TypeInfo_Struct)ti) 4565 name = si.mangledName; // .name() might GC-allocate, avoid deadlock 4566 else if (auto ci = cast(TypeInfo_Const)ti) 4567 static if (__traits(compiles,ci.base)) // different whether compiled with object.di or object.d 4568 return debugTypeName(ci.base); 4569 else 4570 return debugTypeName(ci.next); 4571 else 4572 name = typeid(ti).name; 4573 return name; 4574} 4575 4576/* ======================= Leak Detector =========================== */ 4577 4578debug (LOGGING) 4579{ 4580 struct Log 4581 { 4582 void* p; 4583 size_t size; 4584 size_t line; 4585 char* file; 4586 void* parent; 4587 4588 void print() nothrow 4589 { 4590 printf(" p = %p, size = %lld, parent = %p ", p, cast(ulong)size, parent); 4591 if (file) 4592 { 4593 printf("%s(%u)", file, cast(uint)line); 4594 } 4595 printf("\n"); 4596 } 4597 } 4598 4599 4600 struct LogArray 4601 { 4602 size_t dim; 4603 size_t allocdim; 4604 Log *data; 4605 4606 void Dtor() nothrow @nogc 4607 { 4608 if (data) 4609 cstdlib.free(data); 4610 data = null; 4611 } 4612 4613 void reserve(size_t nentries) nothrow @nogc 4614 { 4615 assert(dim <= allocdim); 4616 if (allocdim - dim < nentries) 4617 { 4618 allocdim = (dim + nentries) * 2; 4619 assert(dim + nentries <= allocdim); 4620 if (!data) 4621 { 4622 data = cast(Log*)cstdlib.malloc(allocdim * Log.sizeof); 4623 if (!data && allocdim) 4624 onOutOfMemoryErrorNoGC(); 4625 } 4626 else 4627 { Log *newdata; 4628 4629 newdata = cast(Log*)cstdlib.malloc(allocdim * Log.sizeof); 4630 if (!newdata && allocdim) 4631 onOutOfMemoryErrorNoGC(); 4632 memcpy(newdata, data, dim * Log.sizeof); 4633 cstdlib.free(data); 4634 data = newdata; 4635 } 4636 } 4637 } 4638 4639 4640 void push(Log log) nothrow @nogc 4641 { 4642 reserve(1); 4643 data[dim++] = log; 4644 } 4645 4646 void remove(size_t i) nothrow @nogc 4647 { 4648 memmove(data + i, data + i + 1, (dim - i) * Log.sizeof); 4649 dim--; 4650 } 4651 4652 4653 size_t find(void *p) nothrow @nogc 4654 { 4655 for (size_t i = 0; i < dim; i++) 4656 { 4657 if (data[i].p == p) 4658 return i; 4659 } 4660 return OPFAIL; // not found 4661 } 4662 4663 4664 void copy(LogArray *from) nothrow @nogc 4665 { 4666 if (allocdim < from.dim) 4667 reserve(from.dim - dim); 4668 assert(from.dim <= allocdim); 4669 memcpy(data, from.data, from.dim * Log.sizeof); 4670 dim = from.dim; 4671 } 4672 } 4673 4674 struct LeakDetector 4675 { 4676 Gcx* gcx; 4677 LogArray current; 4678 LogArray prev; 4679 4680 private void initialize(Gcx* gc) 4681 { 4682 gcx = gc; 4683 //debug(PRINTF) printf("+log_init()\n"); 4684 current.reserve(1000); 4685 prev.reserve(1000); 4686 //debug(PRINTF) printf("-log_init()\n"); 4687 } 4688 4689 4690 private void log_malloc(void *p, size_t size) nothrow 4691 { 4692 //debug(PRINTF) printf("+log_malloc(p = %p, size = %zd)\n", p, size); 4693 Log log; 4694 4695 log.p = p; 4696 log.size = size; 4697 log.line = ConservativeGC.line; 4698 log.file = ConservativeGC.file; 4699 log.parent = null; 4700 4701 ConservativeGC.line = 0; 4702 ConservativeGC.file = null; 4703 4704 current.push(log); 4705 //debug(PRINTF) printf("-log_malloc()\n"); 4706 } 4707 4708 4709 private void log_free(void *p, size_t size) nothrow @nogc 4710 { 4711 //debug(PRINTF) printf("+log_free(%p)\n", p); 4712 auto i = current.find(p); 4713 if (i == OPFAIL) 4714 { 4715 debug(PRINTF) printf("free'ing unallocated memory %p (size %zu)\n", p, size); 4716 } 4717 else 4718 current.remove(i); 4719 //debug(PRINTF) printf("-log_free()\n"); 4720 } 4721 4722 4723 private void log_collect() nothrow 4724 { 4725 //debug(PRINTF) printf("+log_collect()\n"); 4726 // Print everything in current that is not in prev 4727 4728 debug(PRINTF) printf("New pointers this cycle: --------------------------------\n"); 4729 size_t used = 0; 4730 for (size_t i = 0; i < current.dim; i++) 4731 { 4732 auto j = prev.find(current.data[i].p); 4733 if (j == OPFAIL) 4734 current.data[i].print(); 4735 else 4736 used++; 4737 } 4738 4739 debug(PRINTF) printf("All roots this cycle: --------------------------------\n"); 4740 for (size_t i = 0; i < current.dim; i++) 4741 { 4742 void* p = current.data[i].p; 4743 if (!gcx.findPool(current.data[i].parent)) 4744 { 4745 auto j = prev.find(current.data[i].p); 4746 debug(PRINTF) printf(j == OPFAIL ? "N" : " "); 4747 current.data[i].print(); 4748 } 4749 } 4750 4751 debug(PRINTF) printf("Used = %d-------------------------------------------------\n", used); 4752 prev.copy(¤t); 4753 4754 debug(PRINTF) printf("-log_collect()\n"); 4755 } 4756 4757 4758 private void log_parent(void *p, void *parent) nothrow 4759 { 4760 //debug(PRINTF) printf("+log_parent()\n"); 4761 auto i = current.find(p); 4762 if (i == OPFAIL) 4763 { 4764 debug(PRINTF) printf("parent'ing unallocated memory %p, parent = %p\n", p, parent); 4765 Pool *pool; 4766 pool = gcx.findPool(p); 4767 assert(pool); 4768 size_t offset = cast(size_t)(p - pool.baseAddr); 4769 size_t biti; 4770 size_t pn = offset / PAGESIZE; 4771 Bins bin = pool.pagetable[pn]; 4772 biti = (offset & (PAGESIZE - 1)) >> pool.shiftBy; 4773 debug(PRINTF) printf("\tbin = %d, offset = x%x, biti = x%x\n", bin, offset, biti); 4774 } 4775 else 4776 { 4777 current.data[i].parent = parent; 4778 } 4779 //debug(PRINTF) printf("-log_parent()\n"); 4780 } 4781 } 4782} 4783else 4784{ 4785 struct LeakDetector 4786 { 4787 static void initialize(Gcx* gcx) nothrow { } 4788 static void log_malloc(void *p, size_t size) nothrow { } 4789 static void log_free(void *p, size_t size) nothrow @nogc {} 4790 static void log_collect() nothrow { } 4791 static void log_parent(void *p, void *parent) nothrow { } 4792 } 4793} 4794 4795/* ============================ SENTINEL =============================== */ 4796 4797debug (SENTINEL) 4798{ 4799 // pre-sentinel must be smaller than 16 bytes so that the same GC bits 4800 // are used for the allocated pointer and the user pointer 4801 // so use uint for both 32 and 64 bit platforms, limiting usage to < 4GB 4802 const uint SENTINEL_PRE = 0xF4F4F4F4; 4803 const ubyte SENTINEL_POST = 0xF5; // 8 bits 4804 const uint SENTINEL_EXTRA = 2 * uint.sizeof + 1; 4805 4806 4807 inout(uint*) sentinel_psize(inout void *p) nothrow @nogc { return &(cast(inout uint *)p)[-2]; } 4808 inout(uint*) sentinel_pre(inout void *p) nothrow @nogc { return &(cast(inout uint *)p)[-1]; } 4809 inout(ubyte*) sentinel_post(inout void *p) nothrow @nogc { return &(cast(inout ubyte *)p)[*sentinel_psize(p)]; } 4810 4811 4812 void sentinel_init(void *p, size_t size) nothrow @nogc 4813 { 4814 assert(size <= uint.max); 4815 *sentinel_psize(p) = cast(uint)size; 4816 *sentinel_pre(p) = SENTINEL_PRE; 4817 *sentinel_post(p) = SENTINEL_POST; 4818 } 4819 4820 4821 void sentinel_Invariant(const void *p) nothrow @nogc 4822 { 4823 debug 4824 { 4825 assert(*sentinel_pre(p) == SENTINEL_PRE); 4826 assert(*sentinel_post(p) == SENTINEL_POST); 4827 } 4828 else if (*sentinel_pre(p) != SENTINEL_PRE || *sentinel_post(p) != SENTINEL_POST) 4829 onInvalidMemoryOperationError(); // also trigger in release build 4830 } 4831 4832 size_t sentinel_size(const void *p, size_t alloc_size) nothrow @nogc 4833 { 4834 return *sentinel_psize(p); 4835 } 4836 4837 void *sentinel_add(void *p) nothrow @nogc 4838 { 4839 return p + 2 * uint.sizeof; 4840 } 4841 4842 4843 void *sentinel_sub(void *p) nothrow @nogc 4844 { 4845 return p - 2 * uint.sizeof; 4846 } 4847} 4848else 4849{ 4850 const uint SENTINEL_EXTRA = 0; 4851 4852 4853 void sentinel_init(void *p, size_t size) nothrow @nogc 4854 { 4855 } 4856 4857 4858 void sentinel_Invariant(const void *p) nothrow @nogc 4859 { 4860 } 4861 4862 size_t sentinel_size(const void *p, size_t alloc_size) nothrow @nogc 4863 { 4864 return alloc_size; 4865 } 4866 4867 void *sentinel_add(void *p) nothrow @nogc 4868 { 4869 return p; 4870 } 4871 4872 4873 void *sentinel_sub(void *p) nothrow @nogc 4874 { 4875 return p; 4876 } 4877} 4878 4879debug (MEMSTOMP) 4880unittest 4881{ 4882 import core.memory; 4883 auto p = cast(size_t*)GC.malloc(size_t.sizeof*3); 4884 assert(*p == cast(size_t)0xF0F0F0F0F0F0F0F0); 4885 p[2] = 0; // First two will be used for free list 4886 GC.free(p); 4887 assert(p[2] == cast(size_t)0xF2F2F2F2F2F2F2F2); 4888} 4889 4890debug (SENTINEL) 4891unittest 4892{ 4893 import core.memory; 4894 auto p = cast(ubyte*)GC.malloc(1); 4895 assert(p[-1] == 0xF4); 4896 assert(p[ 1] == 0xF5); 4897 4898 // See also stand-alone tests in test/gc 4899} 4900 4901unittest 4902{ 4903 import core.memory; 4904 4905 // https://issues.dlang.org/show_bug.cgi?id=9275 4906 GC.removeRoot(null); 4907 GC.removeRoot(cast(void*)13); 4908} 4909 4910// improve predictability of coverage of code that is eventually not hit by other tests 4911debug (SENTINEL) {} else // cannot extend with SENTINEL 4912debug (MARK_PRINTF) {} else // takes forever 4913unittest 4914{ 4915 import core.memory; 4916 auto p = GC.malloc(260 << 20); // new pool has 390 MB 4917 auto q = GC.malloc(65 << 20); // next chunk (larger than 64MB to ensure the same pool is used) 4918 auto r = GC.malloc(65 << 20); // another chunk in same pool 4919 assert(p + (260 << 20) == q); 4920 assert(q + (65 << 20) == r); 4921 GC.free(q); 4922 // should trigger "assert(bin == Bins.B_FREE);" in mark due to dangling pointer q: 4923 GC.collect(); 4924 // should trigger "break;" in extendNoSync: 4925 size_t sz = GC.extend(p, 64 << 20, 66 << 20); // trigger size after p large enough (but limited) 4926 assert(sz == 325 << 20); 4927 GC.free(p); 4928 GC.free(r); 4929 r = q; // ensure q is not trashed before collection above 4930 4931 p = GC.malloc(70 << 20); // from the same pool 4932 q = GC.malloc(70 << 20); 4933 r = GC.malloc(70 << 20); 4934 auto s = GC.malloc(70 << 20); 4935 auto t = GC.malloc(70 << 20); // 350 MB of 390 MB used 4936 assert(p + (70 << 20) == q); 4937 assert(q + (70 << 20) == r); 4938 assert(r + (70 << 20) == s); 4939 assert(s + (70 << 20) == t); 4940 GC.free(r); // ensure recalculation of largestFree in nxxt allocPages 4941 auto z = GC.malloc(75 << 20); // needs new pool 4942 4943 GC.free(p); 4944 GC.free(q); 4945 GC.free(s); 4946 GC.free(t); 4947 GC.free(z); 4948 GC.minimize(); // release huge pool 4949} 4950 4951// https://issues.dlang.org/show_bug.cgi?id=19281 4952debug (SENTINEL) {} else // cannot allow >= 4 GB with SENTINEL 4953debug (MEMSTOMP) {} else // might take too long to actually touch the memory 4954version (D_LP64) unittest 4955{ 4956 static if (__traits(compiles, os_physical_mem)) 4957 { 4958 // only run if the system has enough physical memory 4959 size_t sz = 2L^^32; 4960 //import core.stdc.stdio; 4961 //printf("availphys = %lld", os_physical_mem()); 4962 if (os_physical_mem() > sz) 4963 { 4964 import core.memory; 4965 GC.collect(); 4966 GC.minimize(); 4967 auto stats = GC.stats(); 4968 auto ptr = GC.malloc(sz, BlkAttr.NO_SCAN); 4969 auto info = GC.query(ptr); 4970 //printf("info.size = %lld", info.size); 4971 assert(info.size >= sz); 4972 GC.free(ptr); 4973 GC.minimize(); 4974 auto nstats = GC.stats(); 4975 assert(nstats.usedSize == stats.usedSize); 4976 assert(nstats.freeSize == stats.freeSize); 4977 assert(nstats.allocatedInCurrentThread - sz == stats.allocatedInCurrentThread); 4978 } 4979 } 4980} 4981 4982// https://issues.dlang.org/show_bug.cgi?id=19522 4983unittest 4984{ 4985 import core.memory; 4986 4987 void test(void* p) 4988 { 4989 assert(GC.getAttr(p) == BlkAttr.NO_SCAN); 4990 assert(GC.setAttr(p + 4, BlkAttr.NO_SCAN) == 0); // interior pointer should fail 4991 assert(GC.clrAttr(p + 4, BlkAttr.NO_SCAN) == 0); // interior pointer should fail 4992 GC.free(p); 4993 assert(GC.query(p).base == null); 4994 assert(GC.query(p).size == 0); 4995 assert(GC.addrOf(p) == null); 4996 assert(GC.sizeOf(p) == 0); // fails 4997 assert(GC.getAttr(p) == 0); 4998 assert(GC.setAttr(p, BlkAttr.NO_SCAN) == 0); 4999 assert(GC.clrAttr(p, BlkAttr.NO_SCAN) == 0); 5000 } 5001 void* large = GC.malloc(10000, BlkAttr.NO_SCAN); 5002 test(large); 5003 5004 void* small = GC.malloc(100, BlkAttr.NO_SCAN); 5005 test(small); 5006} 5007 5008unittest 5009{ 5010 import core.memory; 5011 5012 auto now = currTime; 5013 GC.ProfileStats stats1 = GC.profileStats(); 5014 GC.collect(); 5015 GC.ProfileStats stats2 = GC.profileStats(); 5016 auto diff = currTime - now; 5017 5018 assert(stats2.totalCollectionTime - stats1.totalCollectionTime <= diff); 5019 assert(stats2.totalPauseTime - stats1.totalPauseTime <= stats2.totalCollectionTime - stats1.totalCollectionTime); 5020 5021 assert(stats2.maxPauseTime >= stats1.maxPauseTime); 5022 assert(stats2.maxCollectionTime >= stats1.maxCollectionTime); 5023} 5024 5025// https://issues.dlang.org/show_bug.cgi?id=20214 5026unittest 5027{ 5028 import core.memory; 5029 import core.stdc.stdio; 5030 5031 // allocate from large pool 5032 auto o = GC.malloc(10); 5033 auto p = (cast(void**)GC.malloc(4096 * (void*).sizeof))[0 .. 4096]; 5034 auto q = (cast(void**)GC.malloc(4096 * (void*).sizeof))[0 .. 4096]; 5035 if (p.ptr + p.length is q.ptr) 5036 { 5037 q[] = o; // fill with pointers 5038 5039 // shrink, unused area cleared? 5040 auto nq = (cast(void**)GC.realloc(q.ptr, 4000 * (void*).sizeof))[0 .. 4000]; 5041 assert(q.ptr is nq.ptr); 5042 assert(q.ptr[4095] !is o); 5043 5044 GC.free(q.ptr); 5045 // expected to extend in place 5046 auto np = (cast(void**)GC.realloc(p.ptr, 4200 * (void*).sizeof))[0 .. 4200]; 5047 assert(p.ptr is np.ptr); 5048 assert(q.ptr[4200] !is o); 5049 } 5050 else 5051 { 5052 // adjacent allocations likely but not guaranteed 5053 printf("unexpected pointers %p and %p\n", p.ptr, q.ptr); 5054 } 5055} 5056