1/* 2 Title: Multi-Threaded Garbage Collector - Data sharing phase 3 4 Copyright (c) 2012, 2017, 2019 David C. J. Matthews 5 6 This library is free software; you can redistribute it and/or 7 modify it under the terms of the GNU Lesser General Public 8 License as published by the Free Software Foundation; either 9 version 2.1 of the License, or (at your option) any later version. 10 11 This library is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 Lesser General Public License for more details. 15 16 You should have received a copy of the GNU Lesser General Public 17 License along with this library; if not, write to the Free Software 18 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 19 20*/ 21 22/* GC Sharing pass. 23 This pass is invoked only if the heap sizing code detects that heap space 24 is running very short because it adds a very considerable overhead to GC. 25 It aims to reduce the size of the live data in a similar way to the data 26 sharing function PolyML.shareCommonData by merging immutable cells that 27 contain data that cannot be distinguished. 28 29 This version of the code now does a deep structure merge in a similar 30 way to the full sharing function. This code first does a full pass 31 over the heap creating lists of cells that could possibly be merged. 32 There are separate lists for byte and word objects up to a fixed 33 size. Larger objects and other objects are not considered. Because 34 all the items in a list have the same length and type (flag bits) 35 we can use the length word to link the items in the list. A 36 consequence of this is that positive long precision values can be 37 shared but negative values cannot. 38 39 There is a sharing function that first distributes items into a 40 hash table. Then each hash table is sorted and as part of the 41 sorting process cells with the same contents are merged. One 42 cell is chosen and the length words on the others are set to be 43 forwarding pointers to the chosen cell. Hashing allows for easy 44 parallel processing. 45 46 The structure sharing code works by first sharing the byte 47 data which cannot contain pointers. Then the word data is processed 48 to separate out "tail" cells that contain only tagged integers or 49 pointers to cells that either cannot be merged, such as mutables, 50 or those that have already been processed, such as the byte data. 51 Any pointers to shared data are updated to point to the merged cell. 52 The tail cells are then sorted and shared using the sharing function 53 and become part of the "processed" set. This process is repeated to 54 find cells that are now tails and so on. 55 56 Compared with the full sharing code this is expensive since it 57 requires repeated scans of the list of unprocessed cells. In 58 particular there may be cells that form loops (basically closures 59 for mutually recusive functions) and if they are present they and 60 anything that points directly or indirectly at them will never 61 be removed from the list. We stop when it appears that we are 62 not making progress and simply do a final bit-wise share of the 63 remainder. 64 65 This now uses the forwarding pointer both to indicate that a cell 66 shares with another and also to link together cells that have yet 67 to be tested for sharing. To detect the difference the bitmap is 68 used. The initial scan to create the sharing chains sets the bit 69 for each visited cell so at the start of the sharing phase all 70 reachable cells will be marked. We remove the mark if the cell 71 is to be removed. This requires the bitmap to be locked. 72*/ 73#ifdef HAVE_CONFIG_H 74#include "config.h" 75#elif defined(_WIN32) 76#include "winconfig.h" 77#else 78#error "No configuration file" 79#endif 80 81#ifdef HAVE_ASSERT_H 82#include <assert.h> 83#define ASSERT(x) assert(x) 84#else 85#define ASSERT(x) 86#endif 87 88#ifdef HAVE_STRING_H 89#include <string.h> 90#endif 91 92#ifdef HAVE_STDLIB_H 93#include <stdlib.h> 94#endif 95 96#include "globals.h" 97#include "processes.h" 98#include "gc.h" 99#include "scanaddrs.h" 100#include "bitmap.h" 101#include "memmgr.h" 102#include "diagnostics.h" 103#include "gctaskfarm.h" 104#include "heapsizing.h" 105#include "gc_progress.h" 106 107#ifdef POLYML32IN64 108#define ENDOFLIST ((PolyObject*)globalHeapBase) 109#else 110#define ENDOFLIST 0 111#endif 112 113// Set the forwarding so that references to objToSet will be forwarded to 114// objToShare. objToSet will be garbage. 115void shareWith(PolyObject *objToSet, PolyObject *objToShare) 116{ 117 // We need to remove the bit from this so that we know it's not 118 // a share chain. 119 PolyWord *lengthWord = ((PolyWord*)objToSet) - 1; 120 LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord); 121 ASSERT(space); 122 PLocker locker(&space->bitmapLock); 123 ASSERT(space->bitmap.TestBit(space->wordNo(lengthWord))); 124 space->bitmap.ClearBit(space->wordNo(lengthWord)); 125 // Actually do the forwarding 126 objToSet->SetForwardingPtr(objToShare); 127} 128 129// When we find an address it could be a cell that: 130// 1. is never processed or one that is the copy to be retained, 131// 2. has been merged with another and contains a forwarding pointer or 132// 3. has not yet been processed. 133typedef enum { REALOBJECT, FORWARDED, CHAINED } objectState; 134 135objectState getObjectState(PolyObject *p) 136{ 137 PolyWord *lengthWord = ((PolyWord*)p) - 1; 138 LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord); 139 if (space == 0) 140 return REALOBJECT; // May be the address of a permanent or something else. 141 PLocker locker(&space->bitmapLock); 142 if (!p->ContainsForwardingPtr()) 143 return REALOBJECT; 144 if (space->bitmap.TestBit(space->wordNo(lengthWord))) 145 return CHAINED; 146 else return FORWARDED; 147 148} 149 150class ObjEntry 151{ 152public: 153 ObjEntry(): objList(ENDOFLIST), objCount(0), shareCount(0) {} 154 PolyObject *objList; 155 POLYUNSIGNED objCount; 156 POLYUNSIGNED shareCount; 157}; 158 159// There is an instance of this class for each combination of size and 160// word/byte. 161class SortVector 162{ 163public: 164 SortVector(): totalCount(0), carryOver(0) {} 165 void AddToVector(PolyObject *obj, POLYUNSIGNED length); 166 void SortData(void); 167 POLYUNSIGNED TotalCount() const { return totalCount; } 168 POLYUNSIGNED CurrentCount() const { return baseObject.objCount; } 169 POLYUNSIGNED Shared() const; 170 void SetLengthWord(POLYUNSIGNED l) { lengthWord = l; } 171 POLYUNSIGNED CarryOver() const { return carryOver; } 172 173 static void hashAndSortAllTask(GCTaskId*, void *a, void *b); 174 static void sharingTask(GCTaskId*, void *a, void *b); 175 static void wordDataTask(GCTaskId*, void *a, void *b); 176 177private: 178 void sortList(PolyObject *head, POLYUNSIGNED nItems, POLYUNSIGNED &count); 179 180 ObjEntry baseObject, processObjects[256]; 181 POLYUNSIGNED totalCount; 182 POLYUNSIGNED lengthWord; 183 POLYUNSIGNED carryOver; 184}; 185 186POLYUNSIGNED SortVector::Shared() const 187{ 188 // Add all the sharing counts 189 POLYUNSIGNED shareCount = baseObject.shareCount; 190 for (unsigned i = 0; i < 256; i++) 191 shareCount += processObjects[i].shareCount; 192 return shareCount; 193} 194 195void SortVector::AddToVector(PolyObject *obj, POLYUNSIGNED length) 196{ 197 obj->SetForwardingPtr(baseObject.objList); 198 baseObject.objList = obj; 199 baseObject.objCount++; 200 totalCount++; 201} 202 203// The number of byte and word entries. 204// Objects of up to and including this size are shared. 205// Byte objects include strings so it is more likely that 206// larger objects will share. Word objects that share 207// are much more likely to be 2 or 3 words. 208#define NUM_BYTE_VECTORS 23 209#define NUM_WORD_VECTORS 11 210 211// The stack is allocated as a series of blocks chained together. 212#define RSTACK_SEGMENT_SIZE 1000 213 214class RScanStack { 215public: 216 RScanStack() : nextStack(0), lastStack(0), sp(0) {} 217 ~RScanStack() { delete(nextStack); } 218 219 RScanStack *nextStack; 220 RScanStack *lastStack; 221 unsigned sp; 222 struct { PolyObject *obj; PolyWord *base; } stack[RSTACK_SEGMENT_SIZE]; 223}; 224 225class RecursiveScanWithStack : public ScanAddress 226{ 227public: 228 RecursiveScanWithStack() : stack(0) {} 229 ~RecursiveScanWithStack() { delete(stack); } 230 231public: 232 virtual PolyObject *ScanObjectAddress(PolyObject *base); 233 virtual void ScanAddressesInObject(PolyObject *base, POLYUNSIGNED lengthWord); 234 // Have to redefine this for some reason. 235 void ScanAddressesInObject(PolyObject *base) 236 { 237 ScanAddressesInObject(base, base->LengthWord()); 238 } 239 240protected: 241 // Test the word at the location to see if it points to 242 // something that may have to be scanned. We pass in the 243 // pointer here because the called may side-effect it. 244 virtual bool TestForScan(PolyWord *) = 0; 245 // If we are definitely scanning the address we mark it. 246 virtual void MarkAsScanning(PolyObject *) = 0; 247 // Called when the object has been completed. 248 virtual void Completed(PolyObject *) {} 249 250protected: 251 void PushToStack(PolyObject *obj, PolyWord *base); 252 void PopFromStack(PolyObject *&obj, PolyWord *&base); 253 254 bool StackIsEmpty(void) 255 { 256 return stack == 0 || (stack->sp == 0 && stack->lastStack == 0); 257 } 258 259 RScanStack *stack; 260}; 261 262// This gets called in two circumstances. It may be called for the roots 263// in which case the stack will be empty and we want to process it completely 264// or it is called for a constant address in which case it will have been 265// called from RecursiveScan::ScanAddressesInObject and that can process 266// any addresses. 267PolyObject *RecursiveScanWithStack::ScanObjectAddress(PolyObject *obj) 268{ 269 PolyWord pWord = obj; 270 // Test to see if this needs to be scanned. 271 // It may update the word. 272 bool test = TestForScan(&pWord); 273 obj = pWord.AsObjPtr(); 274 275 if (test) 276 { 277 MarkAsScanning(obj); 278 if (obj->IsByteObject()) 279 Completed(obj); // Don't need to put it on the stack 280 // If we already have something on the stack we must being called 281 // recursively to process a constant in a code segment. Just push 282 // it on the stack and let the caller deal with it. 283 else if (StackIsEmpty()) 284 RecursiveScanWithStack::ScanAddressesInObject(obj, obj->LengthWord()); 285 else 286 PushToStack(obj, (PolyWord*)obj); 287 } 288 289 return obj; 290} 291 292// This is called via ScanAddressesInRegion to process the permanent mutables. It is 293// also called from ScanObjectAddress to process root addresses. 294// It processes all the addresses reachable from the object. 295// This is almost the same as MTGCProcessMarkPointers::ScanAddressesInObject. 296void RecursiveScanWithStack::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord) 297{ 298 if (OBJ_IS_BYTE_OBJECT(lengthWord)) 299 return; // Ignore byte cells and don't call Completed on them 300 301 PolyWord *baseAddr = (PolyWord*)obj; 302 303 while (true) 304 { 305 ASSERT(OBJ_IS_LENGTH(lengthWord)); 306 307 // Get the length and base address. N.B. If this is a code segment 308 // these will be side-effected by GetConstSegmentForCode. 309 POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord); 310 311 if (OBJ_IS_CODE_OBJECT(lengthWord) || OBJ_IS_CLOSURE_OBJECT(lengthWord)) 312 { 313 // It's better to process the whole code object in one go. 314 // For the moment do that for closure objects as well. 315 ScanAddress::ScanAddressesInObject(obj, lengthWord); 316 length = 0; // Finished 317 } 318 319 // else it's a normal object, 320 321 // If there are only two addresses in this cell that need to be 322 // followed we follow them immediately and treat this cell as done. 323 // If there are more than two we push the address of this cell on 324 // the stack, follow the first address and then rescan it. That way 325 // list cells are processed once only but we don't overflow the 326 // stack by pushing all the addresses in a very large vector. 327 PolyWord *endWord = (PolyWord*)obj + length; 328 PolyObject *firstWord = 0; 329 PolyObject *secondWord = 0; 330 PolyWord *restartFrom = baseAddr; 331 332 while (baseAddr != endWord) 333 { 334 PolyWord wordAt = *baseAddr; 335 336 if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0)) 337 { 338 // Normal address. We can have words of all zeros at least in the 339 // situation where we have a partially constructed code segment where 340 // the constants at the end of the code have not yet been filled in. 341 if (TestForScan(baseAddr)) // Test value at baseAddr (may side-effect it) 342 { 343 PolyObject *wObj = (*baseAddr).AsObjPtr(); 344 if (wObj->IsByteObject()) 345 { 346 // Can do this now - don't need to push it 347 MarkAsScanning(wObj); 348 Completed(wObj); 349 } 350 else if (firstWord == 0) 351 { 352 firstWord = wObj; 353 // We mark the word immediately. We can have 354 // two words in an object that are the same 355 // and we don't want to process it again. 356 MarkAsScanning(firstWord); 357 } 358 else if (secondWord == 0) 359 { 360 secondWord = wObj; 361 restartFrom = baseAddr; 362 } 363 else break; // More than two words. 364 } 365 } 366 baseAddr++; 367 } 368 369 if (baseAddr == endWord) 370 { 371 // We have done everything except possibly firstWord and secondWord. 372 // Note: Unfortunately the way that ScanAddressesInRegion works means that 373 // we call Completed on the addresses of cells in the permanent areas without 374 // having called TestForScan. 375 Completed(obj); 376 if (secondWord != 0) 377 { 378 MarkAsScanning(secondWord); 379 // Put this on the stack. If this is a list node we will be 380 // pushing the tail. 381 PushToStack(secondWord, (PolyWord*)secondWord); 382 } 383 } 384 else // Put this back on the stack while we process the first word 385 PushToStack(obj, restartFrom); 386 387 if (firstWord != 0) 388 { 389 // Process it immediately. 390 obj = firstWord; 391 baseAddr = (PolyWord*)obj; 392 } 393 else if (StackIsEmpty()) 394 return; 395 else 396 PopFromStack(obj, baseAddr); 397 398 lengthWord = obj->LengthWord(); 399 } 400} 401 402void RecursiveScanWithStack::PushToStack(PolyObject *obj, PolyWord *base) 403{ 404 if (stack == 0 || stack->sp == RSTACK_SEGMENT_SIZE) 405 { 406 if (stack != 0 && stack->nextStack != 0) 407 stack = stack->nextStack; 408 else 409 { 410 // Need a new segment 411 try { 412 RScanStack *s = new RScanStack; 413 s->lastStack = stack; 414 if (stack != 0) 415 stack->nextStack = s; 416 stack = s; 417 } 418 catch (std::bad_alloc &) { 419 // Ignore stack overflow 420 return; 421 } 422 } 423 } 424 stack->stack[stack->sp].obj = obj; 425 stack->stack[stack->sp].base = base; 426 stack->sp++; 427} 428 429void RecursiveScanWithStack::PopFromStack(PolyObject *&obj, PolyWord *&base) 430{ 431 if (stack->sp == 0) 432 { 433 // Chain to the previous stack if any 434 ASSERT(stack->lastStack != 0); 435 // Before we do, delete any further one to free some memory 436 delete(stack->nextStack); 437 stack->nextStack = 0; 438 stack = stack->lastStack; 439 ASSERT(stack->sp == RSTACK_SEGMENT_SIZE); 440 } 441 --stack->sp; 442 obj = stack->stack[stack->sp].obj; 443 base = stack->stack[stack->sp].base; 444} 445 446class GetSharing: public RecursiveScanWithStack 447{ 448public: 449 GetSharing(); 450 void SortData(void); 451 static void shareByteData(GCTaskId *, void *, void *); 452 static void shareWordData(GCTaskId *, void *, void *); 453 static void shareRemainingWordData(GCTaskId *, void *, void *); 454 455 virtual PolyObject *ScanObjectAddress(PolyObject *obj); 456 457protected: 458 virtual bool TestForScan(PolyWord *); 459 virtual void MarkAsScanning(PolyObject *); 460 virtual void Completed(PolyObject *); 461 462private: 463 // The head of chains of cells of the same size 464 SortVector byteVectors[NUM_BYTE_VECTORS]; 465 SortVector wordVectors[NUM_WORD_VECTORS]; 466 467 POLYUNSIGNED largeWordCount, largeByteCount, excludedCount; 468public: 469 POLYUNSIGNED totalVisited, byteAdded, wordAdded, totalSize; 470}; 471 472GetSharing::GetSharing() 473{ 474 for (unsigned i = 0; i < NUM_BYTE_VECTORS; i++) 475 byteVectors[i].SetLengthWord((POLYUNSIGNED)i | _OBJ_BYTE_OBJ); 476 477 for (unsigned j = 0; j < NUM_WORD_VECTORS; j++) 478 wordVectors[j].SetLengthWord(j); 479 480 largeWordCount = largeByteCount = excludedCount = 0; 481 totalVisited = byteAdded = wordAdded = totalSize = 0; 482} 483 484// This is called for roots and also for constants in the constant area. 485// If we have a code address we MUSTN't call RecursiveScan::ScanObjectAddress 486// because that turns the address into a PolyWord and doesn't work in 32-in-64. 487// We process the code area explicitly so we can simply skip code addresses. 488PolyObject *GetSharing::ScanObjectAddress(PolyObject *obj) 489{ 490 LocalMemSpace *sp = gMem.LocalSpaceForAddress((PolyWord*)obj - 1); 491 492 if (sp == 0) 493 return obj; 494 495 return RecursiveScanWithStack::ScanObjectAddress(obj); 496} 497 498bool GetSharing::TestForScan(PolyWord *pt) 499{ 500 PolyObject *obj; 501 502 // This may be a forwarding pointer left over from a minor GC that did 503 // not complete or it may be a sharing chain pointer that we've set up. 504 while (1) 505 { 506 PolyWord p = *pt; 507 ASSERT(p.IsDataPtr()); 508 obj = p.AsObjPtr(); 509 PolyWord *lengthWord = ((PolyWord*)obj) - 1; 510 LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord); 511 if (space == 0) 512 return false; // Ignore it if it points to a permanent area 513 514 if (space->bitmap.TestBit(space->wordNo(lengthWord))) 515 return false; 516 517 // Wasn't marked - must be a forwarding pointer. 518 if (obj->ContainsForwardingPtr()) 519 { 520 obj = obj->GetForwardingPtr(); 521 *pt = obj; 522 } 523 else break; 524 } 525 526 ASSERT(obj->ContainsNormalLengthWord()); 527 528 totalVisited += 1; 529 totalSize += obj->Length() + 1; 530 531 return true; 532} 533 534void GetSharing::MarkAsScanning(PolyObject *obj) 535{ 536 ASSERT(obj->ContainsNormalLengthWord()); 537 PolyWord *lengthWord = ((PolyWord*)obj) - 1; 538 LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord); 539 ASSERT(! space->bitmap.TestBit(space->wordNo(lengthWord))); 540 space->bitmap.SetBit(space->wordNo(lengthWord)); 541} 542 543void GetSharing::Completed(PolyObject *obj) 544{ 545 // We mustn't include cells in the permanent area. 546 // We scan the permanent mutable areas for local addresses 547 // but we mustn't add the cells themselves. Normally they 548 // will be mutable so would be ignored but cells that have been 549 // locked will now be immutable. The test in TestForScan is bypassed 550 // by ScanAddressesInRegion. 551 PolyWord *lengthWord = ((PolyWord*)obj) - 1; 552 if (gMem.LocalSpaceForAddress(lengthWord) == 0) 553 return; 554 555 POLYUNSIGNED L = obj->LengthWord(); 556 // We have tables for word objects and byte objects 557 // We chain entries together using the length word so it 558 // is important that we only do this for objects that 559 // have no other bits in the header, such as the sign bit. 560 if ((L & _OBJ_PRIVATE_FLAGS_MASK) == 0) 561 { 562 POLYUNSIGNED length = obj->Length(); 563 if (length < NUM_WORD_VECTORS) 564 wordVectors[length].AddToVector(obj, length); 565 else largeWordCount++; 566 wordAdded++; 567 } 568 else if ((L & _OBJ_PRIVATE_FLAGS_MASK) == _OBJ_BYTE_OBJ) 569 { 570 POLYUNSIGNED length = obj->Length(); 571 if (length < NUM_BYTE_VECTORS) 572 byteVectors[length].AddToVector(obj, length); 573 else largeByteCount++; 574 byteAdded++; 575 } 576 else if (! OBJ_IS_CODE_OBJECT(L) && ! OBJ_IS_MUTABLE_OBJECT(L)) 577 excludedCount++; // Code and mutables can't be shared - see what could be 578 // TODO: We don't attempt to share closure cells in 32-in-64. 579} 580 581// Quicksort the list to detect cells with the same content. These are made 582// to share and removed from further sorting. 583void SortVector::sortList(PolyObject *head, POLYUNSIGNED nItems, POLYUNSIGNED &shareCount) 584{ 585 while (nItems > 2) 586 { 587 size_t bytesToCompare = OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord); 588 PolyObject *median = head; 589 head = head->GetForwardingPtr(); 590 median->SetLengthWord(lengthWord); 591 PolyObject *left = ENDOFLIST, *right = ENDOFLIST; 592 POLYUNSIGNED leftCount = 0, rightCount = 0; 593 while (head != ENDOFLIST) 594 { 595 PolyObject *next = head->GetForwardingPtr(); 596 int res = memcmp(median, head, bytesToCompare); 597 if (res == 0) 598 { 599 // Equal - they can share 600 shareWith(head, median); 601 shareCount++; 602 } 603 else if (res < 0) 604 { 605 head->SetForwardingPtr(left); 606 left = head; 607 leftCount++; 608 } 609 else 610 { 611 head->SetForwardingPtr(right); 612 right = head; 613 rightCount++; 614 } 615 head = next; 616 } 617 // We can now drop the median and anything that shares with it. 618 // Process the smaller partition recursively and the larger by 619 // tail recursion. 620 if (leftCount < rightCount) 621 { 622 sortList(left, leftCount, shareCount); 623 head = right; 624 nItems = rightCount; 625 } 626 else 627 { 628 sortList(right, rightCount, shareCount); 629 head = left; 630 nItems = leftCount; 631 } 632 } 633 if (nItems == 1) 634 head->SetLengthWord(lengthWord); 635 else if (nItems == 2) 636 { 637 PolyObject *next = head->GetForwardingPtr(); 638 head->SetLengthWord(lengthWord); 639 if (memcmp(head, next, OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord)) == 0) 640 { 641 shareWith(next, head); 642 shareCount++; 643 } 644 else next->SetLengthWord(lengthWord); 645 } 646} 647 648void SortVector::sharingTask(GCTaskId*, void *a, void *b) 649{ 650 SortVector *s = (SortVector *)a; 651 ObjEntry *o = (ObjEntry*)b; 652 s->sortList(o->objList, o->objCount, o->shareCount); 653} 654 655// Process one level of the word data. 656// N.B. The length words are updated without any locking. This is safe 657// because all length words are initially chain entries and a chain entry 658// can be replaced by another chain entry, a forwarding pointer or a normal 659// length word. Forwarding pointers and normal length words are only ever 660// set once. There is a small chance that we could lose some sharing as a 661// result of a race condition if a thread defers an object because it 662// contains a pointer with a chain entry and later sees an otherwise 663// equal object where another thread has replaced the chain with a 664// normal address, adds it to the list for immediate processing and 665// so never compares the two. 666void SortVector::wordDataTask(GCTaskId*, void *a, void *) 667{ 668 SortVector *s = (SortVector*)a; 669 // Partition the objects between those that have pointers to objects that are 670 // still to be processed and those that have been processed. 671 if (s->baseObject.objList == ENDOFLIST) 672 return; 673 PolyObject *h = s->baseObject.objList; 674 s->baseObject.objList = ENDOFLIST; 675 s->baseObject.objCount = 0; 676 POLYUNSIGNED words = OBJ_OBJECT_LENGTH(s->lengthWord); 677 s->carryOver = 0; 678 679 for (unsigned i = 0; i < 256; i++) 680 { 681 // Clear the entries in the hash table but not the sharing count. 682 s->processObjects[i].objList = ENDOFLIST; 683 s->processObjects[i].objCount = 0; 684 } 685 686 while (h != ENDOFLIST) 687 { 688 PolyObject *next = h->GetForwardingPtr(); 689 bool deferred = false; 690 for (POLYUNSIGNED i = 0; i < words; i++) 691 { 692 PolyWord w = h->Get(i); 693 if (w.IsDataPtr()) 694 { 695 PolyObject *p = w.AsObjPtr(); 696 objectState state = getObjectState(p); 697 if (state == FORWARDED) 698 { 699 // Update the addresses of objects that have been merged 700 h->Set(i, p->GetForwardingPtr()); 701 s->carryOver++; 702 break; 703 } 704 else if (state == CHAINED) 705 { 706 // If it is still to be shared leave it 707 deferred = true; 708 break; // from the loop 709 } 710 } 711 } 712 if (deferred) 713 { 714 // We can't do it yet: add it back to the list 715 h->SetForwardingPtr(s->baseObject.objList); 716 s->baseObject.objList = h; 717 s->baseObject.objCount++; 718 } 719 else 720 { 721 // Add it to the hash table. 722 unsigned char hash = 0; 723 for (POLYUNSIGNED i = 0; i < words*sizeof(PolyWord); i++) 724 hash += h->AsBytePtr()[i]; 725 h->SetForwardingPtr(s->processObjects[hash].objList); 726 s->processObjects[hash].objList = h; 727 s->processObjects[hash].objCount++; 728 } 729 h = next; 730 } 731 s->SortData(); 732} 733 734// Sort the entries in the hash table. 735void SortVector::SortData() 736{ 737 for (unsigned j = 0; j < 256; j++) 738 { 739 ObjEntry *oentry = &processObjects[j]; 740 // Sort this entry. If it's very small just process it now. 741 switch (oentry->objCount) 742 { 743 case 0: break; // Nothing there 744 745 case 1: // Singleton - just restore the length word 746 oentry->objList->SetLengthWord(lengthWord); 747 break; 748 749 case 2: 750 { 751 // Two items - process now 752 PolyObject *obj1 = oentry->objList; 753 PolyObject *obj2 = obj1->GetForwardingPtr(); 754 obj1->SetLengthWord(lengthWord); 755 if (memcmp(obj1, obj2, OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord)) == 0) 756 { 757 shareWith(obj2, obj1); 758 oentry->shareCount++; 759 } 760 else obj2->SetLengthWord(lengthWord); 761 break; 762 } 763 764 default: 765 gpTaskFarm->AddWorkOrRunNow(sharingTask, this, oentry); 766 } 767 } 768} 769 770void SortVector::hashAndSortAllTask(GCTaskId*, void *a, void *b) 771{ 772 SortVector *s = (SortVector *)a; 773 // Hash the contents of the base object then sort them. 774 for (unsigned i = 0; i < 256; i++) 775 { 776 // Clear the entries in the hash table but not the sharing count. 777 s->processObjects[i].objList = ENDOFLIST; 778 s->processObjects[i].objCount = 0; 779 } 780 PolyObject *h = s->baseObject.objList; 781 POLYUNSIGNED bytes = OBJ_OBJECT_LENGTH(s->lengthWord)*sizeof(PolyWord); 782 while (h != ENDOFLIST) 783 { 784 PolyObject *next = h->GetForwardingPtr(); 785 unsigned char hash = 0; 786 for (POLYUNSIGNED j = 0; j < bytes; j++) 787 hash += h->AsBytePtr()[j]; 788 h->SetForwardingPtr(s->processObjects[hash].objList); 789 s->processObjects[hash].objList = h; 790 s->processObjects[hash].objCount++; 791 h = next; 792 } 793 s->SortData(); 794} 795 796// Look for sharing between byte data. These cannot contain pointers 797// so they can all be processed together. 798void GetSharing::shareByteData(GCTaskId *, void *a, void *) 799{ 800 GetSharing *s = (GetSharing*)a; 801 for (unsigned i = 0; i < NUM_BYTE_VECTORS; i++) 802 { 803 if (s->byteVectors[i].CurrentCount() != 0) 804 gpTaskFarm->AddWorkOrRunNow(SortVector::hashAndSortAllTask, &(s->byteVectors[i]), 0); 805 } 806} 807 808// Process word data at this particular level 809void GetSharing::shareWordData(GCTaskId *, void *a, void *) 810{ 811 GetSharing *s = (GetSharing*)a; 812 for (unsigned i = 0; i < NUM_WORD_VECTORS; i++) 813 { 814 if (s->wordVectors[i].CurrentCount() != 0) 815 gpTaskFarm->AddWorkOrRunNow(SortVector::wordDataTask, &(s->wordVectors[i]), 0); 816 } 817} 818 819// Share any entries left. 820void GetSharing::shareRemainingWordData(GCTaskId *, void *a, void *) 821{ 822 GetSharing *s = (GetSharing*)a; 823 for (unsigned i = 0; i < NUM_WORD_VECTORS; i++) 824 { 825 if (s->wordVectors[i].CurrentCount() != 0) 826 gpTaskFarm->AddWorkOrRunNow(SortVector::hashAndSortAllTask, &(s->wordVectors[i]), 0); 827 } 828} 829 830void GetSharing::SortData() 831{ 832 // First process the byte objects. They cannot contain pointers. 833 // We create a task to do this so that we never have more threads 834 // running than given with --gcthreads. 835 gpTaskFarm->AddWorkOrRunNow(shareByteData, this, 0); 836 gpTaskFarm->WaitForCompletion(); 837 838 // Word data may contain pointers to other objects. If an object 839 // has been processed its header will contain either a normal length 840 // word or a forwarding pointer if it shares. We can process an 841 // object if every word in it is either a tagged integer or an 842 // address we have already processed. This works provided there 843 // are no loops so when we reach a stage where we are unable to 844 // process anything we simply run a final scan on the remainder. 845 // Loops can arise from the closures of mutually recursive functions. 846 847 // Now process the word entries until we have nothing left apart from loops. 848 POLYUNSIGNED lastCount = 0, lastShared = 0; 849 for (unsigned n = 0; n < NUM_WORD_VECTORS; n++) 850 lastCount += wordVectors[n].CurrentCount(); 851 852 for(unsigned pass = 1; lastCount != 0; pass++) 853 { 854 gpTaskFarm->AddWorkOrRunNow(shareWordData, this, 0); 855 gpTaskFarm->WaitForCompletion(); 856 857 // At each stage check that we have removed some items 858 // from the lists. 859 POLYUNSIGNED postCount = 0, postShared = 0, carryOver = 0; 860 for (unsigned i = 0; i < NUM_WORD_VECTORS; i++) 861 { 862 postCount += wordVectors[i].CurrentCount(); 863 postShared += wordVectors[i].Shared(); 864 carryOver += wordVectors[i].CarryOver(); 865 } 866 867 if (debugOptions & DEBUG_GC) 868 Log("GC: Share: Pass %u: %" POLYUFMT " removed (%1.1f%%) %" POLYUFMT " shared (%1.1f%%) %" POLYUFMT " remain. %" POLYUFMT " entries updated (%1.1f%%).\n", 869 pass, lastCount-postCount, (double)(lastCount-postCount) / (double) lastCount * 100.0, 870 postShared - lastShared, (double)(postShared - lastShared) / (double) (lastCount-postCount) * 100.0, 871 postCount, carryOver, (double)carryOver / (double)(lastCount-postCount) * 100.0); 872 873 gcProgressSetPercent((unsigned)((double)(totalVisited - postCount) / (double)totalVisited * 100.0)); 874 875 // Condition for exiting the loop. There are some heuristics here. 876 // If we remove less than 10% in a pass it's probably not worth continuing 877 // unless the carry over is large. The "carry over" is the number of words updated as 878 // a result of the last pass. It represents the extra sharing we gained in this pass 879 // as a result of the last pass. If there are deep data structures that can be shared 880 // we get better sharing with more passes. If the data structures are shallow we will 881 // get as much sharing by just running the final pass. The first pass only carries 882 // over any sharing from the byte objects so we need to run at least one more before 883 // checking the carry over. 884 if (pass > 1 && (lastCount - postCount) * 10 < lastCount && (carryOver*2 < (lastCount-postCount) || (lastCount - postCount) * 1000 < lastCount )) 885 break; 886 887 lastCount = postCount; 888 lastShared = postShared; 889 } 890 891 // Process any remaining entries. There may be loops. 892 gpTaskFarm->AddWorkOrRunNow(shareRemainingWordData, this, 0); 893 gpTaskFarm->WaitForCompletion(); 894 895 if (debugOptions & DEBUG_GC) 896 { 897 POLYUNSIGNED postShared = 0; 898 for (unsigned i = 0; i < NUM_WORD_VECTORS; i++) 899 postShared += wordVectors[i].Shared(); 900 if (debugOptions & DEBUG_GC) 901 Log("GC: Share: Final pass %" POLYUFMT " removed %" POLYUFMT " shared (%1.1f%%).\n", 902 lastCount, postShared - lastShared, 903 (double)(postShared - lastShared) / (double) lastCount * 100.0); 904 } 905 906 // Calculate the totals. 907 POLYUNSIGNED totalSize = 0, totalShared = 0, totalRecovered = 0; 908 for (unsigned k = 0; k < NUM_BYTE_VECTORS; k++) 909 { 910 totalSize += byteVectors[k].TotalCount(); 911 POLYUNSIGNED shared = byteVectors[k].Shared(); 912 totalShared += shared; 913 totalRecovered += shared * (k+1); // Add 1 for the length word. 914 if (debugOptions & DEBUG_GC) 915 Log("GC: Share: Byte objects of size %u: %" POLYUFMT " objects %" POLYUFMT " shared\n", 916 k, byteVectors[k].TotalCount(), byteVectors[k].Shared()); 917 } 918 919 for (unsigned l = 0; l < NUM_WORD_VECTORS; l++) 920 { 921 totalSize += wordVectors[l].TotalCount(); 922 POLYUNSIGNED shared = wordVectors[l].Shared(); 923 totalShared += shared; 924 totalRecovered += shared * (l+1); 925 if (debugOptions & DEBUG_GC) 926 Log("GC: Share: Word objects of size %u: %" POLYUFMT " objects %" POLYUFMT " shared\n", 927 l, wordVectors[l].TotalCount(), wordVectors[l].Shared()); 928 } 929 930 if (debugOptions & DEBUG_GC) 931 { 932 Log("GC: Share: Total %" POLYUFMT " objects, %" POLYUFMT " shared (%1.0f%%). %" POLYUFMT " words recovered.\n", 933 totalSize, totalShared, (double)totalShared / (double)totalSize * 100.0, totalRecovered); 934 Log("GC: Share: Excluding %" POLYUFMT " large word objects %" POLYUFMT " large byte objects and %" POLYUFMT " others\n", 935 largeWordCount, largeByteCount, excludedCount); 936 } 937 938 gHeapSizeParameters.RecordSharingData(totalRecovered); 939} 940 941void GCSharingPhase(void) 942{ 943 mainThreadPhase = MTP_GCPHASESHARING; 944 gcProgressBeginSharingGC(); 945 946 GetSharing sharer; 947 948 for (std::vector<LocalMemSpace*>::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++) 949 { 950 LocalMemSpace *lSpace = *i; 951 lSpace->bitmap.ClearBits(0, lSpace->spaceSize()); 952 } 953 954 // Scan the code areas to share any constants. We don't share the code 955 // cells themselves. 956 for (std::vector<CodeSpace *>::iterator i = gMem.cSpaces.begin(); i < gMem.cSpaces.end(); i++) 957 { 958 CodeSpace *space = *i; 959 sharer.ScanAddressesInRegion(space->bottom, space->top); 960 } 961 962 if (debugOptions & DEBUG_GC) 963 Log("GC: Share: After scanning code: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n", 964 sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded); 965 966 // Process the permanent mutable areas and the code areas 967 for (std::vector<PermanentMemSpace*>::iterator i = gMem.pSpaces.begin(); i < gMem.pSpaces.end(); i++) 968 { 969 PermanentMemSpace *space = *i; 970 if (space->isMutable && ! space->byteOnly) 971 sharer.ScanAddressesInRegion(space->bottom, space->top); 972 } 973 974 if (debugOptions & DEBUG_GC) 975 Log("GC: Share: After scanning permanent: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n", 976 sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded); 977 978 // Process the RTS roots. 979 GCModules(&sharer); 980 981 if (debugOptions & DEBUG_GC) 982 Log("GC: Share: After scanning other roots: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n", 983 sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded); 984 985 gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Table"); 986 987 // Sort and merge the data. 988 sharer.SortData(); 989 990 gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Sort"); 991} 992