1/*
2    Title:      Multi-Threaded Garbage Collector - Data sharing phase
3
4    Copyright (c) 2012, 2017, 2019 David C. J. Matthews
5
6    This library is free software; you can redistribute it and/or
7    modify it under the terms of the GNU Lesser General Public
8    License as published by the Free Software Foundation; either
9    version 2.1 of the License, or (at your option) any later version.
10
11    This library is distributed in the hope that it will be useful,
12    but WITHOUT ANY WARRANTY; without even the implied warranty of
13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14    Lesser General Public License for more details.
15
16    You should have received a copy of the GNU Lesser General Public
17    License along with this library; if not, write to the Free Software
18    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19
20*/
21
22/*  GC Sharing pass.
23    This pass is invoked only if the heap sizing code detects that heap space
24    is running very short because it adds a very considerable overhead to GC.
25    It aims to reduce the size of the live data in a similar way to the data
26    sharing function PolyML.shareCommonData by merging immutable cells that
27    contain data that cannot be distinguished.
28
29    This version of the code now does a deep structure merge in a similar
30    way to the full sharing function. This code first does a full pass
31    over the heap creating lists of cells that could possibly be merged.
32    There are separate lists for byte and word objects up to a fixed
33    size.  Larger objects and other objects are not considered. Because
34    all the items in a list have the same length and type (flag bits)
35    we can use the length word to link the items in the list.  A
36    consequence of this is that positive long precision values can be
37    shared but negative values cannot.
38
39    There is a sharing function that first distributes items into a
40    hash table.  Then each hash table is sorted and as part of the
41    sorting process cells with the same contents are merged.  One
42    cell is chosen and the length words on the others are set to be
43    forwarding pointers to the chosen cell.  Hashing allows for easy
44    parallel processing.
45
46    The structure sharing code works by first sharing the byte
47    data which cannot contain pointers.  Then the word data is processed
48    to separate out "tail" cells that contain only tagged integers or
49    pointers to cells that either cannot be merged, such as mutables,
50    or those that have already been processed, such as the byte data.
51    Any pointers to shared data are updated to point to the merged cell.
52    The tail cells are then sorted and shared using the sharing function
53    and become part of the "processed" set.  This process is repeated to
54    find cells that are now tails and so on.
55
56    Compared with the full sharing code this is expensive since it
57    requires repeated scans of the list of unprocessed cells.  In
58    particular there may be cells that form loops (basically closures
59    for mutually recusive functions) and if they are present they and
60    anything that points directly or indirectly at them will never
61    be removed from the list.  We stop when it appears that we are
62    not making progress and simply do a final bit-wise share of the
63    remainder.
64
65    This now uses the forwarding pointer both to indicate that a cell
66    shares with another and also to link together cells that have yet
67    to be tested for sharing.  To detect the difference the bitmap is
68    used.  The initial scan to create the sharing chains sets the bit
69    for each visited cell so at the start of the sharing phase all
70    reachable cells will be marked.  We remove the mark if the cell
71    is to be removed.  This requires the bitmap to be locked.
72*/
73#ifdef HAVE_CONFIG_H
74#include "config.h"
75#elif defined(_WIN32)
76#include "winconfig.h"
77#else
78#error "No configuration file"
79#endif
80
81#ifdef HAVE_ASSERT_H
82#include <assert.h>
83#define ASSERT(x)   assert(x)
84#else
85#define ASSERT(x)
86#endif
87
88#ifdef HAVE_STRING_H
89#include <string.h>
90#endif
91
92#ifdef HAVE_STDLIB_H
93#include <stdlib.h>
94#endif
95
96#include "globals.h"
97#include "processes.h"
98#include "gc.h"
99#include "scanaddrs.h"
100#include "bitmap.h"
101#include "memmgr.h"
102#include "diagnostics.h"
103#include "gctaskfarm.h"
104#include "heapsizing.h"
105#include "gc_progress.h"
106
107#ifdef POLYML32IN64
108#define ENDOFLIST ((PolyObject*)globalHeapBase)
109#else
110#define ENDOFLIST 0
111#endif
112
113// Set the forwarding so that references to objToSet will be forwarded to
114// objToShare.  objToSet will be garbage.
115void shareWith(PolyObject *objToSet, PolyObject *objToShare)
116{
117    // We need to remove the bit from this so that we know it's not
118    // a share chain.
119    PolyWord *lengthWord = ((PolyWord*)objToSet) - 1;
120    LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
121    ASSERT(space);
122    PLocker locker(&space->bitmapLock);
123    ASSERT(space->bitmap.TestBit(space->wordNo(lengthWord)));
124    space->bitmap.ClearBit(space->wordNo(lengthWord));
125    // Actually do the forwarding
126    objToSet->SetForwardingPtr(objToShare);
127}
128
129// When we find an address it could be a cell that:
130// 1. is never processed or one that is the copy to be retained,
131// 2. has been merged with another and contains a forwarding pointer or
132// 3. has not yet been processed.
133typedef enum { REALOBJECT, FORWARDED, CHAINED } objectState;
134
135objectState getObjectState(PolyObject *p)
136{
137    PolyWord *lengthWord = ((PolyWord*)p) - 1;
138    LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
139    if (space == 0)
140        return REALOBJECT; // May be the address of a permanent or something else.
141    PLocker locker(&space->bitmapLock);
142    if (!p->ContainsForwardingPtr())
143        return REALOBJECT;
144    if (space->bitmap.TestBit(space->wordNo(lengthWord)))
145        return CHAINED;
146    else return FORWARDED;
147
148}
149
150class ObjEntry
151{
152public:
153    ObjEntry(): objList(ENDOFLIST), objCount(0), shareCount(0) {}
154    PolyObject *objList;
155    POLYUNSIGNED objCount;
156    POLYUNSIGNED shareCount;
157};
158
159// There is an instance of this class for each combination of size and
160// word/byte.
161class SortVector
162{
163public:
164    SortVector(): totalCount(0), carryOver(0) {}
165    void AddToVector(PolyObject *obj, POLYUNSIGNED length);
166    void SortData(void);
167    POLYUNSIGNED TotalCount() const { return totalCount; }
168    POLYUNSIGNED CurrentCount() const { return baseObject.objCount; }
169    POLYUNSIGNED Shared() const;
170    void SetLengthWord(POLYUNSIGNED l) { lengthWord = l; }
171    POLYUNSIGNED CarryOver() const { return carryOver; }
172
173    static void hashAndSortAllTask(GCTaskId*, void *a, void *b);
174    static void sharingTask(GCTaskId*, void *a, void *b);
175    static void wordDataTask(GCTaskId*, void *a, void *b);
176
177private:
178    void sortList(PolyObject *head, POLYUNSIGNED nItems, POLYUNSIGNED &count);
179
180    ObjEntry baseObject, processObjects[256];
181    POLYUNSIGNED totalCount;
182    POLYUNSIGNED lengthWord;
183    POLYUNSIGNED carryOver;
184};
185
186POLYUNSIGNED SortVector::Shared() const
187{
188    // Add all the sharing counts
189    POLYUNSIGNED shareCount = baseObject.shareCount;
190    for (unsigned i = 0; i < 256; i++)
191        shareCount += processObjects[i].shareCount;
192    return shareCount;
193}
194
195void SortVector::AddToVector(PolyObject *obj, POLYUNSIGNED length)
196{
197    obj->SetForwardingPtr(baseObject.objList);
198    baseObject.objList = obj;
199    baseObject.objCount++;
200    totalCount++;
201}
202
203// The number of byte and word entries.
204// Objects of up to and including this size are shared.
205// Byte objects include strings so it is more likely that
206// larger objects will share.  Word objects that share
207// are much more likely to be 2 or 3 words.
208#define NUM_BYTE_VECTORS    23
209#define NUM_WORD_VECTORS    11
210
211// The stack is allocated as a series of blocks chained together.
212#define RSTACK_SEGMENT_SIZE 1000
213
214class RScanStack {
215public:
216    RScanStack() : nextStack(0), lastStack(0), sp(0) {}
217    ~RScanStack() { delete(nextStack); }
218
219    RScanStack *nextStack;
220    RScanStack *lastStack;
221    unsigned sp;
222    struct { PolyObject *obj; PolyWord *base; } stack[RSTACK_SEGMENT_SIZE];
223};
224
225class RecursiveScanWithStack : public ScanAddress
226{
227public:
228    RecursiveScanWithStack() : stack(0) {}
229    ~RecursiveScanWithStack() { delete(stack); }
230
231public:
232    virtual PolyObject *ScanObjectAddress(PolyObject *base);
233    virtual void ScanAddressesInObject(PolyObject *base, POLYUNSIGNED lengthWord);
234    // Have to redefine this for some reason.
235    void ScanAddressesInObject(PolyObject *base)
236    {
237        ScanAddressesInObject(base, base->LengthWord());
238    }
239
240protected:
241    // Test the word at the location to see if it points to
242    // something that may have to be scanned.  We pass in the
243    // pointer here because the called may side-effect it.
244    virtual bool TestForScan(PolyWord *) = 0;
245    // If we are definitely scanning the address we mark it.
246    virtual void MarkAsScanning(PolyObject *) = 0;
247    // Called when the object has been completed.
248    virtual void Completed(PolyObject *) {}
249
250protected:
251    void PushToStack(PolyObject *obj, PolyWord *base);
252    void PopFromStack(PolyObject *&obj, PolyWord *&base);
253
254    bool StackIsEmpty(void)
255    {
256        return stack == 0 || (stack->sp == 0 && stack->lastStack == 0);
257    }
258
259    RScanStack *stack;
260};
261
262// This gets called in two circumstances.  It may be called for the roots
263// in which case the stack will be empty and we want to process it completely
264// or it is called for a constant address in which case it will have been
265// called from RecursiveScan::ScanAddressesInObject and that can process
266// any addresses.
267PolyObject *RecursiveScanWithStack::ScanObjectAddress(PolyObject *obj)
268{
269    PolyWord pWord = obj;
270    // Test to see if this needs to be scanned.
271    // It may update the word.
272    bool test = TestForScan(&pWord);
273    obj = pWord.AsObjPtr();
274
275    if (test)
276    {
277        MarkAsScanning(obj);
278        if (obj->IsByteObject())
279            Completed(obj); // Don't need to put it on the stack
280                            // If we already have something on the stack we must being called
281                            // recursively to process a constant in a code segment.  Just push
282                            // it on the stack and let the caller deal with it.
283        else if (StackIsEmpty())
284            RecursiveScanWithStack::ScanAddressesInObject(obj, obj->LengthWord());
285        else
286            PushToStack(obj, (PolyWord*)obj);
287    }
288
289    return obj;
290}
291
292// This is called via ScanAddressesInRegion to process the permanent mutables.  It is
293// also called from ScanObjectAddress to process root addresses.
294// It processes all the addresses reachable from the object.
295// This is almost the same as MTGCProcessMarkPointers::ScanAddressesInObject.
296void RecursiveScanWithStack::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
297{
298    if (OBJ_IS_BYTE_OBJECT(lengthWord))
299        return; // Ignore byte cells and don't call Completed on them
300
301    PolyWord *baseAddr = (PolyWord*)obj;
302
303    while (true)
304    {
305        ASSERT(OBJ_IS_LENGTH(lengthWord));
306
307        // Get the length and base address.  N.B.  If this is a code segment
308        // these will be side-effected by GetConstSegmentForCode.
309        POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
310
311        if (OBJ_IS_CODE_OBJECT(lengthWord) || OBJ_IS_CLOSURE_OBJECT(lengthWord))
312        {
313            // It's better to process the whole code object in one go.
314            // For the moment do that for closure objects as well.
315            ScanAddress::ScanAddressesInObject(obj, lengthWord);
316            length = 0; // Finished
317        }
318
319        // else it's a normal object,
320
321        // If there are only two addresses in this cell that need to be
322        // followed we follow them immediately and treat this cell as done.
323        // If there are more than two we push the address of this cell on
324        // the stack, follow the first address and then rescan it.  That way
325        // list cells are processed once only but we don't overflow the
326        // stack by pushing all the addresses in a very large vector.
327        PolyWord *endWord = (PolyWord*)obj + length;
328        PolyObject *firstWord = 0;
329        PolyObject *secondWord = 0;
330        PolyWord *restartFrom = baseAddr;
331
332        while (baseAddr != endWord)
333        {
334            PolyWord wordAt = *baseAddr;
335
336            if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0))
337            {
338                // Normal address.  We can have words of all zeros at least in the
339                // situation where we have a partially constructed code segment where
340                // the constants at the end of the code have not yet been filled in.
341                if (TestForScan(baseAddr)) // Test value at baseAddr (may side-effect it)
342                {
343                    PolyObject *wObj = (*baseAddr).AsObjPtr();
344                    if (wObj->IsByteObject())
345                    {
346                        // Can do this now - don't need to push it
347                        MarkAsScanning(wObj);
348                        Completed(wObj);
349                    }
350                    else if (firstWord == 0)
351                    {
352                        firstWord = wObj;
353                        // We mark the word immediately.  We can have
354                        // two words in an object that are the same
355                        // and we don't want to process it again.
356                        MarkAsScanning(firstWord);
357                    }
358                    else if (secondWord == 0)
359                    {
360                        secondWord = wObj;
361                        restartFrom = baseAddr;
362                    }
363                    else break;  // More than two words.
364                }
365            }
366            baseAddr++;
367        }
368
369        if (baseAddr == endWord)
370        {
371            // We have done everything except possibly firstWord and secondWord.
372            // Note: Unfortunately the way that ScanAddressesInRegion works means that
373            // we call Completed on the addresses of cells in the permanent areas without
374            // having called TestForScan.
375            Completed(obj);
376            if (secondWord != 0)
377            {
378                MarkAsScanning(secondWord);
379                // Put this on the stack.  If this is a list node we will be
380                // pushing the tail.
381                PushToStack(secondWord, (PolyWord*)secondWord);
382            }
383        }
384        else // Put this back on the stack while we process the first word
385            PushToStack(obj, restartFrom);
386
387        if (firstWord != 0)
388        {
389            // Process it immediately.
390            obj = firstWord;
391            baseAddr = (PolyWord*)obj;
392        }
393        else if (StackIsEmpty())
394            return;
395        else
396            PopFromStack(obj, baseAddr);
397
398        lengthWord = obj->LengthWord();
399    }
400}
401
402void RecursiveScanWithStack::PushToStack(PolyObject *obj, PolyWord *base)
403{
404    if (stack == 0 || stack->sp == RSTACK_SEGMENT_SIZE)
405    {
406        if (stack != 0 && stack->nextStack != 0)
407            stack = stack->nextStack;
408        else
409        {
410            // Need a new segment
411            try {
412                RScanStack *s = new RScanStack;
413                s->lastStack = stack;
414                if (stack != 0)
415                    stack->nextStack = s;
416                stack = s;
417            }
418            catch (std::bad_alloc &) {
419                // Ignore stack overflow
420                return;
421            }
422        }
423    }
424    stack->stack[stack->sp].obj = obj;
425    stack->stack[stack->sp].base = base;
426    stack->sp++;
427}
428
429void RecursiveScanWithStack::PopFromStack(PolyObject *&obj, PolyWord *&base)
430{
431    if (stack->sp == 0)
432    {
433        // Chain to the previous stack if any
434        ASSERT(stack->lastStack != 0);
435        // Before we do, delete any further one to free some memory
436        delete(stack->nextStack);
437        stack->nextStack = 0;
438        stack = stack->lastStack;
439        ASSERT(stack->sp == RSTACK_SEGMENT_SIZE);
440    }
441    --stack->sp;
442    obj = stack->stack[stack->sp].obj;
443    base = stack->stack[stack->sp].base;
444}
445
446class GetSharing: public RecursiveScanWithStack
447{
448public:
449    GetSharing();
450    void SortData(void);
451    static void shareByteData(GCTaskId *, void *, void *);
452    static void shareWordData(GCTaskId *, void *, void *);
453    static void shareRemainingWordData(GCTaskId *, void *, void *);
454
455    virtual PolyObject *ScanObjectAddress(PolyObject *obj);
456
457protected:
458    virtual bool TestForScan(PolyWord *);
459    virtual void MarkAsScanning(PolyObject *);
460    virtual void Completed(PolyObject *);
461
462private:
463    // The head of chains of cells of the same size
464    SortVector byteVectors[NUM_BYTE_VECTORS];
465    SortVector wordVectors[NUM_WORD_VECTORS];
466
467    POLYUNSIGNED largeWordCount, largeByteCount, excludedCount;
468public:
469    POLYUNSIGNED totalVisited, byteAdded, wordAdded, totalSize;
470};
471
472GetSharing::GetSharing()
473{
474    for (unsigned i = 0; i < NUM_BYTE_VECTORS; i++)
475        byteVectors[i].SetLengthWord((POLYUNSIGNED)i | _OBJ_BYTE_OBJ);
476
477    for (unsigned j = 0; j < NUM_WORD_VECTORS; j++)
478        wordVectors[j].SetLengthWord(j);
479
480    largeWordCount = largeByteCount = excludedCount = 0;
481    totalVisited = byteAdded = wordAdded = totalSize = 0;
482}
483
484// This is called for roots and also for constants in the constant area.
485// If we have a code address we MUSTN't call RecursiveScan::ScanObjectAddress
486// because that turns the address into a PolyWord and doesn't work in 32-in-64.
487// We process the code area explicitly so we can simply skip code addresses.
488PolyObject *GetSharing::ScanObjectAddress(PolyObject *obj)
489{
490    LocalMemSpace *sp = gMem.LocalSpaceForAddress((PolyWord*)obj - 1);
491
492    if (sp == 0)
493        return obj;
494
495    return RecursiveScanWithStack::ScanObjectAddress(obj);
496}
497
498bool GetSharing::TestForScan(PolyWord *pt)
499{
500    PolyObject *obj;
501
502    // This may be a forwarding pointer left over from a minor GC that did
503    // not complete or it may be a sharing chain pointer that we've set up.
504    while (1)
505    {
506        PolyWord p = *pt;
507        ASSERT(p.IsDataPtr());
508        obj = p.AsObjPtr();
509        PolyWord *lengthWord = ((PolyWord*)obj) - 1;
510        LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
511        if (space == 0)
512            return false; // Ignore it if it points to a permanent area
513
514        if (space->bitmap.TestBit(space->wordNo(lengthWord)))
515            return false;
516
517        // Wasn't marked - must be a forwarding pointer.
518        if (obj->ContainsForwardingPtr())
519        {
520            obj = obj->GetForwardingPtr();
521            *pt = obj;
522        }
523        else break;
524    }
525
526    ASSERT(obj->ContainsNormalLengthWord());
527
528    totalVisited += 1;
529    totalSize += obj->Length() + 1;
530
531    return true;
532}
533
534void GetSharing::MarkAsScanning(PolyObject *obj)
535{
536    ASSERT(obj->ContainsNormalLengthWord());
537    PolyWord *lengthWord = ((PolyWord*)obj) - 1;
538    LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
539    ASSERT(! space->bitmap.TestBit(space->wordNo(lengthWord)));
540    space->bitmap.SetBit(space->wordNo(lengthWord));
541}
542
543void GetSharing::Completed(PolyObject *obj)
544{
545    // We mustn't include cells in the permanent area.
546    // We scan the permanent mutable areas for local addresses
547    // but we mustn't add the cells themselves.  Normally they
548    // will be mutable so would be ignored but cells that have been
549    // locked will now be immutable.  The test in TestForScan is bypassed
550    // by ScanAddressesInRegion.
551    PolyWord *lengthWord = ((PolyWord*)obj) - 1;
552    if (gMem.LocalSpaceForAddress(lengthWord) == 0)
553        return;
554
555    POLYUNSIGNED L = obj->LengthWord();
556    // We have tables for word objects and byte objects
557    // We chain entries together using the length word so it
558    // is important that we only do this for objects that
559    // have no other bits in the header, such as the sign bit.
560    if ((L & _OBJ_PRIVATE_FLAGS_MASK) == 0)
561    {
562        POLYUNSIGNED length = obj->Length();
563        if (length < NUM_WORD_VECTORS)
564            wordVectors[length].AddToVector(obj, length);
565        else largeWordCount++;
566        wordAdded++;
567    }
568    else if ((L & _OBJ_PRIVATE_FLAGS_MASK) == _OBJ_BYTE_OBJ)
569    {
570        POLYUNSIGNED length = obj->Length();
571        if (length < NUM_BYTE_VECTORS)
572            byteVectors[length].AddToVector(obj, length);
573        else largeByteCount++;
574        byteAdded++;
575    }
576    else if (! OBJ_IS_CODE_OBJECT(L) && ! OBJ_IS_MUTABLE_OBJECT(L))
577        excludedCount++; // Code and mutables can't be shared - see what could be
578    // TODO: We don't attempt to share closure cells in 32-in-64.
579}
580
581// Quicksort the list to detect cells with the same content.  These are made
582// to share and removed from further sorting.
583void SortVector::sortList(PolyObject *head, POLYUNSIGNED nItems, POLYUNSIGNED &shareCount)
584{
585    while (nItems > 2)
586    {
587        size_t bytesToCompare = OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord);
588        PolyObject *median = head;
589        head = head->GetForwardingPtr();
590        median->SetLengthWord(lengthWord);
591        PolyObject *left = ENDOFLIST, *right = ENDOFLIST;
592        POLYUNSIGNED leftCount = 0, rightCount = 0;
593        while (head != ENDOFLIST)
594        {
595            PolyObject *next = head->GetForwardingPtr();
596            int res = memcmp(median, head, bytesToCompare);
597            if (res == 0)
598            {
599                // Equal - they can share
600                shareWith(head, median);
601                shareCount++;
602            }
603            else if (res < 0)
604            {
605                head->SetForwardingPtr(left);
606                left = head;
607                leftCount++;
608            }
609            else
610            {
611                head->SetForwardingPtr(right);
612                right = head;
613                rightCount++;
614            }
615            head = next;
616        }
617        // We can now drop the median and anything that shares with it.
618        // Process the smaller partition recursively and the larger by
619        // tail recursion.
620        if (leftCount < rightCount)
621        {
622            sortList(left, leftCount, shareCount);
623            head = right;
624            nItems = rightCount;
625        }
626        else
627        {
628            sortList(right, rightCount, shareCount);
629            head = left;
630            nItems = leftCount;
631        }
632    }
633    if (nItems == 1)
634        head->SetLengthWord(lengthWord);
635    else if (nItems == 2)
636    {
637        PolyObject *next = head->GetForwardingPtr();
638        head->SetLengthWord(lengthWord);
639        if (memcmp(head, next, OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord)) == 0)
640        {
641            shareWith(next, head);
642            shareCount++;
643        }
644        else next->SetLengthWord(lengthWord);
645    }
646}
647
648void SortVector::sharingTask(GCTaskId*, void *a, void *b)
649{
650    SortVector *s = (SortVector *)a;
651    ObjEntry *o = (ObjEntry*)b;
652    s->sortList(o->objList, o->objCount, o->shareCount);
653}
654
655// Process one level of the word data.
656// N.B.  The length words are updated without any locking.  This is safe
657// because all length words are initially chain entries and a chain entry
658// can be replaced by another chain entry, a forwarding pointer or a normal
659// length word.  Forwarding pointers and normal length words are only ever
660// set once.  There is a small chance that we could lose some sharing as a
661// result of a race condition if a thread defers an object because it
662// contains a pointer with a chain entry and later sees an otherwise
663// equal object where another thread has replaced the chain with a
664// normal address, adds it to the list for immediate processing and
665// so never compares the two.
666void SortVector::wordDataTask(GCTaskId*, void *a, void *)
667{
668    SortVector *s = (SortVector*)a;
669    // Partition the objects between those that have pointers to objects that are
670    // still to be processed and those that have been processed.
671    if (s->baseObject.objList == ENDOFLIST)
672        return;
673    PolyObject *h = s->baseObject.objList;
674    s->baseObject.objList = ENDOFLIST;
675    s->baseObject.objCount = 0;
676    POLYUNSIGNED words = OBJ_OBJECT_LENGTH(s->lengthWord);
677    s->carryOver = 0;
678
679    for (unsigned i = 0; i < 256; i++)
680    {
681        // Clear the entries in the hash table but not the sharing count.
682        s->processObjects[i].objList = ENDOFLIST;
683        s->processObjects[i].objCount = 0;
684    }
685
686    while (h != ENDOFLIST)
687    {
688        PolyObject *next = h->GetForwardingPtr();
689        bool deferred = false;
690        for (POLYUNSIGNED i = 0; i < words; i++)
691        {
692            PolyWord w = h->Get(i);
693            if (w.IsDataPtr())
694            {
695                PolyObject *p = w.AsObjPtr();
696                objectState state = getObjectState(p);
697                if (state == FORWARDED)
698                {
699                    // Update the addresses of objects that have been merged
700                    h->Set(i, p->GetForwardingPtr());
701                    s->carryOver++;
702                    break;
703                }
704                else if (state == CHAINED)
705                {
706                    // If it is still to be shared leave it
707                    deferred = true;
708                    break; // from the loop
709                }
710            }
711        }
712        if (deferred)
713        {
714            // We can't do it yet: add it back to the list
715            h->SetForwardingPtr(s->baseObject.objList);
716            s->baseObject.objList = h;
717            s->baseObject.objCount++;
718        }
719        else
720        {
721            // Add it to the hash table.
722            unsigned char hash = 0;
723            for (POLYUNSIGNED i = 0; i < words*sizeof(PolyWord); i++)
724                hash += h->AsBytePtr()[i];
725            h->SetForwardingPtr(s->processObjects[hash].objList);
726            s->processObjects[hash].objList = h;
727            s->processObjects[hash].objCount++;
728        }
729        h = next;
730    }
731    s->SortData();
732}
733
734// Sort the entries in the hash table.
735void SortVector::SortData()
736{
737    for (unsigned j = 0; j < 256; j++)
738    {
739        ObjEntry *oentry = &processObjects[j];
740        // Sort this entry.  If it's very small just process it now.
741        switch (oentry->objCount)
742        {
743        case 0: break; // Nothing there
744
745        case 1: // Singleton - just restore the length word
746            oentry->objList->SetLengthWord(lengthWord);
747            break;
748
749        case 2:
750            {
751                // Two items - process now
752                PolyObject *obj1 = oentry->objList;
753                PolyObject *obj2 = obj1->GetForwardingPtr();
754                obj1->SetLengthWord(lengthWord);
755                if (memcmp(obj1, obj2, OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord)) == 0)
756                {
757                    shareWith(obj2, obj1);
758                    oentry->shareCount++;
759                }
760                else obj2->SetLengthWord(lengthWord);
761                break;
762            }
763
764        default:
765            gpTaskFarm->AddWorkOrRunNow(sharingTask, this, oentry);
766        }
767    }
768}
769
770void SortVector::hashAndSortAllTask(GCTaskId*, void *a, void *b)
771{
772    SortVector *s = (SortVector *)a;
773    // Hash the contents of the base object then sort them.
774    for (unsigned i = 0; i < 256; i++)
775    {
776        // Clear the entries in the hash table but not the sharing count.
777        s->processObjects[i].objList = ENDOFLIST;
778        s->processObjects[i].objCount = 0;
779    }
780    PolyObject *h = s->baseObject.objList;
781    POLYUNSIGNED bytes = OBJ_OBJECT_LENGTH(s->lengthWord)*sizeof(PolyWord);
782    while (h != ENDOFLIST)
783    {
784        PolyObject *next = h->GetForwardingPtr();
785        unsigned char hash = 0;
786        for (POLYUNSIGNED j = 0; j < bytes; j++)
787            hash += h->AsBytePtr()[j];
788        h->SetForwardingPtr(s->processObjects[hash].objList);
789        s->processObjects[hash].objList = h;
790        s->processObjects[hash].objCount++;
791        h = next;
792    }
793    s->SortData();
794}
795
796// Look for sharing between byte data.  These cannot contain pointers
797// so they can all be processed together.
798void GetSharing::shareByteData(GCTaskId *, void *a, void *)
799{
800    GetSharing *s = (GetSharing*)a;
801    for (unsigned i = 0; i < NUM_BYTE_VECTORS; i++)
802    {
803        if (s->byteVectors[i].CurrentCount() != 0)
804            gpTaskFarm->AddWorkOrRunNow(SortVector::hashAndSortAllTask, &(s->byteVectors[i]), 0);
805    }
806}
807
808// Process word data at this particular level
809void GetSharing::shareWordData(GCTaskId *, void *a, void *)
810{
811    GetSharing *s = (GetSharing*)a;
812    for (unsigned i = 0; i < NUM_WORD_VECTORS; i++)
813    {
814        if (s->wordVectors[i].CurrentCount() != 0)
815            gpTaskFarm->AddWorkOrRunNow(SortVector::wordDataTask, &(s->wordVectors[i]), 0);
816    }
817}
818
819// Share any entries left.
820void GetSharing::shareRemainingWordData(GCTaskId *, void *a, void *)
821{
822    GetSharing *s = (GetSharing*)a;
823    for (unsigned i = 0; i < NUM_WORD_VECTORS; i++)
824    {
825        if (s->wordVectors[i].CurrentCount() != 0)
826            gpTaskFarm->AddWorkOrRunNow(SortVector::hashAndSortAllTask, &(s->wordVectors[i]), 0);
827    }
828}
829
830void GetSharing::SortData()
831{
832    // First process the byte objects.  They cannot contain pointers.
833    // We create a task to do this so that we never have more threads
834    // running than given with --gcthreads.
835    gpTaskFarm->AddWorkOrRunNow(shareByteData, this, 0);
836    gpTaskFarm->WaitForCompletion();
837
838    // Word data may contain pointers to other objects.  If an object
839    // has been processed its header will contain either a normal length
840    // word or a forwarding pointer if it shares.  We can process an
841    // object if every word in it is either a tagged integer or an
842    // address we have already processed.  This works provided there
843    // are no loops so when we reach a stage where we are unable to
844    // process anything we simply run a final scan on the remainder.
845    // Loops can arise from the closures of mutually recursive functions.
846
847    // Now process the word entries until we have nothing left apart from loops.
848    POLYUNSIGNED lastCount = 0, lastShared = 0;
849    for (unsigned n = 0; n < NUM_WORD_VECTORS; n++)
850        lastCount += wordVectors[n].CurrentCount();
851
852    for(unsigned pass = 1; lastCount != 0; pass++)
853    {
854        gpTaskFarm->AddWorkOrRunNow(shareWordData, this, 0);
855        gpTaskFarm->WaitForCompletion();
856
857        // At each stage check that we have removed some items
858        // from the lists.
859        POLYUNSIGNED postCount = 0, postShared = 0, carryOver = 0;
860        for (unsigned i = 0; i < NUM_WORD_VECTORS; i++)
861        {
862            postCount += wordVectors[i].CurrentCount();
863            postShared += wordVectors[i].Shared();
864            carryOver += wordVectors[i].CarryOver();
865        }
866
867        if (debugOptions & DEBUG_GC)
868            Log("GC: Share: Pass %u: %" POLYUFMT " removed (%1.1f%%) %" POLYUFMT " shared (%1.1f%%) %" POLYUFMT " remain. %" POLYUFMT " entries updated (%1.1f%%).\n",
869                pass, lastCount-postCount, (double)(lastCount-postCount) / (double) lastCount * 100.0,
870                postShared - lastShared, (double)(postShared - lastShared) / (double) (lastCount-postCount) * 100.0,
871                postCount, carryOver, (double)carryOver / (double)(lastCount-postCount) * 100.0);
872
873		gcProgressSetPercent((unsigned)((double)(totalVisited - postCount) / (double)totalVisited * 100.0));
874
875        // Condition for exiting the loop.  There are some heuristics here.
876        // If we remove less than 10% in a pass it's probably not worth continuing
877        // unless the carry over is large.  The "carry over" is the number of words updated as
878        // a result of the last pass.  It represents the extra sharing we gained in this pass
879        // as a result of the last pass.  If there are deep data structures that can be shared
880        // we get better sharing with more passes.  If the data structures are shallow we will
881        // get as much sharing by just running the final pass.  The first pass only carries
882        // over any sharing from the byte objects so we need to run at least one more before
883        // checking the carry over.
884        if (pass > 1 && (lastCount - postCount) * 10 < lastCount && (carryOver*2 < (lastCount-postCount) || (lastCount - postCount) * 1000 < lastCount ))
885            break;
886
887        lastCount = postCount;
888        lastShared = postShared;
889    }
890
891    // Process any remaining entries.  There may be loops.
892    gpTaskFarm->AddWorkOrRunNow(shareRemainingWordData, this, 0);
893    gpTaskFarm->WaitForCompletion();
894
895    if (debugOptions & DEBUG_GC)
896    {
897        POLYUNSIGNED postShared = 0;
898        for (unsigned i = 0; i < NUM_WORD_VECTORS; i++)
899            postShared += wordVectors[i].Shared();
900        if (debugOptions & DEBUG_GC)
901            Log("GC: Share: Final pass %" POLYUFMT " removed %" POLYUFMT " shared (%1.1f%%).\n",
902                lastCount, postShared - lastShared,
903                (double)(postShared - lastShared) / (double) lastCount * 100.0);
904    }
905
906    // Calculate the totals.
907    POLYUNSIGNED totalSize = 0, totalShared = 0, totalRecovered = 0;
908    for (unsigned k = 0; k < NUM_BYTE_VECTORS; k++)
909    {
910        totalSize += byteVectors[k].TotalCount();
911        POLYUNSIGNED shared = byteVectors[k].Shared();
912        totalShared += shared;
913        totalRecovered += shared * (k+1); // Add 1 for the length word.
914        if (debugOptions & DEBUG_GC)
915            Log("GC: Share: Byte objects of size %u: %" POLYUFMT " objects %" POLYUFMT " shared\n",
916                k, byteVectors[k].TotalCount(), byteVectors[k].Shared());
917    }
918
919    for (unsigned l = 0; l < NUM_WORD_VECTORS; l++)
920    {
921        totalSize += wordVectors[l].TotalCount();
922        POLYUNSIGNED shared = wordVectors[l].Shared();
923        totalShared += shared;
924        totalRecovered += shared * (l+1);
925        if (debugOptions & DEBUG_GC)
926            Log("GC: Share: Word objects of size %u: %" POLYUFMT " objects %" POLYUFMT " shared\n",
927                l, wordVectors[l].TotalCount(), wordVectors[l].Shared());
928    }
929
930    if (debugOptions & DEBUG_GC)
931    {
932        Log("GC: Share: Total %" POLYUFMT " objects, %" POLYUFMT " shared (%1.0f%%).  %" POLYUFMT " words recovered.\n",
933            totalSize, totalShared, (double)totalShared / (double)totalSize * 100.0, totalRecovered);
934        Log("GC: Share: Excluding %" POLYUFMT " large word objects %" POLYUFMT " large byte objects and %" POLYUFMT " others\n",
935            largeWordCount, largeByteCount, excludedCount);
936    }
937
938    gHeapSizeParameters.RecordSharingData(totalRecovered);
939}
940
941void GCSharingPhase(void)
942{
943    mainThreadPhase = MTP_GCPHASESHARING;
944    gcProgressBeginSharingGC();
945
946    GetSharing sharer;
947
948    for (std::vector<LocalMemSpace*>::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++)
949    {
950        LocalMemSpace *lSpace = *i;
951        lSpace->bitmap.ClearBits(0, lSpace->spaceSize());
952    }
953
954    // Scan the code areas to share any constants.  We don't share the code
955    // cells themselves.
956    for (std::vector<CodeSpace *>::iterator i = gMem.cSpaces.begin(); i < gMem.cSpaces.end(); i++)
957    {
958        CodeSpace *space = *i;
959        sharer.ScanAddressesInRegion(space->bottom, space->top);
960    }
961
962    if (debugOptions & DEBUG_GC)
963        Log("GC: Share: After scanning code: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n",
964            sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded);
965
966    // Process the permanent mutable areas and the code areas
967    for (std::vector<PermanentMemSpace*>::iterator i = gMem.pSpaces.begin(); i < gMem.pSpaces.end(); i++)
968    {
969        PermanentMemSpace *space = *i;
970        if (space->isMutable && ! space->byteOnly)
971            sharer.ScanAddressesInRegion(space->bottom, space->top);
972    }
973
974    if (debugOptions & DEBUG_GC)
975        Log("GC: Share: After scanning permanent: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n",
976            sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded);
977
978    // Process the RTS roots.
979    GCModules(&sharer);
980
981    if (debugOptions & DEBUG_GC)
982        Log("GC: Share: After scanning other roots: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n",
983            sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded);
984
985    gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Table");
986
987    // Sort and merge the data.
988    sharer.SortData();
989
990    gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Sort");
991}
992