1/*
2 * Copyright (C) 2011 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#ifndef CopiedSpaceInlines_h
27#define CopiedSpaceInlines_h
28
29#include "CopiedBlock.h"
30#include "CopiedSpace.h"
31#include "Heap.h"
32#include "HeapBlock.h"
33#include "VM.h"
34#include <wtf/CheckedBoolean.h>
35
36namespace JSC {
37
38inline bool CopiedSpace::contains(CopiedBlock* block)
39{
40    return (!m_newGen.blockFilter.ruleOut(reinterpret_cast<Bits>(block)) || !m_oldGen.blockFilter.ruleOut(reinterpret_cast<Bits>(block)))
41        && m_blockSet.contains(block);
42}
43
44inline bool CopiedSpace::contains(void* ptr, CopiedBlock*& result)
45{
46    CopiedBlock* block = blockFor(ptr);
47    if (contains(block)) {
48        result = block;
49        return true;
50    }
51    result = 0;
52    return false;
53}
54
55inline void CopiedSpace::pin(CopiedBlock* block)
56{
57    block->pin();
58}
59
60inline void CopiedSpace::pinIfNecessary(void* opaquePointer)
61{
62    // Pointers into the copied space come in the following varieties:
63    // 1)  Pointers to the start of a span of memory. This is the most
64    //     natural though not necessarily the most common.
65    // 2)  Pointers to one value-sized (8 byte) word past the end of
66    //     a span of memory. This currently occurs with semi-butterflies
67    //     and should be fixed soon, once the other half of the
68    //     butterfly lands.
69    // 3)  Pointers to the innards arising from loop induction variable
70    //     optimizations (either manual ones or automatic, by the
71    //     compiler).
72    // 4)  Pointers to the end of a span of memory in arising from
73    //     induction variable optimizations combined with the
74    //     GC-to-compiler contract laid out in the C spec: a pointer to
75    //     the end of a span of memory must be considered to be a
76    //     pointer to that memory.
77
78    EncodedJSValue* pointer = reinterpret_cast<EncodedJSValue*>(opaquePointer);
79    CopiedBlock* block;
80
81    // Handle (1) and (3).
82    if (contains(pointer, block))
83        pin(block);
84
85    // Handle (4). We don't have to explicitly check and pin the block under this
86    // pointer because it cannot possibly point to something that cases (1) and
87    // (3) above or case (2) below wouldn't already catch.
88    pointer--;
89
90    // Handle (2)
91    pointer--;
92    if (contains(pointer, block))
93        pin(block);
94}
95
96inline void CopiedSpace::recycleEvacuatedBlock(CopiedBlock* block, HeapOperation collectionType)
97{
98    ASSERT(block);
99    ASSERT(block->canBeRecycled());
100    ASSERT(!block->m_isPinned);
101    {
102        SpinLockHolder locker(&m_toSpaceLock);
103        m_blockSet.remove(block);
104        if (collectionType == EdenCollection)
105            m_newGen.fromSpace->remove(block);
106        else
107            m_oldGen.fromSpace->remove(block);
108    }
109    m_heap->blockAllocator().deallocate(CopiedBlock::destroy(block));
110}
111
112inline void CopiedSpace::recycleBorrowedBlock(CopiedBlock* block)
113{
114    m_heap->blockAllocator().deallocate(CopiedBlock::destroy(block));
115
116    {
117        MutexLocker locker(m_loanedBlocksLock);
118        ASSERT(m_numberOfLoanedBlocks > 0);
119        ASSERT(m_inCopyingPhase);
120        m_numberOfLoanedBlocks--;
121        if (!m_numberOfLoanedBlocks)
122            m_loanedBlocksCondition.signal();
123    }
124}
125
126inline CopiedBlock* CopiedSpace::allocateBlockForCopyingPhase()
127{
128    ASSERT(m_inCopyingPhase);
129    CopiedBlock* block = CopiedBlock::createNoZeroFill(m_heap->blockAllocator().allocate<CopiedBlock>());
130
131    {
132        MutexLocker locker(m_loanedBlocksLock);
133        m_numberOfLoanedBlocks++;
134    }
135
136    ASSERT(!block->dataSize());
137    return block;
138}
139
140inline void CopiedSpace::allocateBlock()
141{
142    m_heap->collectIfNecessaryOrDefer();
143
144    m_allocator.resetCurrentBlock();
145
146    CopiedBlock* block = CopiedBlock::create(m_heap->blockAllocator().allocate<CopiedBlock>());
147
148    m_newGen.toSpace->push(block);
149    m_newGen.blockFilter.add(reinterpret_cast<Bits>(block));
150    m_blockSet.add(block);
151    m_allocator.setCurrentBlock(block);
152}
153
154inline CheckedBoolean CopiedSpace::tryAllocate(size_t bytes, void** outPtr)
155{
156    ASSERT(!m_heap->vm()->isInitializingObject());
157    ASSERT(bytes);
158
159    if (!m_allocator.tryAllocate(bytes, outPtr))
160        return tryAllocateSlowCase(bytes, outPtr);
161
162    ASSERT(*outPtr);
163    return true;
164}
165
166inline bool CopiedSpace::isOversize(size_t bytes)
167{
168    return bytes > s_maxAllocationSize;
169}
170
171inline bool CopiedSpace::isPinned(void* ptr)
172{
173    return blockFor(ptr)->m_isPinned;
174}
175
176inline CopiedBlock* CopiedSpace::blockFor(void* ptr)
177{
178    return reinterpret_cast<CopiedBlock*>(reinterpret_cast<size_t>(ptr) & s_blockMask);
179}
180
181template <HeapOperation collectionType>
182inline void CopiedSpace::startedCopying()
183{
184    DoublyLinkedList<CopiedBlock>* fromSpace;
185    DoublyLinkedList<CopiedBlock>* oversizeBlocks;
186    TinyBloomFilter* blockFilter;
187    if (collectionType == FullCollection) {
188        ASSERT(m_oldGen.fromSpace->isEmpty());
189        ASSERT(m_newGen.fromSpace->isEmpty());
190
191        m_oldGen.toSpace->append(*m_newGen.toSpace);
192        m_oldGen.oversizeBlocks.append(m_newGen.oversizeBlocks);
193
194        ASSERT(m_newGen.toSpace->isEmpty());
195        ASSERT(m_newGen.fromSpace->isEmpty());
196        ASSERT(m_newGen.oversizeBlocks.isEmpty());
197
198        std::swap(m_oldGen.fromSpace, m_oldGen.toSpace);
199        fromSpace = m_oldGen.fromSpace;
200        oversizeBlocks = &m_oldGen.oversizeBlocks;
201        blockFilter = &m_oldGen.blockFilter;
202    } else {
203        std::swap(m_newGen.fromSpace, m_newGen.toSpace);
204        fromSpace = m_newGen.fromSpace;
205        oversizeBlocks = &m_newGen.oversizeBlocks;
206        blockFilter = &m_newGen.blockFilter;
207    }
208
209    blockFilter->reset();
210    m_allocator.resetCurrentBlock();
211
212    CopiedBlock* next = 0;
213    size_t totalLiveBytes = 0;
214    size_t totalUsableBytes = 0;
215    for (CopiedBlock* block = fromSpace->head(); block; block = next) {
216        next = block->next();
217        if (!block->isPinned() && block->canBeRecycled()) {
218            recycleEvacuatedBlock(block, collectionType);
219            continue;
220        }
221        ASSERT(block->liveBytes() <= CopiedBlock::blockSize);
222        totalLiveBytes += block->liveBytes();
223        totalUsableBytes += block->payloadCapacity();
224        block->didPromote();
225    }
226
227    CopiedBlock* block = oversizeBlocks->head();
228    while (block) {
229        CopiedBlock* next = block->next();
230        if (block->isPinned()) {
231            blockFilter->add(reinterpret_cast<Bits>(block));
232            totalLiveBytes += block->payloadCapacity();
233            totalUsableBytes += block->payloadCapacity();
234            block->didPromote();
235        } else {
236            oversizeBlocks->remove(block);
237            m_blockSet.remove(block);
238            m_heap->blockAllocator().deallocateCustomSize(CopiedBlock::destroy(block));
239        }
240        block = next;
241    }
242
243    double markedSpaceBytes = m_heap->objectSpace().capacity();
244    double totalFragmentation = static_cast<double>(totalLiveBytes + markedSpaceBytes) / static_cast<double>(totalUsableBytes + markedSpaceBytes);
245    m_shouldDoCopyPhase = m_heap->operationInProgress() == EdenCollection || totalFragmentation <= Options::minHeapUtilization();
246    if (!m_shouldDoCopyPhase) {
247        if (Options::logGC())
248            dataLog("Skipped copying, ");
249        return;
250    }
251
252    if (Options::logGC())
253        dataLogF("Did copy, ");
254    ASSERT(m_shouldDoCopyPhase);
255    ASSERT(!m_numberOfLoanedBlocks);
256    ASSERT(!m_inCopyingPhase);
257    m_inCopyingPhase = true;
258}
259
260} // namespace JSC
261
262#endif // CopiedSpaceInlines_h
263
264