NewInstanceStub.java revision 12968:4d8a004e5c6d 
1/*
2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23package org.graalvm.compiler.hotspot.stubs;
24
25import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
27import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
28import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
29import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
30import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
31import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
32import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
33import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
34import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
35import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
36import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
37import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
38import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
39import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
40import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
41import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
42import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
43import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
44import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
45import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
46import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
47import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
48import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
49import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
50import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
51import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
52import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
53import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
54import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
55import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
56import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
57import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
58import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
59import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
60import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
61import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
62import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
63import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
64import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
65import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
66import org.graalvm.compiler.api.replacements.Fold;
67import org.graalvm.compiler.api.replacements.Snippet;
68import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
69import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
70import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
71import org.graalvm.compiler.graph.Node.NodeIntrinsic;
72import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
73import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
74import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
75import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
76import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
77import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
78import org.graalvm.compiler.hotspot.word.KlassPointer;
79import org.graalvm.compiler.nodes.ConstantNode;
80import org.graalvm.compiler.options.OptionValues;
81import org.graalvm.compiler.word.Word;
82
83import jdk.vm.ci.code.Register;
84import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
85import jdk.vm.ci.meta.JavaKind;
86
87/**
88 * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
89 * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
90 * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
91 * runtime for to complete the allocation.
92 */
93public class NewInstanceStub extends SnippetStub {
94
95    public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
96        super("newInstance", options, providers, linkage);
97    }
98
99    @Override
100    protected Object[] makeConstArgs() {
101        HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
102        int count = method.getSignature().getParameterCount(false);
103        Object[] args = new Object[count];
104        assert checkConstArg(1, "intArrayHub");
105        assert checkConstArg(2, "threadRegister");
106        assert checkConstArg(3, "options");
107        args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
108        args[2] = providers.getRegisters().getThreadRegister();
109        args[3] = options;
110        return args;
111    }
112
113    private static Word allocate(Word thread, int size) {
114        Word top = readTlabTop(thread);
115        Word end = readTlabEnd(thread);
116        Word newTop = top.add(size);
117        /*
118         * this check might lead to problems if the TLAB is within 16GB of the address space end
119         * (checked in c++ code)
120         */
121        if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
122            writeTlabTop(thread, newTop);
123            return top;
124        }
125        return Word.zero();
126    }
127
128    @Fold
129    static boolean logging(OptionValues options) {
130        return StubOptions.TraceNewInstanceStub.getValue(options);
131    }
132
133    /**
134     * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
135     * -XX:-UseTLAB).
136     *
137     * @param hub the hub of the object to be allocated
138     * @param intArrayHub the hub for {@code int[].class}
139     */
140    @Snippet
141    private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) {
142        /*
143         * The type is known to be an instance so Klass::_layout_helper is the instance size as a
144         * raw number
145         */
146        Word thread = registerAsWord(threadRegister);
147        boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
148        if (!forceSlowPath(options) && inlineContiguousAllocationSupported) {
149            if (isInstanceKlassFullyInitialized(hub)) {
150                int sizeInBytes = readLayoutHelper(hub);
151                Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
152                if (memory.notEqual(0)) {
153                    Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
154                    NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
155                    return verifyObject(memory.toObject());
156                }
157            }
158        }
159
160        if (logging(options)) {
161            printf("newInstance: calling new_instance_c\n");
162        }
163
164        newInstanceC(NEW_INSTANCE_C, thread, hub);
165        handlePendingException(thread, true);
166        return verifyObject(getAndClearObjectResult(thread));
167    }
168
169    /**
170     * Attempts to refill the current thread's TLAB and retries the allocation.
171     *
172     * @param intArrayHub the hub for {@code int[].class}
173     * @param sizeInBytes the size of the allocation
174     * @param log specifies if logging is enabled
175     *
176     * @return the newly allocated, uninitialized chunk of memory, or {@link Word#zero()} if the
177     *         operation was unsuccessful
178     */
179    static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
180        // If G1 is enabled, the "eden" allocation space is not the same always
181        // and therefore we have to go to slowpath to allocate a new TLAB.
182        if (useG1GC(INJECTED_VMCONFIG)) {
183            return Word.zero();
184        }
185        if (!useTLAB(INJECTED_VMCONFIG)) {
186            return edenAllocate(Word.unsigned(sizeInBytes), log);
187        }
188        Word intArrayMarkWord = Word.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
189        int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
190
191        Word top = readTlabTop(thread);
192        Word end = readTlabEnd(thread);
193
194        // calculate amount of free space
195        long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
196
197        if (log) {
198            printf("refillTLAB: thread=%p\n", thread.rawValue());
199            printf("refillTLAB: top=%p\n", top.rawValue());
200            printf("refillTLAB: end=%p\n", end.rawValue());
201            printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
202        }
203
204        long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
205
206        // Retain TLAB and allocate object in shared space if
207        // the amount free in the TLAB is too large to discard.
208        Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
209        if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
210            if (tlabStats(INJECTED_VMCONFIG)) {
211                // increment number of refills
212                thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
213                if (log) {
214                    printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION));
215                }
216                // accumulate wastage
217                int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
218                if (log) {
219                    printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
220                }
221                thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
222            }
223
224            // if TLAB is currently allocated (top or end != null) then
225            // fill [top, end + alignment_reserve) with array object
226            if (top.notEqual(0)) {
227                int headerSize = arrayBaseOffset(JavaKind.Int);
228                // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
229                // an int
230                int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
231                int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
232                NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null);
233
234                long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
235                allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
236                thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
237            }
238
239            // refill the TLAB with an eden allocation
240            Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION);
241            Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
242            // allocate new TLAB, address returned in top
243            top = edenAllocate(tlabRefillSizeInBytes, log);
244            if (top.notEqual(0)) {
245                end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
246                initializeTlab(thread, top, end);
247
248                return NewInstanceStub.allocate(thread, sizeInBytes);
249            } else {
250                return Word.zero();
251            }
252        } else {
253            // Retain TLAB
254            Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG));
255            thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
256            if (log) {
257                printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
258            }
259
260            if (tlabStats(INJECTED_VMCONFIG)) {
261                thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1,
262                                TLAB_SLOW_ALLOCATIONS_LOCATION);
263            }
264
265            return edenAllocate(Word.unsigned(sizeInBytes), log);
266        }
267    }
268
269    /**
270     * Attempts to allocate a chunk of memory from Eden space.
271     *
272     * @param sizeInBytes the size of the chunk to allocate
273     * @param log specifies if logging is enabled
274     * @return the allocated chunk or {@link Word#zero()} if allocation fails
275     */
276    public static Word edenAllocate(Word sizeInBytes, boolean log) {
277        final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress();
278        final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress();
279
280        Word heapTopAddress = Word.unsigned(heapTopRawAddress);
281        Word heapEndAddress = Word.unsigned(heapEndRawAddress);
282
283        while (true) {
284            Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
285            Word newHeapTop = heapTop.add(sizeInBytes);
286            if (newHeapTop.belowOrEqual(heapTop)) {
287                return Word.zero();
288            }
289
290            Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
291            if (newHeapTop.aboveThan(heapEnd)) {
292                return Word.zero();
293            }
294            if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) {
295                return heapTop;
296            }
297        }
298    }
299
300    @Fold
301    static boolean forceSlowPath(OptionValues options) {
302        return StubOptions.ForceUseOfNewInstanceStub.getValue(options);
303    }
304
305    public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
306
307    @NodeIntrinsic(StubForeignCallNode.class)
308    public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
309}
310