NewObjectSnippets.java revision 12995:5e441a7ec5e3 
1226031Sstas/*
2226031Sstas * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3226031Sstas * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4226031Sstas *
5226031Sstas * This code is free software; you can redistribute it and/or modify it
6226031Sstas * under the terms of the GNU General Public License version 2 only, as
7226031Sstas * published by the Free Software Foundation.
8226031Sstas *
9226031Sstas * This code is distributed in the hope that it will be useful, but WITHOUT
10226031Sstas * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11226031Sstas * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12226031Sstas * version 2 for more details (a copy is included in the LICENSE file that
13226031Sstas * accompanied this code).
14226031Sstas *
15226031Sstas * You should have received a copy of the GNU General Public License version
16226031Sstas * 2 along with this work; if not, write to the Free Software Foundation,
17226031Sstas * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18226031Sstas *
19226031Sstas * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20226031Sstas * or visit www.oracle.com if you need additional information or have any
21226031Sstas * questions.
22226031Sstas */
23226031Sstaspackage org.graalvm.compiler.hotspot.replacements;
24226031Sstas
25226031Sstasimport static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26226031Sstasimport static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize;
27226031Sstasimport static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
28226031Sstasimport static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
29226031Sstasimport static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
30226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
31226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
32226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
33226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
34226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
35226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
36226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
37226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
38226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
39226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
40226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
41226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
43226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
44226031Sstasimport static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
45import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
46import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
47import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
48import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
49import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
50import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
51import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
52import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
53import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
54import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
55import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
56import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
57import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
58import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
59import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
60import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
61import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
62import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
63import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
64import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
65import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
66import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
67import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
68import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
69import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
70
71import org.graalvm.compiler.api.replacements.Fold;
72import org.graalvm.compiler.api.replacements.Snippet;
73import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
74import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
75import org.graalvm.compiler.core.common.LocationIdentity;
76import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
77import org.graalvm.compiler.core.common.type.StampFactory;
78import org.graalvm.compiler.debug.Debug;
79import org.graalvm.compiler.debug.GraalError;
80import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
81import org.graalvm.compiler.graph.Node.NodeIntrinsic;
82import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
83import org.graalvm.compiler.hotspot.HotSpotBackend;
84import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
85import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
86import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
87import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
88import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
89import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
90import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets;
91import org.graalvm.compiler.hotspot.word.KlassPointer;
92import org.graalvm.compiler.nodes.ConstantNode;
93import org.graalvm.compiler.nodes.DeoptimizeNode;
94import org.graalvm.compiler.nodes.PiNode;
95import org.graalvm.compiler.nodes.PrefetchAllocateNode;
96import org.graalvm.compiler.nodes.SnippetAnchorNode;
97import org.graalvm.compiler.nodes.StructuredGraph;
98import org.graalvm.compiler.nodes.ValueNode;
99import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
100import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
101import org.graalvm.compiler.nodes.extended.ForeignCallNode;
102import org.graalvm.compiler.nodes.extended.MembarNode;
103import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
104import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
105import org.graalvm.compiler.nodes.java.NewArrayNode;
106import org.graalvm.compiler.nodes.java.NewInstanceNode;
107import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
108import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
109import org.graalvm.compiler.nodes.spi.LoweringTool;
110import org.graalvm.compiler.nodes.util.GraphUtil;
111import org.graalvm.compiler.options.OptionValues;
112import org.graalvm.compiler.replacements.ReplacementsUtil;
113import org.graalvm.compiler.replacements.SnippetCounter;
114import org.graalvm.compiler.replacements.SnippetCounter.Group;
115import org.graalvm.compiler.replacements.SnippetTemplate;
116import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
117import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
118import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
119import org.graalvm.compiler.replacements.Snippets;
120import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
121import org.graalvm.compiler.word.Word;
122
123import jdk.vm.ci.code.CodeUtil;
124import jdk.vm.ci.code.MemoryBarriers;
125import jdk.vm.ci.code.Register;
126import jdk.vm.ci.code.TargetDescription;
127import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
128import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
129import jdk.vm.ci.meta.DeoptimizationAction;
130import jdk.vm.ci.meta.DeoptimizationReason;
131import jdk.vm.ci.meta.JavaKind;
132import jdk.vm.ci.meta.ResolvedJavaType;
133
134/**
135 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
136 */
137public class NewObjectSnippets implements Snippets {
138
139    enum ProfileContext {
140        AllocatingMethod,
141        InstanceOrArray,
142        AllocatedType,
143        AllocatedTypesInMethod,
144        Total
145    }
146
147    @Fold
148    static String createName(String path, String typeContext, OptionValues options) {
149        switch (ProfileAllocationsContext.getValue(options)) {
150            case AllocatingMethod:
151                return "";
152            case InstanceOrArray:
153                return path;
154            case AllocatedType:
155            case AllocatedTypesInMethod:
156                return typeContext;
157            case Total:
158                return "bytes";
159            default:
160                throw GraalError.shouldNotReachHere();
161        }
162    }
163
164    @Fold
165    static boolean doProfile(OptionValues options) {
166        return ProfileAllocations.getValue(options);
167    }
168
169    @Fold
170    static boolean withContext(OptionValues options) {
171        ProfileContext context = ProfileAllocationsContext.getValue(options);
172        return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
173    }
174
175    protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
176        if (doProfile(options)) {
177            String name = createName(path, typeContext, options);
178
179            boolean context = withContext(options);
180            DynamicCounterNode.counter(name, "number of bytes allocated", size, context);
181            DynamicCounterNode.counter(name, "number of allocations", 1, context);
182        }
183    }
184
185    public static void emitPrefetchAllocate(Word address, boolean isArray) {
186        GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
187        if (config.allocatePrefetchStyle > 0) {
188            // Insert a prefetch for each allocation only on the fast-path
189            // Generate several prefetch instructions.
190            int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
191            int stepSize = config.allocatePrefetchStepSize;
192            int distance = config.allocatePrefetchDistance;
193            ExplodeLoopNode.explodeLoop();
194            for (int i = 0; i < lines; i++) {
195                PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
196                distance += stepSize;
197            }
198        }
199    }
200
201    @Snippet
202    public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
203                    @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
204                    @ConstantParameter Counters counters) {
205        return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
206    }
207
208    public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
209                    Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
210        Object result;
211        Word thread = registerAsWord(threadRegister);
212        Word top = readTlabTop(thread);
213        Word end = readTlabEnd(thread);
214        Word newTop = top.add(size);
215        if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
216            writeTlabTop(thread, newTop);
217            emitPrefetchAllocate(newTop, false);
218            result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
219        } else {
220            if (counters != null && counters.stub != null) {
221                counters.stub.inc();
222            }
223            result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
224        }
225        profileAllocation("instance", size, typeContext, options);
226        return verifyOop(result);
227    }
228
229    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
230    public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
231
232    @Snippet
233    public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
234                    @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
235                    @ConstantParameter Counters counters) {
236        // Klass must be initialized by the time the first instance is allocated, therefore we can
237        // just load it from the corresponding cell and avoid the resolution check. We have to use a
238        // fixed load though, to prevent it from floating above the initialization.
239        KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
240        return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
241    }
242
243    @Snippet
244    public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
245                    @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
246        if (probability(SLOW_PATH_PROBABILITY, type == null)) {
247            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
248        }
249        Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
250
251        if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
252            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
253        }
254
255        return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
256    }
257
258    private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
259        KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
260        if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
261            KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
262
263            if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
264                int layoutHelper = readLayoutHelper(nonNullHub);
265                /*
266                 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
267                 * the instance size. This size is already passed through align_object_size and
268                 * scaled to bytes. The low order bit is set if instances of this class cannot be
269                 * allocated using the fastpath.
270                 */
271                if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
272                    Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
273                    /*
274                     * FIXME(je,ds): we should actually pass typeContext instead of "" but late
275                     * binding of parameters is not yet supported by the GraphBuilderPlugin system.
276                     */
277                    return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
278                }
279            }
280        }
281        return dynamicNewInstanceStub(type);
282    }
283
284    /**
285     * Maximum array length for which fast path allocation is used.
286     */
287    public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
288
289    @Snippet
290    public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
291                    @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
292                    @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
293        KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
294        return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
295    }
296
297    @Snippet
298    public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
299                    @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
300                    @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
301        KlassPointer picHub = ResolveConstantSnippets.resolveKlassConstant(hub);
302        return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
303    }
304
305    @Snippet
306    public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
307                    @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
308                    @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
309        Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
310        return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
311    }
312
313    private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
314                    boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
315        Object result;
316        int alignment = wordSize();
317        int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize);
318        Word thread = registerAsWord(threadRegister);
319        Word top = readTlabTop(thread);
320        Word end = readTlabEnd(thread);
321        Word newTop = top.add(allocationSize);
322        if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
323                        probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
324            writeTlabTop(thread, newTop);
325            emitPrefetchAllocate(newTop, true);
326            if (counters != null && counters.arrayLoopInit != null) {
327                counters.arrayLoopInit.inc();
328            }
329            result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
330        } else {
331            result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
332        }
333        profileAllocation("array", allocationSize, typeContext, options);
334        return result;
335    }
336
337    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
338    public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
339
340    public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
341    public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
342
343    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
344    public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
345
346    public static Object dynamicNewInstanceStub(Class<?> elementType) {
347        return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);
348    }
349
350    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
351    public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
352
353    @Snippet
354    public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
355                    @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
356                    @ConstantParameter Counters counters) {
357        Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
358        return result;
359    }
360
361    private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
362                    int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
363        /*
364         * We only need the dynamic check for void when we have no static information from
365         * knownElementKind.
366         */
367        staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
368        if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
369            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
370        }
371
372        KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
373        if (klass.isNull()) {
374            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
375        }
376        KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
377
378        if (length < 0) {
379            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
380        }
381        int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(nonNullKlass);
382        //@formatter:off
383        // from src/share/vm/oops/klass.hpp:
384        //
385        // For arrays, layout helper is a negative number, containing four
386        // distinct bytes, as follows:
387        //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
388        // where:
389        //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
390        //    hsz is array header size in bytes (i.e., offset of first element)
391        //    ebt is the BasicType of the elements
392        //    esz is the element size in bytes
393        //@formatter:on
394
395        int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
396        int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
397
398        Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
399        return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
400    }
401
402    /**
403     * Calls the runtime stub for implementing MULTIANEWARRAY.
404     */
405    @Snippet
406    public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
407        Word dims = DimensionsNode.allocaDimsArray(rank);
408        ExplodeLoopNode.explodeLoop();
409        for (int i = 0; i < rank; i++) {
410            dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
411        }
412        return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);
413    }
414
415    @Snippet
416    public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
417        KlassPointer hubPIC = ResolveConstantSnippets.resolveKlassConstant(hub);
418        return newmultiarray(hubPIC, rank, dimensions);
419    }
420
421    @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
422    public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
423
424    /**
425     * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
426     * objects have their bodies initialized in a loop.
427     */
428    private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
429
430    /**
431     * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
432     * that stores are aligned.
433     *
434     * @param size number of bytes to zero
435     * @param memory beginning of object which is being zeroed
436     * @param constantSize is {@code size} known to be constant in the snippet
437     * @param startOffset offset to begin zeroing. May not be word aligned.
438     * @param manualUnroll maximally unroll zeroing
439     */
440    private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
441        fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
442    }
443
444    private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
445        ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
446        int offset = startOffset;
447        if ((offset & 0x7) != 0) {
448            memory.writeInt(offset, (int) value, LocationIdentity.init());
449            offset += 4;
450        }
451        ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
452        if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
453            ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
454            // This case handles arrays of constant length. Instead of having a snippet variant for
455            // each length, generate a chain of stores of maximum length. Once it's inlined the
456            // break statement will trim excess stores.
457            if (counters != null && counters.instanceSeqInit != null) {
458                counters.instanceSeqInit.inc();
459            }
460
461            explodeLoop();
462            for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
463                if (offset == size) {
464                    break;
465                }
466                memory.initializeLong(offset, value, LocationIdentity.init());
467            }
468        } else {
469            // Use Word instead of int to avoid extension to long in generated code
470            Word off = Word.signed(offset);
471            if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
472                if (counters != null && counters.instanceSeqInit != null) {
473                    counters.instanceSeqInit.inc();
474                }
475                explodeLoop();
476            } else {
477                if (counters != null && counters.instanceLoopInit != null) {
478                    counters.instanceLoopInit.inc();
479                }
480            }
481            for (; off.rawValue() < size; off = off.add(8)) {
482                memory.initializeLong(off, value, LocationIdentity.init());
483            }
484        }
485    }
486
487    /**
488     * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
489     * necessary and ensuring that stores are aligned.
490     *
491     * @param size number of bytes to zero
492     * @param memory beginning of object which is being zeroed
493     * @param constantSize is {@code  size} known to be constant in the snippet
494     * @param startOffset offset to begin zeroing. May not be word aligned.
495     * @param manualUnroll maximally unroll zeroing
496     */
497    private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
498        fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
499    }
500
501    /**
502     * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
503     * since they can't be compiled in stubs.
504     */
505    public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
506        return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null);
507    }
508
509    /**
510     * Formats some allocated memory with an object header and zeroes out the rest.
511     */
512    protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
513        Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
514        initializeObjectHeader(memory, prototypeMarkWord, hub);
515        if (fillContents) {
516            zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
517        } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
518            fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
519        }
520        MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
521        return memory.toObjectNonNull();
522    }
523
524    @Snippet
525    protected static void verifyHeap(@ConstantParameter Register threadRegister) {
526        Word thread = registerAsWord(threadRegister);
527        Word topValue = readTlabTop(thread);
528        if (!topValue.equal(Word.zero())) {
529            Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
530            if (topValueContents.equal(Word.zero())) {
531                AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
532            }
533        }
534    }
535
536    /**
537     * Formats some allocated memory with an object header and zeroes out the rest.
538     */
539    public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
540                    Counters counters) {
541        memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
542        /*
543         * store hub last as the concurrent garbage collectors assume length is valid if hub field
544         * is not null
545         */
546        initializeObjectHeader(memory, prototypeMarkWord, hub);
547        if (fillContents) {
548            zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
549        } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
550            fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
551        }
552        MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
553        return memory.toObjectNonNull();
554    }
555
556    static class Counters {
557        Counters(SnippetCounter.Group.Factory factory) {
558            Group newInstance = factory.createSnippetCounterGroup("NewInstance");
559            Group newArray = factory.createSnippetCounterGroup("NewArray");
560            instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
561            instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
562            arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
563            stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
564        }
565
566        final SnippetCounter instanceSeqInit;
567        final SnippetCounter instanceLoopInit;
568        final SnippetCounter arrayLoopInit;
569        final SnippetCounter stub;
570    }
571
572    public static class Templates extends AbstractTemplates {
573
574        private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
575        private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
576                        TLAB_END_LOCATION);
577        private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
578        private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
579        private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
580                        TLAB_END_LOCATION);
581        private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
582                        TLAB_END_LOCATION);
583        private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
584                        TLAB_END_LOCATION);
585        private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
586        private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
587        private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
588        private final GraalHotSpotVMConfig config;
589        private final Counters counters;
590
591        public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target, GraalHotSpotVMConfig config) {
592            super(options, providers, providers.getSnippetReflection(), target);
593            this.config = config;
594            counters = new Counters(factory);
595        }
596
597        /**
598         * Lowers a {@link NewInstanceNode}.
599         */
600        public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
601            StructuredGraph graph = newInstanceNode.graph();
602            HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
603            assert !type.isArray();
604            ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
605            int size = instanceSize(type);
606
607            OptionValues localOptions = graph.getOptions();
608            SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
609            Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
610            args.addConst("size", size);
611            args.add("hub", hub);
612            args.add("prototypeMarkWord", type.prototypeMarkWord());
613            args.addConst("fillContents", newInstanceNode.fillContents());
614            args.addConst("threadRegister", registers.getThreadRegister());
615            args.addConst("constantSize", true);
616            args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
617            args.addConst("options", localOptions);
618            args.addConst("counters", counters);
619
620            SnippetTemplate template = template(args);
621            Debug.log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
622            template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
623        }
624
625        /**
626         * Lowers a {@link NewArrayNode}.
627         */
628        public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
629            StructuredGraph graph = newArrayNode.graph();
630            ResolvedJavaType elementType = newArrayNode.elementType();
631            HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
632            JavaKind elementKind = elementType.getJavaKind();
633            ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
634            final int headerSize = getArrayBaseOffset(elementKind);
635            int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind));
636
637            OptionValues localOptions = graph.getOptions();
638            SnippetInfo snippet;
639            if (GeneratePIC.getValue(localOptions)) {
640                if (elementType.isPrimitive()) {
641                    snippet = allocatePrimitiveArrayPIC;
642                } else {
643                    snippet = allocateArrayPIC;
644                }
645            } else {
646                snippet = allocateArray;
647            }
648
649            Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
650            args.add("hub", hub);
651            ValueNode length = newArrayNode.length();
652            args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
653            assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
654            args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
655            args.addConst("headerSize", headerSize);
656            args.addConst("log2ElementSize", log2ElementSize);
657            args.addConst("fillContents", newArrayNode.fillContents());
658            args.addConst("threadRegister", registers.getThreadRegister());
659            args.addConst("maybeUnroll", length.isConstant());
660            args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
661            args.addConst("options", localOptions);
662            args.addConst("counters", counters);
663            SnippetTemplate template = template(args);
664            Debug.log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
665            template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
666        }
667
668        public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
669            Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
670            OptionValues localOptions = newInstanceNode.getOptions();
671            args.add("type", newInstanceNode.getInstanceType());
672            ValueNode classClass = newInstanceNode.getClassClass();
673            assert classClass != null;
674            args.add("classClass", classClass);
675            args.addConst("fillContents", newInstanceNode.fillContents());
676            args.addConst("threadRegister", registers.getThreadRegister());
677            args.addConst("options", localOptions);
678            args.addConst("counters", counters);
679
680            SnippetTemplate template = template(args);
681            template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
682        }
683
684        public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
685            StructuredGraph graph = newArrayNode.graph();
686            OptionValues localOptions = graph.getOptions();
687            Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
688            args.add("elementType", newArrayNode.getElementType());
689            ValueNode voidClass = newArrayNode.getVoidClass();
690            assert voidClass != null;
691            args.add("voidClass", voidClass);
692            ValueNode length = newArrayNode.length();
693            args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
694            args.addConst("fillContents", newArrayNode.fillContents());
695            args.addConst("threadRegister", registers.getThreadRegister());
696            /*
697             * We use Kind.Illegal as a marker value instead of null because constant snippet
698             * parameters cannot be null.
699             */
700            args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
701            if (newArrayNode.getKnownElementKind() != null) {
702                args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
703            } else {
704                args.addConst("knownLayoutHelper", 0);
705            }
706            args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
707            args.addConst("options", localOptions);
708            args.addConst("counters", counters);
709            SnippetTemplate template = template(args);
710            template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
711        }
712
713        private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
714            return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
715        }
716
717        public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
718            StructuredGraph graph = newmultiarrayNode.graph();
719            OptionValues localOptions = graph.getOptions();
720            int rank = newmultiarrayNode.dimensionCount();
721            ValueNode[] dims = new ValueNode[rank];
722            for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
723                dims[i] = newmultiarrayNode.dimension(i);
724            }
725            HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
726            ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
727
728            SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
729            Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
730            args.add("hub", hub);
731            args.addConst("rank", rank);
732            args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
733            template(args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
734        }
735
736        private static int instanceSize(HotSpotResolvedObjectType type) {
737            int size = type.instanceSize();
738            assert size >= 0;
739            return size;
740        }
741
742        public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
743            if (config.cAssertions) {
744                Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
745                args.addConst("threadRegister", registers.getThreadRegister());
746
747                SnippetTemplate template = template(args);
748                template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
749            } else {
750                GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
751            }
752        }
753    }
754}
755