ArrayCopySnippets.java revision 12651:6ef01bd40ce2 
1/*
2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23package org.graalvm.compiler.hotspot.replacements.arraycopy;
24
25import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
26import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
27import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
28import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
29import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
30import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
31import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
32import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
33import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
34import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
35import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
36import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
37import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
38import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
39
40import java.lang.reflect.Method;
41import java.util.EnumMap;
42import java.util.Map;
43
44import org.graalvm.compiler.api.directives.GraalDirectives;
45import org.graalvm.compiler.api.replacements.Fold;
46import org.graalvm.compiler.api.replacements.Snippet;
47import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
48import org.graalvm.compiler.core.common.LocationIdentity;
49import org.graalvm.compiler.debug.GraalError;
50import org.graalvm.compiler.graph.Node;
51import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
52import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
53import org.graalvm.compiler.hotspot.word.KlassPointer;
54import org.graalvm.compiler.nodes.CallTargetNode;
55import org.graalvm.compiler.nodes.ConstantNode;
56import org.graalvm.compiler.nodes.DeoptimizeNode;
57import org.graalvm.compiler.nodes.Invoke;
58import org.graalvm.compiler.nodes.InvokeNode;
59import org.graalvm.compiler.nodes.NamedLocationIdentity;
60import org.graalvm.compiler.nodes.PiNode;
61import org.graalvm.compiler.nodes.StructuredGraph;
62import org.graalvm.compiler.nodes.ValueNode;
63import org.graalvm.compiler.nodes.extended.UnsafeLoadNode;
64import org.graalvm.compiler.nodes.java.ArrayLengthNode;
65import org.graalvm.compiler.nodes.spi.LoweringTool;
66import org.graalvm.compiler.nodes.type.StampTool;
67import org.graalvm.compiler.replacements.SnippetCounter;
68import org.graalvm.compiler.replacements.SnippetTemplate;
69import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
70import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
71import org.graalvm.compiler.replacements.Snippets;
72import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
73import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode;
74import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
75import org.graalvm.compiler.word.Word;
76
77import jdk.vm.ci.code.TargetDescription;
78import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
79import jdk.vm.ci.meta.DeoptimizationAction;
80import jdk.vm.ci.meta.DeoptimizationReason;
81import jdk.vm.ci.meta.JavaConstant;
82import jdk.vm.ci.meta.JavaKind;
83import jdk.vm.ci.meta.ResolvedJavaMethod;
84import jdk.vm.ci.meta.ResolvedJavaType;
85
86public class ArrayCopySnippets implements Snippets {
87
88    private static int checkArrayType(KlassPointer hub) {
89        int layoutHelper = readLayoutHelper(hub);
90        if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
91            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
92        }
93        return layoutHelper;
94    }
95
96    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) {
97        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
98            checkAIOOBECounter.inc();
99            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
100        }
101        if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
102            checkAIOOBECounter.inc();
103            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
104        }
105        if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
106            checkAIOOBECounter.inc();
107            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
108        }
109        if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
110            checkAIOOBECounter.inc();
111            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
112        }
113        if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
114            checkAIOOBECounter.inc();
115            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
116        }
117        checkSuccessCounter.inc();
118    }
119
120    @Snippet
121    public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) {
122        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
123        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
124        KlassPointer srcHub = loadHub(nonNullSrc);
125        KlassPointer destHub = loadHub(nonNullDest);
126        checkArrayType(srcHub);
127        checkArrayType(destHub);
128        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
129        zeroLengthStaticCounter.inc();
130    }
131
132    @Snippet
133    public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
134                    @ConstantParameter SnippetCounter copiedCounter) {
135        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
136        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
137        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
138        counter.inc();
139        copiedCounter.add(length);
140        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
141        if (length == 0) {
142            zeroLengthDynamicCounter.inc();
143        } else {
144            nonZeroLengthDynamicCounter.inc();
145            nonZeroLengthDynamicCopiedCounter.add(length);
146        }
147    }
148
149    /**
150     * This intrinsic is useful for the case where we know something statically about one of the
151     * inputs but not the other.
152     */
153    @Snippet
154    public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
155                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
156        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
157        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
158        KlassPointer srcHub = loadHub(nonNullSrc);
159        KlassPointer destHub = loadHub(nonNullDest);
160        if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
161            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
162        }
163        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
164        counter.inc();
165        copiedCounter.add(length);
166        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
167        if (length == 0) {
168            zeroLengthDynamicCounter.inc();
169        } else {
170            nonZeroLengthDynamicCounter.inc();
171            nonZeroLengthDynamicCopiedCounter.add(length);
172        }
173    }
174
175    @Snippet
176    public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
177                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
178        if (length > 0) {
179            KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
180            KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
181            if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
182                counter.inc();
183                copiedCounter.add(length);
184                predictedObjectArrayCopyFastPathCounter.inc();
185                predictedObjectArrayCopyFastPathCopiedCounter.add(length);
186                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
187            } else {
188                predictedObjectArrayCopySlowPathCounter.inc();
189                predictedObjectArrayCopySlowPathCopiedCounter.add(length);
190                System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
191            }
192        }
193    }
194
195    /**
196     * This is the basic template for the full arraycopy checks, including a check that the
197     * underlying type is really an array type.
198     */
199    @Snippet
200    public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
201                    @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument) {
202        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
203        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
204        KlassPointer srcHub = loadHub(nonNullSrc);
205        KlassPointer destHub = loadHub(nonNullDest);
206        checkArrayType(srcHub);
207        checkArrayType(destHub);
208        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
209        if (length == 0) {
210            zeroLengthDynamicCounter.inc();
211        } else {
212            nonZeroLengthDynamicCounter.inc();
213            nonZeroLengthDynamicCopiedCounter.add(length);
214        }
215        ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
216    }
217
218    /**
219     * Snippet for unrolled arraycopy.
220     */
221    @Snippet
222    public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind) {
223        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
224        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
225        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
226        if (length == 0) {
227            zeroLengthDynamicCounter.inc();
228        } else {
229            nonZeroLengthDynamicCounter.inc();
230            nonZeroLengthDynamicCopiedCounter.add(length);
231        }
232        ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
233    }
234
235    @Snippet
236    public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) {
237        if (length > 0) {
238            KlassPointer destKlass = loadHub(nonNullDest);
239            KlassPointer srcKlass = loadHub(nonNullSrc);
240            if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
241                // no storecheck required.
242                objectCheckcastSameTypeCounter.inc();
243                objectCheckcastSameTypeCopiedCounter.add(length);
244                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
245            } else {
246                KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
247                Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
248                objectCheckcastCounter.inc();
249                objectCheckcastCopiedCounter.add(length);
250                int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
251                if (copiedElements != 0) {
252                    /*
253                     * the checkcast stub doesn't throw the ArrayStoreException, but returns the
254                     * number of copied elements (xor'd with -1).
255                     */
256                    copiedElements ^= -1;
257                    System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
258                }
259            }
260        }
261    }
262
263    @Snippet
264    public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) {
265        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
266        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
267        KlassPointer srcHub = loadHub(nonNullSrc);
268        KlassPointer destHub = loadHub(nonNullDest);
269        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
270            int layoutHelper = checkArrayType(srcHub);
271            final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
272            checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
273            if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
274                genericObjectExactCallCounter.inc();
275                genericObjectExactCallCopiedCounter.add(length);
276                ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
277            } else {
278                genericPrimitiveCallCounter.inc();
279                genericPrimitiveCallCopiedCounter.add(length);
280                UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
281            }
282        } else {
283            SystemArraycopyCounter.inc();
284            SystemArraycopyCopiedCounter.add(length);
285            System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
286        }
287    }
288
289    @Fold
290    static LocationIdentity getArrayLocation(JavaKind kind) {
291        return NamedLocationIdentity.getArrayLocation(kind);
292    }
293
294    @Snippet
295    public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
296        final int scale = arrayIndexScale(elementKind);
297        int arrayBaseOffset = arrayBaseOffset(elementKind);
298        LocationIdentity arrayLocation = getArrayLocation(elementKind);
299        if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
300            long start = (long) (length - 1) * scale;
301            long i = start;
302            ExplodeLoopNode.explodeLoop();
303            for (int iteration = 0; iteration < length; iteration++) {
304                if (i >= 0) {
305                    Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
306                    DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
307                    i -= scale;
308                }
309            }
310        } else {
311            long end = (long) length * scale;
312            long i = 0;
313            ExplodeLoopNode.explodeLoop();
314            for (int iteration = 0; iteration < length; iteration++) {
315                if (i < end) {
316                    Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
317                    DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
318                    i += scale;
319                }
320            }
321        }
322    }
323
324    private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null;
325    private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
326    private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
327
328    private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null;
329
330    private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
331    private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
332    private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
333    private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
334
335    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
336    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
337
338    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
339    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
340
341    static void createArraycopyCounter(JavaKind kind) {
342        arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
343        arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
344
345        arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
346        arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
347    }
348
349    static {
350        createArraycopyCounter(JavaKind.Byte);
351        createArraycopyCounter(JavaKind.Boolean);
352        createArraycopyCounter(JavaKind.Char);
353        createArraycopyCounter(JavaKind.Short);
354        createArraycopyCounter(JavaKind.Int);
355        createArraycopyCounter(JavaKind.Long);
356        createArraycopyCounter(JavaKind.Float);
357        createArraycopyCounter(JavaKind.Double);
358        createArraycopyCounter(JavaKind.Object);
359    }
360
361    private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
362    private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
363    private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
364
365    private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null;
366
367    private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
368    private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
369    private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
370
371    private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null;
372
373    private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
374    private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
375    private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
376    private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
377
378    private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
379    private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
380    private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
381                    "used System.arraycopy slow path for predicted Object[] arrays");
382    private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
383
384    public static class Templates extends SnippetTemplate.AbstractTemplates {
385
386        public Templates(HotSpotProviders providers, TargetDescription target) {
387            super(providers, providers.getSnippetReflection(), target);
388        }
389
390        private ResolvedJavaMethod originalArraycopy() throws GraalError {
391            if (originalArraycopy == null) {
392                Method method;
393                try {
394                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
395                } catch (NoSuchMethodException | SecurityException e) {
396                    throw new GraalError(e);
397                }
398                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
399            }
400            return originalArraycopy;
401        }
402
403        private ResolvedJavaMethod originalArraycopy;
404
405        private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
406        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
407
408        private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
409        private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
410        private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
411        private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
412        private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
413        private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
414
415        private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
416
417        protected SnippetInfo snippet(String methodName) {
418            SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
419            info.setOriginalMethod(originalArraycopy());
420            return info;
421        }
422
423        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
424            return selectComponentKind(arraycopy, true);
425        }
426
427        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
428            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
429            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
430
431            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
432                if (!exact) {
433                    JavaKind component = getComponentKind(srcType);
434                    if (component != null) {
435                        return component;
436                    }
437                    return getComponentKind(destType);
438                }
439                return null;
440            }
441            if (exact) {
442                if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
443                    return null;
444                }
445                if (!arraycopy.isExact()) {
446                    return null;
447                }
448            }
449            return srcType.getComponentType().getJavaKind();
450        }
451
452        private static JavaKind getComponentKind(ResolvedJavaType type) {
453            if (type != null && type.isArray()) {
454                return type.getComponentType().getJavaKind();
455            }
456            return null;
457        }
458
459        private static boolean shouldUnroll(ValueNode length) {
460            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
461        }
462
463        public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
464            JavaKind componentKind = selectComponentKind(arraycopy);
465            SnippetInfo snippetInfo = null;
466            SnippetInfo slowPathSnippetInfo = null;
467            Object slowPathArgument = null;
468
469            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
470                snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
471            } else if (arraycopy.isExact()) {
472                snippetInfo = arraycopyExactIntrinsicSnippet;
473                if (shouldUnroll(arraycopy.getLength())) {
474                    snippetInfo = arraycopyUnrolledIntrinsicSnippet;
475                }
476            } else {
477                if (componentKind == JavaKind.Object) {
478                    ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
479                    ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
480                    ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
481                    ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
482                    if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
483                        snippetInfo = arraycopySlowPathIntrinsicSnippet;
484                        slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
485                        slowPathArgument = LocationIdentity.any();
486                        /*
487                         * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
488                         * pretend to kill any() so clear the componentKind.
489                         */
490                        componentKind = null;
491                    }
492                }
493                if (componentKind == null && snippetInfo == null) {
494                    JavaKind predictedKind = selectComponentKind(arraycopy, false);
495                    if (predictedKind != null) {
496                        /*
497                         * At least one array is of a known type requiring no store checks, so
498                         * assume the other is of the same type. Generally this is working around
499                         * deficiencies in our propagation of type information.
500                         */
501                        componentKind = predictedKind;
502                        if (predictedKind == JavaKind.Object) {
503                            snippetInfo = arraycopySlowPathIntrinsicSnippet;
504                            slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
505                            slowPathArgument = predictedKind;
506                            componentKind = null;
507                        } else {
508                            snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
509                        }
510                    }
511                }
512                if (snippetInfo == null) {
513                    snippetInfo = arraycopyGenericSnippet;
514                }
515            }
516            Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
517            args.add("src", arraycopy.getSource());
518            args.add("srcPos", arraycopy.getSourcePosition());
519            args.add("dest", arraycopy.getDestination());
520            args.add("destPos", arraycopy.getDestinationPosition());
521            args.add("length", arraycopy.getLength());
522            if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
523                args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
524                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
525            } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
526                ValueNode predictedKlass = null;
527                if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
528                    HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
529                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
530                } else {
531                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
532                }
533                args.add("predictedKlass", predictedKlass);
534                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
535                args.addConst("slowPath", slowPathSnippetInfo);
536                assert slowPathArgument != null;
537                args.addConst("slowPathArgument", slowPathArgument);
538            } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
539                assert componentKind != null;
540                args.addConst("elementKind", componentKind);
541                args.addConst("counter", arraycopyCallCounters.get(componentKind));
542                args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind));
543            }
544            instantiate(args, arraycopy);
545        }
546
547        public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
548            StructuredGraph graph = arraycopy.graph();
549            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
550                // Can't be lowered yet
551                return;
552            }
553            SnippetInfo snippetInfo = arraycopy.getSnippet();
554            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
555            args.add("nonNullSrc", arraycopy.getSource());
556            args.add("srcPos", arraycopy.getSourcePosition());
557            args.add("nonNullDest", arraycopy.getDestination());
558            args.add("destPos", arraycopy.getDestinationPosition());
559            if (snippetInfo == arraycopyUnrolledWorkSnippet) {
560                args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
561                args.addConst("elementKind", arraycopy.getElementKind());
562            } else {
563                args.add("length", arraycopy.getLength());
564            }
565            if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
566                args.add("objectArrayKlass", arraycopy.getPredictedKlass());
567                args.addConst("counter", arraycopyCallCounters.get(JavaKind.Object));
568                args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(JavaKind.Object));
569            }
570            instantiate(args, arraycopy);
571        }
572
573        public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
574            StructuredGraph graph = arraycopy.graph();
575            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
576                // Can't be lowered yet
577                return;
578            }
579            SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
580            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
581            args.add("nonNullSrc", arraycopy.getSource());
582            args.add("srcPos", arraycopy.getSourcePosition());
583            args.add("nonNullDest", arraycopy.getDestination());
584            args.add("destPos", arraycopy.getDestinationPosition());
585            args.addConst("length", arraycopy.getUnrollLength());
586            args.addConst("elementKind", arraycopy.getElementKind());
587            template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
588        }
589
590        /**
591         * Instantiate the snippet template and fix up the FrameState of any Invokes of
592         * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
593         *
594         * @param args
595         * @param arraycopy
596         */
597        private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
598            StructuredGraph graph = arraycopy.graph();
599            SnippetTemplate template = template(args);
600            Map<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
601            for (Node originalNode : replacements.keySet()) {
602                if (originalNode instanceof Invoke) {
603                    Invoke invoke = (Invoke) replacements.get(originalNode);
604                    assert invoke.asNode().graph() == graph;
605                    CallTargetNode call = invoke.callTarget();
606
607                    if (!call.targetMethod().equals(originalArraycopy)) {
608                        throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
609                    }
610                    // Here we need to fix the bci of the invoke
611                    InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
612                    if (arraycopy.stateDuring() != null) {
613                        newInvoke.setStateDuring(arraycopy.stateDuring());
614                    } else {
615                        assert arraycopy.stateAfter() != null;
616                        newInvoke.setStateAfter(arraycopy.stateAfter());
617                    }
618                    graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
619                } else if (originalNode instanceof ArrayCopySlowPathNode) {
620                    ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
621                    assert arraycopy.stateAfter() != null;
622                    slowPath.setStateAfter(arraycopy.stateAfter());
623                    slowPath.setBci(arraycopy.getBci());
624                }
625            }
626        }
627    }
628}
629