DefaultHotSpotLoweringProvider.java revision 12995:5e441a7ec5e3 
1/*
2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23package org.graalvm.compiler.hotspot.meta;
24
25import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26import static org.graalvm.compiler.core.common.GraalOptions.AlwaysInlineVTableStubs;
27import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
28import static org.graalvm.compiler.core.common.GraalOptions.InlineVTableStubs;
29import static org.graalvm.compiler.core.common.GraalOptions.OmitHotExceptionStacktrace;
30import static org.graalvm.compiler.core.common.LocationIdentity.any;
31import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.OSR_MIGRATION_END;
32import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_KLASS_LOCATION;
33import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_MIRROR_LOCATION;
34import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_MIRROR_HANDLE_LOCATION;
35import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.COMPRESSED_HUB_LOCATION;
36import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION;
37import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_LOCATION;
38import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
39import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_LAYOUT_HELPER_LOCATION;
40import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
41import java.lang.ref.Reference;
42
43import org.graalvm.compiler.api.directives.GraalDirectives;
44import org.graalvm.compiler.core.common.GraalOptions;
45import org.graalvm.compiler.core.common.LocationIdentity;
46import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
47import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
48import org.graalvm.compiler.core.common.type.ObjectStamp;
49import org.graalvm.compiler.core.common.type.Stamp;
50import org.graalvm.compiler.core.common.type.StampFactory;
51import org.graalvm.compiler.core.common.type.StampPair;
52import org.graalvm.compiler.debug.GraalError;
53import org.graalvm.compiler.graph.Node;
54import org.graalvm.compiler.graph.NodeInputList;
55import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
56import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
57import org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode;
58import org.graalvm.compiler.hotspot.nodes.CompressionNode;
59import org.graalvm.compiler.hotspot.nodes.CompressionNode.CompressionOp;
60import org.graalvm.compiler.hotspot.nodes.ComputeObjectAddressNode;
61import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
62import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier;
63import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier;
64import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier;
65import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier;
66import org.graalvm.compiler.hotspot.nodes.GetObjectAddressNode;
67import org.graalvm.compiler.hotspot.nodes.HotSpotDirectCallTargetNode;
68import org.graalvm.compiler.hotspot.nodes.HotSpotIndirectCallTargetNode;
69import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
70import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
71import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode;
72import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantNode;
73import org.graalvm.compiler.hotspot.nodes.aot.ResolveMethodAndLoadCountersNode;
74import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode;
75import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
76import org.graalvm.compiler.hotspot.nodes.type.MethodPointerStamp;
77import org.graalvm.compiler.hotspot.nodes.type.NarrowOopStamp;
78import org.graalvm.compiler.hotspot.replacements.AssertionSnippets;
79import org.graalvm.compiler.hotspot.replacements.ClassGetHubNode;
80import org.graalvm.compiler.hotspot.replacements.HashCodeSnippets;
81import org.graalvm.compiler.hotspot.replacements.HubGetClassNode;
82import org.graalvm.compiler.hotspot.replacements.IdentityHashCodeNode;
83import org.graalvm.compiler.hotspot.replacements.InstanceOfSnippets;
84import org.graalvm.compiler.hotspot.replacements.KlassLayoutHelperNode;
85import org.graalvm.compiler.hotspot.replacements.LoadExceptionObjectSnippets;
86import org.graalvm.compiler.hotspot.replacements.MonitorSnippets;
87import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
88import org.graalvm.compiler.hotspot.replacements.StringToBytesSnippets;
89import org.graalvm.compiler.hotspot.replacements.UnsafeLoadSnippets;
90import org.graalvm.compiler.hotspot.replacements.WriteBarrierSnippets;
91import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets;
92import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyNode;
93import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopySlowPathNode;
94import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopySnippets;
95import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyUnrollNode;
96import org.graalvm.compiler.hotspot.replacements.arraycopy.UnsafeArrayCopySnippets;
97import org.graalvm.compiler.hotspot.replacements.profiling.ProfileSnippets;
98import org.graalvm.compiler.hotspot.word.KlassPointer;
99import org.graalvm.compiler.nodes.AbstractBeginNode;
100import org.graalvm.compiler.nodes.AbstractDeoptimizeNode;
101import org.graalvm.compiler.nodes.ConstantNode;
102import org.graalvm.compiler.nodes.FixedNode;
103import org.graalvm.compiler.nodes.Invoke;
104import org.graalvm.compiler.nodes.LogicNode;
105import org.graalvm.compiler.nodes.LoweredCallTargetNode;
106import org.graalvm.compiler.nodes.ParameterNode;
107import org.graalvm.compiler.nodes.SafepointNode;
108import org.graalvm.compiler.nodes.StartNode;
109import org.graalvm.compiler.nodes.StructuredGraph;
110import org.graalvm.compiler.nodes.UnwindNode;
111import org.graalvm.compiler.nodes.ValueNode;
112import org.graalvm.compiler.nodes.calc.AddNode;
113import org.graalvm.compiler.nodes.calc.FloatingNode;
114import org.graalvm.compiler.nodes.calc.IntegerDivRemNode;
115import org.graalvm.compiler.nodes.calc.IsNullNode;
116import org.graalvm.compiler.nodes.calc.RemNode;
117import org.graalvm.compiler.nodes.debug.StringToBytesNode;
118import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
119import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode;
120import org.graalvm.compiler.nodes.extended.ForeignCallNode;
121import org.graalvm.compiler.nodes.extended.GetClassNode;
122import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode;
123import org.graalvm.compiler.nodes.extended.LoadHubNode;
124import org.graalvm.compiler.nodes.extended.LoadMethodNode;
125import org.graalvm.compiler.nodes.extended.OSRLocalNode;
126import org.graalvm.compiler.nodes.extended.OSRLockNode;
127import org.graalvm.compiler.nodes.extended.OSRMonitorEnterNode;
128import org.graalvm.compiler.nodes.extended.OSRStartNode;
129import org.graalvm.compiler.nodes.extended.StoreHubNode;
130import org.graalvm.compiler.nodes.extended.RawLoadNode;
131import org.graalvm.compiler.nodes.java.ClassIsAssignableFromNode;
132import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
133import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
134import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode;
135import org.graalvm.compiler.nodes.java.InstanceOfNode;
136import org.graalvm.compiler.nodes.java.LoadExceptionObjectNode;
137import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
138import org.graalvm.compiler.nodes.java.MonitorExitNode;
139import org.graalvm.compiler.nodes.java.MonitorIdNode;
140import org.graalvm.compiler.nodes.java.NewArrayNode;
141import org.graalvm.compiler.nodes.java.NewInstanceNode;
142import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
143import org.graalvm.compiler.nodes.java.RawMonitorEnterNode;
144import org.graalvm.compiler.nodes.memory.FloatingReadNode;
145import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
146import org.graalvm.compiler.nodes.memory.ReadNode;
147import org.graalvm.compiler.nodes.memory.WriteNode;
148import org.graalvm.compiler.nodes.memory.address.AddressNode;
149import org.graalvm.compiler.nodes.spi.LoweringProvider;
150import org.graalvm.compiler.nodes.spi.LoweringTool;
151import org.graalvm.compiler.nodes.spi.StampProvider;
152import org.graalvm.compiler.nodes.type.StampTool;
153import org.graalvm.compiler.nodes.util.GraphUtil;
154import org.graalvm.compiler.options.OptionValues;
155import org.graalvm.compiler.replacements.DefaultJavaLoweringProvider;
156import org.graalvm.compiler.replacements.nodes.AssertionNode;
157
158import jdk.vm.ci.code.TargetDescription;
159import jdk.vm.ci.hotspot.HotSpotCallingConventionType;
160import jdk.vm.ci.hotspot.HotSpotConstantReflectionProvider;
161import jdk.vm.ci.hotspot.HotSpotResolvedJavaField;
162import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
163import jdk.vm.ci.meta.JavaConstant;
164import jdk.vm.ci.meta.JavaKind;
165import jdk.vm.ci.meta.JavaType;
166import jdk.vm.ci.meta.MetaAccessProvider;
167import jdk.vm.ci.meta.ResolvedJavaField;
168import jdk.vm.ci.meta.ResolvedJavaType;
169
170/**
171 * HotSpot implementation of {@link LoweringProvider}.
172 */
173public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider implements HotSpotLoweringProvider {
174
175    protected final HotSpotGraalRuntimeProvider runtime;
176    protected final HotSpotRegistersProvider registers;
177    protected final HotSpotConstantReflectionProvider constantReflection;
178
179    protected InstanceOfSnippets.Templates instanceofSnippets;
180    protected NewObjectSnippets.Templates newObjectSnippets;
181    protected MonitorSnippets.Templates monitorSnippets;
182    protected WriteBarrierSnippets.Templates writeBarrierSnippets;
183    protected LoadExceptionObjectSnippets.Templates exceptionObjectSnippets;
184    protected UnsafeLoadSnippets.Templates unsafeLoadSnippets;
185    protected AssertionSnippets.Templates assertionSnippets;
186    protected ArrayCopySnippets.Templates arraycopySnippets;
187    protected StringToBytesSnippets.Templates stringToBytesSnippets;
188    protected HashCodeSnippets.Templates hashCodeSnippets;
189    protected ResolveConstantSnippets.Templates resolveConstantSnippets;
190    protected ProfileSnippets.Templates profileSnippets;
191
192    public DefaultHotSpotLoweringProvider(HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, HotSpotRegistersProvider registers,
193                    HotSpotConstantReflectionProvider constantReflection, TargetDescription target) {
194        super(metaAccess, foreignCalls, target);
195        this.runtime = runtime;
196        this.registers = registers;
197        this.constantReflection = constantReflection;
198    }
199
200    @Override
201    public void initialize(OptionValues options, HotSpotProviders providers, GraalHotSpotVMConfig config) {
202        super.initialize(options, runtime, providers, providers.getSnippetReflection());
203
204        assert target == providers.getCodeCache().getTarget();
205        instanceofSnippets = new InstanceOfSnippets.Templates(options, runtime, providers, target);
206        newObjectSnippets = new NewObjectSnippets.Templates(options, runtime, providers, target, config);
207        monitorSnippets = new MonitorSnippets.Templates(options, runtime, providers, target, config.useFastLocking);
208        writeBarrierSnippets = new WriteBarrierSnippets.Templates(options, runtime, providers, target, config.useCompressedOops ? config.getOopEncoding() : null);
209        exceptionObjectSnippets = new LoadExceptionObjectSnippets.Templates(options, providers, target);
210        unsafeLoadSnippets = new UnsafeLoadSnippets.Templates(options, providers, target);
211        assertionSnippets = new AssertionSnippets.Templates(options, providers, target);
212        arraycopySnippets = new ArrayCopySnippets.Templates(options, runtime, providers, target);
213        stringToBytesSnippets = new StringToBytesSnippets.Templates(options, providers, target);
214        hashCodeSnippets = new HashCodeSnippets.Templates(options, providers, target);
215        if (GeneratePIC.getValue(options)) {
216            resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, providers, target);
217            profileSnippets = new ProfileSnippets.Templates(options, providers, target);
218        }
219        providers.getReplacements().registerSnippetTemplateCache(new UnsafeArrayCopySnippets.Templates(options, providers, target));
220    }
221
222    public MonitorSnippets.Templates getMonitorSnippets() {
223        return monitorSnippets;
224    }
225
226    @Override
227    public void lower(Node n, LoweringTool tool) {
228        StructuredGraph graph = (StructuredGraph) n.graph();
229        if (n instanceof Invoke) {
230            lowerInvoke((Invoke) n, tool, graph);
231        } else if (n instanceof LoadMethodNode) {
232            lowerLoadMethodNode((LoadMethodNode) n);
233        } else if (n instanceof GetClassNode) {
234            lowerGetClassNode((GetClassNode) n, tool, graph);
235        } else if (n instanceof StoreHubNode) {
236            lowerStoreHubNode((StoreHubNode) n, graph);
237        } else if (n instanceof OSRStartNode) {
238            lowerOSRStartNode((OSRStartNode) n);
239        } else if (n instanceof BytecodeExceptionNode) {
240            lowerBytecodeExceptionNode((BytecodeExceptionNode) n);
241        } else if (n instanceof InstanceOfNode) {
242            InstanceOfNode instanceOfNode = (InstanceOfNode) n;
243            if (graph.getGuardsStage().areDeoptsFixed()) {
244                instanceofSnippets.lower(instanceOfNode, tool);
245            } else {
246                if (instanceOfNode.allowsNull()) {
247                    ValueNode object = instanceOfNode.getValue();
248                    LogicNode newTypeCheck = graph.addOrUniqueWithInputs(InstanceOfNode.create(instanceOfNode.type(), object, instanceOfNode.profile(), instanceOfNode.getAnchor()));
249                    LogicNode newNode = LogicNode.or(graph.unique(IsNullNode.create(object)), newTypeCheck, GraalDirectives.UNLIKELY_PROBABILITY);
250                    instanceOfNode.replaceAndDelete(newNode);
251                }
252            }
253        } else if (n instanceof InstanceOfDynamicNode) {
254            InstanceOfDynamicNode instanceOfDynamicNode = (InstanceOfDynamicNode) n;
255            if (graph.getGuardsStage().areDeoptsFixed()) {
256                instanceofSnippets.lower(instanceOfDynamicNode, tool);
257            } else {
258                ValueNode mirror = instanceOfDynamicNode.getMirrorOrHub();
259                if (mirror.stamp().getStackKind() == JavaKind.Object) {
260                    ClassGetHubNode classGetHub = graph.unique(new ClassGetHubNode(mirror));
261                    instanceOfDynamicNode.setMirror(classGetHub);
262                }
263
264                if (instanceOfDynamicNode.allowsNull()) {
265                    ValueNode object = instanceOfDynamicNode.getObject();
266                    LogicNode newTypeCheck = graph.addOrUniqueWithInputs(
267                                    InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), instanceOfDynamicNode.getMirrorOrHub(), object, false));
268                    LogicNode newNode = LogicNode.or(graph.unique(IsNullNode.create(object)), newTypeCheck, GraalDirectives.UNLIKELY_PROBABILITY);
269                    instanceOfDynamicNode.replaceAndDelete(newNode);
270                }
271            }
272        } else if (n instanceof ClassIsAssignableFromNode) {
273            if (graph.getGuardsStage().areDeoptsFixed()) {
274                instanceofSnippets.lower((ClassIsAssignableFromNode) n, tool);
275            }
276        } else if (n instanceof NewInstanceNode) {
277            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
278                newObjectSnippets.lower((NewInstanceNode) n, registers, tool);
279            }
280        } else if (n instanceof DynamicNewInstanceNode) {
281            DynamicNewInstanceNode newInstanceNode = (DynamicNewInstanceNode) n;
282            if (newInstanceNode.getClassClass() == null) {
283                JavaConstant classClassMirror = constantReflection.forObject(Class.class);
284                ConstantNode classClass = ConstantNode.forConstant(classClassMirror, tool.getMetaAccess(), graph);
285                newInstanceNode.setClassClass(classClass);
286            }
287            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
288                newObjectSnippets.lower(newInstanceNode, registers, tool);
289            }
290        } else if (n instanceof NewArrayNode) {
291            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
292                newObjectSnippets.lower((NewArrayNode) n, registers, tool);
293            }
294        } else if (n instanceof DynamicNewArrayNode) {
295            DynamicNewArrayNode dynamicNewArrayNode = (DynamicNewArrayNode) n;
296            if (dynamicNewArrayNode.getVoidClass() == null) {
297                JavaConstant voidClassMirror = constantReflection.forObject(void.class);
298                ConstantNode voidClass = ConstantNode.forConstant(voidClassMirror, tool.getMetaAccess(), graph);
299                dynamicNewArrayNode.setVoidClass(voidClass);
300            }
301            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
302                newObjectSnippets.lower(dynamicNewArrayNode, registers, tool);
303            }
304        } else if (n instanceof VerifyHeapNode) {
305            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
306                newObjectSnippets.lower((VerifyHeapNode) n, registers, tool);
307            }
308        } else if (n instanceof RawMonitorEnterNode) {
309            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
310                monitorSnippets.lower((RawMonitorEnterNode) n, registers, tool);
311            }
312        } else if (n instanceof MonitorExitNode) {
313            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
314                monitorSnippets.lower((MonitorExitNode) n, registers, tool);
315            }
316        } else if (n instanceof ArrayCopyNode) {
317            arraycopySnippets.lower((ArrayCopyNode) n, tool);
318        } else if (n instanceof ArrayCopySlowPathNode) {
319            arraycopySnippets.lower((ArrayCopySlowPathNode) n, tool);
320        } else if (n instanceof ArrayCopyUnrollNode) {
321            arraycopySnippets.lower((ArrayCopyUnrollNode) n, tool);
322        } else if (n instanceof G1PreWriteBarrier) {
323            writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool);
324        } else if (n instanceof G1PostWriteBarrier) {
325            writeBarrierSnippets.lower((G1PostWriteBarrier) n, registers, tool);
326        } else if (n instanceof G1ReferentFieldReadBarrier) {
327            writeBarrierSnippets.lower((G1ReferentFieldReadBarrier) n, registers, tool);
328        } else if (n instanceof SerialWriteBarrier) {
329            writeBarrierSnippets.lower((SerialWriteBarrier) n, tool);
330        } else if (n instanceof SerialArrayRangeWriteBarrier) {
331            writeBarrierSnippets.lower((SerialArrayRangeWriteBarrier) n, tool);
332        } else if (n instanceof G1ArrayRangePreWriteBarrier) {
333            writeBarrierSnippets.lower((G1ArrayRangePreWriteBarrier) n, registers, tool);
334        } else if (n instanceof G1ArrayRangePostWriteBarrier) {
335            writeBarrierSnippets.lower((G1ArrayRangePostWriteBarrier) n, registers, tool);
336        } else if (n instanceof NewMultiArrayNode) {
337            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
338                newObjectSnippets.lower((NewMultiArrayNode) n, tool);
339            }
340        } else if (n instanceof LoadExceptionObjectNode) {
341            exceptionObjectSnippets.lower((LoadExceptionObjectNode) n, registers, tool);
342        } else if (n instanceof AssertionNode) {
343            assertionSnippets.lower((AssertionNode) n, tool);
344        } else if (n instanceof StringToBytesNode) {
345            if (graph.getGuardsStage().areDeoptsFixed()) {
346                stringToBytesSnippets.lower((StringToBytesNode) n, tool);
347            }
348        } else if (n instanceof IntegerDivRemNode) {
349            // Nothing to do for division nodes. The HotSpot signal handler catches divisions by
350            // zero and the MIN_VALUE / -1 cases.
351        } else if (n instanceof AbstractDeoptimizeNode || n instanceof UnwindNode || n instanceof RemNode || n instanceof SafepointNode) {
352            /* No lowering, we generate LIR directly for these nodes. */
353        } else if (n instanceof ClassGetHubNode) {
354            lowerClassGetHubNode((ClassGetHubNode) n, tool);
355        } else if (n instanceof HubGetClassNode) {
356            lowerHubGetClassNode((HubGetClassNode) n, tool);
357        } else if (n instanceof KlassLayoutHelperNode) {
358            lowerKlassLayoutHelperNode((KlassLayoutHelperNode) n, tool);
359        } else if (n instanceof ComputeObjectAddressNode) {
360            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
361                lowerComputeObjectAddressNode((ComputeObjectAddressNode) n);
362            }
363        } else if (n instanceof IdentityHashCodeNode) {
364            hashCodeSnippets.lower((IdentityHashCodeNode) n, tool);
365        } else if (n instanceof ResolveConstantNode) {
366            resolveConstantSnippets.lower((ResolveConstantNode) n, tool);
367        } else if (n instanceof ResolveMethodAndLoadCountersNode) {
368            resolveConstantSnippets.lower((ResolveMethodAndLoadCountersNode) n, tool);
369        } else if (n instanceof InitializeKlassNode) {
370            resolveConstantSnippets.lower((InitializeKlassNode) n, tool);
371        } else if (n instanceof ProfileNode) {
372            profileSnippets.lower((ProfileNode) n, tool);
373        } else {
374            super.lower(n, tool);
375        }
376    }
377
378    private static void lowerComputeObjectAddressNode(ComputeObjectAddressNode n) {
379        /*
380         * Lower the node into a ComputeObjectAddress node and an Add but ensure that it's below any
381         * potential safepoints and above it's uses.
382         */
383        for (Node use : n.usages().snapshot()) {
384            if (use instanceof FixedNode) {
385                FixedNode fixed = (FixedNode) use;
386                StructuredGraph graph = n.graph();
387                GetObjectAddressNode address = graph.add(new GetObjectAddressNode(n.getObject()));
388                graph.addBeforeFixed(fixed, address);
389                AddNode add = graph.addOrUnique(new AddNode(address, n.getOffset()));
390                use.replaceFirstInput(n, add);
391            } else {
392                throw GraalError.shouldNotReachHere("Unexpected floating use of ComputeObjectAddressNode " + n);
393            }
394        }
395        GraphUtil.unlinkFixedNode(n);
396        n.safeDelete();
397    }
398
399    private void lowerKlassLayoutHelperNode(KlassLayoutHelperNode n, LoweringTool tool) {
400        if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
401            return;
402        }
403        StructuredGraph graph = n.graph();
404        assert !n.getHub().isConstant();
405        AddressNode address = createOffsetAddress(graph, n.getHub(), runtime.getVMConfig().klassLayoutHelperOffset);
406        n.replaceAtUsagesAndDelete(graph.unique(new FloatingReadNode(address, KLASS_LAYOUT_HELPER_LOCATION, null, n.stamp(), null, BarrierType.NONE)));
407    }
408
409    private void lowerHubGetClassNode(HubGetClassNode n, LoweringTool tool) {
410        if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
411            return;
412        }
413
414        ValueNode hub = n.getHub();
415        GraalHotSpotVMConfig vmConfig = runtime.getVMConfig();
416        StructuredGraph graph = n.graph();
417        assert !hub.isConstant() || GraalOptions.ImmutableCode.getValue(graph.getOptions());
418        AddressNode mirrorAddress = createOffsetAddress(graph, hub, vmConfig.classMirrorOffset);
419        FloatingReadNode read = graph.unique(new FloatingReadNode(mirrorAddress, CLASS_MIRROR_LOCATION, null, vmConfig.classMirrorIsHandle ? StampFactory.forKind(target.wordJavaKind) : n.stamp(),
420                        null, BarrierType.NONE));
421        if (vmConfig.classMirrorIsHandle) {
422            AddressNode address = createOffsetAddress(graph, read, 0);
423            read = graph.unique(new FloatingReadNode(address, CLASS_MIRROR_HANDLE_LOCATION, null, n.stamp(), null, BarrierType.NONE));
424        }
425        n.replaceAtUsagesAndDelete(read);
426    }
427
428    private void lowerClassGetHubNode(ClassGetHubNode n, LoweringTool tool) {
429        if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
430            return;
431        }
432
433        StructuredGraph graph = n.graph();
434        assert !n.getValue().isConstant();
435        AddressNode address = createOffsetAddress(graph, n.getValue(), runtime.getVMConfig().klassOffset);
436        FloatingReadNode read = graph.unique(new FloatingReadNode(address, CLASS_KLASS_LOCATION, null, n.stamp(), null, BarrierType.NONE));
437        n.replaceAtUsagesAndDelete(read);
438    }
439
440    private void lowerInvoke(Invoke invoke, LoweringTool tool, StructuredGraph graph) {
441        if (invoke.callTarget() instanceof MethodCallTargetNode) {
442            MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget();
443            NodeInputList<ValueNode> parameters = callTarget.arguments();
444            ValueNode receiver = parameters.size() <= 0 ? null : parameters.get(0);
445            if (!callTarget.isStatic() && receiver.stamp() instanceof ObjectStamp && !StampTool.isPointerNonNull(receiver)) {
446                ValueNode nonNullReceiver = createNullCheckedValue(receiver, invoke.asNode(), tool);
447                parameters.set(0, nonNullReceiver);
448                receiver = nonNullReceiver;
449            }
450            JavaType[] signature = callTarget.targetMethod().getSignature().toParameterTypes(callTarget.isStatic() ? null : callTarget.targetMethod().getDeclaringClass());
451
452            LoweredCallTargetNode loweredCallTarget = null;
453            OptionValues options = graph.getOptions();
454            if (InlineVTableStubs.getValue(options) && callTarget.invokeKind().isIndirect() && (AlwaysInlineVTableStubs.getValue(options) || invoke.isPolymorphic())) {
455                HotSpotResolvedJavaMethod hsMethod = (HotSpotResolvedJavaMethod) callTarget.targetMethod();
456                ResolvedJavaType receiverType = invoke.getReceiverType();
457                if (hsMethod.isInVirtualMethodTable(receiverType)) {
458                    JavaKind wordKind = runtime.getTarget().wordJavaKind;
459                    ValueNode hub = createReadHub(graph, receiver, tool);
460
461                    ReadNode metaspaceMethod = createReadVirtualMethod(graph, hub, hsMethod, receiverType);
462                    // We use LocationNode.ANY_LOCATION for the reads that access the
463                    // compiled code entry as HotSpot does not guarantee they are final
464                    // values.
465                    int methodCompiledEntryOffset = runtime.getVMConfig().methodCompiledEntryOffset;
466                    AddressNode address = createOffsetAddress(graph, metaspaceMethod, methodCompiledEntryOffset);
467                    ReadNode compiledEntry = graph.add(new ReadNode(address, any(), StampFactory.forKind(wordKind), BarrierType.NONE));
468
469                    loweredCallTarget = graph.add(new HotSpotIndirectCallTargetNode(metaspaceMethod, compiledEntry, parameters.toArray(new ValueNode[parameters.size()]), callTarget.returnStamp(),
470                                    signature, callTarget.targetMethod(),
471                                    HotSpotCallingConventionType.JavaCall, callTarget.invokeKind()));
472
473                    graph.addBeforeFixed(invoke.asNode(), metaspaceMethod);
474                    graph.addAfterFixed(metaspaceMethod, compiledEntry);
475                }
476            }
477
478            if (loweredCallTarget == null) {
479                loweredCallTarget = graph.add(new HotSpotDirectCallTargetNode(parameters.toArray(new ValueNode[parameters.size()]), callTarget.returnStamp(),
480                                signature, callTarget.targetMethod(),
481                                HotSpotCallingConventionType.JavaCall,
482                                callTarget.invokeKind()));
483            }
484            callTarget.replaceAndDelete(loweredCallTarget);
485        }
486    }
487
488    @Override
489    protected Stamp loadStamp(Stamp stamp, JavaKind kind, boolean compressible) {
490        if (kind == JavaKind.Object && compressible && runtime.getVMConfig().useCompressedOops) {
491            return NarrowOopStamp.compressed((ObjectStamp) stamp, runtime.getVMConfig().getOopEncoding());
492        }
493        return super.loadStamp(stamp, kind, compressible);
494    }
495
496    @Override
497    protected ValueNode implicitLoadConvert(JavaKind kind, ValueNode value, boolean compressible) {
498        if (kind == JavaKind.Object && compressible && runtime.getVMConfig().useCompressedOops) {
499            return new CompressionNode(CompressionOp.Uncompress, value, runtime.getVMConfig().getOopEncoding());
500        }
501        return super.implicitLoadConvert(kind, value, compressible);
502    }
503
504    @Override
505    public ValueNode staticFieldBase(StructuredGraph graph, ResolvedJavaField f) {
506        HotSpotResolvedJavaField field = (HotSpotResolvedJavaField) f;
507        JavaConstant base = constantReflection.asJavaClass(field.getDeclaringClass());
508        return ConstantNode.forConstant(base, metaAccess, graph);
509    }
510
511    @Override
512    protected ValueNode implicitStoreConvert(JavaKind kind, ValueNode value, boolean compressible) {
513        if (kind == JavaKind.Object && compressible && runtime.getVMConfig().useCompressedOops) {
514            return new CompressionNode(CompressionOp.Compress, value, runtime.getVMConfig().getOopEncoding());
515        }
516        return super.implicitStoreConvert(kind, value, compressible);
517    }
518
519    @Override
520    protected ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode arrayHub, FixedNode anchor) {
521        /*
522         * Anchor the read of the element klass to the cfg, because it is only valid when arrayClass
523         * is an object class, which might not be the case in other parts of the compiled method.
524         */
525        AddressNode address = createOffsetAddress(graph, arrayHub, runtime.getVMConfig().arrayClassElementOffset);
526        return graph.unique(new FloatingReadNode(address, OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION, null, KlassPointerStamp.klassNonNull(), AbstractBeginNode.prevBegin(anchor)));
527    }
528
529    @Override
530    protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) {
531        StructuredGraph graph = load.graph();
532        if (!(load instanceof GuardedUnsafeLoadNode) && !graph.getGuardsStage().allowsFloatingGuards() && addReadBarrier(load)) {
533            unsafeLoadSnippets.lower(load, tool);
534        } else {
535            super.lowerUnsafeLoadNode(load, tool);
536        }
537    }
538
539    private void lowerLoadMethodNode(LoadMethodNode loadMethodNode) {
540        StructuredGraph graph = loadMethodNode.graph();
541        HotSpotResolvedJavaMethod method = (HotSpotResolvedJavaMethod) loadMethodNode.getMethod();
542        ReadNode metaspaceMethod = createReadVirtualMethod(graph, loadMethodNode.getHub(), method, loadMethodNode.getReceiverType());
543        graph.replaceFixed(loadMethodNode, metaspaceMethod);
544    }
545
546    private static void lowerGetClassNode(GetClassNode getClass, LoweringTool tool, StructuredGraph graph) {
547        StampProvider stampProvider = tool.getStampProvider();
548        LoadHubNode hub = graph.unique(new LoadHubNode(stampProvider, getClass.getObject()));
549        HubGetClassNode hubGetClass = graph.unique(new HubGetClassNode(tool.getMetaAccess(), hub));
550        getClass.replaceAtUsagesAndDelete(hubGetClass);
551        hub.lower(tool);
552        hubGetClass.lower(tool);
553    }
554
555    private void lowerStoreHubNode(StoreHubNode storeHub, StructuredGraph graph) {
556        WriteNode hub = createWriteHub(graph, storeHub.getObject(), storeHub.getValue());
557        graph.replaceFixed(storeHub, hub);
558    }
559
560    @Override
561    public BarrierType fieldInitializationBarrier(JavaKind entryKind) {
562        return (entryKind == JavaKind.Object && !runtime.getVMConfig().useDeferredInitBarriers) ? BarrierType.IMPRECISE : BarrierType.NONE;
563    }
564
565    @Override
566    public BarrierType arrayInitializationBarrier(JavaKind entryKind) {
567        return (entryKind == JavaKind.Object && !runtime.getVMConfig().useDeferredInitBarriers) ? BarrierType.PRECISE : BarrierType.NONE;
568    }
569
570    private void lowerOSRStartNode(OSRStartNode osrStart) {
571        StructuredGraph graph = osrStart.graph();
572        if (graph.getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) {
573            StartNode newStart = graph.add(new StartNode());
574            ParameterNode buffer = graph.addWithoutUnique(new ParameterNode(0, StampPair.createSingle(StampFactory.forKind(runtime.getTarget().wordJavaKind))));
575            ForeignCallNode migrationEnd = graph.add(new ForeignCallNode(foreignCalls, OSR_MIGRATION_END, buffer));
576            migrationEnd.setStateAfter(osrStart.stateAfter());
577            newStart.setNext(migrationEnd);
578            FixedNode next = osrStart.next();
579            osrStart.setNext(null);
580            migrationEnd.setNext(next);
581            graph.setStart(newStart);
582
583            final int wordSize = target.wordSize;
584
585            // @formatter:off
586            // taken from c2 locals_addr = osr_buf + (max_locals-1)*wordSize)
587            // @formatter:on
588            int localsOffset = (graph.method().getMaxLocals() - 1) * wordSize;
589            for (OSRLocalNode osrLocal : graph.getNodes(OSRLocalNode.TYPE)) {
590                int size = osrLocal.getStackKind().getSlotCount();
591                int offset = localsOffset - (osrLocal.index() + size - 1) * wordSize;
592                AddressNode address = createOffsetAddress(graph, buffer, offset);
593                ReadNode load = graph.add(new ReadNode(address, any(), osrLocal.stamp(), BarrierType.NONE));
594                osrLocal.replaceAndDelete(load);
595                graph.addBeforeFixed(migrationEnd, load);
596            }
597
598            // @formatter:off
599            // taken from c2 monitors_addr = osr_buf + (max_locals+mcnt*2-1)*wordSize);
600            // @formatter:on
601            final int lockCount = osrStart.stateAfter().locksSize();
602            final int locksOffset = (graph.method().getMaxLocals() + lockCount * 2 - 1) * wordSize;
603
604            // first initialize the lock slots for all enters with the displaced marks read from the
605            // buffer
606            for (OSRMonitorEnterNode osrMonitorEnter : graph.getNodes(OSRMonitorEnterNode.TYPE)) {
607                MonitorIdNode monitorID = osrMonitorEnter.getMonitorId();
608                OSRLockNode lock = (OSRLockNode) osrMonitorEnter.object();
609                final int index = lock.index();
610
611                final int offsetDisplacedHeader = locksOffset - ((index * 2) + 1) * wordSize;
612                final int offsetLockObject = locksOffset - index * 2 * wordSize;
613
614                // load the displaced mark from the osr buffer
615                AddressNode addressDisplacedHeader = createOffsetAddress(graph, buffer, offsetDisplacedHeader);
616                ReadNode loadDisplacedHeader = graph.add(new ReadNode(addressDisplacedHeader, any(), lock.stamp(), BarrierType.NONE));
617                graph.addBeforeFixed(migrationEnd, loadDisplacedHeader);
618
619                // we need to initialize the stack slot for the lock
620                BeginLockScopeNode beginLockScope = graph.add(new BeginLockScopeNode(lock.getStackKind(), monitorID.getLockDepth()));
621                graph.addBeforeFixed(migrationEnd, beginLockScope);
622
623                // write the displaced mark to the correct stack slot
624                AddressNode addressDisplacedMark = createOffsetAddress(graph, beginLockScope, runtime.getVMConfig().basicLockDisplacedHeaderOffset);
625                WriteNode writeStackSlot = graph.add(new WriteNode(addressDisplacedMark, DISPLACED_MARK_WORD_LOCATION, loadDisplacedHeader, BarrierType.NONE));
626                graph.addBeforeFixed(migrationEnd, writeStackSlot);
627
628                // load the lock object from the osr buffer
629                AddressNode addressLockObject = createOffsetAddress(graph, buffer, offsetLockObject);
630                ReadNode loadObject = graph.add(new ReadNode(addressLockObject, any(), lock.stamp(), BarrierType.NONE));
631                lock.replaceAndDelete(loadObject);
632                graph.addBeforeFixed(migrationEnd, loadObject);
633            }
634
635            osrStart.replaceAtUsagesAndDelete(newStart);
636        }
637    }
638
639    static final class Exceptions {
640        protected static final ArrayIndexOutOfBoundsException cachedArrayIndexOutOfBoundsException;
641        protected static final NullPointerException cachedNullPointerException;
642
643        static {
644            cachedArrayIndexOutOfBoundsException = new ArrayIndexOutOfBoundsException();
645            cachedArrayIndexOutOfBoundsException.setStackTrace(new StackTraceElement[0]);
646            cachedNullPointerException = new NullPointerException();
647            cachedNullPointerException.setStackTrace(new StackTraceElement[0]);
648        }
649    }
650
651    public static final class RuntimeCalls {
652        public static final ForeignCallDescriptor CREATE_ARRAY_STORE_EXCEPTION = new ForeignCallDescriptor("createArrayStoreException", ArrayStoreException.class, Object.class);
653        public static final ForeignCallDescriptor CREATE_CLASS_CAST_EXCEPTION = new ForeignCallDescriptor("createClassCastException", ClassCastException.class, Object.class, KlassPointer.class);
654        public static final ForeignCallDescriptor CREATE_NULL_POINTER_EXCEPTION = new ForeignCallDescriptor("createNullPointerException", NullPointerException.class);
655        public static final ForeignCallDescriptor CREATE_OUT_OF_BOUNDS_EXCEPTION = new ForeignCallDescriptor("createOutOfBoundsException", ArrayIndexOutOfBoundsException.class, int.class);
656    }
657
658    private boolean throwCachedException(BytecodeExceptionNode node) {
659        Throwable exception;
660        if (node.getExceptionClass() == NullPointerException.class) {
661            exception = Exceptions.cachedNullPointerException;
662        } else if (node.getExceptionClass() == ArrayIndexOutOfBoundsException.class) {
663            exception = Exceptions.cachedArrayIndexOutOfBoundsException;
664        } else {
665            return false;
666        }
667
668        StructuredGraph graph = node.graph();
669        FloatingNode exceptionNode = ConstantNode.forConstant(constantReflection.forObject(exception), metaAccess, graph);
670        graph.replaceFixedWithFloating(node, exceptionNode);
671        return true;
672    }
673
674    private void lowerBytecodeExceptionNode(BytecodeExceptionNode node) {
675        if (OmitHotExceptionStacktrace.getValue(node.getOptions())) {
676            if (throwCachedException(node)) {
677                return;
678            }
679        }
680
681        ForeignCallDescriptor descriptor;
682        if (node.getExceptionClass() == NullPointerException.class) {
683            descriptor = RuntimeCalls.CREATE_NULL_POINTER_EXCEPTION;
684        } else if (node.getExceptionClass() == ArrayIndexOutOfBoundsException.class) {
685            descriptor = RuntimeCalls.CREATE_OUT_OF_BOUNDS_EXCEPTION;
686        } else if (node.getExceptionClass() == ArrayStoreException.class) {
687            descriptor = RuntimeCalls.CREATE_ARRAY_STORE_EXCEPTION;
688        } else if (node.getExceptionClass() == ClassCastException.class) {
689            descriptor = RuntimeCalls.CREATE_CLASS_CAST_EXCEPTION;
690        } else {
691            throw GraalError.shouldNotReachHere();
692        }
693
694        StructuredGraph graph = node.graph();
695        ForeignCallNode foreignCallNode = graph.add(new ForeignCallNode(foreignCalls, descriptor, node.stamp(), node.getArguments()));
696        graph.replaceFixedWithFixed(node, foreignCallNode);
697    }
698
699    private boolean addReadBarrier(RawLoadNode load) {
700        if (runtime.getVMConfig().useG1GC && load.graph().getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS && load.object().getStackKind() == JavaKind.Object &&
701                        load.accessKind() == JavaKind.Object && !StampTool.isPointerAlwaysNull(load.object())) {
702            ResolvedJavaType type = StampTool.typeOrNull(load.object());
703            if (type != null && !type.isArray()) {
704                return true;
705            }
706        }
707        return false;
708    }
709
710    private ReadNode createReadVirtualMethod(StructuredGraph graph, ValueNode hub, HotSpotResolvedJavaMethod method, ResolvedJavaType receiverType) {
711        return createReadVirtualMethod(graph, hub, method.vtableEntryOffset(receiverType));
712    }
713
714    private ReadNode createReadVirtualMethod(StructuredGraph graph, ValueNode hub, int vtableEntryOffset) {
715        assert vtableEntryOffset > 0;
716        // We use LocationNode.ANY_LOCATION for the reads that access the vtable
717        // entry as HotSpot does not guarantee that this is a final value.
718        Stamp methodStamp = MethodPointerStamp.methodNonNull();
719        AddressNode address = createOffsetAddress(graph, hub, vtableEntryOffset);
720        ReadNode metaspaceMethod = graph.add(new ReadNode(address, any(), methodStamp, BarrierType.NONE));
721        return metaspaceMethod;
722    }
723
724    @Override
725    protected ValueNode createReadHub(StructuredGraph graph, ValueNode object, LoweringTool tool) {
726        if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
727            return graph.unique(new LoadHubNode(tool.getStampProvider(), object));
728        }
729        assert !object.isConstant() || object.isNullConstant();
730
731        KlassPointerStamp hubStamp = KlassPointerStamp.klassNonNull();
732        if (runtime.getVMConfig().useCompressedClassPointers) {
733            hubStamp = hubStamp.compressed(runtime.getVMConfig().getKlassEncoding());
734        }
735
736        AddressNode address = createOffsetAddress(graph, object, runtime.getVMConfig().hubOffset);
737        LocationIdentity hubLocation = runtime.getVMConfig().useCompressedClassPointers ? COMPRESSED_HUB_LOCATION : HUB_LOCATION;
738        FloatingReadNode memoryRead = graph.unique(new FloatingReadNode(address, hubLocation, null, hubStamp, null, BarrierType.NONE));
739        if (runtime.getVMConfig().useCompressedClassPointers) {
740            return CompressionNode.uncompress(memoryRead, runtime.getVMConfig().getKlassEncoding());
741        } else {
742            return memoryRead;
743        }
744    }
745
746    private WriteNode createWriteHub(StructuredGraph graph, ValueNode object, ValueNode value) {
747        assert !object.isConstant() || object.asConstant().isDefaultForKind();
748
749        ValueNode writeValue = value;
750        if (runtime.getVMConfig().useCompressedClassPointers) {
751            writeValue = CompressionNode.compress(value, runtime.getVMConfig().getKlassEncoding());
752        }
753
754        AddressNode address = createOffsetAddress(graph, object, runtime.getVMConfig().hubOffset);
755        return graph.add(new WriteNode(address, HUB_WRITE_LOCATION, writeValue, BarrierType.NONE));
756    }
757
758    @Override
759    protected BarrierType fieldLoadBarrierType(ResolvedJavaField f) {
760        HotSpotResolvedJavaField loadField = (HotSpotResolvedJavaField) f;
761        BarrierType barrierType = BarrierType.NONE;
762        if (runtime.getVMConfig().useG1GC && loadField.getJavaKind() == JavaKind.Object && metaAccess.lookupJavaType(Reference.class).equals(loadField.getDeclaringClass()) &&
763                        loadField.getName().equals("referent")) {
764            barrierType = BarrierType.PRECISE;
765        }
766        return barrierType;
767    }
768
769    @Override
770    public int fieldOffset(ResolvedJavaField f) {
771        HotSpotResolvedJavaField field = (HotSpotResolvedJavaField) f;
772        return field.offset();
773    }
774
775    @Override
776    public int arrayScalingFactor(JavaKind kind) {
777        if (runtime.getVMConfig().useCompressedOops && kind == JavaKind.Object) {
778            return super.arrayScalingFactor(JavaKind.Int);
779        } else {
780            return super.arrayScalingFactor(kind);
781        }
782    }
783
784    @Override
785    public int arrayBaseOffset(JavaKind kind) {
786        return getArrayBaseOffset(kind);
787    }
788
789    @Override
790    public int arrayLengthOffset() {
791        return runtime.getVMConfig().arrayOopDescLengthOffset();
792    }
793}
794