SchedulePhase.java revision 12968:4d8a004e5c6d
1/* 2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23package org.graalvm.compiler.phases.schedule; 24 25import static org.graalvm.compiler.core.common.GraalOptions.OptScheduleOutOfLoops; 26import static org.graalvm.compiler.core.common.cfg.AbstractControlFlowGraph.strictlyDominates; 27 28import java.util.ArrayList; 29import java.util.Arrays; 30import java.util.Formatter; 31import java.util.List; 32 33import org.graalvm.compiler.core.common.GraalOptions; 34import org.graalvm.compiler.core.common.LocationIdentity; 35import org.graalvm.compiler.core.common.SuppressFBWarnings; 36import org.graalvm.compiler.core.common.cfg.AbstractControlFlowGraph; 37import org.graalvm.compiler.core.common.cfg.BlockMap; 38import org.graalvm.compiler.debug.Assertions; 39import org.graalvm.compiler.debug.Debug; 40import org.graalvm.compiler.graph.Graph.NodeEvent; 41import org.graalvm.compiler.graph.Graph.NodeEventListener; 42import org.graalvm.compiler.graph.Graph.NodeEventScope; 43import org.graalvm.compiler.graph.Node; 44import org.graalvm.compiler.graph.NodeBitMap; 45import org.graalvm.compiler.graph.NodeMap; 46import org.graalvm.compiler.graph.NodeStack; 47import org.graalvm.compiler.nodes.AbstractBeginNode; 48import org.graalvm.compiler.nodes.AbstractEndNode; 49import org.graalvm.compiler.nodes.AbstractMergeNode; 50import org.graalvm.compiler.nodes.ControlSinkNode; 51import org.graalvm.compiler.nodes.ControlSplitNode; 52import org.graalvm.compiler.nodes.DeoptimizeNode; 53import org.graalvm.compiler.nodes.FixedNode; 54import org.graalvm.compiler.nodes.GuardNode; 55import org.graalvm.compiler.nodes.IfNode; 56import org.graalvm.compiler.nodes.KillingBeginNode; 57import org.graalvm.compiler.nodes.LoopBeginNode; 58import org.graalvm.compiler.nodes.LoopExitNode; 59import org.graalvm.compiler.nodes.PhiNode; 60import org.graalvm.compiler.nodes.ProxyNode; 61import org.graalvm.compiler.nodes.StartNode; 62import org.graalvm.compiler.nodes.StructuredGraph; 63import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; 64import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult; 65import org.graalvm.compiler.nodes.ValueNode; 66import org.graalvm.compiler.nodes.VirtualState; 67import org.graalvm.compiler.nodes.calc.ConvertNode; 68import org.graalvm.compiler.nodes.calc.IsNullNode; 69import org.graalvm.compiler.nodes.cfg.Block; 70import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; 71import org.graalvm.compiler.nodes.cfg.HIRLoop; 72import org.graalvm.compiler.nodes.cfg.LocationSet; 73import org.graalvm.compiler.nodes.memory.FloatingReadNode; 74import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; 75import org.graalvm.compiler.nodes.spi.ValueProxy; 76import org.graalvm.compiler.options.OptionValues; 77import org.graalvm.compiler.phases.Phase; 78import org.graalvm.compiler.nodes.FixedWithNextNode; 79 80public final class SchedulePhase extends Phase { 81 82 public enum SchedulingStrategy { 83 EARLIEST, 84 LATEST, 85 LATEST_OUT_OF_LOOPS, 86 FINAL_SCHEDULE 87 } 88 89 private final SchedulingStrategy selectedStrategy; 90 91 private final boolean immutableGraph; 92 93 public SchedulePhase(OptionValues options) { 94 this(false, options); 95 } 96 97 public SchedulePhase(boolean immutableGraph, OptionValues options) { 98 this(OptScheduleOutOfLoops.getValue(options) ? SchedulingStrategy.LATEST_OUT_OF_LOOPS : SchedulingStrategy.LATEST, immutableGraph); 99 } 100 101 public SchedulePhase(SchedulingStrategy strategy) { 102 this(strategy, false); 103 } 104 105 public SchedulePhase(SchedulingStrategy strategy, boolean immutableGraph) { 106 this.selectedStrategy = strategy; 107 this.immutableGraph = immutableGraph; 108 } 109 110 private NodeEventScope verifyImmutableGraph(StructuredGraph graph) { 111 if (immutableGraph && Assertions.ENABLED) { 112 return graph.trackNodeEvents(new NodeEventListener() { 113 @Override 114 public void event(NodeEvent e, Node node) { 115 assert false : "graph changed: " + e + " on node " + node; 116 } 117 }); 118 } else { 119 return null; 120 } 121 } 122 123 @Override 124 @SuppressWarnings("try") 125 protected void run(StructuredGraph graph) { 126 try (NodeEventScope scope = verifyImmutableGraph(graph)) { 127 Instance inst = new Instance(); 128 inst.run(graph, selectedStrategy, immutableGraph); 129 } 130 } 131 132 public static void run(StructuredGraph graph, SchedulingStrategy strategy, ControlFlowGraph cfg) { 133 Instance inst = new Instance(cfg); 134 inst.run(graph, strategy, false); 135 } 136 137 public static class Instance { 138 139 private static final double IMPLICIT_NULL_CHECK_OPPORTUNITY_PROBABILITY_FACTOR = 2; 140 /** 141 * Map from blocks to the nodes in each block. 142 */ 143 protected ControlFlowGraph cfg; 144 protected BlockMap<List<Node>> blockToNodesMap; 145 protected NodeMap<Block> nodeToBlockMap; 146 147 public Instance() { 148 this(null); 149 } 150 151 public Instance(ControlFlowGraph cfg) { 152 this.cfg = cfg; 153 } 154 155 @SuppressWarnings("try") 156 public void run(StructuredGraph graph, SchedulingStrategy selectedStrategy, boolean immutableGraph) { 157 // assert GraphOrder.assertNonCyclicGraph(graph); 158 159 if (this.cfg == null) { 160 this.cfg = ControlFlowGraph.compute(graph, true, true, true, false); 161 } 162 163 NodeMap<Block> currentNodeMap = graph.createNodeMap(); 164 NodeBitMap visited = graph.createNodeBitMap(); 165 BlockMap<List<Node>> earliestBlockToNodesMap = new BlockMap<>(cfg); 166 this.nodeToBlockMap = currentNodeMap; 167 this.blockToNodesMap = earliestBlockToNodesMap; 168 169 scheduleEarliestIterative(earliestBlockToNodesMap, currentNodeMap, visited, graph, immutableGraph); 170 171 if (selectedStrategy != SchedulingStrategy.EARLIEST) { 172 // For non-earliest schedules, we need to do a second pass. 173 BlockMap<List<Node>> latestBlockToNodesMap = new BlockMap<>(cfg); 174 for (Block b : cfg.getBlocks()) { 175 latestBlockToNodesMap.put(b, new ArrayList<Node>()); 176 } 177 178 BlockMap<ArrayList<FloatingReadNode>> watchListMap = calcLatestBlocks(selectedStrategy, currentNodeMap, earliestBlockToNodesMap, visited, latestBlockToNodesMap, immutableGraph); 179 sortNodesLatestWithinBlock(cfg, earliestBlockToNodesMap, latestBlockToNodesMap, currentNodeMap, watchListMap, visited); 180 181 assert verifySchedule(cfg, latestBlockToNodesMap, currentNodeMap); 182 assert (!GraalOptions.DetailedAsserts.getValue(graph.getOptions())) || MemoryScheduleVerification.check(cfg.getStartBlock(), latestBlockToNodesMap); 183 184 this.blockToNodesMap = latestBlockToNodesMap; 185 186 cfg.setNodeToBlock(currentNodeMap); 187 } 188 189 graph.setLastSchedule(new ScheduleResult(this.cfg, this.nodeToBlockMap, this.blockToNodesMap)); 190 } 191 192 @SuppressFBWarnings(value = "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE", justification = "false positive found by findbugs") 193 private BlockMap<ArrayList<FloatingReadNode>> calcLatestBlocks(SchedulingStrategy strategy, NodeMap<Block> currentNodeMap, BlockMap<List<Node>> earliestBlockToNodesMap, NodeBitMap visited, 194 BlockMap<List<Node>> latestBlockToNodesMap, boolean immutableGraph) { 195 BlockMap<ArrayList<FloatingReadNode>> watchListMap = new BlockMap<>(cfg); 196 Block[] reversePostOrder = cfg.reversePostOrder(); 197 for (int j = reversePostOrder.length - 1; j >= 0; --j) { 198 Block currentBlock = reversePostOrder[j]; 199 List<Node> blockToNodes = earliestBlockToNodesMap.get(currentBlock); 200 LocationSet killed = null; 201 int previousIndex = blockToNodes.size(); 202 for (int i = blockToNodes.size() - 1; i >= 0; --i) { 203 Node currentNode = blockToNodes.get(i); 204 assert currentNodeMap.get(currentNode) == currentBlock; 205 assert !(currentNode instanceof PhiNode) && !(currentNode instanceof ProxyNode); 206 assert visited.isMarked(currentNode); 207 if (currentNode instanceof FixedNode) { 208 // For these nodes, the earliest is at the same time the latest block. 209 } else { 210 Block latestBlock = null; 211 212 LocationIdentity constrainingLocation = null; 213 if (currentNode instanceof FloatingReadNode) { 214 // We are scheduling a floating read node => check memory 215 // anti-dependencies. 216 FloatingReadNode floatingReadNode = (FloatingReadNode) currentNode; 217 LocationIdentity location = floatingReadNode.getLocationIdentity(); 218 if (location.isMutable()) { 219 // Location can be killed. 220 constrainingLocation = location; 221 if (currentBlock.canKill(location)) { 222 if (killed == null) { 223 killed = new LocationSet(); 224 } 225 fillKillSet(killed, blockToNodes.subList(i + 1, previousIndex)); 226 previousIndex = i; 227 if (killed.contains(location)) { 228 // Earliest block kills location => we need to stay within 229 // earliest block. 230 latestBlock = currentBlock; 231 } 232 } 233 } 234 } 235 236 if (latestBlock == null) { 237 // We are not constraint within earliest block => calculate optimized 238 // schedule. 239 calcLatestBlock(currentBlock, strategy, currentNode, currentNodeMap, constrainingLocation, watchListMap, latestBlockToNodesMap, visited, immutableGraph); 240 } else { 241 selectLatestBlock(currentNode, currentBlock, latestBlock, currentNodeMap, watchListMap, constrainingLocation, latestBlockToNodesMap); 242 } 243 } 244 } 245 } 246 return watchListMap; 247 } 248 249 protected static void selectLatestBlock(Node currentNode, Block currentBlock, Block latestBlock, NodeMap<Block> currentNodeMap, BlockMap<ArrayList<FloatingReadNode>> watchListMap, 250 LocationIdentity constrainingLocation, BlockMap<List<Node>> latestBlockToNodesMap) { 251 252 assert checkLatestEarliestRelation(currentNode, currentBlock, latestBlock); 253 if (currentBlock != latestBlock) { 254 255 currentNodeMap.setAndGrow(currentNode, latestBlock); 256 257 if (constrainingLocation != null && latestBlock.canKill(constrainingLocation)) { 258 if (watchListMap.get(latestBlock) == null) { 259 watchListMap.put(latestBlock, new ArrayList<>()); 260 } 261 watchListMap.get(latestBlock).add((FloatingReadNode) currentNode); 262 } 263 } 264 265 latestBlockToNodesMap.get(latestBlock).add(currentNode); 266 } 267 268 private static boolean checkLatestEarliestRelation(Node currentNode, Block earliestBlock, Block latestBlock) { 269 assert AbstractControlFlowGraph.dominates(earliestBlock, latestBlock) || (currentNode instanceof VirtualState && latestBlock == earliestBlock.getDominator()) : String.format( 270 "%s %s (%s) %s (%s)", currentNode, earliestBlock, earliestBlock.getBeginNode(), latestBlock, latestBlock.getBeginNode()); 271 return true; 272 } 273 274 private static boolean verifySchedule(ControlFlowGraph cfg, BlockMap<List<Node>> blockToNodesMap, NodeMap<Block> nodeMap) { 275 for (Block b : cfg.getBlocks()) { 276 List<Node> nodes = blockToNodesMap.get(b); 277 for (Node n : nodes) { 278 assert n.isAlive(); 279 assert nodeMap.get(n) == b; 280 StructuredGraph g = (StructuredGraph) n.graph(); 281 if (g.hasLoops() && g.getGuardsStage() == GuardsStage.AFTER_FSA && n instanceof DeoptimizeNode) { 282 assert b.getLoopDepth() == 0 : n; 283 } 284 } 285 } 286 return true; 287 } 288 289 public static Block checkKillsBetween(Block earliestBlock, Block latestBlock, LocationIdentity location) { 290 assert strictlyDominates(earliestBlock, latestBlock); 291 Block current = latestBlock.getDominator(); 292 293 // Collect dominator chain that needs checking. 294 List<Block> dominatorChain = new ArrayList<>(); 295 dominatorChain.add(latestBlock); 296 while (current != earliestBlock) { 297 // Current is an intermediate dominator between earliestBlock and latestBlock. 298 assert strictlyDominates(earliestBlock, current) && strictlyDominates(current, latestBlock); 299 if (current.canKill(location)) { 300 dominatorChain.clear(); 301 } 302 dominatorChain.add(current); 303 current = current.getDominator(); 304 } 305 306 // The first element of dominatorChain now contains the latest possible block. 307 assert dominatorChain.size() >= 1; 308 assert dominatorChain.get(dominatorChain.size() - 1).getDominator() == earliestBlock; 309 310 Block lastBlock = earliestBlock; 311 for (int i = dominatorChain.size() - 1; i >= 0; --i) { 312 Block currentBlock = dominatorChain.get(i); 313 if (currentBlock.getLoopDepth() > lastBlock.getLoopDepth()) { 314 // We are entering a loop boundary. The new loops must not kill the location for 315 // the crossing to be safe. 316 if (currentBlock.getLoop() != null && ((HIRLoop) currentBlock.getLoop()).canKill(location)) { 317 break; 318 } 319 } 320 321 if (currentBlock.canKillBetweenThisAndDominator(location)) { 322 break; 323 } 324 lastBlock = currentBlock; 325 } 326 327 if (lastBlock.getBeginNode() instanceof KillingBeginNode) { 328 LocationIdentity locationIdentity = ((KillingBeginNode) lastBlock.getBeginNode()).getLocationIdentity(); 329 if ((locationIdentity.isAny() || locationIdentity.equals(location)) && lastBlock != earliestBlock) { 330 // The begin of this block kills the location, so we *have* to schedule the node 331 // in the dominating block. 332 lastBlock = lastBlock.getDominator(); 333 } 334 } 335 336 return lastBlock; 337 } 338 339 private static void fillKillSet(LocationSet killed, List<Node> subList) { 340 if (!killed.isAny()) { 341 for (Node n : subList) { 342 // Check if this node kills a node in the watch list. 343 if (n instanceof MemoryCheckpoint.Single) { 344 LocationIdentity identity = ((MemoryCheckpoint.Single) n).getLocationIdentity(); 345 killed.add(identity); 346 if (killed.isAny()) { 347 return; 348 } 349 } else if (n instanceof MemoryCheckpoint.Multi) { 350 for (LocationIdentity identity : ((MemoryCheckpoint.Multi) n).getLocationIdentities()) { 351 killed.add(identity); 352 if (killed.isAny()) { 353 return; 354 } 355 } 356 } 357 } 358 } 359 } 360 361 private static void sortNodesLatestWithinBlock(ControlFlowGraph cfg, BlockMap<List<Node>> earliestBlockToNodesMap, BlockMap<List<Node>> latestBlockToNodesMap, NodeMap<Block> currentNodeMap, 362 BlockMap<ArrayList<FloatingReadNode>> watchListMap, NodeBitMap visited) { 363 for (Block b : cfg.getBlocks()) { 364 sortNodesLatestWithinBlock(b, earliestBlockToNodesMap, latestBlockToNodesMap, currentNodeMap, watchListMap, visited); 365 } 366 } 367 368 private static void sortNodesLatestWithinBlock(Block b, BlockMap<List<Node>> earliestBlockToNodesMap, BlockMap<List<Node>> latestBlockToNodesMap, NodeMap<Block> nodeMap, 369 BlockMap<ArrayList<FloatingReadNode>> watchListMap, NodeBitMap unprocessed) { 370 List<Node> earliestSorting = earliestBlockToNodesMap.get(b); 371 ArrayList<Node> result = new ArrayList<>(earliestSorting.size()); 372 ArrayList<FloatingReadNode> watchList = null; 373 if (watchListMap != null) { 374 watchList = watchListMap.get(b); 375 assert watchList == null || !b.getKillLocations().isEmpty(); 376 } 377 AbstractBeginNode beginNode = b.getBeginNode(); 378 if (beginNode instanceof LoopExitNode) { 379 LoopExitNode loopExitNode = (LoopExitNode) beginNode; 380 for (ProxyNode proxy : loopExitNode.proxies()) { 381 unprocessed.clear(proxy); 382 ValueNode value = proxy.value(); 383 // if multiple proxies reference the same value, schedule the value of a 384 // proxy once 385 if (value != null && nodeMap.get(value) == b && unprocessed.isMarked(value)) { 386 sortIntoList(value, b, result, nodeMap, unprocessed, null); 387 } 388 } 389 } 390 FixedNode endNode = b.getEndNode(); 391 FixedNode fixedEndNode = null; 392 if (isFixedEnd(endNode)) { 393 // Only if the end node is either a control split or an end node, we need to force 394 // it to be the last node in the schedule. 395 fixedEndNode = endNode; 396 } 397 for (Node n : earliestSorting) { 398 if (n != fixedEndNode) { 399 if (n instanceof FixedNode) { 400 assert nodeMap.get(n) == b; 401 checkWatchList(b, nodeMap, unprocessed, result, watchList, n); 402 sortIntoList(n, b, result, nodeMap, unprocessed, null); 403 } else if (nodeMap.get(n) == b && n instanceof FloatingReadNode) { 404 FloatingReadNode floatingReadNode = (FloatingReadNode) n; 405 if (isImplicitNullOpportunity(floatingReadNode, b)) { 406 // Schedule at the beginning of the block. 407 sortIntoList(floatingReadNode, b, result, nodeMap, unprocessed, null); 408 } else { 409 LocationIdentity location = floatingReadNode.getLocationIdentity(); 410 if (b.canKill(location)) { 411 // This read can be killed in this block, add to watch list. 412 if (watchList == null) { 413 watchList = new ArrayList<>(); 414 } 415 watchList.add(floatingReadNode); 416 } 417 } 418 } 419 } 420 } 421 422 for (Node n : latestBlockToNodesMap.get(b)) { 423 assert nodeMap.get(n) == b : n; 424 assert !(n instanceof FixedNode); 425 if (unprocessed.isMarked(n)) { 426 sortIntoList(n, b, result, nodeMap, unprocessed, fixedEndNode); 427 } 428 } 429 430 if (endNode != null && unprocessed.isMarked(endNode)) { 431 sortIntoList(endNode, b, result, nodeMap, unprocessed, null); 432 } 433 434 latestBlockToNodesMap.put(b, result); 435 } 436 437 private static void checkWatchList(Block b, NodeMap<Block> nodeMap, NodeBitMap unprocessed, ArrayList<Node> result, ArrayList<FloatingReadNode> watchList, Node n) { 438 if (watchList != null && !watchList.isEmpty()) { 439 // Check if this node kills a node in the watch list. 440 if (n instanceof MemoryCheckpoint.Single) { 441 LocationIdentity identity = ((MemoryCheckpoint.Single) n).getLocationIdentity(); 442 checkWatchList(watchList, identity, b, result, nodeMap, unprocessed); 443 } else if (n instanceof MemoryCheckpoint.Multi) { 444 for (LocationIdentity identity : ((MemoryCheckpoint.Multi) n).getLocationIdentities()) { 445 checkWatchList(watchList, identity, b, result, nodeMap, unprocessed); 446 } 447 } 448 } 449 } 450 451 private static void checkWatchList(ArrayList<FloatingReadNode> watchList, LocationIdentity identity, Block b, ArrayList<Node> result, NodeMap<Block> nodeMap, NodeBitMap unprocessed) { 452 if (identity.isImmutable()) { 453 // Nothing to do. This can happen for an initialization write. 454 } else if (identity.isAny()) { 455 for (FloatingReadNode r : watchList) { 456 if (unprocessed.isMarked(r)) { 457 sortIntoList(r, b, result, nodeMap, unprocessed, null); 458 } 459 } 460 watchList.clear(); 461 } else { 462 int index = 0; 463 while (index < watchList.size()) { 464 FloatingReadNode r = watchList.get(index); 465 LocationIdentity locationIdentity = r.getLocationIdentity(); 466 assert locationIdentity.isMutable(); 467 if (unprocessed.isMarked(r)) { 468 if (identity.overlaps(locationIdentity)) { 469 sortIntoList(r, b, result, nodeMap, unprocessed, null); 470 } else { 471 ++index; 472 continue; 473 } 474 } 475 int lastIndex = watchList.size() - 1; 476 watchList.set(index, watchList.get(lastIndex)); 477 watchList.remove(lastIndex); 478 } 479 } 480 } 481 482 private static void sortIntoList(Node n, Block b, ArrayList<Node> result, NodeMap<Block> nodeMap, NodeBitMap unprocessed, Node excludeNode) { 483 assert unprocessed.isMarked(n) : n; 484 assert nodeMap.get(n) == b; 485 486 if (n instanceof PhiNode) { 487 return; 488 } 489 490 unprocessed.clear(n); 491 492 for (Node input : n.inputs()) { 493 if (nodeMap.get(input) == b && unprocessed.isMarked(input) && input != excludeNode) { 494 sortIntoList(input, b, result, nodeMap, unprocessed, excludeNode); 495 } 496 } 497 498 if (n instanceof ProxyNode) { 499 // Skip proxy nodes. 500 } else { 501 result.add(n); 502 } 503 504 } 505 506 protected void calcLatestBlock(Block earliestBlock, SchedulingStrategy strategy, Node currentNode, NodeMap<Block> currentNodeMap, LocationIdentity constrainingLocation, 507 BlockMap<ArrayList<FloatingReadNode>> watchListMap, BlockMap<List<Node>> latestBlockToNodesMap, NodeBitMap visited, boolean immutableGraph) { 508 Block latestBlock = null; 509 if (!currentNode.hasUsages()) { 510 assert currentNode instanceof GuardNode; 511 latestBlock = earliestBlock; 512 } else { 513 assert currentNode.hasUsages(); 514 for (Node usage : currentNode.usages()) { 515 if (immutableGraph && !visited.contains(usage)) { 516 /* 517 * Normally, dead nodes are deleted by the scheduler before we reach this 518 * point. Only when the scheduler is asked to not modify a graph, we can see 519 * dead nodes here. 520 */ 521 continue; 522 } 523 latestBlock = calcBlockForUsage(currentNode, usage, latestBlock, currentNodeMap); 524 } 525 526 assert latestBlock != null : currentNode; 527 528 if (strategy == SchedulingStrategy.FINAL_SCHEDULE || strategy == SchedulingStrategy.LATEST_OUT_OF_LOOPS) { 529 assert latestBlock != null; 530 Block currentBlock = latestBlock; 531 while (currentBlock.getLoopDepth() > earliestBlock.getLoopDepth() && currentBlock != earliestBlock.getDominator()) { 532 Block previousCurrentBlock = currentBlock; 533 currentBlock = currentBlock.getDominator(); 534 if (previousCurrentBlock.isLoopHeader()) { 535 if (currentBlock.probability() < latestBlock.probability() || ((StructuredGraph) currentNode.graph()).hasValueProxies()) { 536 // Only assign new latest block if frequency is actually lower or if 537 // loop proxies would be required otherwise. 538 latestBlock = currentBlock; 539 } 540 } 541 } 542 } 543 544 if (latestBlock != earliestBlock && latestBlock != earliestBlock.getDominator() && constrainingLocation != null) { 545 latestBlock = checkKillsBetween(earliestBlock, latestBlock, constrainingLocation); 546 } 547 } 548 549 if (latestBlock != earliestBlock && currentNode instanceof FloatingReadNode) { 550 551 FloatingReadNode floatingReadNode = (FloatingReadNode) currentNode; 552 if (isImplicitNullOpportunity(floatingReadNode, earliestBlock) && earliestBlock.probability() < latestBlock.probability() * IMPLICIT_NULL_CHECK_OPPORTUNITY_PROBABILITY_FACTOR) { 553 latestBlock = earliestBlock; 554 } 555 } 556 557 selectLatestBlock(currentNode, earliestBlock, latestBlock, currentNodeMap, watchListMap, constrainingLocation, latestBlockToNodesMap); 558 } 559 560 private static boolean isImplicitNullOpportunity(FloatingReadNode floatingReadNode, Block block) { 561 562 Node pred = block.getBeginNode().predecessor(); 563 if (pred instanceof IfNode) { 564 IfNode ifNode = (IfNode) pred; 565 if (ifNode.condition() instanceof IsNullNode) { 566 IsNullNode isNullNode = (IsNullNode) ifNode.condition(); 567 if (getUnproxifiedUncompressed(floatingReadNode.getAddress().getBase()) == getUnproxifiedUncompressed(isNullNode.getValue())) { 568 return true; 569 } 570 } 571 } 572 return false; 573 } 574 575 private static Node getUnproxifiedUncompressed(Node node) { 576 Node result = node; 577 while (true) { 578 if (result instanceof ValueProxy) { 579 ValueProxy valueProxy = (ValueProxy) result; 580 result = valueProxy.getOriginalNode(); 581 } else if (result instanceof ConvertNode) { 582 ConvertNode convertNode = (ConvertNode) result; 583 if (convertNode.mayNullCheckSkipConversion()) { 584 result = convertNode.getValue(); 585 } else { 586 break; 587 } 588 } else { 589 break; 590 } 591 } 592 return result; 593 } 594 595 private static Block calcBlockForUsage(Node node, Node usage, Block startBlock, NodeMap<Block> currentNodeMap) { 596 assert !(node instanceof PhiNode); 597 Block currentBlock = startBlock; 598 if (usage instanceof PhiNode) { 599 // An input to a PhiNode is used at the end of the predecessor block that 600 // corresponds to the PhiNode input. One PhiNode can use an input multiple times. 601 PhiNode phi = (PhiNode) usage; 602 AbstractMergeNode merge = phi.merge(); 603 Block mergeBlock = currentNodeMap.get(merge); 604 for (int i = 0; i < phi.valueCount(); ++i) { 605 if (phi.valueAt(i) == node) { 606 Block otherBlock = mergeBlock.getPredecessors()[i]; 607 currentBlock = AbstractControlFlowGraph.commonDominatorTyped(currentBlock, otherBlock); 608 } 609 } 610 } else if (usage instanceof AbstractBeginNode) { 611 AbstractBeginNode abstractBeginNode = (AbstractBeginNode) usage; 612 if (abstractBeginNode instanceof StartNode) { 613 currentBlock = AbstractControlFlowGraph.commonDominatorTyped(currentBlock, currentNodeMap.get(abstractBeginNode)); 614 } else { 615 Block otherBlock = currentNodeMap.get(abstractBeginNode).getDominator(); 616 currentBlock = AbstractControlFlowGraph.commonDominatorTyped(currentBlock, otherBlock); 617 } 618 } else { 619 // All other types of usages: Put the input into the same block as the usage. 620 Block otherBlock = currentNodeMap.get(usage); 621 if (usage instanceof ProxyNode) { 622 ProxyNode proxyNode = (ProxyNode) usage; 623 otherBlock = currentNodeMap.get(proxyNode.proxyPoint()); 624 625 } 626 currentBlock = AbstractControlFlowGraph.commonDominatorTyped(currentBlock, otherBlock); 627 } 628 return currentBlock; 629 } 630 631 /** 632 * Micro block that is allocated for each fixed node and captures all floating nodes that 633 * need to be scheduled immediately after the corresponding fixed node. 634 */ 635 private static class MicroBlock { 636 private final int id; 637 private int nodeCount; 638 private NodeEntry head; 639 private NodeEntry tail; 640 641 MicroBlock(int id) { 642 this.id = id; 643 } 644 645 /** 646 * Adds a new floating node into the micro block. 647 */ 648 public void add(Node node) { 649 assert !(node instanceof FixedNode) : node; 650 NodeEntry newTail = new NodeEntry(node, null); 651 if (tail == null) { 652 tail = head = newTail; 653 } else { 654 tail.next = newTail; 655 tail = newTail; 656 } 657 nodeCount++; 658 } 659 660 /** 661 * Number of nodes in this micro block. 662 */ 663 public int getNodeCount() { 664 return nodeCount; 665 } 666 667 /** 668 * The id of the micro block, with a block always associated with a lower id than its 669 * successors. 670 */ 671 public int getId() { 672 return id; 673 } 674 675 /** 676 * First node of the linked list of nodes of this micro block. 677 */ 678 public NodeEntry getFirstNode() { 679 return head; 680 } 681 682 /** 683 * Takes all nodes in this micro blocks and prepends them to the nodes of the given 684 * parameter. 685 * 686 * @param newBlock the new block for the nodes 687 */ 688 public void prependChildrenTo(MicroBlock newBlock) { 689 if (tail != null) { 690 tail.next = newBlock.head; 691 newBlock.head = head; 692 head = tail = null; 693 newBlock.nodeCount += nodeCount; 694 nodeCount = 0; 695 } 696 } 697 698 @Override 699 public String toString() { 700 return String.format("MicroBlock[id=%d]", id); 701 } 702 } 703 704 /** 705 * Entry in the linked list of nodes. 706 */ 707 private static class NodeEntry { 708 private final Node node; 709 private NodeEntry next; 710 711 NodeEntry(Node node, NodeEntry next) { 712 this.node = node; 713 this.next = next; 714 } 715 716 public NodeEntry getNext() { 717 return next; 718 } 719 720 public Node getNode() { 721 return node; 722 } 723 } 724 725 private void scheduleEarliestIterative(BlockMap<List<Node>> blockToNodes, NodeMap<Block> nodeToBlock, NodeBitMap visited, StructuredGraph graph, boolean immutableGraph) { 726 727 NodeMap<MicroBlock> entries = graph.createNodeMap(); 728 NodeStack stack = new NodeStack(); 729 730 // Initialize with fixed nodes. 731 MicroBlock startBlock = null; 732 int nextId = 1; 733 for (Block b : cfg.reversePostOrder()) { 734 FixedNode current = b.getBeginNode(); 735 while (true) { 736 MicroBlock microBlock = new MicroBlock(nextId++); 737 entries.put(current, microBlock); 738 visited.checkAndMarkInc(current); 739 740 if (startBlock == null) { 741 startBlock = microBlock; 742 } 743 744 // Process inputs of this fixed node. 745 for (Node input : current.inputs()) { 746 if (entries.get(input) == null) { 747 processStack(input, startBlock, entries, visited, stack); 748 } 749 } 750 751 if (current == b.getEndNode()) { 752 // Break loop when reaching end node. 753 break; 754 } 755 756 current = ((FixedWithNextNode) current).next(); 757 } 758 } 759 760 // Now process guards. 761 for (GuardNode guardNode : graph.getNodes(GuardNode.TYPE)) { 762 if (entries.get(guardNode) == null) { 763 processStack(guardNode, startBlock, entries, visited, stack); 764 } 765 } 766 767 // Now process inputs of fixed nodes. 768 for (Block b : cfg.reversePostOrder()) { 769 FixedNode current = b.getBeginNode(); 770 while (true) { 771 772 // Process inputs of this fixed node. 773 for (Node input : current.inputs()) { 774 if (entries.get(input) == null) { 775 processStack(input, startBlock, entries, visited, stack); 776 } 777 } 778 779 if (current == b.getEndNode()) { 780 // Break loop when reaching end node. 781 break; 782 } 783 784 current = ((FixedWithNextNode) current).next(); 785 } 786 } 787 788 if (visited.getCounter() < graph.getNodeCount()) { 789 790 // Visit back input edges of loop phis. 791 boolean changed; 792 boolean unmarkedPhi; 793 do { 794 changed = false; 795 unmarkedPhi = false; 796 for (LoopBeginNode loopBegin : graph.getNodes(LoopBeginNode.TYPE)) { 797 for (PhiNode phi : loopBegin.phis()) { 798 if (visited.isMarked(phi)) { 799 for (int i = 0; i < loopBegin.getLoopEndCount(); ++i) { 800 Node node = phi.valueAt(i + loopBegin.forwardEndCount()); 801 if (node != null && entries.get(node) == null) { 802 changed = true; 803 processStack(node, startBlock, entries, visited, stack); 804 } 805 } 806 } else { 807 unmarkedPhi = true; 808 } 809 } 810 } 811 812 /* 813 * the processing of one loop phi could have marked a previously checked loop 814 * phi, therefore this needs to be iterative. 815 */ 816 } while (unmarkedPhi && changed); 817 } 818 819 // Check for dead nodes. 820 if (!immutableGraph && visited.getCounter() < graph.getNodeCount()) { 821 for (Node n : graph.getNodes()) { 822 if (!visited.isMarked(n)) { 823 n.clearInputs(); 824 n.markDeleted(); 825 } 826 } 827 } 828 829 for (Block b : cfg.reversePostOrder()) { 830 FixedNode fixedNode = b.getEndNode(); 831 if (fixedNode instanceof ControlSplitNode) { 832 ControlSplitNode controlSplitNode = (ControlSplitNode) fixedNode; 833 MicroBlock endBlock = entries.get(fixedNode); 834 MicroBlock primarySuccessor = entries.get(controlSplitNode.getPrimarySuccessor()); 835 endBlock.prependChildrenTo(primarySuccessor); 836 } 837 } 838 839 // Initialize with begin nodes 840 for (Block b : cfg.reversePostOrder()) { 841 842 FixedNode current = b.getBeginNode(); 843 int totalCount = 0; 844 while (true) { 845 846 MicroBlock microBlock = entries.get(current); 847 totalCount += microBlock.getNodeCount() + 1; 848 849 if (current == b.getEndNode()) { 850 // Break loop when reaching end node. 851 break; 852 } 853 854 current = ((FixedWithNextNode) current).next(); 855 } 856 857 // Initialize with begin node, it is always the first node. 858 ArrayList<Node> nodes = new ArrayList<>(totalCount); 859 blockToNodes.put(b, nodes); 860 861 current = b.getBeginNode(); 862 while (true) { 863 864 MicroBlock microBlock = entries.get(current); 865 nodeToBlock.set(current, b); 866 nodes.add(current); 867 NodeEntry next = microBlock.getFirstNode(); 868 while (next != null) { 869 Node nextNode = next.getNode(); 870 nodeToBlock.set(nextNode, b); 871 nodes.add(nextNode); 872 next = next.getNext(); 873 } 874 875 if (current == b.getEndNode()) { 876 // Break loop when reaching end node. 877 break; 878 } 879 880 current = ((FixedWithNextNode) current).next(); 881 } 882 } 883 884 assert (!GraalOptions.DetailedAsserts.getValue(cfg.graph.getOptions())) || MemoryScheduleVerification.check(cfg.getStartBlock(), blockToNodes); 885 } 886 887 private static void processStackPhi(NodeStack stack, PhiNode phiNode, NodeMap<MicroBlock> nodeToBlock, NodeBitMap visited) { 888 stack.pop(); 889 if (visited.checkAndMarkInc(phiNode)) { 890 MicroBlock mergeBlock = nodeToBlock.get(phiNode.merge()); 891 assert mergeBlock != null : phiNode; 892 nodeToBlock.set(phiNode, mergeBlock); 893 AbstractMergeNode merge = phiNode.merge(); 894 for (int i = 0; i < merge.forwardEndCount(); ++i) { 895 Node input = phiNode.valueAt(i); 896 if (input != null && nodeToBlock.get(input) == null) { 897 stack.push(input); 898 } 899 } 900 } 901 } 902 903 private static void processStackProxy(NodeStack stack, ProxyNode proxyNode, NodeMap<MicroBlock> nodeToBlock, NodeBitMap visited) { 904 stack.pop(); 905 if (visited.checkAndMarkInc(proxyNode)) { 906 nodeToBlock.set(proxyNode, nodeToBlock.get(proxyNode.proxyPoint())); 907 Node input = proxyNode.value(); 908 if (input != null && nodeToBlock.get(input) == null) { 909 stack.push(input); 910 } 911 } 912 } 913 914 private static void processStack(Node first, MicroBlock startBlock, NodeMap<MicroBlock> nodeToMicroBlock, NodeBitMap visited, NodeStack stack) { 915 assert stack.isEmpty(); 916 assert !visited.isMarked(first); 917 stack.push(first); 918 Node current = first; 919 while (true) { 920 if (current instanceof PhiNode) { 921 processStackPhi(stack, (PhiNode) current, nodeToMicroBlock, visited); 922 } else if (current instanceof ProxyNode) { 923 processStackProxy(stack, (ProxyNode) current, nodeToMicroBlock, visited); 924 } else { 925 MicroBlock currentBlock = nodeToMicroBlock.get(current); 926 if (currentBlock == null) { 927 MicroBlock earliestBlock = processInputs(nodeToMicroBlock, stack, startBlock, current); 928 if (earliestBlock == null) { 929 // We need to delay until inputs are processed. 930 } else { 931 // Can immediately process and pop. 932 stack.pop(); 933 visited.checkAndMarkInc(current); 934 nodeToMicroBlock.set(current, earliestBlock); 935 earliestBlock.add(current); 936 } 937 } else { 938 stack.pop(); 939 } 940 } 941 942 if (stack.isEmpty()) { 943 break; 944 } 945 current = stack.peek(); 946 } 947 } 948 949 /** 950 * Processes the inputs of given block. Pushes unprocessed inputs onto the stack. Returns 951 * null if there were still unprocessed inputs, otherwise returns the earliest block given 952 * node can be scheduled in. 953 */ 954 private static MicroBlock processInputs(NodeMap<MicroBlock> nodeToBlock, NodeStack stack, MicroBlock startBlock, Node current) { 955 if (current.getNodeClass().isLeafNode()) { 956 return startBlock; 957 } 958 959 MicroBlock earliestBlock = startBlock; 960 for (Node input : current.inputs()) { 961 MicroBlock inputBlock = nodeToBlock.get(input); 962 if (inputBlock == null) { 963 earliestBlock = null; 964 stack.push(input); 965 } else if (earliestBlock != null && inputBlock.getId() >= earliestBlock.getId()) { 966 earliestBlock = inputBlock; 967 } 968 } 969 return earliestBlock; 970 } 971 972 private static boolean isFixedEnd(FixedNode endNode) { 973 return endNode instanceof ControlSplitNode || endNode instanceof ControlSinkNode || endNode instanceof AbstractEndNode; 974 } 975 976 public String printScheduleHelper(String desc) { 977 Formatter buf = new Formatter(); 978 buf.format("=== %s / %s ===%n", getCFG().getStartBlock().getBeginNode().graph(), desc); 979 for (Block b : getCFG().getBlocks()) { 980 buf.format("==== b: %s (loopDepth: %s). ", b, b.getLoopDepth()); 981 buf.format("dom: %s. ", b.getDominator()); 982 buf.format("preds: %s. ", Arrays.toString(b.getPredecessors())); 983 buf.format("succs: %s ====%n", Arrays.toString(b.getSuccessors())); 984 985 if (blockToNodesMap.get(b) != null) { 986 for (Node n : nodesFor(b)) { 987 printNode(n); 988 } 989 } else { 990 for (Node n : b.getNodes()) { 991 printNode(n); 992 } 993 } 994 } 995 buf.format("%n"); 996 return buf.toString(); 997 } 998 999 private static void printNode(Node n) { 1000 Formatter buf = new Formatter(); 1001 buf.format("%s", n); 1002 if (n instanceof MemoryCheckpoint.Single) { 1003 buf.format(" // kills %s", ((MemoryCheckpoint.Single) n).getLocationIdentity()); 1004 } else if (n instanceof MemoryCheckpoint.Multi) { 1005 buf.format(" // kills "); 1006 for (LocationIdentity locid : ((MemoryCheckpoint.Multi) n).getLocationIdentities()) { 1007 buf.format("%s, ", locid); 1008 } 1009 } else if (n instanceof FloatingReadNode) { 1010 FloatingReadNode frn = (FloatingReadNode) n; 1011 buf.format(" // from %s", frn.getLocationIdentity()); 1012 buf.format(", lastAccess: %s", frn.getLastLocationAccess()); 1013 buf.format(", address: %s", frn.getAddress()); 1014 } else if (n instanceof GuardNode) { 1015 buf.format(", anchor: %s", ((GuardNode) n).getAnchor()); 1016 } 1017 Debug.log("%s", buf); 1018 } 1019 1020 public ControlFlowGraph getCFG() { 1021 return cfg; 1022 } 1023 1024 /** 1025 * Gets the nodes in a given block. 1026 */ 1027 public List<Node> nodesFor(Block block) { 1028 return blockToNodesMap.get(block); 1029 } 1030 } 1031 1032} 1033