1//===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements a fast scheduler.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/SchedulerRegistry.h"
15#include "InstrEmitter.h"
16#include "ScheduleDAGSDNodes.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/CodeGen/SelectionDAGISel.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/IR/InlineAsm.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/raw_ostream.h"
26#include "llvm/Target/TargetInstrInfo.h"
27#include "llvm/Target/TargetRegisterInfo.h"
28using namespace llvm;
29
30#define DEBUG_TYPE "pre-RA-sched"
31
32STATISTIC(NumUnfolds,    "Number of nodes unfolded");
33STATISTIC(NumDups,       "Number of duplicated nodes");
34STATISTIC(NumPRCopies,   "Number of physical copies");
35
36static RegisterScheduler
37  fastDAGScheduler("fast", "Fast suboptimal list scheduling",
38                   createFastDAGScheduler);
39static RegisterScheduler
40  linearizeDAGScheduler("linearize", "Linearize DAG, no scheduling",
41                        createDAGLinearizer);
42
43
44namespace {
45  /// FastPriorityQueue - A degenerate priority queue that considers
46  /// all nodes to have the same priority.
47  ///
48  struct FastPriorityQueue {
49    SmallVector<SUnit *, 16> Queue;
50
51    bool empty() const { return Queue.empty(); }
52
53    void push(SUnit *U) {
54      Queue.push_back(U);
55    }
56
57    SUnit *pop() {
58      if (empty()) return nullptr;
59      SUnit *V = Queue.back();
60      Queue.pop_back();
61      return V;
62    }
63  };
64
65//===----------------------------------------------------------------------===//
66/// ScheduleDAGFast - The actual "fast" list scheduler implementation.
67///
68class ScheduleDAGFast : public ScheduleDAGSDNodes {
69private:
70  /// AvailableQueue - The priority queue to use for the available SUnits.
71  FastPriorityQueue AvailableQueue;
72
73  /// LiveRegDefs - A set of physical registers and their definition
74  /// that are "live". These nodes must be scheduled before any other nodes that
75  /// modifies the registers can be scheduled.
76  unsigned NumLiveRegs;
77  std::vector<SUnit*> LiveRegDefs;
78  std::vector<unsigned> LiveRegCycles;
79
80public:
81  ScheduleDAGFast(MachineFunction &mf)
82    : ScheduleDAGSDNodes(mf) {}
83
84  void Schedule() override;
85
86  /// AddPred - adds a predecessor edge to SUnit SU.
87  /// This returns true if this is a new predecessor.
88  void AddPred(SUnit *SU, const SDep &D) {
89    SU->addPred(D);
90  }
91
92  /// RemovePred - removes a predecessor edge from SUnit SU.
93  /// This returns true if an edge was removed.
94  void RemovePred(SUnit *SU, const SDep &D) {
95    SU->removePred(D);
96  }
97
98private:
99  void ReleasePred(SUnit *SU, SDep *PredEdge);
100  void ReleasePredecessors(SUnit *SU, unsigned CurCycle);
101  void ScheduleNodeBottomUp(SUnit*, unsigned);
102  SUnit *CopyAndMoveSuccessors(SUnit*);
103  void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
104                                const TargetRegisterClass*,
105                                const TargetRegisterClass*,
106                                SmallVectorImpl<SUnit*>&);
107  bool DelayForLiveRegsBottomUp(SUnit*, SmallVectorImpl<unsigned>&);
108  void ListScheduleBottomUp();
109
110  /// forceUnitLatencies - The fast scheduler doesn't care about real latencies.
111  bool forceUnitLatencies() const override { return true; }
112};
113}  // end anonymous namespace
114
115
116/// Schedule - Schedule the DAG using list scheduling.
117void ScheduleDAGFast::Schedule() {
118  DEBUG(dbgs() << "********** List Scheduling **********\n");
119
120  NumLiveRegs = 0;
121  LiveRegDefs.resize(TRI->getNumRegs(), nullptr);
122  LiveRegCycles.resize(TRI->getNumRegs(), 0);
123
124  // Build the scheduling graph.
125  BuildSchedGraph(nullptr);
126
127  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
128          SUnits[su].dumpAll(this));
129
130  // Execute the actual scheduling loop.
131  ListScheduleBottomUp();
132}
133
134//===----------------------------------------------------------------------===//
135//  Bottom-Up Scheduling
136//===----------------------------------------------------------------------===//
137
138/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
139/// the AvailableQueue if the count reaches zero. Also update its cycle bound.
140void ScheduleDAGFast::ReleasePred(SUnit *SU, SDep *PredEdge) {
141  SUnit *PredSU = PredEdge->getSUnit();
142
143#ifndef NDEBUG
144  if (PredSU->NumSuccsLeft == 0) {
145    dbgs() << "*** Scheduling failed! ***\n";
146    PredSU->dump(this);
147    dbgs() << " has been released too many times!\n";
148    llvm_unreachable(nullptr);
149  }
150#endif
151  --PredSU->NumSuccsLeft;
152
153  // If all the node's successors are scheduled, this node is ready
154  // to be scheduled. Ignore the special EntrySU node.
155  if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
156    PredSU->isAvailable = true;
157    AvailableQueue.push(PredSU);
158  }
159}
160
161void ScheduleDAGFast::ReleasePredecessors(SUnit *SU, unsigned CurCycle) {
162  // Bottom up: release predecessors
163  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
164       I != E; ++I) {
165    ReleasePred(SU, &*I);
166    if (I->isAssignedRegDep()) {
167      // This is a physical register dependency and it's impossible or
168      // expensive to copy the register. Make sure nothing that can
169      // clobber the register is scheduled between the predecessor and
170      // this node.
171      if (!LiveRegDefs[I->getReg()]) {
172        ++NumLiveRegs;
173        LiveRegDefs[I->getReg()] = I->getSUnit();
174        LiveRegCycles[I->getReg()] = CurCycle;
175      }
176    }
177  }
178}
179
180/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
181/// count of its predecessors. If a predecessor pending count is zero, add it to
182/// the Available queue.
183void ScheduleDAGFast::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) {
184  DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: ");
185  DEBUG(SU->dump(this));
186
187  assert(CurCycle >= SU->getHeight() && "Node scheduled below its height!");
188  SU->setHeightToAtLeast(CurCycle);
189  Sequence.push_back(SU);
190
191  ReleasePredecessors(SU, CurCycle);
192
193  // Release all the implicit physical register defs that are live.
194  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
195       I != E; ++I) {
196    if (I->isAssignedRegDep()) {
197      if (LiveRegCycles[I->getReg()] == I->getSUnit()->getHeight()) {
198        assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
199        assert(LiveRegDefs[I->getReg()] == SU &&
200               "Physical register dependency violated?");
201        --NumLiveRegs;
202        LiveRegDefs[I->getReg()] = nullptr;
203        LiveRegCycles[I->getReg()] = 0;
204      }
205    }
206  }
207
208  SU->isScheduled = true;
209}
210
211/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
212/// successors to the newly created node.
213SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
214  if (SU->getNode()->getGluedNode())
215    return nullptr;
216
217  SDNode *N = SU->getNode();
218  if (!N)
219    return nullptr;
220
221  SUnit *NewSU;
222  bool TryUnfold = false;
223  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
224    MVT VT = N->getSimpleValueType(i);
225    if (VT == MVT::Glue)
226      return nullptr;
227    else if (VT == MVT::Other)
228      TryUnfold = true;
229  }
230  for (const SDValue &Op : N->op_values()) {
231    MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
232    if (VT == MVT::Glue)
233      return nullptr;
234  }
235
236  if (TryUnfold) {
237    SmallVector<SDNode*, 2> NewNodes;
238    if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
239      return nullptr;
240
241    DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n");
242    assert(NewNodes.size() == 2 && "Expected a load folding node!");
243
244    N = NewNodes[1];
245    SDNode *LoadNode = NewNodes[0];
246    unsigned NumVals = N->getNumValues();
247    unsigned OldNumVals = SU->getNode()->getNumValues();
248    for (unsigned i = 0; i != NumVals; ++i)
249      DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
250    DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
251                                   SDValue(LoadNode, 1));
252
253    SUnit *NewSU = newSUnit(N);
254    assert(N->getNodeId() == -1 && "Node already inserted!");
255    N->setNodeId(NewSU->NodeNum);
256
257    const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
258    for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
259      if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
260        NewSU->isTwoAddress = true;
261        break;
262      }
263    }
264    if (MCID.isCommutable())
265      NewSU->isCommutable = true;
266
267    // LoadNode may already exist. This can happen when there is another
268    // load from the same location and producing the same type of value
269    // but it has different alignment or volatileness.
270    bool isNewLoad = true;
271    SUnit *LoadSU;
272    if (LoadNode->getNodeId() != -1) {
273      LoadSU = &SUnits[LoadNode->getNodeId()];
274      isNewLoad = false;
275    } else {
276      LoadSU = newSUnit(LoadNode);
277      LoadNode->setNodeId(LoadSU->NodeNum);
278    }
279
280    SDep ChainPred;
281    SmallVector<SDep, 4> ChainSuccs;
282    SmallVector<SDep, 4> LoadPreds;
283    SmallVector<SDep, 4> NodePreds;
284    SmallVector<SDep, 4> NodeSuccs;
285    for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
286         I != E; ++I) {
287      if (I->isCtrl())
288        ChainPred = *I;
289      else if (I->getSUnit()->getNode() &&
290               I->getSUnit()->getNode()->isOperandOf(LoadNode))
291        LoadPreds.push_back(*I);
292      else
293        NodePreds.push_back(*I);
294    }
295    for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
296         I != E; ++I) {
297      if (I->isCtrl())
298        ChainSuccs.push_back(*I);
299      else
300        NodeSuccs.push_back(*I);
301    }
302
303    if (ChainPred.getSUnit()) {
304      RemovePred(SU, ChainPred);
305      if (isNewLoad)
306        AddPred(LoadSU, ChainPred);
307    }
308    for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
309      const SDep &Pred = LoadPreds[i];
310      RemovePred(SU, Pred);
311      if (isNewLoad) {
312        AddPred(LoadSU, Pred);
313      }
314    }
315    for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
316      const SDep &Pred = NodePreds[i];
317      RemovePred(SU, Pred);
318      AddPred(NewSU, Pred);
319    }
320    for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
321      SDep D = NodeSuccs[i];
322      SUnit *SuccDep = D.getSUnit();
323      D.setSUnit(SU);
324      RemovePred(SuccDep, D);
325      D.setSUnit(NewSU);
326      AddPred(SuccDep, D);
327    }
328    for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
329      SDep D = ChainSuccs[i];
330      SUnit *SuccDep = D.getSUnit();
331      D.setSUnit(SU);
332      RemovePred(SuccDep, D);
333      if (isNewLoad) {
334        D.setSUnit(LoadSU);
335        AddPred(SuccDep, D);
336      }
337    }
338    if (isNewLoad) {
339      SDep D(LoadSU, SDep::Barrier);
340      D.setLatency(LoadSU->Latency);
341      AddPred(NewSU, D);
342    }
343
344    ++NumUnfolds;
345
346    if (NewSU->NumSuccsLeft == 0) {
347      NewSU->isAvailable = true;
348      return NewSU;
349    }
350    SU = NewSU;
351  }
352
353  DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n");
354  NewSU = Clone(SU);
355
356  // New SUnit has the exact same predecessors.
357  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
358       I != E; ++I)
359    if (!I->isArtificial())
360      AddPred(NewSU, *I);
361
362  // Only copy scheduled successors. Cut them from old node's successor
363  // list and move them over.
364  SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
365  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
366       I != E; ++I) {
367    if (I->isArtificial())
368      continue;
369    SUnit *SuccSU = I->getSUnit();
370    if (SuccSU->isScheduled) {
371      SDep D = *I;
372      D.setSUnit(NewSU);
373      AddPred(SuccSU, D);
374      D.setSUnit(SU);
375      DelDeps.push_back(std::make_pair(SuccSU, D));
376    }
377  }
378  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
379    RemovePred(DelDeps[i].first, DelDeps[i].second);
380
381  ++NumDups;
382  return NewSU;
383}
384
385/// InsertCopiesAndMoveSuccs - Insert register copies and move all
386/// scheduled successors of the given SUnit to the last copy.
387void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
388                                              const TargetRegisterClass *DestRC,
389                                              const TargetRegisterClass *SrcRC,
390                                              SmallVectorImpl<SUnit*> &Copies) {
391  SUnit *CopyFromSU = newSUnit(static_cast<SDNode *>(nullptr));
392  CopyFromSU->CopySrcRC = SrcRC;
393  CopyFromSU->CopyDstRC = DestRC;
394
395  SUnit *CopyToSU = newSUnit(static_cast<SDNode *>(nullptr));
396  CopyToSU->CopySrcRC = DestRC;
397  CopyToSU->CopyDstRC = SrcRC;
398
399  // Only copy scheduled successors. Cut them from old node's successor
400  // list and move them over.
401  SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
402  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
403       I != E; ++I) {
404    if (I->isArtificial())
405      continue;
406    SUnit *SuccSU = I->getSUnit();
407    if (SuccSU->isScheduled) {
408      SDep D = *I;
409      D.setSUnit(CopyToSU);
410      AddPred(SuccSU, D);
411      DelDeps.push_back(std::make_pair(SuccSU, *I));
412    }
413  }
414  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
415    RemovePred(DelDeps[i].first, DelDeps[i].second);
416  }
417  SDep FromDep(SU, SDep::Data, Reg);
418  FromDep.setLatency(SU->Latency);
419  AddPred(CopyFromSU, FromDep);
420  SDep ToDep(CopyFromSU, SDep::Data, 0);
421  ToDep.setLatency(CopyFromSU->Latency);
422  AddPred(CopyToSU, ToDep);
423
424  Copies.push_back(CopyFromSU);
425  Copies.push_back(CopyToSU);
426
427  ++NumPRCopies;
428}
429
430/// getPhysicalRegisterVT - Returns the ValueType of the physical register
431/// definition of the specified node.
432/// FIXME: Move to SelectionDAG?
433static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
434                                 const TargetInstrInfo *TII) {
435  unsigned NumRes;
436  if (N->getOpcode() == ISD::CopyFromReg) {
437    // CopyFromReg has: "chain, Val, glue" so operand 1 gives the type.
438    NumRes = 1;
439  } else {
440    const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
441    assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!");
442    NumRes = MCID.getNumDefs();
443    for (const MCPhysReg *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) {
444      if (Reg == *ImpDef)
445        break;
446      ++NumRes;
447    }
448  }
449  return N->getSimpleValueType(NumRes);
450}
451
452/// CheckForLiveRegDef - Return true and update live register vector if the
453/// specified register def of the specified SUnit clobbers any "live" registers.
454static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg,
455                               std::vector<SUnit*> &LiveRegDefs,
456                               SmallSet<unsigned, 4> &RegAdded,
457                               SmallVectorImpl<unsigned> &LRegs,
458                               const TargetRegisterInfo *TRI) {
459  bool Added = false;
460  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
461    if (LiveRegDefs[*AI] && LiveRegDefs[*AI] != SU) {
462      if (RegAdded.insert(*AI).second) {
463        LRegs.push_back(*AI);
464        Added = true;
465      }
466    }
467  }
468  return Added;
469}
470
471/// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay
472/// scheduling of the given node to satisfy live physical register dependencies.
473/// If the specific node is the last one that's available to schedule, do
474/// whatever is necessary (i.e. backtracking or cloning) to make it possible.
475bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU,
476                                              SmallVectorImpl<unsigned> &LRegs){
477  if (NumLiveRegs == 0)
478    return false;
479
480  SmallSet<unsigned, 4> RegAdded;
481  // If this node would clobber any "live" register, then it's not ready.
482  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
483       I != E; ++I) {
484    if (I->isAssignedRegDep()) {
485      CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs,
486                         RegAdded, LRegs, TRI);
487    }
488  }
489
490  for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
491    if (Node->getOpcode() == ISD::INLINEASM) {
492      // Inline asm can clobber physical defs.
493      unsigned NumOps = Node->getNumOperands();
494      if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
495        --NumOps;  // Ignore the glue operand.
496
497      for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
498        unsigned Flags =
499          cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
500        unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
501
502        ++i; // Skip the ID value.
503        if (InlineAsm::isRegDefKind(Flags) ||
504            InlineAsm::isRegDefEarlyClobberKind(Flags) ||
505            InlineAsm::isClobberKind(Flags)) {
506          // Check for def of register or earlyclobber register.
507          for (; NumVals; --NumVals, ++i) {
508            unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
509            if (TargetRegisterInfo::isPhysicalRegister(Reg))
510              CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI);
511          }
512        } else
513          i += NumVals;
514      }
515      continue;
516    }
517    if (!Node->isMachineOpcode())
518      continue;
519    const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode());
520    if (!MCID.ImplicitDefs)
521      continue;
522    for (const MCPhysReg *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) {
523      CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI);
524    }
525  }
526  return !LRegs.empty();
527}
528
529
530/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
531/// schedulers.
532void ScheduleDAGFast::ListScheduleBottomUp() {
533  unsigned CurCycle = 0;
534
535  // Release any predecessors of the special Exit node.
536  ReleasePredecessors(&ExitSU, CurCycle);
537
538  // Add root to Available queue.
539  if (!SUnits.empty()) {
540    SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
541    assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!");
542    RootSU->isAvailable = true;
543    AvailableQueue.push(RootSU);
544  }
545
546  // While Available queue is not empty, grab the node with the highest
547  // priority. If it is not ready put it back.  Schedule the node.
548  SmallVector<SUnit*, 4> NotReady;
549  DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMap;
550  Sequence.reserve(SUnits.size());
551  while (!AvailableQueue.empty()) {
552    bool Delayed = false;
553    LRegsMap.clear();
554    SUnit *CurSU = AvailableQueue.pop();
555    while (CurSU) {
556      SmallVector<unsigned, 4> LRegs;
557      if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
558        break;
559      Delayed = true;
560      LRegsMap.insert(std::make_pair(CurSU, LRegs));
561
562      CurSU->isPending = true;  // This SU is not in AvailableQueue right now.
563      NotReady.push_back(CurSU);
564      CurSU = AvailableQueue.pop();
565    }
566
567    // All candidates are delayed due to live physical reg dependencies.
568    // Try code duplication or inserting cross class copies
569    // to resolve it.
570    if (Delayed && !CurSU) {
571      if (!CurSU) {
572        // Try duplicating the nodes that produces these
573        // "expensive to copy" values to break the dependency. In case even
574        // that doesn't work, insert cross class copies.
575        SUnit *TrySU = NotReady[0];
576        SmallVectorImpl<unsigned> &LRegs = LRegsMap[TrySU];
577        assert(LRegs.size() == 1 && "Can't handle this yet!");
578        unsigned Reg = LRegs[0];
579        SUnit *LRDef = LiveRegDefs[Reg];
580        MVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII);
581        const TargetRegisterClass *RC =
582          TRI->getMinimalPhysRegClass(Reg, VT);
583        const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC);
584
585        // If cross copy register class is the same as RC, then it must be
586        // possible copy the value directly. Do not try duplicate the def.
587        // If cross copy register class is not the same as RC, then it's
588        // possible to copy the value but it require cross register class copies
589        // and it is expensive.
590        // If cross copy register class is null, then it's not possible to copy
591        // the value at all.
592        SUnit *NewDef = nullptr;
593        if (DestRC != RC) {
594          NewDef = CopyAndMoveSuccessors(LRDef);
595          if (!DestRC && !NewDef)
596            report_fatal_error("Can't handle live physical "
597                               "register dependency!");
598        }
599        if (!NewDef) {
600          // Issue copies, these can be expensive cross register class copies.
601          SmallVector<SUnit*, 2> Copies;
602          InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
603          DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
604                       << " to SU #" << Copies.front()->NodeNum << "\n");
605          AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
606          NewDef = Copies.back();
607        }
608
609        DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
610                     << " to SU #" << TrySU->NodeNum << "\n");
611        LiveRegDefs[Reg] = NewDef;
612        AddPred(NewDef, SDep(TrySU, SDep::Artificial));
613        TrySU->isAvailable = false;
614        CurSU = NewDef;
615      }
616
617      if (!CurSU) {
618        llvm_unreachable("Unable to resolve live physical register dependencies!");
619      }
620    }
621
622    // Add the nodes that aren't ready back onto the available list.
623    for (unsigned i = 0, e = NotReady.size(); i != e; ++i) {
624      NotReady[i]->isPending = false;
625      // May no longer be available due to backtracking.
626      if (NotReady[i]->isAvailable)
627        AvailableQueue.push(NotReady[i]);
628    }
629    NotReady.clear();
630
631    if (CurSU)
632      ScheduleNodeBottomUp(CurSU, CurCycle);
633    ++CurCycle;
634  }
635
636  // Reverse the order since it is bottom up.
637  std::reverse(Sequence.begin(), Sequence.end());
638
639#ifndef NDEBUG
640  VerifyScheduledSequence(/*isBottomUp=*/true);
641#endif
642}
643
644
645namespace {
646//===----------------------------------------------------------------------===//
647// ScheduleDAGLinearize - No scheduling scheduler, it simply linearize the
648// DAG in topological order.
649// IMPORTANT: this may not work for targets with phyreg dependency.
650//
651class ScheduleDAGLinearize : public ScheduleDAGSDNodes {
652public:
653  ScheduleDAGLinearize(MachineFunction &mf) : ScheduleDAGSDNodes(mf) {}
654
655  void Schedule() override;
656
657  MachineBasicBlock *
658    EmitSchedule(MachineBasicBlock::iterator &InsertPos) override;
659
660private:
661  std::vector<SDNode*> Sequence;
662  DenseMap<SDNode*, SDNode*> GluedMap;  // Cache glue to its user
663
664  void ScheduleNode(SDNode *N);
665};
666} // end anonymous namespace
667
668void ScheduleDAGLinearize::ScheduleNode(SDNode *N) {
669  if (N->getNodeId() != 0)
670    llvm_unreachable(nullptr);
671
672  if (!N->isMachineOpcode() &&
673      (N->getOpcode() == ISD::EntryToken || isPassiveNode(N)))
674    // These nodes do not need to be translated into MIs.
675    return;
676
677  DEBUG(dbgs() << "\n*** Scheduling: ");
678  DEBUG(N->dump(DAG));
679  Sequence.push_back(N);
680
681  unsigned NumOps = N->getNumOperands();
682  if (unsigned NumLeft = NumOps) {
683    SDNode *GluedOpN = nullptr;
684    do {
685      const SDValue &Op = N->getOperand(NumLeft-1);
686      SDNode *OpN = Op.getNode();
687
688      if (NumLeft == NumOps && Op.getValueType() == MVT::Glue) {
689        // Schedule glue operand right above N.
690        GluedOpN = OpN;
691        assert(OpN->getNodeId() != 0 && "Glue operand not ready?");
692        OpN->setNodeId(0);
693        ScheduleNode(OpN);
694        continue;
695      }
696
697      if (OpN == GluedOpN)
698        // Glue operand is already scheduled.
699        continue;
700
701      DenseMap<SDNode*, SDNode*>::iterator DI = GluedMap.find(OpN);
702      if (DI != GluedMap.end() && DI->second != N)
703        // Users of glues are counted against the glued users.
704        OpN = DI->second;
705
706      unsigned Degree = OpN->getNodeId();
707      assert(Degree > 0 && "Predecessor over-released!");
708      OpN->setNodeId(--Degree);
709      if (Degree == 0)
710        ScheduleNode(OpN);
711    } while (--NumLeft);
712  }
713}
714
715/// findGluedUser - Find the representative use of a glue value by walking
716/// the use chain.
717static SDNode *findGluedUser(SDNode *N) {
718  while (SDNode *Glued = N->getGluedUser())
719    N = Glued;
720  return N;
721}
722
723void ScheduleDAGLinearize::Schedule() {
724  DEBUG(dbgs() << "********** DAG Linearization **********\n");
725
726  SmallVector<SDNode*, 8> Glues;
727  unsigned DAGSize = 0;
728  for (SDNode &Node : DAG->allnodes()) {
729    SDNode *N = &Node;
730
731    // Use node id to record degree.
732    unsigned Degree = N->use_size();
733    N->setNodeId(Degree);
734    unsigned NumVals = N->getNumValues();
735    if (NumVals && N->getValueType(NumVals-1) == MVT::Glue &&
736        N->hasAnyUseOfValue(NumVals-1)) {
737      SDNode *User = findGluedUser(N);
738      if (User) {
739        Glues.push_back(N);
740        GluedMap.insert(std::make_pair(N, User));
741      }
742    }
743
744    if (N->isMachineOpcode() ||
745        (N->getOpcode() != ISD::EntryToken && !isPassiveNode(N)))
746      ++DAGSize;
747  }
748
749  for (unsigned i = 0, e = Glues.size(); i != e; ++i) {
750    SDNode *Glue = Glues[i];
751    SDNode *GUser = GluedMap[Glue];
752    unsigned Degree = Glue->getNodeId();
753    unsigned UDegree = GUser->getNodeId();
754
755    // Glue user must be scheduled together with the glue operand. So other
756    // users of the glue operand must be treated as its users.
757    SDNode *ImmGUser = Glue->getGluedUser();
758    for (SDNode::use_iterator ui = Glue->use_begin(), ue = Glue->use_end();
759         ui != ue; ++ui)
760      if (*ui == ImmGUser)
761        --Degree;
762    GUser->setNodeId(UDegree + Degree);
763    Glue->setNodeId(1);
764  }
765
766  Sequence.reserve(DAGSize);
767  ScheduleNode(DAG->getRoot().getNode());
768}
769
770MachineBasicBlock*
771ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
772  InstrEmitter Emitter(BB, InsertPos);
773  DenseMap<SDValue, unsigned> VRBaseMap;
774
775  DEBUG({
776      dbgs() << "\n*** Final schedule ***\n";
777    });
778
779  // FIXME: Handle dbg_values.
780  unsigned NumNodes = Sequence.size();
781  for (unsigned i = 0; i != NumNodes; ++i) {
782    SDNode *N = Sequence[NumNodes-i-1];
783    DEBUG(N->dump(DAG));
784    Emitter.EmitNode(N, false, false, VRBaseMap);
785  }
786
787  DEBUG(dbgs() << '\n');
788
789  InsertPos = Emitter.getInsertPos();
790  return Emitter.getBlock();
791}
792
793//===----------------------------------------------------------------------===//
794//                         Public Constructor Functions
795//===----------------------------------------------------------------------===//
796
797llvm::ScheduleDAGSDNodes *
798llvm::createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
799  return new ScheduleDAGFast(*IS->MF);
800}
801
802llvm::ScheduleDAGSDNodes *
803llvm::createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level) {
804  return new ScheduleDAGLinearize(*IS->MF);
805}
806