Lines Matching refs:SU
211 /// IsReachable - Checks if SU is reachable from TargetSU.
212 bool IsReachable(const SUnit *SU, const SUnit *TargetSU) {
213 return Topo.IsReachable(SU, TargetSU);
216 /// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will
218 bool WillCreateCycle(SUnit *SU, SUnit *TargetSU) {
219 return Topo.WillCreateCycle(SU, TargetSU);
222 /// AddPredQueued - Queues and update to add a predecessor edge to SUnit SU.
225 void AddPredQueued(SUnit *SU, const SDep &D) {
226 Topo.AddPredQueued(SU, D.getSUnit());
227 SU->addPred(D);
230 /// AddPred - adds a predecessor edge to SUnit SU.
233 void AddPred(SUnit *SU, const SDep &D) {
234 Topo.AddPred(SU, D.getSUnit());
235 SU->addPred(D);
238 /// RemovePred - removes a predecessor edge from SUnit SU.
241 void RemovePred(SUnit *SU, const SDep &D) {
242 Topo.RemovePred(SU, D.getSUnit());
243 SU->removePred(D);
247 bool isReady(SUnit *SU) {
249 AvailableQueue->isReady(SU);
252 void ReleasePred(SUnit *SU, const SDep *PredEdge);
253 void ReleasePredecessors(SUnit *SU);
256 void AdvancePastStalls(SUnit *SU);
257 void EmitNode(SUnit *SU);
398 void ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) {
414 PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency());
538 /// Always update LiveRegDefs for a register dependence even if the current SU
554 void ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) {
556 for (SDep &Pred : SU->Preds) {
557 ReleasePred(SU, &Pred);
564 assert((!RegDef || RegDef == SU || RegDef == Pred.getSUnit()) &&
569 LiveRegGens[Pred.getReg()] = SU;
579 for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode())
588 CallSeqEndForStart[Def] = SU;
592 LiveRegGens[CallResource] = SU;
651 void ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) {
660 // if (!SU->getNode() || !SU->getNode()->isMachineOpcode()) return;
662 unsigned ReadyCycle = SU->getHeight();
673 if (SU->isCall)
681 HazardRec->getHazardType(SU, -Stalls);
693 void ScheduleDAGRRList::EmitNode(SUnit *SU) {
698 if (!SU->getNode())
701 switch (SU->getNode()->getOpcode()) {
703 assert(SU->getNode()->isMachineOpcode() &&
722 if (SU->isCall) {
728 HazardRec->EmitInstruction(SU);
731 static void resetVRegCycle(SUnit *SU);
736 void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) {
738 LLVM_DEBUG(dumpNode(*SU));
741 if (CurCycle < SU->getHeight())
742 LLVM_DEBUG(dbgs() << " Height [" << SU->getHeight()
750 SU->setHeightToAtLeast(CurCycle);
753 EmitNode(SU);
755 Sequence.push_back(SU);
757 AvailableQueue->scheduledNode(SU);
767 ReleasePredecessors(SU);
770 for (SDep &Succ : SU->Succs) {
771 // LiveRegDegs[Succ.getReg()] != SU when SU is a two-address node.
772 if (Succ.isAssignedRegDep() && LiveRegDefs[Succ.getReg()] == SU) {
783 if (LiveRegDefs[CallResource] == SU)
784 for (const SDNode *SUNode = SU->getNode(); SUNode;
796 resetVRegCycle(SU);
798 SU->isScheduled = true;
809 if (SU->getNode() && SU->getNode()->isMachineOpcode())
817 /// CapturePred - This does the opposite of ReleasePred. Since SU is being
835 void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) {
836 LLVM_DEBUG(dbgs() << "*** Unscheduling [" << SU->getHeight() << "]: ");
837 LLVM_DEBUG(dumpNode(*SU));
839 for (SDep &Pred : SU->Preds) {
841 if (Pred.isAssignedRegDep() && SU == LiveRegGens[Pred.getReg()]){
855 for (const SDNode *SUNode = SU->getNode(); SUNode;
859 SUnit *SeqEnd = CallSeqEndForStart[SU];
864 LiveRegDefs[CallResource] = SU;
871 if (LiveRegGens[CallResource] == SU)
872 for (const SDNode *SUNode = SU->getNode(); SUNode;
886 for (auto &Succ : SU->Succs) {
893 LiveRegDefs[Reg] = SU;
900 for (auto &Succ2 : SU->Succs) {
908 if (SU->getHeight() < MinAvailableCycle)
909 MinAvailableCycle = SU->getHeight();
911 SU->setHeightDirty();
912 SU->isScheduled = false;
913 SU->isAvailable = true;
916 SU->isPending = true;
917 PendingQueue.push_back(SU);
920 AvailableQueue->push(SU);
922 AvailableQueue->unscheduledNode(SU);
938 SUnit *SU = *I;
939 for (; SU->getHeight() > HazardCycle; ++HazardCycle) {
942 EmitNode(SU);
948 void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) {
961 assert(!SU->isSucc(OldSU) && "Something is wrong!");
970 static bool isOperandOf(const SUnit *SU, SDNode *N) {
971 for (const SDNode *SUNode = SU->getNode(); SUNode;
980 SUnit *ScheduleDAGRRList::TryUnfoldSU(SUnit *SU) {
981 SDNode *N = SU->getNode();
997 unsigned OldNumVals = SU->getNode()->getNumValues();
1007 // this would negate the benefit to unfolding so just return SU.
1009 return SU;
1025 // negates the benefit to unfolding so just return SU.
1027 return SU;
1048 LLVM_DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n");
1052 DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
1053 DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals - 1),
1056 // Record all the edges to and from the old SU, by category.
1062 for (SDep &Pred : SU->Preds) {
1070 for (SDep &Succ : SU->Succs) {
1079 RemovePred(SU, Pred);
1084 RemovePred(SU, Pred);
1089 RemovePred(SU, Pred);
1094 D.setSUnit(SU);
1105 D.setSUnit(SU);
1134 SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) {
1135 SDNode *N = SU->getNode();
1139 LLVM_DEBUG(dbgs() << "Considering duplicating the SU\n");
1140 LLVM_DEBUG(dumpNode(*SU));
1173 SUnit *UnfoldSU = TryUnfoldSU(SU);
1176 SU = UnfoldSU;
1177 N = SU->getNode();
1179 if (SU->NumSuccsLeft == 0)
1180 return SU;
1183 LLVM_DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n");
1184 NewSU = CreateClone(SU);
1187 for (SDep &Pred : SU->Preds)
1193 AddPredQueued(NewSU, SDep(SU, SDep::Artificial));
1198 for (SDep &Succ : SU->Succs) {
1206 D.setSUnit(SU);
1213 AvailableQueue->updateNode(SU);
1222 void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
1237 for (SDep &Succ : SU->Succs) {
1257 SDep FromDep(SU, SDep::Data, Reg);
1258 FromDep.setLatency(SU->Latency);
1264 AvailableQueue->updateNode(SU);
1297 static void CheckForLiveRegDef(SUnit *SU, unsigned Reg,
1308 if (LiveRegDefs[*AliasI] == SU) continue;
1319 static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask,
1326 if (LiveRegDefs[i] == SU) continue;
1346 DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl<unsigned> &LRegs) {
1353 // If SU is the currently live definition of the same register that it uses,
1355 for (SDep &Pred : SU->Preds) {
1356 if (Pred.isAssignedRegDep() && LiveRegDefs[Pred.getReg()] != SU)
1361 for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
1382 CheckForLiveRegDef(SU, Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI);
1408 CheckForLiveRegDefMasked(SU, RegMask,
1422 CheckForLiveRegDef(SU, Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI);
1428 CheckForLiveRegDef(SU, *Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI);
1437 SUnit *SU = Interferences[i-1];
1438 LRegsMapT::iterator LRegsPos = LRegsMap.find(SU);
1444 SU->isPending = false;
1448 if (SU->isAvailable && !SU->NodeQueueId) {
1449 LLVM_DEBUG(dbgs() << " Repushing SU #" << SU->NodeNum << '\n');
1450 AvailableQueue->push(SU);
1473 dbgs() << " SU #" << CurSU->NodeNum << '\n');
1477 CurSU->isPending = true; // This SU is not in AvailableQueue right now.
1524 LLVM_DEBUG(dbgs() << "ARTIFICIAL edge from SU(" << BtSU->NodeNum
1525 << ") to SU(" << TrySU->NodeNum << ")\n");
1578 LLVM_DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum
1579 << " to SU #" << Copies.front()->NodeNum << "\n");
1584 LLVM_DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum
1585 << " to SU #" << TrySU->NodeNum << "\n");
1618 SUnit *SU = PickNodeToScheduleBottomUp();
1620 AdvancePastStalls(SU);
1622 ScheduleNodeBottomUp(SU);
1625 // Advance the cycle to free resources. Skip ahead to the next ready SU.
1645 bool isReady(SUnit* SU, unsigned CurCycle) const { return true; }
1703 bool isReady(SUnit *SU, unsigned CurCycle) const;
1720 bool isReady(SUnit *SU, unsigned CurCycle) const;
1782 void addNode(const SUnit *SU) override;
1784 void updateNode(const SUnit *SU) override;
1792 unsigned getNodePriority(const SUnit *SU) const;
1794 unsigned getNodeOrdering(const SUnit *SU) const {
1795 if (!SU->getNode()) return 0;
1797 return SU->getNode()->getIROrder();
1808 void remove(SUnit *SU) override {
1810 assert(SU->NodeQueueId != 0 && "Not in queue!");
1811 std::vector<SUnit *>::iterator I = llvm::find(Queue, SU);
1815 SU->NodeQueueId = 0;
1822 bool HighRegPressure(const SUnit *SU) const;
1824 bool MayReduceRegPressure(SUnit *SU) const;
1826 int RegPressureDiff(SUnit *SU, unsigned &LiveUses) const;
1828 void scheduledNode(SUnit *SU) override;
1830 void unscheduledNode(SUnit *SU) override;
1833 bool canClobber(const SUnit *SU, const SUnit *Op);
1906 SUnit *SU = popFromQueue(DumpQueue, DumpPicker, scheduleDAG);
1907 dbgs() << "Height " << SU->getHeight() << ": ";
1908 DAG->dumpNode(*SU);
1942 CalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) {
1943 if (SUNumbers[SU->NodeNum] != 0)
1944 return SUNumbers[SU->NodeNum];
1948 WorkState(const SUnit *SU) : SU(SU) {}
1949 const SUnit *SU;
1954 WorkList.push_back(SU);
1957 auto *TempSU = Temp.SU;
1968 assert(It.SU != PredSU && "Trying to push an element twice?");
2003 assert(SUNumbers[SU->NodeNum] > 0 && "SethiUllman should never be zero!");
2004 return SUNumbers[SU->NodeNum];
2012 for (const SUnit &SU : *SUnits)
2013 CalcNodeSethiUllmanNumber(&SU, SethiUllmanNumbers);
2016 void RegReductionPQBase::addNode(const SUnit *SU) {
2020 CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers);
2023 void RegReductionPQBase::updateNode(const SUnit *SU) {
2024 SethiUllmanNumbers[SU->NodeNum] = 0;
2025 CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers);
2030 unsigned RegReductionPQBase::getNodePriority(const SUnit *SU) const {
2031 assert(SU->NodeNum < SethiUllmanNumbers.size());
2032 unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0;
2043 if (SU->NumSuccs == 0 && SU->NumPreds != 0)
2044 // If SU does not have a register use, i.e. it doesn't produce a value
2050 if (SU->NumPreds == 0 && SU->NumSuccs != 0)
2051 // If SU does not have a register def, schedule it close to its uses
2055 return SethiUllmanNumbers[SU->NodeNum];
2057 unsigned Priority = SethiUllmanNumbers[SU->NodeNum];
2058 if (SU->isCallOp) {
2060 int NP = (int)Priority - SU->getNode()->getNumValues();
2083 bool RegReductionPQBase::HighRegPressure(const SUnit *SU) const {
2087 for (const SDep &Pred : SU->Preds) {
2108 bool RegReductionPQBase::MayReduceRegPressure(SUnit *SU) const {
2109 const SDNode *N = SU->getNode();
2111 if (!N->isMachineOpcode() || !SU->NumSuccs)
2133 int RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const {
2136 for (const SDep &Pred : SU->Preds) {
2155 const SDNode *N = SU->getNode();
2157 if (!N || !N->isMachineOpcode() || !SU->NumSuccs)
2172 void RegReductionPQBase::scheduledNode(SUnit *SU) {
2176 if (!SU->getNode())
2179 for (const SDep &Pred : SU->Preds) {
2199 // here with the reduction further below. Note that this SU may use multiple
2219 //assert(SU->NumRegDefsLeft == 0 && "not all regdefs have scheduled uses");
2220 int SkipRegDefs = (int)SU->NumRegDefsLeft;
2221 for (ScheduleDAGSDNodes::RegDefIter RegDefPos(SU, scheduleDAG);
2230 LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum
2241 void RegReductionPQBase::unscheduledNode(SUnit *SU) {
2245 const SDNode *N = SU->getNode();
2261 for (const SDep &Pred : SU->Preds) {
2305 if (SU->NumSuccs && N->isMachineOpcode()) {
2327 static unsigned closestSucc(const SUnit *SU) {
2329 for (const SDep &Succ : SU->Succs) {
2345 static unsigned calcMaxScratches(const SUnit *SU) {
2347 for (const SDep &Pred : SU->Preds) {
2354 /// hasOnlyLiveInOpers - Return true if SU has only value predecessors that are
2356 static bool hasOnlyLiveInOpers(const SUnit *SU) {
2358 for (const SDep &Pred : SU->Preds) {
2375 /// hasOnlyLiveOutUses - Return true if SU has only value successors that are
2376 /// CopyToReg to a virtual register. This SU def is probably a liveout and
2378 static bool hasOnlyLiveOutUses(const SUnit *SU) {
2380 for (const SDep &Succ : SU->Succs) {
2406 static void initVRegCycle(SUnit *SU) {
2410 if (!hasOnlyLiveInOpers(SU) || !hasOnlyLiveOutUses(SU))
2413 LLVM_DEBUG(dbgs() << "VRegCycle: SU(" << SU->NodeNum << ")\n");
2415 SU->isVRegCycle = true;
2417 for (const SDep &Pred : SU->Preds) {
2425 static void resetVRegCycle(SUnit *SU) {
2426 if (!SU->isVRegCycle)
2429 for (const SDep &Pred : SU->Preds) {
2442 static bool hasVRegCycleUse(const SUnit *SU) {
2443 // If this SU also defines the VReg, don't hoist it as a "use".
2444 if (SU->isVRegCycle)
2447 for (const SDep &Pred : SU->Preds) {
2451 LLVM_DEBUG(dbgs() << " VReg cycle use: SU (" << SU->NodeNum << ")\n");
2460 // Note: The ScheduleHazardRecognizer interface requires a non-const SU.
2461 static bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ) {
2463 if (SPQ->getHazardRec()->getHazardType(SU, 0)
2511 LLVM_DEBUG(dbgs() << " Comparing latency of SU (" << left->NodeNum
2512 << ") depth " << LDepth << " vs SU (" << right->NodeNum
2535 LLVM_DEBUG(dbgs() << " SU (" << left->NodeNum << ") "
2536 << PhysRegMsg[LHasPhysReg] << " SU(" << right->NodeNum
2653 bool hybrid_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
2656 if (SPQ->MayReduceRegPressure(SU)) return true;
2658 if (SU->getHeight() > (CurCycle + ReadyDelay)) return false;
2660 if (SPQ->getHazardRec()->getHazardType(SU, -ReadyDelay)
2681 LLVM_DEBUG(dbgs() << " pressure SU(" << left->NodeNum << ") > SU("
2686 LLVM_DEBUG(dbgs() << " pressure SU(" << right->NodeNum << ") > SU("
2700 bool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
2701 if (SU->getHeight() > CurCycle) return false;
2703 if (SPQ->getHazardRec()->getHazardType(SU, 0)
2710 static bool canEnableCoalescing(SUnit *SU) {
2711 unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0;
2724 if (SU->NumPreds == 0 && SU->NumSuccs != 0)
2725 // If SU does not have a register def, schedule it close to its uses
2749 LLVM_DEBUG(dbgs() << "RegPressureDiff SU(" << left->NodeNum
2750 << "): " << LPDiff << " != SU(" << right->NodeNum
2763 LLVM_DEBUG(dbgs() << "Live uses SU(" << left->NodeNum << "): " << LLiveUses
2764 << " != SU(" << right->NodeNum << "): " << RLiveUses
2779 LLVM_DEBUG(dbgs() << "Depth of SU(" << left->NodeNum << "): "
2780 << left->getDepth() << " != SU(" << right->NodeNum
2808 for (SUnit &SU : sunits)
2809 initVRegCycle(&SU);
2816 bool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) {
2817 if (SU->isTwoAddress) {
2818 unsigned Opc = SU->getNode()->getMachineOpcode();
2824 SDNode *DU = SU->getNode()->getOperand(i).getNode();
2834 /// canClobberReachingPhysRegUse - True if SU would clobber one of it's
2836 /// i.e. DepSU should not be scheduled above SU.
2837 static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU,
2842 = TII->get(SU->getNode()->getMachineOpcode()).getImplicitDefs();
2843 const uint32_t *RegMask = getNodeRegMask(SU->getNode());
2847 for (const SDep &Succ : SU->Succs) {
2860 // Return true if SU clobbers this physical register use and the
2871 /// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's
2873 static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU,
2880 for (const SDNode *SUNode = SU->getNode(); SUNode;
2942 for (SUnit &SU : *SUnits) {
2946 if (SU.NumSuccs != 0)
2949 if (SU.NumPreds != 1)
2953 if (SDNode *N = SU.getNode())
2960 for (const SDep &Pred : SU.Preds)
2984 for (const SDep &Pred : SU.Preds)
2995 // Short-circuit the case where SU is PredSU's only data successor.
3000 if (SDNode *N = SU.getNode())
3009 if (PredSuccSU == &SU) continue;
3015 if (SU.hasPhysRegClobbers && PredSuccSU->hasPhysRegDefs)
3016 if (canClobberPhysRegDefs(PredSuccSU, &SU, TII, TRI))
3019 if (scheduleDAG->IsReachable(&SU, PredSuccSU))
3026 dbgs() << " Prescheduling SU #" << SU.NodeNum << " next to PredSU #"
3033 if (SuccSU != &SU) {
3036 scheduleDAG->AddPredQueued(&SU, Edge);
3037 Edge.setSUnit(&SU);
3054 for (SUnit &SU : *SUnits) {
3055 if (!SU.isTwoAddress)
3058 SDNode *Node = SU.getNode();
3059 if (!Node || !Node->isMachineOpcode() || SU.getNode()->getGluedNode())
3062 bool isLiveOut = hasOnlyLiveOutUses(&SU);
3070 SDNode *DU = SU.getNode()->getOperand(j).getNode();
3080 if (SuccSU == &SU)
3084 if (SuccSU->getHeight() < SU.getHeight() &&
3085 (SU.getHeight() - SuccSU->getHeight()) > 1)
3101 if (SuccSU->hasPhysRegDefs && SU.hasPhysRegClobbers) {
3102 if (canClobberPhysRegDefs(SuccSU, &SU, TII, TRI))
3112 if (!canClobberReachingPhysRegUse(SuccSU, &SU, scheduleDAG, TII, TRI) &&
3115 (!SU.isCommutable && SuccSU->isCommutable)) &&
3116 !scheduleDAG->IsReachable(SuccSU, &SU)) {
3118 << " Adding a pseudo-two-addr edge from SU #"
3119 << SU.NodeNum << " to SU #" << SuccSU->NodeNum << "\n");
3120 scheduleDAG->AddPredQueued(&SU, SDep(SuccSU, SDep::Artificial));