IfConversion.cpp revision 314564
1219820Sjeff//===-- IfConversion.cpp - Machine code if conversion pass. ---------------===// 2219820Sjeff// 3219820Sjeff// The LLVM Compiler Infrastructure 4219820Sjeff// 5219820Sjeff// This file is distributed under the University of Illinois Open Source 6219820Sjeff// License. See LICENSE.TXT for details. 7219820Sjeff// 8219820Sjeff//===----------------------------------------------------------------------===// 9219820Sjeff// 10219820Sjeff// This file implements the machine instruction level if-conversion pass, which 11219820Sjeff// tries to convert conditional branches into predicated instructions. 12219820Sjeff// 13219820Sjeff//===----------------------------------------------------------------------===// 14219820Sjeff 15219820Sjeff#include "llvm/CodeGen/Passes.h" 16219820Sjeff#include "BranchFolding.h" 17219820Sjeff#include "llvm/ADT/STLExtras.h" 18219820Sjeff#include "llvm/ADT/ScopeExit.h" 19219820Sjeff#include "llvm/ADT/SmallSet.h" 20219820Sjeff#include "llvm/ADT/Statistic.h" 21219820Sjeff#include "llvm/CodeGen/LivePhysRegs.h" 22219820Sjeff#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 23219820Sjeff#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" 24219820Sjeff#include "llvm/CodeGen/MachineFunctionPass.h" 25219820Sjeff#include "llvm/CodeGen/MachineInstrBuilder.h" 26219820Sjeff#include "llvm/CodeGen/MachineModuleInfo.h" 27219820Sjeff#include "llvm/CodeGen/MachineRegisterInfo.h" 28219820Sjeff#include "llvm/CodeGen/TargetSchedule.h" 29219820Sjeff#include "llvm/Support/CommandLine.h" 30219820Sjeff#include "llvm/Support/Debug.h" 31219820Sjeff#include "llvm/Support/ErrorHandling.h" 32219820Sjeff#include "llvm/Support/raw_ostream.h" 33219820Sjeff#include "llvm/Target/TargetInstrInfo.h" 34219820Sjeff#include "llvm/Target/TargetLowering.h" 35219820Sjeff#include "llvm/Target/TargetRegisterInfo.h" 36219820Sjeff#include "llvm/Target/TargetSubtargetInfo.h" 37219820Sjeff#include <algorithm> 38219820Sjeff#include <utility> 39219820Sjeff 40219820Sjeffusing namespace llvm; 41219820Sjeff 42219820Sjeff#define DEBUG_TYPE "ifcvt" 43219820Sjeff 44219820Sjeff// Hidden options for help debugging. 45219820Sjeffstatic cl::opt<int> IfCvtFnStart("ifcvt-fn-start", cl::init(-1), cl::Hidden); 46219820Sjeffstatic cl::opt<int> IfCvtFnStop("ifcvt-fn-stop", cl::init(-1), cl::Hidden); 47219820Sjeffstatic cl::opt<int> IfCvtLimit("ifcvt-limit", cl::init(-1), cl::Hidden); 48219820Sjeffstatic cl::opt<bool> DisableSimple("disable-ifcvt-simple", 49219820Sjeff cl::init(false), cl::Hidden); 50219820Sjeffstatic cl::opt<bool> DisableSimpleF("disable-ifcvt-simple-false", 51219820Sjeff cl::init(false), cl::Hidden); 52219820Sjeffstatic cl::opt<bool> DisableTriangle("disable-ifcvt-triangle", 53219820Sjeff cl::init(false), cl::Hidden); 54219820Sjeffstatic cl::opt<bool> DisableTriangleR("disable-ifcvt-triangle-rev", 55219820Sjeff cl::init(false), cl::Hidden); 56219820Sjeffstatic cl::opt<bool> DisableTriangleF("disable-ifcvt-triangle-false", 57219820Sjeff cl::init(false), cl::Hidden); 58219820Sjeffstatic cl::opt<bool> DisableTriangleFR("disable-ifcvt-triangle-false-rev", 59219820Sjeff cl::init(false), cl::Hidden); 60219820Sjeffstatic cl::opt<bool> DisableDiamond("disable-ifcvt-diamond", 61219820Sjeff cl::init(false), cl::Hidden); 62219820Sjeffstatic cl::opt<bool> DisableForkedDiamond("disable-ifcvt-forked-diamond", 63219820Sjeff cl::init(false), cl::Hidden); 64219820Sjeffstatic cl::opt<bool> IfCvtBranchFold("ifcvt-branch-fold", 65219820Sjeff cl::init(true), cl::Hidden); 66219820Sjeff 67219820SjeffSTATISTIC(NumSimple, "Number of simple if-conversions performed"); 68219820SjeffSTATISTIC(NumSimpleFalse, "Number of simple (F) if-conversions performed"); 69219820SjeffSTATISTIC(NumTriangle, "Number of triangle if-conversions performed"); 70219820SjeffSTATISTIC(NumTriangleRev, "Number of triangle (R) if-conversions performed"); 71219820SjeffSTATISTIC(NumTriangleFalse,"Number of triangle (F) if-conversions performed"); 72219820SjeffSTATISTIC(NumTriangleFRev, "Number of triangle (F/R) if-conversions performed"); 73219820SjeffSTATISTIC(NumDiamonds, "Number of diamond if-conversions performed"); 74219820SjeffSTATISTIC(NumForkedDiamonds, "Number of forked-diamond if-conversions performed"); 75219820SjeffSTATISTIC(NumIfConvBBs, "Number of if-converted blocks"); 76219820SjeffSTATISTIC(NumDupBBs, "Number of duplicated blocks"); 77219820SjeffSTATISTIC(NumUnpred, "Number of true blocks of diamonds unpredicated"); 78219820Sjeff 79219820Sjeffnamespace { 80219820Sjeff class IfConverter : public MachineFunctionPass { 81219820Sjeff enum IfcvtKind { 82219820Sjeff ICNotClassfied, // BB data valid, but not classified. 83219820Sjeff ICSimpleFalse, // Same as ICSimple, but on the false path. 84219820Sjeff ICSimple, // BB is entry of an one split, no rejoin sub-CFG. 85219820Sjeff ICTriangleFRev, // Same as ICTriangleFalse, but false path rev condition. 86219820Sjeff ICTriangleRev, // Same as ICTriangle, but true path rev condition. 87219820Sjeff ICTriangleFalse, // Same as ICTriangle, but on the false path. 88219820Sjeff ICTriangle, // BB is entry of a triangle sub-CFG. 89219820Sjeff ICDiamond, // BB is entry of a diamond sub-CFG. 90219820Sjeff ICForkedDiamond // BB is entry of an almost diamond sub-CFG, with a 91219820Sjeff // common tail that can be shared. 92219820Sjeff }; 93219820Sjeff 94219820Sjeff /// One per MachineBasicBlock, this is used to cache the result 95219820Sjeff /// if-conversion feasibility analysis. This includes results from 96219820Sjeff /// TargetInstrInfo::analyzeBranch() (i.e. TBB, FBB, and Cond), and its 97219820Sjeff /// classification, and common tail block of its successors (if it's a 98219820Sjeff /// diamond shape), its size, whether it's predicable, and whether any 99219820Sjeff /// instruction can clobber the 'would-be' predicate. 100219820Sjeff /// 101219820Sjeff /// IsDone - True if BB is not to be considered for ifcvt. 102219820Sjeff /// IsBeingAnalyzed - True if BB is currently being analyzed. 103219820Sjeff /// IsAnalyzed - True if BB has been analyzed (info is still valid). 104219820Sjeff /// IsEnqueued - True if BB has been enqueued to be ifcvt'ed. 105219820Sjeff /// IsBrAnalyzable - True if analyzeBranch() returns false. 106219820Sjeff /// HasFallThrough - True if BB may fallthrough to the following BB. 107219820Sjeff /// IsUnpredicable - True if BB is known to be unpredicable. 108219820Sjeff /// ClobbersPred - True if BB could modify predicates (e.g. has 109219820Sjeff /// cmp, call, etc.) 110219820Sjeff /// NonPredSize - Number of non-predicated instructions. 111219820Sjeff /// ExtraCost - Extra cost for multi-cycle instructions. 112219820Sjeff /// ExtraCost2 - Some instructions are slower when predicated 113219820Sjeff /// BB - Corresponding MachineBasicBlock. 114219820Sjeff /// TrueBB / FalseBB- See analyzeBranch(). 115219820Sjeff /// BrCond - Conditions for end of block conditional branches. 116219820Sjeff /// Predicate - Predicate used in the BB. 117219820Sjeff struct BBInfo { 118219820Sjeff bool IsDone : 1; 119219820Sjeff bool IsBeingAnalyzed : 1; 120219820Sjeff bool IsAnalyzed : 1; 121219820Sjeff bool IsEnqueued : 1; 122219820Sjeff bool IsBrAnalyzable : 1; 123219820Sjeff bool IsBrReversible : 1; 124219820Sjeff bool HasFallThrough : 1; 125219820Sjeff bool IsUnpredicable : 1; 126219820Sjeff bool CannotBeCopied : 1; 127219820Sjeff bool ClobbersPred : 1; 128219820Sjeff unsigned NonPredSize; 129219820Sjeff unsigned ExtraCost; 130219820Sjeff unsigned ExtraCost2; 131219820Sjeff MachineBasicBlock *BB; 132219820Sjeff MachineBasicBlock *TrueBB; 133219820Sjeff MachineBasicBlock *FalseBB; 134219820Sjeff SmallVector<MachineOperand, 4> BrCond; 135219820Sjeff SmallVector<MachineOperand, 4> Predicate; 136219820Sjeff BBInfo() : IsDone(false), IsBeingAnalyzed(false), 137219820Sjeff IsAnalyzed(false), IsEnqueued(false), IsBrAnalyzable(false), 138219820Sjeff IsBrReversible(false), HasFallThrough(false), 139219820Sjeff IsUnpredicable(false), CannotBeCopied(false), 140219820Sjeff ClobbersPred(false), NonPredSize(0), ExtraCost(0), 141219820Sjeff ExtraCost2(0), BB(nullptr), TrueBB(nullptr), 142219820Sjeff FalseBB(nullptr) {} 143219820Sjeff }; 144219820Sjeff 145219820Sjeff /// Record information about pending if-conversions to attempt: 146219820Sjeff /// BBI - Corresponding BBInfo. 147219820Sjeff /// Kind - Type of block. See IfcvtKind. 148219820Sjeff /// NeedSubsumption - True if the to-be-predicated BB has already been 149219820Sjeff /// predicated. 150219820Sjeff /// NumDups - Number of instructions that would be duplicated due 151219820Sjeff /// to this if-conversion. (For diamonds, the number of 152219820Sjeff /// identical instructions at the beginnings of both 153219820Sjeff /// paths). 154219820Sjeff /// NumDups2 - For diamonds, the number of identical instructions 155219820Sjeff /// at the ends of both paths. 156219820Sjeff struct IfcvtToken { 157219820Sjeff BBInfo &BBI; 158219820Sjeff IfcvtKind Kind; 159219820Sjeff unsigned NumDups; 160219820Sjeff unsigned NumDups2; 161219820Sjeff bool NeedSubsumption : 1; 162219820Sjeff bool TClobbersPred : 1; 163219820Sjeff bool FClobbersPred : 1; 164219820Sjeff IfcvtToken(BBInfo &b, IfcvtKind k, bool s, unsigned d, unsigned d2 = 0, 165219820Sjeff bool tc = false, bool fc = false) 166219820Sjeff : BBI(b), Kind(k), NumDups(d), NumDups2(d2), NeedSubsumption(s), 167219820Sjeff TClobbersPred(tc), FClobbersPred(fc) {} 168219820Sjeff }; 169219820Sjeff 170219820Sjeff /// Results of if-conversion feasibility analysis indexed by basic block 171219820Sjeff /// number. 172219820Sjeff std::vector<BBInfo> BBAnalysis; 173219820Sjeff TargetSchedModel SchedModel; 174219820Sjeff 175219820Sjeff const TargetLoweringBase *TLI; 176219820Sjeff const TargetInstrInfo *TII; 177219820Sjeff const TargetRegisterInfo *TRI; 178219820Sjeff const MachineBranchProbabilityInfo *MBPI; 179219820Sjeff MachineRegisterInfo *MRI; 180219820Sjeff 181219820Sjeff LivePhysRegs Redefs; 182219820Sjeff LivePhysRegs DontKill; 183219820Sjeff 184219820Sjeff bool PreRegAlloc; 185219820Sjeff bool MadeChange; 186219820Sjeff int FnNum; 187219820Sjeff std::function<bool(const MachineFunction &)> PredicateFtor; 188219820Sjeff 189219820Sjeff public: 190219820Sjeff static char ID; 191219820Sjeff IfConverter(std::function<bool(const MachineFunction &)> Ftor = nullptr) 192219820Sjeff : MachineFunctionPass(ID), FnNum(-1), PredicateFtor(std::move(Ftor)) { 193219820Sjeff initializeIfConverterPass(*PassRegistry::getPassRegistry()); 194219820Sjeff } 195219820Sjeff 196219820Sjeff void getAnalysisUsage(AnalysisUsage &AU) const override { 197219820Sjeff AU.addRequired<MachineBlockFrequencyInfo>(); 198219820Sjeff AU.addRequired<MachineBranchProbabilityInfo>(); 199219820Sjeff MachineFunctionPass::getAnalysisUsage(AU); 200219820Sjeff } 201219820Sjeff 202219820Sjeff bool runOnMachineFunction(MachineFunction &MF) override; 203219820Sjeff 204219820Sjeff MachineFunctionProperties getRequiredProperties() const override { 205219820Sjeff return MachineFunctionProperties().set( 206219820Sjeff MachineFunctionProperties::Property::NoVRegs); 207219820Sjeff } 208219820Sjeff 209219820Sjeff private: 210219820Sjeff bool reverseBranchCondition(BBInfo &BBI) const; 211219820Sjeff bool ValidSimple(BBInfo &TrueBBI, unsigned &Dups, 212219820Sjeff BranchProbability Prediction) const; 213219820Sjeff bool ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI, 214219820Sjeff bool FalseBranch, unsigned &Dups, 215219820Sjeff BranchProbability Prediction) const; 216219820Sjeff bool CountDuplicatedInstructions( 217219820Sjeff MachineBasicBlock::iterator &TIB, MachineBasicBlock::iterator &FIB, 218219820Sjeff MachineBasicBlock::iterator &TIE, MachineBasicBlock::iterator &FIE, 219219820Sjeff unsigned &Dups1, unsigned &Dups2, 220219820Sjeff MachineBasicBlock &TBB, MachineBasicBlock &FBB, 221219820Sjeff bool SkipUnconditionalBranches) const; 222219820Sjeff bool ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI, 223219820Sjeff unsigned &Dups1, unsigned &Dups2, 224219820Sjeff BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const; 225219820Sjeff bool ValidForkedDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI, 226219820Sjeff unsigned &Dups1, unsigned &Dups2, 227219820Sjeff BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const; 228219820Sjeff void AnalyzeBranches(BBInfo &BBI); 229219820Sjeff void ScanInstructions(BBInfo &BBI, 230219820Sjeff MachineBasicBlock::iterator &Begin, 231219820Sjeff MachineBasicBlock::iterator &End, 232219820Sjeff bool BranchUnpredicable = false) const; 233219820Sjeff bool RescanInstructions( 234219820Sjeff MachineBasicBlock::iterator &TIB, MachineBasicBlock::iterator &FIB, 235219820Sjeff MachineBasicBlock::iterator &TIE, MachineBasicBlock::iterator &FIE, 236219820Sjeff BBInfo &TrueBBI, BBInfo &FalseBBI) const; 237219820Sjeff void AnalyzeBlock(MachineBasicBlock &MBB, 238219820Sjeff std::vector<std::unique_ptr<IfcvtToken>> &Tokens); 239219820Sjeff bool FeasibilityAnalysis(BBInfo &BBI, SmallVectorImpl<MachineOperand> &Cond, 240219820Sjeff bool isTriangle = false, bool RevBranch = false, 241219820Sjeff bool hasCommonTail = false); 242219820Sjeff void AnalyzeBlocks(MachineFunction &MF, 243219820Sjeff std::vector<std::unique_ptr<IfcvtToken>> &Tokens); 244219820Sjeff void InvalidatePreds(MachineBasicBlock &MBB); 245219820Sjeff void RemoveExtraEdges(BBInfo &BBI); 246219820Sjeff bool IfConvertSimple(BBInfo &BBI, IfcvtKind Kind); 247219820Sjeff bool IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind); 248219820Sjeff bool IfConvertDiamondCommon(BBInfo &BBI, BBInfo &TrueBBI, BBInfo &FalseBBI, 249219820Sjeff unsigned NumDups1, unsigned NumDups2, 250219820Sjeff bool TClobbersPred, bool FClobbersPred, 251219820Sjeff bool RemoveBranch, bool MergeAddEdges); 252219820Sjeff bool IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind, 253219820Sjeff unsigned NumDups1, unsigned NumDups2, 254219820Sjeff bool TClobbers, bool FClobbers); 255219820Sjeff bool IfConvertForkedDiamond(BBInfo &BBI, IfcvtKind Kind, 256219820Sjeff unsigned NumDups1, unsigned NumDups2, 257219820Sjeff bool TClobbers, bool FClobbers); 258219820Sjeff void PredicateBlock(BBInfo &BBI, 259219820Sjeff MachineBasicBlock::iterator E, 260219820Sjeff SmallVectorImpl<MachineOperand> &Cond, 261219820Sjeff SmallSet<unsigned, 4> *LaterRedefs = nullptr); 262219820Sjeff void CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, 263219820Sjeff SmallVectorImpl<MachineOperand> &Cond, 264219820Sjeff bool IgnoreBr = false); 265219820Sjeff void MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges = true); 266219820Sjeff 267219820Sjeff bool MeetIfcvtSizeLimit(MachineBasicBlock &BB, 268219820Sjeff unsigned Cycle, unsigned Extra, 269219820Sjeff BranchProbability Prediction) const { 270219820Sjeff return Cycle > 0 && TII->isProfitableToIfCvt(BB, Cycle, Extra, 271219820Sjeff Prediction); 272219820Sjeff } 273219820Sjeff 274219820Sjeff bool MeetIfcvtSizeLimit(MachineBasicBlock &TBB, 275219820Sjeff unsigned TCycle, unsigned TExtra, 276219820Sjeff MachineBasicBlock &FBB, 277219820Sjeff unsigned FCycle, unsigned FExtra, 278219820Sjeff BranchProbability Prediction) const { 279219820Sjeff return TCycle > 0 && FCycle > 0 && 280219820Sjeff TII->isProfitableToIfCvt(TBB, TCycle, TExtra, FBB, FCycle, FExtra, 281219820Sjeff Prediction); 282219820Sjeff } 283219820Sjeff 284219820Sjeff /// Returns true if Block ends without a terminator. 285219820Sjeff bool blockAlwaysFallThrough(BBInfo &BBI) const { 286219820Sjeff return BBI.IsBrAnalyzable && BBI.TrueBB == nullptr; 287219820Sjeff } 288219820Sjeff 289219820Sjeff /// Used to sort if-conversion candidates. 290219820Sjeff static bool IfcvtTokenCmp(const std::unique_ptr<IfcvtToken> &C1, 291219820Sjeff const std::unique_ptr<IfcvtToken> &C2) { 292219820Sjeff int Incr1 = (C1->Kind == ICDiamond) 293219820Sjeff ? -(int)(C1->NumDups + C1->NumDups2) : (int)C1->NumDups; 294219820Sjeff int Incr2 = (C2->Kind == ICDiamond) 295219820Sjeff ? -(int)(C2->NumDups + C2->NumDups2) : (int)C2->NumDups; 296219820Sjeff if (Incr1 > Incr2) 297219820Sjeff return true; 298219820Sjeff else if (Incr1 == Incr2) { 299219820Sjeff // Favors subsumption. 300219820Sjeff if (!C1->NeedSubsumption && C2->NeedSubsumption) 301219820Sjeff return true; 302219820Sjeff else if (C1->NeedSubsumption == C2->NeedSubsumption) { 303219820Sjeff // Favors diamond over triangle, etc. 304219820Sjeff if ((unsigned)C1->Kind < (unsigned)C2->Kind) 305219820Sjeff return true; 306219820Sjeff else if (C1->Kind == C2->Kind) 307219820Sjeff return C1->BBI.BB->getNumber() < C2->BBI.BB->getNumber(); 308219820Sjeff } 309219820Sjeff } 310219820Sjeff return false; 311219820Sjeff } 312219820Sjeff }; 313219820Sjeff 314219820Sjeff char IfConverter::ID = 0; 315219820Sjeff} 316219820Sjeff 317219820Sjeffchar &llvm::IfConverterID = IfConverter::ID; 318219820Sjeff 319219820SjeffINITIALIZE_PASS_BEGIN(IfConverter, "if-converter", "If Converter", false, false) 320219820SjeffINITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) 321219820SjeffINITIALIZE_PASS_END(IfConverter, "if-converter", "If Converter", false, false) 322219820Sjeff 323219820Sjeffbool IfConverter::runOnMachineFunction(MachineFunction &MF) { 324219820Sjeff if (skipFunction(*MF.getFunction()) || (PredicateFtor && !PredicateFtor(MF))) 325219820Sjeff return false; 326219820Sjeff 327219820Sjeff const TargetSubtargetInfo &ST = MF.getSubtarget(); 328219820Sjeff TLI = ST.getTargetLowering(); 329219820Sjeff TII = ST.getInstrInfo(); 330219820Sjeff TRI = ST.getRegisterInfo(); 331219820Sjeff BranchFolder::MBFIWrapper MBFI(getAnalysis<MachineBlockFrequencyInfo>()); 332219820Sjeff MBPI = &getAnalysis<MachineBranchProbabilityInfo>(); 333219820Sjeff MRI = &MF.getRegInfo(); 334219820Sjeff SchedModel.init(ST.getSchedModel(), &ST, TII); 335219820Sjeff 336219820Sjeff if (!TII) return false; 337219820Sjeff 338219820Sjeff PreRegAlloc = MRI->isSSA(); 339219820Sjeff 340219820Sjeff bool BFChange = false; 341219820Sjeff if (!PreRegAlloc) { 342219820Sjeff // Tail merge tend to expose more if-conversion opportunities. 343219820Sjeff BranchFolder BF(true, false, MBFI, *MBPI); 344219820Sjeff BFChange = BF.OptimizeFunction(MF, TII, ST.getRegisterInfo(), 345219820Sjeff getAnalysisIfAvailable<MachineModuleInfo>()); 346219820Sjeff } 347219820Sjeff 348219820Sjeff DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'" 349219820Sjeff << MF.getName() << "\'"); 350219820Sjeff 351219820Sjeff if (FnNum < IfCvtFnStart || (IfCvtFnStop != -1 && FnNum > IfCvtFnStop)) { 352219820Sjeff DEBUG(dbgs() << " skipped\n"); 353219820Sjeff return false; 354219820Sjeff } 355219820Sjeff DEBUG(dbgs() << "\n"); 356219820Sjeff 357 MF.RenumberBlocks(); 358 BBAnalysis.resize(MF.getNumBlockIDs()); 359 360 std::vector<std::unique_ptr<IfcvtToken>> Tokens; 361 MadeChange = false; 362 unsigned NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + 363 NumTriangleRev + NumTriangleFalse + NumTriangleFRev + NumDiamonds; 364 while (IfCvtLimit == -1 || (int)NumIfCvts < IfCvtLimit) { 365 // Do an initial analysis for each basic block and find all the potential 366 // candidates to perform if-conversion. 367 bool Change = false; 368 AnalyzeBlocks(MF, Tokens); 369 while (!Tokens.empty()) { 370 std::unique_ptr<IfcvtToken> Token = std::move(Tokens.back()); 371 Tokens.pop_back(); 372 BBInfo &BBI = Token->BBI; 373 IfcvtKind Kind = Token->Kind; 374 unsigned NumDups = Token->NumDups; 375 unsigned NumDups2 = Token->NumDups2; 376 377 // If the block has been evicted out of the queue or it has already been 378 // marked dead (due to it being predicated), then skip it. 379 if (BBI.IsDone) 380 BBI.IsEnqueued = false; 381 if (!BBI.IsEnqueued) 382 continue; 383 384 BBI.IsEnqueued = false; 385 386 bool RetVal = false; 387 switch (Kind) { 388 default: llvm_unreachable("Unexpected!"); 389 case ICSimple: 390 case ICSimpleFalse: { 391 bool isFalse = Kind == ICSimpleFalse; 392 if ((isFalse && DisableSimpleF) || (!isFalse && DisableSimple)) break; 393 DEBUG(dbgs() << "Ifcvt (Simple" << (Kind == ICSimpleFalse ? 394 " false" : "") 395 << "): BB#" << BBI.BB->getNumber() << " (" 396 << ((Kind == ICSimpleFalse) 397 ? BBI.FalseBB->getNumber() 398 : BBI.TrueBB->getNumber()) << ") "); 399 RetVal = IfConvertSimple(BBI, Kind); 400 DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n"); 401 if (RetVal) { 402 if (isFalse) ++NumSimpleFalse; 403 else ++NumSimple; 404 } 405 break; 406 } 407 case ICTriangle: 408 case ICTriangleRev: 409 case ICTriangleFalse: 410 case ICTriangleFRev: { 411 bool isFalse = Kind == ICTriangleFalse; 412 bool isRev = (Kind == ICTriangleRev || Kind == ICTriangleFRev); 413 if (DisableTriangle && !isFalse && !isRev) break; 414 if (DisableTriangleR && !isFalse && isRev) break; 415 if (DisableTriangleF && isFalse && !isRev) break; 416 if (DisableTriangleFR && isFalse && isRev) break; 417 DEBUG(dbgs() << "Ifcvt (Triangle"); 418 if (isFalse) 419 DEBUG(dbgs() << " false"); 420 if (isRev) 421 DEBUG(dbgs() << " rev"); 422 DEBUG(dbgs() << "): BB#" << BBI.BB->getNumber() << " (T:" 423 << BBI.TrueBB->getNumber() << ",F:" 424 << BBI.FalseBB->getNumber() << ") "); 425 RetVal = IfConvertTriangle(BBI, Kind); 426 DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n"); 427 if (RetVal) { 428 if (isFalse) { 429 if (isRev) ++NumTriangleFRev; 430 else ++NumTriangleFalse; 431 } else { 432 if (isRev) ++NumTriangleRev; 433 else ++NumTriangle; 434 } 435 } 436 break; 437 } 438 case ICDiamond: { 439 if (DisableDiamond) break; 440 DEBUG(dbgs() << "Ifcvt (Diamond): BB#" << BBI.BB->getNumber() << " (T:" 441 << BBI.TrueBB->getNumber() << ",F:" 442 << BBI.FalseBB->getNumber() << ") "); 443 RetVal = IfConvertDiamond(BBI, Kind, NumDups, NumDups2, 444 Token->TClobbersPred, 445 Token->FClobbersPred); 446 DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n"); 447 if (RetVal) ++NumDiamonds; 448 break; 449 } 450 case ICForkedDiamond: { 451 if (DisableForkedDiamond) break; 452 DEBUG(dbgs() << "Ifcvt (Forked Diamond): BB#" 453 << BBI.BB->getNumber() << " (T:" 454 << BBI.TrueBB->getNumber() << ",F:" 455 << BBI.FalseBB->getNumber() << ") "); 456 RetVal = IfConvertForkedDiamond(BBI, Kind, NumDups, NumDups2, 457 Token->TClobbersPred, 458 Token->FClobbersPred); 459 DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n"); 460 if (RetVal) ++NumForkedDiamonds; 461 break; 462 } 463 } 464 465 Change |= RetVal; 466 467 NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + NumTriangleRev + 468 NumTriangleFalse + NumTriangleFRev + NumDiamonds; 469 if (IfCvtLimit != -1 && (int)NumIfCvts >= IfCvtLimit) 470 break; 471 } 472 473 if (!Change) 474 break; 475 MadeChange |= Change; 476 } 477 478 Tokens.clear(); 479 BBAnalysis.clear(); 480 481 if (MadeChange && IfCvtBranchFold) { 482 BranchFolder BF(false, false, MBFI, *MBPI); 483 BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(), 484 getAnalysisIfAvailable<MachineModuleInfo>()); 485 } 486 487 MadeChange |= BFChange; 488 return MadeChange; 489} 490 491/// BB has a fallthrough. Find its 'false' successor given its 'true' successor. 492static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB, 493 MachineBasicBlock *TrueBB) { 494 for (MachineBasicBlock *SuccBB : BB->successors()) { 495 if (SuccBB != TrueBB) 496 return SuccBB; 497 } 498 return nullptr; 499} 500 501/// Reverse the condition of the end of the block branch. Swap block's 'true' 502/// and 'false' successors. 503bool IfConverter::reverseBranchCondition(BBInfo &BBI) const { 504 DebugLoc dl; // FIXME: this is nowhere 505 if (!TII->reverseBranchCondition(BBI.BrCond)) { 506 TII->removeBranch(*BBI.BB); 507 TII->insertBranch(*BBI.BB, BBI.FalseBB, BBI.TrueBB, BBI.BrCond, dl); 508 std::swap(BBI.TrueBB, BBI.FalseBB); 509 return true; 510 } 511 return false; 512} 513 514/// Returns the next block in the function blocks ordering. If it is the end, 515/// returns NULL. 516static inline MachineBasicBlock *getNextBlock(MachineBasicBlock &MBB) { 517 MachineFunction::iterator I = MBB.getIterator(); 518 MachineFunction::iterator E = MBB.getParent()->end(); 519 if (++I == E) 520 return nullptr; 521 return &*I; 522} 523 524/// Returns true if the 'true' block (along with its predecessor) forms a valid 525/// simple shape for ifcvt. It also returns the number of instructions that the 526/// ifcvt would need to duplicate if performed in Dups. 527bool IfConverter::ValidSimple(BBInfo &TrueBBI, unsigned &Dups, 528 BranchProbability Prediction) const { 529 Dups = 0; 530 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone) 531 return false; 532 533 if (TrueBBI.IsBrAnalyzable) 534 return false; 535 536 if (TrueBBI.BB->pred_size() > 1) { 537 if (TrueBBI.CannotBeCopied || 538 !TII->isProfitableToDupForIfCvt(*TrueBBI.BB, TrueBBI.NonPredSize, 539 Prediction)) 540 return false; 541 Dups = TrueBBI.NonPredSize; 542 } 543 544 return true; 545} 546 547/// Returns true if the 'true' and 'false' blocks (along with their common 548/// predecessor) forms a valid triangle shape for ifcvt. If 'FalseBranch' is 549/// true, it checks if 'true' block's false branch branches to the 'false' block 550/// rather than the other way around. It also returns the number of instructions 551/// that the ifcvt would need to duplicate if performed in 'Dups'. 552bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI, 553 bool FalseBranch, unsigned &Dups, 554 BranchProbability Prediction) const { 555 Dups = 0; 556 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone) 557 return false; 558 559 if (TrueBBI.BB->pred_size() > 1) { 560 if (TrueBBI.CannotBeCopied) 561 return false; 562 563 unsigned Size = TrueBBI.NonPredSize; 564 if (TrueBBI.IsBrAnalyzable) { 565 if (TrueBBI.TrueBB && TrueBBI.BrCond.empty()) 566 // Ends with an unconditional branch. It will be removed. 567 --Size; 568 else { 569 MachineBasicBlock *FExit = FalseBranch 570 ? TrueBBI.TrueBB : TrueBBI.FalseBB; 571 if (FExit) 572 // Require a conditional branch 573 ++Size; 574 } 575 } 576 if (!TII->isProfitableToDupForIfCvt(*TrueBBI.BB, Size, Prediction)) 577 return false; 578 Dups = Size; 579 } 580 581 MachineBasicBlock *TExit = FalseBranch ? TrueBBI.FalseBB : TrueBBI.TrueBB; 582 if (!TExit && blockAlwaysFallThrough(TrueBBI)) { 583 MachineFunction::iterator I = TrueBBI.BB->getIterator(); 584 if (++I == TrueBBI.BB->getParent()->end()) 585 return false; 586 TExit = &*I; 587 } 588 return TExit && TExit == FalseBBI.BB; 589} 590 591/// Shrink the provided inclusive range by one instruction. 592/// If the range was one instruction (\p It == \p Begin), It is not modified, 593/// but \p Empty is set to true. 594static inline void shrinkInclusiveRange( 595 MachineBasicBlock::iterator &Begin, 596 MachineBasicBlock::iterator &It, 597 bool &Empty) { 598 if (It == Begin) 599 Empty = true; 600 else 601 It--; 602} 603 604/// Count duplicated instructions and move the iterators to show where they 605/// are. 606/// @param TIB True Iterator Begin 607/// @param FIB False Iterator Begin 608/// These two iterators initially point to the first instruction of the two 609/// blocks, and finally point to the first non-shared instruction. 610/// @param TIE True Iterator End 611/// @param FIE False Iterator End 612/// These two iterators initially point to End() for the two blocks() and 613/// finally point to the first shared instruction in the tail. 614/// Upon return [TIB, TIE), and [FIB, FIE) mark the un-duplicated portions of 615/// two blocks. 616/// @param Dups1 count of duplicated instructions at the beginning of the 2 617/// blocks. 618/// @param Dups2 count of duplicated instructions at the end of the 2 blocks. 619/// @param SkipUnconditionalBranches if true, Don't make sure that 620/// unconditional branches at the end of the blocks are the same. True is 621/// passed when the blocks are analyzable to allow for fallthrough to be 622/// handled. 623/// @return false if the shared portion prevents if conversion. 624bool IfConverter::CountDuplicatedInstructions( 625 MachineBasicBlock::iterator &TIB, 626 MachineBasicBlock::iterator &FIB, 627 MachineBasicBlock::iterator &TIE, 628 MachineBasicBlock::iterator &FIE, 629 unsigned &Dups1, unsigned &Dups2, 630 MachineBasicBlock &TBB, MachineBasicBlock &FBB, 631 bool SkipUnconditionalBranches) const { 632 633 while (TIB != TIE && FIB != FIE) { 634 // Skip dbg_value instructions. These do not count. 635 TIB = skipDebugInstructionsForward(TIB, TIE); 636 if(TIB == TIE) 637 break; 638 FIB = skipDebugInstructionsForward(FIB, FIE); 639 if(FIB == FIE) 640 break; 641 if (!TIB->isIdenticalTo(*FIB)) 642 break; 643 // A pred-clobbering instruction in the shared portion prevents 644 // if-conversion. 645 std::vector<MachineOperand> PredDefs; 646 if (TII->DefinesPredicate(*TIB, PredDefs)) 647 return false; 648 // If we get all the way to the branch instructions, don't count them. 649 if (!TIB->isBranch()) 650 ++Dups1; 651 ++TIB; 652 ++FIB; 653 } 654 655 // Check for already containing all of the block. 656 if (TIB == TIE || FIB == FIE) 657 return true; 658 // Now, in preparation for counting duplicate instructions at the ends of the 659 // blocks, move the end iterators up past any branch instructions. 660 --TIE; 661 --FIE; 662 663 // After this point TIB and TIE define an inclusive range, which means that 664 // TIB == TIE is true when there is one more instruction to consider, not at 665 // the end. Because we may not be able to go before TIB, we need a flag to 666 // indicate a completely empty range. 667 bool TEmpty = false, FEmpty = false; 668 669 // Upon exit TIE and FIE will both point at the last non-shared instruction. 670 // They need to be moved forward to point past the last non-shared 671 // instruction if the range they delimit is non-empty. 672 auto IncrementEndIteratorsOnExit = make_scope_exit([&]() { 673 if (!TEmpty) 674 ++TIE; 675 if (!FEmpty) 676 ++FIE; 677 }); 678 679 if (!TBB.succ_empty() || !FBB.succ_empty()) { 680 if (SkipUnconditionalBranches) { 681 while (!TEmpty && TIE->isUnconditionalBranch()) 682 shrinkInclusiveRange(TIB, TIE, TEmpty); 683 while (!FEmpty && FIE->isUnconditionalBranch()) 684 shrinkInclusiveRange(FIB, FIE, FEmpty); 685 } 686 } 687 688 // If Dups1 includes all of a block, then don't count duplicate 689 // instructions at the end of the blocks. 690 if (TEmpty || FEmpty) 691 return true; 692 693 // Count duplicate instructions at the ends of the blocks. 694 while (!TEmpty && !FEmpty) { 695 // Skip dbg_value instructions. These do not count. 696 TIE = skipDebugInstructionsBackward(TIE, TIB); 697 FIE = skipDebugInstructionsBackward(FIE, FIB); 698 TEmpty = TIE == TIB && TIE->isDebugValue(); 699 FEmpty = FIE == FIB && FIE->isDebugValue(); 700 if (TEmpty || FEmpty) 701 break; 702 if (!TIE->isIdenticalTo(*FIE)) 703 break; 704 // We have to verify that any branch instructions are the same, and then we 705 // don't count them toward the # of duplicate instructions. 706 if (!TIE->isBranch()) 707 ++Dups2; 708 shrinkInclusiveRange(TIB, TIE, TEmpty); 709 shrinkInclusiveRange(FIB, FIE, FEmpty); 710 } 711 return true; 712} 713 714/// RescanInstructions - Run ScanInstructions on a pair of blocks. 715/// @param TIB - True Iterator Begin, points to first non-shared instruction 716/// @param FIB - False Iterator Begin, points to first non-shared instruction 717/// @param TIE - True Iterator End, points past last non-shared instruction 718/// @param FIE - False Iterator End, points past last non-shared instruction 719/// @param TrueBBI - BBInfo to update for the true block. 720/// @param FalseBBI - BBInfo to update for the false block. 721/// @returns - false if either block cannot be predicated or if both blocks end 722/// with a predicate-clobbering instruction. 723bool IfConverter::RescanInstructions( 724 MachineBasicBlock::iterator &TIB, MachineBasicBlock::iterator &FIB, 725 MachineBasicBlock::iterator &TIE, MachineBasicBlock::iterator &FIE, 726 BBInfo &TrueBBI, BBInfo &FalseBBI) const { 727 bool BranchUnpredicable = true; 728 TrueBBI.IsUnpredicable = FalseBBI.IsUnpredicable = false; 729 ScanInstructions(TrueBBI, TIB, TIE, BranchUnpredicable); 730 if (TrueBBI.IsUnpredicable) 731 return false; 732 ScanInstructions(FalseBBI, FIB, FIE, BranchUnpredicable); 733 if (FalseBBI.IsUnpredicable) 734 return false; 735 if (TrueBBI.ClobbersPred && FalseBBI.ClobbersPred) 736 return false; 737 return true; 738} 739 740#ifndef NDEBUG 741static void verifySameBranchInstructions( 742 MachineBasicBlock *MBB1, 743 MachineBasicBlock *MBB2) { 744 MachineBasicBlock::iterator B1 = MBB1->begin(); 745 MachineBasicBlock::iterator B2 = MBB2->begin(); 746 MachineBasicBlock::iterator E1 = std::prev(MBB1->end()); 747 MachineBasicBlock::iterator E2 = std::prev(MBB2->end()); 748 bool Empty1 = false, Empty2 = false; 749 while (!Empty1 && !Empty2) { 750 E1 = skipDebugInstructionsBackward(E1, B1); 751 E2 = skipDebugInstructionsBackward(E2, B2); 752 Empty1 = E1 == B1 && E1->isDebugValue(); 753 Empty2 = E2 == B2 && E2->isDebugValue(); 754 755 if (Empty1 && Empty2) 756 break; 757 758 if (Empty1) { 759 assert(!E2->isBranch() && "Branch mis-match, one block is empty."); 760 break; 761 } 762 if (Empty2) { 763 assert(!E1->isBranch() && "Branch mis-match, one block is empty."); 764 break; 765 } 766 767 if (E1->isBranch() || E2->isBranch()) 768 assert(E1->isIdenticalTo(*E2) && 769 "Branch mis-match, branch instructions don't match."); 770 else 771 break; 772 shrinkInclusiveRange(B1, E1, Empty1); 773 shrinkInclusiveRange(B2, E2, Empty2); 774 } 775} 776#endif 777 778/// ValidForkedDiamond - Returns true if the 'true' and 'false' blocks (along 779/// with their common predecessor) form a diamond if a common tail block is 780/// extracted. 781/// While not strictly a diamond, this pattern would form a diamond if 782/// tail-merging had merged the shared tails. 783/// EBB 784/// _/ \_ 785/// | | 786/// TBB FBB 787/// / \ / \ 788/// FalseBB TrueBB FalseBB 789/// Currently only handles analyzable branches. 790/// Specifically excludes actual diamonds to avoid overlap. 791bool IfConverter::ValidForkedDiamond( 792 BBInfo &TrueBBI, BBInfo &FalseBBI, 793 unsigned &Dups1, unsigned &Dups2, 794 BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const { 795 Dups1 = Dups2 = 0; 796 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone || 797 FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone) 798 return false; 799 800 if (!TrueBBI.IsBrAnalyzable || !FalseBBI.IsBrAnalyzable) 801 return false; 802 // Don't IfConvert blocks that can't be folded into their predecessor. 803 if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) 804 return false; 805 806 // This function is specifically looking for conditional tails, as 807 // unconditional tails are already handled by the standard diamond case. 808 if (TrueBBI.BrCond.size() == 0 || 809 FalseBBI.BrCond.size() == 0) 810 return false; 811 812 MachineBasicBlock *TT = TrueBBI.TrueBB; 813 MachineBasicBlock *TF = TrueBBI.FalseBB; 814 MachineBasicBlock *FT = FalseBBI.TrueBB; 815 MachineBasicBlock *FF = FalseBBI.FalseBB; 816 817 if (!TT) 818 TT = getNextBlock(*TrueBBI.BB); 819 if (!TF) 820 TF = getNextBlock(*TrueBBI.BB); 821 if (!FT) 822 FT = getNextBlock(*FalseBBI.BB); 823 if (!FF) 824 FF = getNextBlock(*FalseBBI.BB); 825 826 if (!TT || !TF) 827 return false; 828 829 // Check successors. If they don't match, bail. 830 if (!((TT == FT && TF == FF) || (TF == FT && TT == FF))) 831 return false; 832 833 bool FalseReversed = false; 834 if (TF == FT && TT == FF) { 835 // If the branches are opposing, but we can't reverse, don't do it. 836 if (!FalseBBI.IsBrReversible) 837 return false; 838 FalseReversed = true; 839 reverseBranchCondition(FalseBBI); 840 } 841 auto UnReverseOnExit = make_scope_exit([&]() { 842 if (FalseReversed) 843 reverseBranchCondition(FalseBBI); 844 }); 845 846 // Count duplicate instructions at the beginning of the true and false blocks. 847 MachineBasicBlock::iterator TIB = TrueBBI.BB->begin(); 848 MachineBasicBlock::iterator FIB = FalseBBI.BB->begin(); 849 MachineBasicBlock::iterator TIE = TrueBBI.BB->end(); 850 MachineBasicBlock::iterator FIE = FalseBBI.BB->end(); 851 if(!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2, 852 *TrueBBI.BB, *FalseBBI.BB, 853 /* SkipUnconditionalBranches */ true)) 854 return false; 855 856 TrueBBICalc.BB = TrueBBI.BB; 857 FalseBBICalc.BB = FalseBBI.BB; 858 if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc)) 859 return false; 860 861 // The size is used to decide whether to if-convert, and the shared portions 862 // are subtracted off. Because of the subtraction, we just use the size that 863 // was calculated by the original ScanInstructions, as it is correct. 864 TrueBBICalc.NonPredSize = TrueBBI.NonPredSize; 865 FalseBBICalc.NonPredSize = FalseBBI.NonPredSize; 866 return true; 867} 868 869/// ValidDiamond - Returns true if the 'true' and 'false' blocks (along 870/// with their common predecessor) forms a valid diamond shape for ifcvt. 871bool IfConverter::ValidDiamond( 872 BBInfo &TrueBBI, BBInfo &FalseBBI, 873 unsigned &Dups1, unsigned &Dups2, 874 BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const { 875 Dups1 = Dups2 = 0; 876 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone || 877 FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone) 878 return false; 879 880 MachineBasicBlock *TT = TrueBBI.TrueBB; 881 MachineBasicBlock *FT = FalseBBI.TrueBB; 882 883 if (!TT && blockAlwaysFallThrough(TrueBBI)) 884 TT = getNextBlock(*TrueBBI.BB); 885 if (!FT && blockAlwaysFallThrough(FalseBBI)) 886 FT = getNextBlock(*FalseBBI.BB); 887 if (TT != FT) 888 return false; 889 if (!TT && (TrueBBI.IsBrAnalyzable || FalseBBI.IsBrAnalyzable)) 890 return false; 891 if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) 892 return false; 893 894 // FIXME: Allow true block to have an early exit? 895 if (TrueBBI.FalseBB || FalseBBI.FalseBB) 896 return false; 897 898 // Count duplicate instructions at the beginning and end of the true and 899 // false blocks. 900 // Skip unconditional branches only if we are considering an analyzable 901 // diamond. Otherwise the branches must be the same. 902 bool SkipUnconditionalBranches = 903 TrueBBI.IsBrAnalyzable && FalseBBI.IsBrAnalyzable; 904 MachineBasicBlock::iterator TIB = TrueBBI.BB->begin(); 905 MachineBasicBlock::iterator FIB = FalseBBI.BB->begin(); 906 MachineBasicBlock::iterator TIE = TrueBBI.BB->end(); 907 MachineBasicBlock::iterator FIE = FalseBBI.BB->end(); 908 if(!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2, 909 *TrueBBI.BB, *FalseBBI.BB, 910 SkipUnconditionalBranches)) 911 return false; 912 913 TrueBBICalc.BB = TrueBBI.BB; 914 FalseBBICalc.BB = FalseBBI.BB; 915 if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc)) 916 return false; 917 // The size is used to decide whether to if-convert, and the shared portions 918 // are subtracted off. Because of the subtraction, we just use the size that 919 // was calculated by the original ScanInstructions, as it is correct. 920 TrueBBICalc.NonPredSize = TrueBBI.NonPredSize; 921 FalseBBICalc.NonPredSize = FalseBBI.NonPredSize; 922 return true; 923} 924 925/// AnalyzeBranches - Look at the branches at the end of a block to determine if 926/// the block is predicable. 927void IfConverter::AnalyzeBranches(BBInfo &BBI) { 928 if (BBI.IsDone) 929 return; 930 931 BBI.TrueBB = BBI.FalseBB = nullptr; 932 BBI.BrCond.clear(); 933 BBI.IsBrAnalyzable = 934 !TII->analyzeBranch(*BBI.BB, BBI.TrueBB, BBI.FalseBB, BBI.BrCond); 935 SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end()); 936 BBI.IsBrReversible = (RevCond.size() == 0) || 937 !TII->reverseBranchCondition(RevCond); 938 BBI.HasFallThrough = BBI.IsBrAnalyzable && BBI.FalseBB == nullptr; 939 940 if (BBI.BrCond.size()) { 941 // No false branch. This BB must end with a conditional branch and a 942 // fallthrough. 943 if (!BBI.FalseBB) 944 BBI.FalseBB = findFalseBlock(BBI.BB, BBI.TrueBB); 945 if (!BBI.FalseBB) { 946 // Malformed bcc? True and false blocks are the same? 947 BBI.IsUnpredicable = true; 948 } 949 } 950} 951 952/// ScanInstructions - Scan all the instructions in the block to determine if 953/// the block is predicable. In most cases, that means all the instructions 954/// in the block are isPredicable(). Also checks if the block contains any 955/// instruction which can clobber a predicate (e.g. condition code register). 956/// If so, the block is not predicable unless it's the last instruction. 957void IfConverter::ScanInstructions(BBInfo &BBI, 958 MachineBasicBlock::iterator &Begin, 959 MachineBasicBlock::iterator &End, 960 bool BranchUnpredicable) const { 961 if (BBI.IsDone || BBI.IsUnpredicable) 962 return; 963 964 bool AlreadyPredicated = !BBI.Predicate.empty(); 965 966 BBI.NonPredSize = 0; 967 BBI.ExtraCost = 0; 968 BBI.ExtraCost2 = 0; 969 BBI.ClobbersPred = false; 970 for (MachineInstr &MI : make_range(Begin, End)) { 971 if (MI.isDebugValue()) 972 continue; 973 974 // It's unsafe to duplicate convergent instructions in this context, so set 975 // BBI.CannotBeCopied to true if MI is convergent. To see why, consider the 976 // following CFG, which is subject to our "simple" transformation. 977 // 978 // BB0 // if (c1) goto BB1; else goto BB2; 979 // / \ 980 // BB1 | 981 // | BB2 // if (c2) goto TBB; else goto FBB; 982 // | / | 983 // | / | 984 // TBB | 985 // | | 986 // | FBB 987 // | 988 // exit 989 // 990 // Suppose we want to move TBB's contents up into BB1 and BB2 (in BB1 they'd 991 // be unconditional, and in BB2, they'd be predicated upon c2), and suppose 992 // TBB contains a convergent instruction. This is safe iff doing so does 993 // not add a control-flow dependency to the convergent instruction -- i.e., 994 // it's safe iff the set of control flows that leads us to the convergent 995 // instruction does not get smaller after the transformation. 996 // 997 // Originally we executed TBB if c1 || c2. After the transformation, there 998 // are two copies of TBB's instructions. We get to the first if c1, and we 999 // get to the second if !c1 && c2. 1000 // 1001 // There are clearly fewer ways to satisfy the condition "c1" than 1002 // "c1 || c2". Since we've shrunk the set of control flows which lead to 1003 // our convergent instruction, the transformation is unsafe. 1004 if (MI.isNotDuplicable() || MI.isConvergent()) 1005 BBI.CannotBeCopied = true; 1006 1007 bool isPredicated = TII->isPredicated(MI); 1008 bool isCondBr = BBI.IsBrAnalyzable && MI.isConditionalBranch(); 1009 1010 if (BranchUnpredicable && MI.isBranch()) { 1011 BBI.IsUnpredicable = true; 1012 return; 1013 } 1014 1015 // A conditional branch is not predicable, but it may be eliminated. 1016 if (isCondBr) 1017 continue; 1018 1019 if (!isPredicated) { 1020 BBI.NonPredSize++; 1021 unsigned ExtraPredCost = TII->getPredicationCost(MI); 1022 unsigned NumCycles = SchedModel.computeInstrLatency(&MI, false); 1023 if (NumCycles > 1) 1024 BBI.ExtraCost += NumCycles-1; 1025 BBI.ExtraCost2 += ExtraPredCost; 1026 } else if (!AlreadyPredicated) { 1027 // FIXME: This instruction is already predicated before the 1028 // if-conversion pass. It's probably something like a conditional move. 1029 // Mark this block unpredicable for now. 1030 BBI.IsUnpredicable = true; 1031 return; 1032 } 1033 1034 if (BBI.ClobbersPred && !isPredicated) { 1035 // Predicate modification instruction should end the block (except for 1036 // already predicated instructions and end of block branches). 1037 // Predicate may have been modified, the subsequent (currently) 1038 // unpredicated instructions cannot be correctly predicated. 1039 BBI.IsUnpredicable = true; 1040 return; 1041 } 1042 1043 // FIXME: Make use of PredDefs? e.g. ADDC, SUBC sets predicates but are 1044 // still potentially predicable. 1045 std::vector<MachineOperand> PredDefs; 1046 if (TII->DefinesPredicate(MI, PredDefs)) 1047 BBI.ClobbersPred = true; 1048 1049 if (!TII->isPredicable(MI)) { 1050 BBI.IsUnpredicable = true; 1051 return; 1052 } 1053 } 1054} 1055 1056/// Determine if the block is a suitable candidate to be predicated by the 1057/// specified predicate. 1058/// @param BBI BBInfo for the block to check 1059/// @param Pred Predicate array for the branch that leads to BBI 1060/// @param isTriangle true if the Analysis is for a triangle 1061/// @param RevBranch true if Reverse(Pred) leads to BBI (e.g. BBI is the false 1062/// case 1063/// @param hasCommonTail true if BBI shares a tail with a sibling block that 1064/// contains any instruction that would make the block unpredicable. 1065bool IfConverter::FeasibilityAnalysis(BBInfo &BBI, 1066 SmallVectorImpl<MachineOperand> &Pred, 1067 bool isTriangle, bool RevBranch, 1068 bool hasCommonTail) { 1069 // If the block is dead or unpredicable, then it cannot be predicated. 1070 // Two blocks may share a common unpredicable tail, but this doesn't prevent 1071 // them from being if-converted. The non-shared portion is assumed to have 1072 // been checked 1073 if (BBI.IsDone || (BBI.IsUnpredicable && !hasCommonTail)) 1074 return false; 1075 1076 // If it is already predicated but we couldn't analyze its terminator, the 1077 // latter might fallthrough, but we can't determine where to. 1078 // Conservatively avoid if-converting again. 1079 if (BBI.Predicate.size() && !BBI.IsBrAnalyzable) 1080 return false; 1081 1082 // If it is already predicated, check if the new predicate subsumes 1083 // its predicate. 1084 if (BBI.Predicate.size() && !TII->SubsumesPredicate(Pred, BBI.Predicate)) 1085 return false; 1086 1087 if (!hasCommonTail && BBI.BrCond.size()) { 1088 if (!isTriangle) 1089 return false; 1090 1091 // Test predicate subsumption. 1092 SmallVector<MachineOperand, 4> RevPred(Pred.begin(), Pred.end()); 1093 SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end()); 1094 if (RevBranch) { 1095 if (TII->reverseBranchCondition(Cond)) 1096 return false; 1097 } 1098 if (TII->reverseBranchCondition(RevPred) || 1099 !TII->SubsumesPredicate(Cond, RevPred)) 1100 return false; 1101 } 1102 1103 return true; 1104} 1105 1106/// Analyze the structure of the sub-CFG starting from the specified block. 1107/// Record its successors and whether it looks like an if-conversion candidate. 1108void IfConverter::AnalyzeBlock( 1109 MachineBasicBlock &MBB, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) { 1110 struct BBState { 1111 BBState(MachineBasicBlock &MBB) : MBB(&MBB), SuccsAnalyzed(false) {} 1112 MachineBasicBlock *MBB; 1113 1114 /// This flag is true if MBB's successors have been analyzed. 1115 bool SuccsAnalyzed; 1116 }; 1117 1118 // Push MBB to the stack. 1119 SmallVector<BBState, 16> BBStack(1, MBB); 1120 1121 while (!BBStack.empty()) { 1122 BBState &State = BBStack.back(); 1123 MachineBasicBlock *BB = State.MBB; 1124 BBInfo &BBI = BBAnalysis[BB->getNumber()]; 1125 1126 if (!State.SuccsAnalyzed) { 1127 if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed) { 1128 BBStack.pop_back(); 1129 continue; 1130 } 1131 1132 BBI.BB = BB; 1133 BBI.IsBeingAnalyzed = true; 1134 1135 AnalyzeBranches(BBI); 1136 MachineBasicBlock::iterator Begin = BBI.BB->begin(); 1137 MachineBasicBlock::iterator End = BBI.BB->end(); 1138 ScanInstructions(BBI, Begin, End); 1139 1140 // Unanalyzable or ends with fallthrough or unconditional branch, or if is 1141 // not considered for ifcvt anymore. 1142 if (!BBI.IsBrAnalyzable || BBI.BrCond.empty() || BBI.IsDone) { 1143 BBI.IsBeingAnalyzed = false; 1144 BBI.IsAnalyzed = true; 1145 BBStack.pop_back(); 1146 continue; 1147 } 1148 1149 // Do not ifcvt if either path is a back edge to the entry block. 1150 if (BBI.TrueBB == BB || BBI.FalseBB == BB) { 1151 BBI.IsBeingAnalyzed = false; 1152 BBI.IsAnalyzed = true; 1153 BBStack.pop_back(); 1154 continue; 1155 } 1156 1157 // Do not ifcvt if true and false fallthrough blocks are the same. 1158 if (!BBI.FalseBB) { 1159 BBI.IsBeingAnalyzed = false; 1160 BBI.IsAnalyzed = true; 1161 BBStack.pop_back(); 1162 continue; 1163 } 1164 1165 // Push the False and True blocks to the stack. 1166 State.SuccsAnalyzed = true; 1167 BBStack.push_back(*BBI.FalseBB); 1168 BBStack.push_back(*BBI.TrueBB); 1169 continue; 1170 } 1171 1172 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; 1173 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; 1174 1175 if (TrueBBI.IsDone && FalseBBI.IsDone) { 1176 BBI.IsBeingAnalyzed = false; 1177 BBI.IsAnalyzed = true; 1178 BBStack.pop_back(); 1179 continue; 1180 } 1181 1182 SmallVector<MachineOperand, 4> 1183 RevCond(BBI.BrCond.begin(), BBI.BrCond.end()); 1184 bool CanRevCond = !TII->reverseBranchCondition(RevCond); 1185 1186 unsigned Dups = 0; 1187 unsigned Dups2 = 0; 1188 bool TNeedSub = !TrueBBI.Predicate.empty(); 1189 bool FNeedSub = !FalseBBI.Predicate.empty(); 1190 bool Enqueued = false; 1191 1192 BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB); 1193 1194 if (CanRevCond) { 1195 BBInfo TrueBBICalc, FalseBBICalc; 1196 auto feasibleDiamond = [&]() { 1197 bool MeetsSize = MeetIfcvtSizeLimit( 1198 *TrueBBI.BB, (TrueBBICalc.NonPredSize - (Dups + Dups2) + 1199 TrueBBICalc.ExtraCost), TrueBBICalc.ExtraCost2, 1200 *FalseBBI.BB, (FalseBBICalc.NonPredSize - (Dups + Dups2) + 1201 FalseBBICalc.ExtraCost), FalseBBICalc.ExtraCost2, 1202 Prediction); 1203 bool TrueFeasible = FeasibilityAnalysis(TrueBBI, BBI.BrCond, 1204 /* IsTriangle */ false, /* RevCond */ false, 1205 /* hasCommonTail */ true); 1206 bool FalseFeasible = FeasibilityAnalysis(FalseBBI, RevCond, 1207 /* IsTriangle */ false, /* RevCond */ false, 1208 /* hasCommonTail */ true); 1209 return MeetsSize && TrueFeasible && FalseFeasible; 1210 }; 1211 1212 if (ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2, 1213 TrueBBICalc, FalseBBICalc)) { 1214 if (feasibleDiamond()) { 1215 // Diamond: 1216 // EBB 1217 // / \_ 1218 // | | 1219 // TBB FBB 1220 // \ / 1221 // TailBB 1222 // Note TailBB can be empty. 1223 Tokens.push_back(llvm::make_unique<IfcvtToken>( 1224 BBI, ICDiamond, TNeedSub | FNeedSub, Dups, Dups2, 1225 (bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred)); 1226 Enqueued = true; 1227 } 1228 } else if (ValidForkedDiamond(TrueBBI, FalseBBI, Dups, Dups2, 1229 TrueBBICalc, FalseBBICalc)) { 1230 if (feasibleDiamond()) { 1231 // ForkedDiamond: 1232 // if TBB and FBB have a common tail that includes their conditional 1233 // branch instructions, then we can If Convert this pattern. 1234 // EBB 1235 // _/ \_ 1236 // | | 1237 // TBB FBB 1238 // / \ / \ 1239 // FalseBB TrueBB FalseBB 1240 // 1241 Tokens.push_back(llvm::make_unique<IfcvtToken>( 1242 BBI, ICForkedDiamond, TNeedSub | FNeedSub, Dups, Dups2, 1243 (bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred)); 1244 Enqueued = true; 1245 } 1246 } 1247 } 1248 1249 if (ValidTriangle(TrueBBI, FalseBBI, false, Dups, Prediction) && 1250 MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost, 1251 TrueBBI.ExtraCost2, Prediction) && 1252 FeasibilityAnalysis(TrueBBI, BBI.BrCond, true)) { 1253 // Triangle: 1254 // EBB 1255 // | \_ 1256 // | | 1257 // | TBB 1258 // | / 1259 // FBB 1260 Tokens.push_back( 1261 llvm::make_unique<IfcvtToken>(BBI, ICTriangle, TNeedSub, Dups)); 1262 Enqueued = true; 1263 } 1264 1265 if (ValidTriangle(TrueBBI, FalseBBI, true, Dups, Prediction) && 1266 MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost, 1267 TrueBBI.ExtraCost2, Prediction) && 1268 FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) { 1269 Tokens.push_back( 1270 llvm::make_unique<IfcvtToken>(BBI, ICTriangleRev, TNeedSub, Dups)); 1271 Enqueued = true; 1272 } 1273 1274 if (ValidSimple(TrueBBI, Dups, Prediction) && 1275 MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost, 1276 TrueBBI.ExtraCost2, Prediction) && 1277 FeasibilityAnalysis(TrueBBI, BBI.BrCond)) { 1278 // Simple (split, no rejoin): 1279 // EBB 1280 // | \_ 1281 // | | 1282 // | TBB---> exit 1283 // | 1284 // FBB 1285 Tokens.push_back( 1286 llvm::make_unique<IfcvtToken>(BBI, ICSimple, TNeedSub, Dups)); 1287 Enqueued = true; 1288 } 1289 1290 if (CanRevCond) { 1291 // Try the other path... 1292 if (ValidTriangle(FalseBBI, TrueBBI, false, Dups, 1293 Prediction.getCompl()) && 1294 MeetIfcvtSizeLimit(*FalseBBI.BB, 1295 FalseBBI.NonPredSize + FalseBBI.ExtraCost, 1296 FalseBBI.ExtraCost2, Prediction.getCompl()) && 1297 FeasibilityAnalysis(FalseBBI, RevCond, true)) { 1298 Tokens.push_back(llvm::make_unique<IfcvtToken>(BBI, ICTriangleFalse, 1299 FNeedSub, Dups)); 1300 Enqueued = true; 1301 } 1302 1303 if (ValidTriangle(FalseBBI, TrueBBI, true, Dups, 1304 Prediction.getCompl()) && 1305 MeetIfcvtSizeLimit(*FalseBBI.BB, 1306 FalseBBI.NonPredSize + FalseBBI.ExtraCost, 1307 FalseBBI.ExtraCost2, Prediction.getCompl()) && 1308 FeasibilityAnalysis(FalseBBI, RevCond, true, true)) { 1309 Tokens.push_back( 1310 llvm::make_unique<IfcvtToken>(BBI, ICTriangleFRev, FNeedSub, Dups)); 1311 Enqueued = true; 1312 } 1313 1314 if (ValidSimple(FalseBBI, Dups, Prediction.getCompl()) && 1315 MeetIfcvtSizeLimit(*FalseBBI.BB, 1316 FalseBBI.NonPredSize + FalseBBI.ExtraCost, 1317 FalseBBI.ExtraCost2, Prediction.getCompl()) && 1318 FeasibilityAnalysis(FalseBBI, RevCond)) { 1319 Tokens.push_back( 1320 llvm::make_unique<IfcvtToken>(BBI, ICSimpleFalse, FNeedSub, Dups)); 1321 Enqueued = true; 1322 } 1323 } 1324 1325 BBI.IsEnqueued = Enqueued; 1326 BBI.IsBeingAnalyzed = false; 1327 BBI.IsAnalyzed = true; 1328 BBStack.pop_back(); 1329 } 1330} 1331 1332/// Analyze all blocks and find entries for all if-conversion candidates. 1333void IfConverter::AnalyzeBlocks( 1334 MachineFunction &MF, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) { 1335 for (MachineBasicBlock &MBB : MF) 1336 AnalyzeBlock(MBB, Tokens); 1337 1338 // Sort to favor more complex ifcvt scheme. 1339 std::stable_sort(Tokens.begin(), Tokens.end(), IfcvtTokenCmp); 1340} 1341 1342/// Returns true either if ToMBB is the next block after MBB or that all the 1343/// intervening blocks are empty (given MBB can fall through to its next block). 1344static bool canFallThroughTo(MachineBasicBlock &MBB, MachineBasicBlock &ToMBB) { 1345 MachineFunction::iterator PI = MBB.getIterator(); 1346 MachineFunction::iterator I = std::next(PI); 1347 MachineFunction::iterator TI = ToMBB.getIterator(); 1348 MachineFunction::iterator E = MBB.getParent()->end(); 1349 while (I != TI) { 1350 // Check isSuccessor to avoid case where the next block is empty, but 1351 // it's not a successor. 1352 if (I == E || !I->empty() || !PI->isSuccessor(&*I)) 1353 return false; 1354 PI = I++; 1355 } 1356 return true; 1357} 1358 1359/// Invalidate predecessor BB info so it would be re-analyzed to determine if it 1360/// can be if-converted. If predecessor is already enqueued, dequeue it! 1361void IfConverter::InvalidatePreds(MachineBasicBlock &MBB) { 1362 for (const MachineBasicBlock *Predecessor : MBB.predecessors()) { 1363 BBInfo &PBBI = BBAnalysis[Predecessor->getNumber()]; 1364 if (PBBI.IsDone || PBBI.BB == &MBB) 1365 continue; 1366 PBBI.IsAnalyzed = false; 1367 PBBI.IsEnqueued = false; 1368 } 1369} 1370 1371/// Inserts an unconditional branch from \p MBB to \p ToMBB. 1372static void InsertUncondBranch(MachineBasicBlock &MBB, MachineBasicBlock &ToMBB, 1373 const TargetInstrInfo *TII) { 1374 DebugLoc dl; // FIXME: this is nowhere 1375 SmallVector<MachineOperand, 0> NoCond; 1376 TII->insertBranch(MBB, &ToMBB, nullptr, NoCond, dl); 1377} 1378 1379/// Remove true / false edges if either / both are no longer successors. 1380void IfConverter::RemoveExtraEdges(BBInfo &BBI) { 1381 MachineBasicBlock *TBB = nullptr, *FBB = nullptr; 1382 SmallVector<MachineOperand, 4> Cond; 1383 if (!TII->analyzeBranch(*BBI.BB, TBB, FBB, Cond)) 1384 BBI.BB->CorrectExtraCFGEdges(TBB, FBB, !Cond.empty()); 1385} 1386 1387/// Behaves like LiveRegUnits::StepForward() but also adds implicit uses to all 1388/// values defined in MI which are also live/used by MI. 1389static void UpdatePredRedefs(MachineInstr &MI, LivePhysRegs &Redefs) { 1390 const TargetRegisterInfo *TRI = MI.getParent()->getParent() 1391 ->getSubtarget().getRegisterInfo(); 1392 1393 // Before stepping forward past MI, remember which regs were live 1394 // before MI. This is needed to set the Undef flag only when reg is 1395 // dead. 1396 SparseSet<unsigned> LiveBeforeMI; 1397 LiveBeforeMI.setUniverse(TRI->getNumRegs()); 1398 for (unsigned Reg : Redefs) 1399 LiveBeforeMI.insert(Reg); 1400 1401 SmallVector<std::pair<unsigned, const MachineOperand*>, 4> Clobbers; 1402 Redefs.stepForward(MI, Clobbers); 1403 1404 // Now add the implicit uses for each of the clobbered values. 1405 for (auto Clobber : Clobbers) { 1406 // FIXME: Const cast here is nasty, but better than making StepForward 1407 // take a mutable instruction instead of const. 1408 unsigned Reg = Clobber.first; 1409 MachineOperand &Op = const_cast<MachineOperand&>(*Clobber.second); 1410 MachineInstr *OpMI = Op.getParent(); 1411 MachineInstrBuilder MIB(*OpMI->getParent()->getParent(), OpMI); 1412 if (Op.isRegMask()) { 1413 // First handle regmasks. They clobber any entries in the mask which 1414 // means that we need a def for those registers. 1415 if (LiveBeforeMI.count(Reg)) 1416 MIB.addReg(Reg, RegState::Implicit); 1417 1418 // We also need to add an implicit def of this register for the later 1419 // use to read from. 1420 // For the register allocator to have allocated a register clobbered 1421 // by the call which is used later, it must be the case that 1422 // the call doesn't return. 1423 MIB.addReg(Reg, RegState::Implicit | RegState::Define); 1424 continue; 1425 } 1426 assert(Op.isReg() && "Register operand required"); 1427 if (Op.isDead()) { 1428 // If we found a dead def, but it needs to be live, then remove the dead 1429 // flag. 1430 if (Redefs.contains(Op.getReg())) 1431 Op.setIsDead(false); 1432 } 1433 if (LiveBeforeMI.count(Reg)) 1434 MIB.addReg(Reg, RegState::Implicit); 1435 else { 1436 bool HasLiveSubReg = false; 1437 for (MCSubRegIterator S(Reg, TRI); S.isValid(); ++S) { 1438 if (!LiveBeforeMI.count(*S)) 1439 continue; 1440 HasLiveSubReg = true; 1441 break; 1442 } 1443 if (HasLiveSubReg) 1444 MIB.addReg(Reg, RegState::Implicit); 1445 } 1446 } 1447} 1448 1449/// Remove kill flags from operands with a registers in the \p DontKill set. 1450static void RemoveKills(MachineInstr &MI, const LivePhysRegs &DontKill) { 1451 for (MIBundleOperands O(MI); O.isValid(); ++O) { 1452 if (!O->isReg() || !O->isKill()) 1453 continue; 1454 if (DontKill.contains(O->getReg())) 1455 O->setIsKill(false); 1456 } 1457} 1458 1459/// Walks a range of machine instructions and removes kill flags for registers 1460/// in the \p DontKill set. 1461static void RemoveKills(MachineBasicBlock::iterator I, 1462 MachineBasicBlock::iterator E, 1463 const LivePhysRegs &DontKill, 1464 const MCRegisterInfo &MCRI) { 1465 for (MachineInstr &MI : make_range(I, E)) 1466 RemoveKills(MI, DontKill); 1467} 1468 1469/// If convert a simple (split, no rejoin) sub-CFG. 1470bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind Kind) { 1471 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; 1472 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; 1473 BBInfo *CvtBBI = &TrueBBI; 1474 BBInfo *NextBBI = &FalseBBI; 1475 1476 SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end()); 1477 if (Kind == ICSimpleFalse) 1478 std::swap(CvtBBI, NextBBI); 1479 1480 MachineBasicBlock &CvtMBB = *CvtBBI->BB; 1481 MachineBasicBlock &NextMBB = *NextBBI->BB; 1482 if (CvtBBI->IsDone || 1483 (CvtBBI->CannotBeCopied && CvtMBB.pred_size() > 1)) { 1484 // Something has changed. It's no longer safe to predicate this block. 1485 BBI.IsAnalyzed = false; 1486 CvtBBI->IsAnalyzed = false; 1487 return false; 1488 } 1489 1490 if (CvtMBB.hasAddressTaken()) 1491 // Conservatively abort if-conversion if BB's address is taken. 1492 return false; 1493 1494 if (Kind == ICSimpleFalse) 1495 if (TII->reverseBranchCondition(Cond)) 1496 llvm_unreachable("Unable to reverse branch condition!"); 1497 1498 Redefs.init(*TRI); 1499 DontKill.init(*TRI); 1500 1501 if (MRI->tracksLiveness()) { 1502 // Initialize liveins to the first BB. These are potentiall redefined by 1503 // predicated instructions. 1504 Redefs.addLiveIns(CvtMBB); 1505 Redefs.addLiveIns(NextMBB); 1506 // Compute a set of registers which must not be killed by instructions in 1507 // BB1: This is everything live-in to BB2. 1508 DontKill.addLiveIns(NextMBB); 1509 } 1510 1511 if (CvtMBB.pred_size() > 1) { 1512 BBI.NonPredSize -= TII->removeBranch(*BBI.BB); 1513 // Copy instructions in the true block, predicate them, and add them to 1514 // the entry block. 1515 CopyAndPredicateBlock(BBI, *CvtBBI, Cond); 1516 1517 // RemoveExtraEdges won't work if the block has an unanalyzable branch, so 1518 // explicitly remove CvtBBI as a successor. 1519 BBI.BB->removeSuccessor(&CvtMBB, true); 1520 } else { 1521 RemoveKills(CvtMBB.begin(), CvtMBB.end(), DontKill, *TRI); 1522 PredicateBlock(*CvtBBI, CvtMBB.end(), Cond); 1523 1524 // Merge converted block into entry block. 1525 BBI.NonPredSize -= TII->removeBranch(*BBI.BB); 1526 MergeBlocks(BBI, *CvtBBI); 1527 } 1528 1529 bool IterIfcvt = true; 1530 if (!canFallThroughTo(*BBI.BB, NextMBB)) { 1531 InsertUncondBranch(*BBI.BB, NextMBB, TII); 1532 BBI.HasFallThrough = false; 1533 // Now ifcvt'd block will look like this: 1534 // BB: 1535 // ... 1536 // t, f = cmp 1537 // if t op 1538 // b BBf 1539 // 1540 // We cannot further ifcvt this block because the unconditional branch 1541 // will have to be predicated on the new condition, that will not be 1542 // available if cmp executes. 1543 IterIfcvt = false; 1544 } 1545 1546 RemoveExtraEdges(BBI); 1547 1548 // Update block info. BB can be iteratively if-converted. 1549 if (!IterIfcvt) 1550 BBI.IsDone = true; 1551 InvalidatePreds(*BBI.BB); 1552 CvtBBI->IsDone = true; 1553 1554 // FIXME: Must maintain LiveIns. 1555 return true; 1556} 1557 1558/// If convert a triangle sub-CFG. 1559bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) { 1560 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; 1561 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; 1562 BBInfo *CvtBBI = &TrueBBI; 1563 BBInfo *NextBBI = &FalseBBI; 1564 DebugLoc dl; // FIXME: this is nowhere 1565 1566 SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end()); 1567 if (Kind == ICTriangleFalse || Kind == ICTriangleFRev) 1568 std::swap(CvtBBI, NextBBI); 1569 1570 MachineBasicBlock &CvtMBB = *CvtBBI->BB; 1571 MachineBasicBlock &NextMBB = *NextBBI->BB; 1572 if (CvtBBI->IsDone || 1573 (CvtBBI->CannotBeCopied && CvtMBB.pred_size() > 1)) { 1574 // Something has changed. It's no longer safe to predicate this block. 1575 BBI.IsAnalyzed = false; 1576 CvtBBI->IsAnalyzed = false; 1577 return false; 1578 } 1579 1580 if (CvtMBB.hasAddressTaken()) 1581 // Conservatively abort if-conversion if BB's address is taken. 1582 return false; 1583 1584 if (Kind == ICTriangleFalse || Kind == ICTriangleFRev) 1585 if (TII->reverseBranchCondition(Cond)) 1586 llvm_unreachable("Unable to reverse branch condition!"); 1587 1588 if (Kind == ICTriangleRev || Kind == ICTriangleFRev) { 1589 if (reverseBranchCondition(*CvtBBI)) { 1590 // BB has been changed, modify its predecessors (except for this 1591 // one) so they don't get ifcvt'ed based on bad intel. 1592 for (MachineBasicBlock *PBB : CvtMBB.predecessors()) { 1593 if (PBB == BBI.BB) 1594 continue; 1595 BBInfo &PBBI = BBAnalysis[PBB->getNumber()]; 1596 if (PBBI.IsEnqueued) { 1597 PBBI.IsAnalyzed = false; 1598 PBBI.IsEnqueued = false; 1599 } 1600 } 1601 } 1602 } 1603 1604 // Initialize liveins to the first BB. These are potentially redefined by 1605 // predicated instructions. 1606 Redefs.init(*TRI); 1607 if (MRI->tracksLiveness()) { 1608 Redefs.addLiveIns(CvtMBB); 1609 Redefs.addLiveIns(NextMBB); 1610 } 1611 1612 DontKill.clear(); 1613 1614 bool HasEarlyExit = CvtBBI->FalseBB != nullptr; 1615 BranchProbability CvtNext, CvtFalse, BBNext, BBCvt; 1616 1617 if (HasEarlyExit) { 1618 // Get probabilities before modifying CvtMBB and BBI.BB. 1619 CvtNext = MBPI->getEdgeProbability(&CvtMBB, &NextMBB); 1620 CvtFalse = MBPI->getEdgeProbability(&CvtMBB, CvtBBI->FalseBB); 1621 BBNext = MBPI->getEdgeProbability(BBI.BB, &NextMBB); 1622 BBCvt = MBPI->getEdgeProbability(BBI.BB, &CvtMBB); 1623 } 1624 1625 if (CvtMBB.pred_size() > 1) { 1626 BBI.NonPredSize -= TII->removeBranch(*BBI.BB); 1627 // Copy instructions in the true block, predicate them, and add them to 1628 // the entry block. 1629 CopyAndPredicateBlock(BBI, *CvtBBI, Cond, true); 1630 1631 // RemoveExtraEdges won't work if the block has an unanalyzable branch, so 1632 // explicitly remove CvtBBI as a successor. 1633 BBI.BB->removeSuccessor(&CvtMBB, true); 1634 } else { 1635 // Predicate the 'true' block after removing its branch. 1636 CvtBBI->NonPredSize -= TII->removeBranch(CvtMBB); 1637 PredicateBlock(*CvtBBI, CvtMBB.end(), Cond); 1638 1639 // Now merge the entry of the triangle with the true block. 1640 BBI.NonPredSize -= TII->removeBranch(*BBI.BB); 1641 MergeBlocks(BBI, *CvtBBI, false); 1642 } 1643 1644 // If 'true' block has a 'false' successor, add an exit branch to it. 1645 if (HasEarlyExit) { 1646 SmallVector<MachineOperand, 4> RevCond(CvtBBI->BrCond.begin(), 1647 CvtBBI->BrCond.end()); 1648 if (TII->reverseBranchCondition(RevCond)) 1649 llvm_unreachable("Unable to reverse branch condition!"); 1650 1651 // Update the edge probability for both CvtBBI->FalseBB and NextBBI. 1652 // NewNext = New_Prob(BBI.BB, NextMBB) = 1653 // Prob(BBI.BB, NextMBB) + 1654 // Prob(BBI.BB, CvtMBB) * Prob(CvtMBB, NextMBB) 1655 // NewFalse = New_Prob(BBI.BB, CvtBBI->FalseBB) = 1656 // Prob(BBI.BB, CvtMBB) * Prob(CvtMBB, CvtBBI->FalseBB) 1657 auto NewTrueBB = getNextBlock(*BBI.BB); 1658 auto NewNext = BBNext + BBCvt * CvtNext; 1659 auto NewTrueBBIter = find(BBI.BB->successors(), NewTrueBB); 1660 if (NewTrueBBIter != BBI.BB->succ_end()) 1661 BBI.BB->setSuccProbability(NewTrueBBIter, NewNext); 1662 1663 auto NewFalse = BBCvt * CvtFalse; 1664 TII->insertBranch(*BBI.BB, CvtBBI->FalseBB, nullptr, RevCond, dl); 1665 BBI.BB->addSuccessor(CvtBBI->FalseBB, NewFalse); 1666 } 1667 1668 // Merge in the 'false' block if the 'false' block has no other 1669 // predecessors. Otherwise, add an unconditional branch to 'false'. 1670 bool FalseBBDead = false; 1671 bool IterIfcvt = true; 1672 bool isFallThrough = canFallThroughTo(*BBI.BB, NextMBB); 1673 if (!isFallThrough) { 1674 // Only merge them if the true block does not fallthrough to the false 1675 // block. By not merging them, we make it possible to iteratively 1676 // ifcvt the blocks. 1677 if (!HasEarlyExit && 1678 NextMBB.pred_size() == 1 && !NextBBI->HasFallThrough && 1679 !NextMBB.hasAddressTaken()) { 1680 MergeBlocks(BBI, *NextBBI); 1681 FalseBBDead = true; 1682 } else { 1683 InsertUncondBranch(*BBI.BB, NextMBB, TII); 1684 BBI.HasFallThrough = false; 1685 } 1686 // Mixed predicated and unpredicated code. This cannot be iteratively 1687 // predicated. 1688 IterIfcvt = false; 1689 } 1690 1691 RemoveExtraEdges(BBI); 1692 1693 // Update block info. BB can be iteratively if-converted. 1694 if (!IterIfcvt) 1695 BBI.IsDone = true; 1696 InvalidatePreds(*BBI.BB); 1697 CvtBBI->IsDone = true; 1698 if (FalseBBDead) 1699 NextBBI->IsDone = true; 1700 1701 // FIXME: Must maintain LiveIns. 1702 return true; 1703} 1704 1705/// Common code shared between diamond conversions. 1706/// \p BBI, \p TrueBBI, and \p FalseBBI form the diamond shape. 1707/// \p NumDups1 - number of shared instructions at the beginning of \p TrueBBI 1708/// and FalseBBI 1709/// \p NumDups2 - number of shared instructions at the end of \p TrueBBI 1710/// and \p FalseBBI 1711/// \p RemoveBranch - Remove the common branch of the two blocks before 1712/// predicating. Only false for unanalyzable fallthrough 1713/// cases. The caller will replace the branch if necessary. 1714/// \p MergeAddEdges - Add successor edges when merging blocks. Only false for 1715/// unanalyzable fallthrough 1716bool IfConverter::IfConvertDiamondCommon( 1717 BBInfo &BBI, BBInfo &TrueBBI, BBInfo &FalseBBI, 1718 unsigned NumDups1, unsigned NumDups2, 1719 bool TClobbersPred, bool FClobbersPred, 1720 bool RemoveBranch, bool MergeAddEdges) { 1721 1722 if (TrueBBI.IsDone || FalseBBI.IsDone || 1723 TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) { 1724 // Something has changed. It's no longer safe to predicate these blocks. 1725 BBI.IsAnalyzed = false; 1726 TrueBBI.IsAnalyzed = false; 1727 FalseBBI.IsAnalyzed = false; 1728 return false; 1729 } 1730 1731 if (TrueBBI.BB->hasAddressTaken() || FalseBBI.BB->hasAddressTaken()) 1732 // Conservatively abort if-conversion if either BB has its address taken. 1733 return false; 1734 1735 // Put the predicated instructions from the 'true' block before the 1736 // instructions from the 'false' block, unless the true block would clobber 1737 // the predicate, in which case, do the opposite. 1738 BBInfo *BBI1 = &TrueBBI; 1739 BBInfo *BBI2 = &FalseBBI; 1740 SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end()); 1741 if (TII->reverseBranchCondition(RevCond)) 1742 llvm_unreachable("Unable to reverse branch condition!"); 1743 SmallVector<MachineOperand, 4> *Cond1 = &BBI.BrCond; 1744 SmallVector<MachineOperand, 4> *Cond2 = &RevCond; 1745 1746 // Figure out the more profitable ordering. 1747 bool DoSwap = false; 1748 if (TClobbersPred && !FClobbersPred) 1749 DoSwap = true; 1750 else if (!TClobbersPred && !FClobbersPred) { 1751 if (TrueBBI.NonPredSize > FalseBBI.NonPredSize) 1752 DoSwap = true; 1753 } else if (TClobbersPred && FClobbersPred) 1754 llvm_unreachable("Predicate info cannot be clobbered by both sides."); 1755 if (DoSwap) { 1756 std::swap(BBI1, BBI2); 1757 std::swap(Cond1, Cond2); 1758 } 1759 1760 // Remove the conditional branch from entry to the blocks. 1761 BBI.NonPredSize -= TII->removeBranch(*BBI.BB); 1762 1763 MachineBasicBlock &MBB1 = *BBI1->BB; 1764 MachineBasicBlock &MBB2 = *BBI2->BB; 1765 1766 // Initialize the Redefs: 1767 // - BB2 live-in regs need implicit uses before being redefined by BB1 1768 // instructions. 1769 // - BB1 live-out regs need implicit uses before being redefined by BB2 1770 // instructions. We start with BB1 live-ins so we have the live-out regs 1771 // after tracking the BB1 instructions. 1772 Redefs.init(*TRI); 1773 if (MRI->tracksLiveness()) { 1774 Redefs.addLiveIns(MBB1); 1775 Redefs.addLiveIns(MBB2); 1776 } 1777 1778 // Remove the duplicated instructions at the beginnings of both paths. 1779 // Skip dbg_value instructions 1780 MachineBasicBlock::iterator DI1 = MBB1.getFirstNonDebugInstr(); 1781 MachineBasicBlock::iterator DI2 = MBB2.getFirstNonDebugInstr(); 1782 BBI1->NonPredSize -= NumDups1; 1783 BBI2->NonPredSize -= NumDups1; 1784 1785 // Skip past the dups on each side separately since there may be 1786 // differing dbg_value entries. 1787 for (unsigned i = 0; i < NumDups1; ++DI1) { 1788 if (!DI1->isDebugValue()) 1789 ++i; 1790 } 1791 while (NumDups1 != 0) { 1792 ++DI2; 1793 if (!DI2->isDebugValue()) 1794 --NumDups1; 1795 } 1796 1797 // Compute a set of registers which must not be killed by instructions in BB1: 1798 // This is everything used+live in BB2 after the duplicated instructions. We 1799 // can compute this set by simulating liveness backwards from the end of BB2. 1800 DontKill.init(*TRI); 1801 if (MRI->tracksLiveness()) { 1802 for (const MachineInstr &MI : make_range(MBB2.rbegin(), ++DI2.getReverse())) 1803 DontKill.stepBackward(MI); 1804 1805 for (const MachineInstr &MI : make_range(MBB1.begin(), DI1)) { 1806 SmallVector<std::pair<unsigned, const MachineOperand*>, 4> Dummy; 1807 Redefs.stepForward(MI, Dummy); 1808 } 1809 } 1810 BBI.BB->splice(BBI.BB->end(), &MBB1, MBB1.begin(), DI1); 1811 MBB2.erase(MBB2.begin(), DI2); 1812 1813 // The branches have been checked to match, so it is safe to remove the branch 1814 // in BB1 and rely on the copy in BB2 1815#ifndef NDEBUG 1816 // Unanalyzable branches must match exactly. Check that now. 1817 if (!BBI1->IsBrAnalyzable) 1818 verifySameBranchInstructions(&MBB1, &MBB2); 1819#endif 1820 BBI1->NonPredSize -= TII->removeBranch(*BBI1->BB); 1821 // Remove duplicated instructions. 1822 DI1 = MBB1.end(); 1823 for (unsigned i = 0; i != NumDups2; ) { 1824 // NumDups2 only counted non-dbg_value instructions, so this won't 1825 // run off the head of the list. 1826 assert(DI1 != MBB1.begin()); 1827 --DI1; 1828 // skip dbg_value instructions 1829 if (!DI1->isDebugValue()) 1830 ++i; 1831 } 1832 MBB1.erase(DI1, MBB1.end()); 1833 1834 // Kill flags in the true block for registers living into the false block 1835 // must be removed. 1836 RemoveKills(MBB1.begin(), MBB1.end(), DontKill, *TRI); 1837 1838 DI2 = BBI2->BB->end(); 1839 // The branches have been checked to match. Skip over the branch in the false 1840 // block so that we don't try to predicate it. 1841 if (RemoveBranch) 1842 BBI2->NonPredSize -= TII->removeBranch(*BBI2->BB); 1843 else { 1844 do { 1845 assert(DI2 != MBB2.begin()); 1846 DI2--; 1847 } while (DI2->isBranch() || DI2->isDebugValue()); 1848 DI2++; 1849 } 1850 while (NumDups2 != 0) { 1851 // NumDups2 only counted non-dbg_value instructions, so this won't 1852 // run off the head of the list. 1853 assert(DI2 != MBB2.begin()); 1854 --DI2; 1855 // skip dbg_value instructions 1856 if (!DI2->isDebugValue()) 1857 --NumDups2; 1858 } 1859 1860 // Remember which registers would later be defined by the false block. 1861 // This allows us not to predicate instructions in the true block that would 1862 // later be re-defined. That is, rather than 1863 // subeq r0, r1, #1 1864 // addne r0, r1, #1 1865 // generate: 1866 // sub r0, r1, #1 1867 // addne r0, r1, #1 1868 SmallSet<unsigned, 4> RedefsByFalse; 1869 SmallSet<unsigned, 4> ExtUses; 1870 if (TII->isProfitableToUnpredicate(MBB1, MBB2)) { 1871 for (const MachineInstr &FI : make_range(MBB2.begin(), DI2)) { 1872 if (FI.isDebugValue()) 1873 continue; 1874 SmallVector<unsigned, 4> Defs; 1875 for (const MachineOperand &MO : FI.operands()) { 1876 if (!MO.isReg()) 1877 continue; 1878 unsigned Reg = MO.getReg(); 1879 if (!Reg) 1880 continue; 1881 if (MO.isDef()) { 1882 Defs.push_back(Reg); 1883 } else if (!RedefsByFalse.count(Reg)) { 1884 // These are defined before ctrl flow reach the 'false' instructions. 1885 // They cannot be modified by the 'true' instructions. 1886 for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); 1887 SubRegs.isValid(); ++SubRegs) 1888 ExtUses.insert(*SubRegs); 1889 } 1890 } 1891 1892 for (unsigned Reg : Defs) { 1893 if (!ExtUses.count(Reg)) { 1894 for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); 1895 SubRegs.isValid(); ++SubRegs) 1896 RedefsByFalse.insert(*SubRegs); 1897 } 1898 } 1899 } 1900 } 1901 1902 // Predicate the 'true' block. 1903 PredicateBlock(*BBI1, MBB1.end(), *Cond1, &RedefsByFalse); 1904 1905 // After predicating BBI1, if there is a predicated terminator in BBI1 and 1906 // a non-predicated in BBI2, then we don't want to predicate the one from 1907 // BBI2. The reason is that if we merged these blocks, we would end up with 1908 // two predicated terminators in the same block. 1909 if (!MBB2.empty() && (DI2 == MBB2.end())) { 1910 MachineBasicBlock::iterator BBI1T = MBB1.getFirstTerminator(); 1911 MachineBasicBlock::iterator BBI2T = MBB2.getFirstTerminator(); 1912 if (BBI1T != MBB1.end() && TII->isPredicated(*BBI1T) && 1913 BBI2T != MBB2.end() && !TII->isPredicated(*BBI2T)) 1914 --DI2; 1915 } 1916 1917 // Predicate the 'false' block. 1918 PredicateBlock(*BBI2, DI2, *Cond2); 1919 1920 // Merge the true block into the entry of the diamond. 1921 MergeBlocks(BBI, *BBI1, MergeAddEdges); 1922 MergeBlocks(BBI, *BBI2, MergeAddEdges); 1923 return true; 1924} 1925 1926/// If convert an almost-diamond sub-CFG where the true 1927/// and false blocks share a common tail. 1928bool IfConverter::IfConvertForkedDiamond( 1929 BBInfo &BBI, IfcvtKind Kind, 1930 unsigned NumDups1, unsigned NumDups2, 1931 bool TClobbersPred, bool FClobbersPred) { 1932 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; 1933 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; 1934 1935 // Save the debug location for later. 1936 DebugLoc dl; 1937 MachineBasicBlock::iterator TIE = TrueBBI.BB->getFirstTerminator(); 1938 if (TIE != TrueBBI.BB->end()) 1939 dl = TIE->getDebugLoc(); 1940 // Removing branches from both blocks is safe, because we have already 1941 // determined that both blocks have the same branch instructions. The branch 1942 // will be added back at the end, unpredicated. 1943 if (!IfConvertDiamondCommon( 1944 BBI, TrueBBI, FalseBBI, 1945 NumDups1, NumDups2, 1946 TClobbersPred, FClobbersPred, 1947 /* RemoveBranch */ true, /* MergeAddEdges */ true)) 1948 return false; 1949 1950 // Add back the branch. 1951 // Debug location saved above when removing the branch from BBI2 1952 TII->insertBranch(*BBI.BB, TrueBBI.TrueBB, TrueBBI.FalseBB, 1953 TrueBBI.BrCond, dl); 1954 1955 RemoveExtraEdges(BBI); 1956 1957 // Update block info. 1958 BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true; 1959 InvalidatePreds(*BBI.BB); 1960 1961 // FIXME: Must maintain LiveIns. 1962 return true; 1963} 1964 1965/// If convert a diamond sub-CFG. 1966bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind, 1967 unsigned NumDups1, unsigned NumDups2, 1968 bool TClobbersPred, bool FClobbersPred) { 1969 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; 1970 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; 1971 MachineBasicBlock *TailBB = TrueBBI.TrueBB; 1972 1973 // True block must fall through or end with an unanalyzable terminator. 1974 if (!TailBB) { 1975 if (blockAlwaysFallThrough(TrueBBI)) 1976 TailBB = FalseBBI.TrueBB; 1977 assert((TailBB || !TrueBBI.IsBrAnalyzable) && "Unexpected!"); 1978 } 1979 1980 if (!IfConvertDiamondCommon( 1981 BBI, TrueBBI, FalseBBI, 1982 NumDups1, NumDups2, 1983 TClobbersPred, FClobbersPred, 1984 /* RemoveBranch */ TrueBBI.IsBrAnalyzable, 1985 /* MergeAddEdges */ TailBB == nullptr)) 1986 return false; 1987 1988 // If the if-converted block falls through or unconditionally branches into 1989 // the tail block, and the tail block does not have other predecessors, then 1990 // fold the tail block in as well. Otherwise, unless it falls through to the 1991 // tail, add a unconditional branch to it. 1992 if (TailBB) { 1993 BBInfo &TailBBI = BBAnalysis[TailBB->getNumber()]; 1994 bool CanMergeTail = !TailBBI.HasFallThrough && 1995 !TailBBI.BB->hasAddressTaken(); 1996 // The if-converted block can still have a predicated terminator 1997 // (e.g. a predicated return). If that is the case, we cannot merge 1998 // it with the tail block. 1999 MachineBasicBlock::const_iterator TI = BBI.BB->getFirstTerminator(); 2000 if (TI != BBI.BB->end() && TII->isPredicated(*TI)) 2001 CanMergeTail = false; 2002 // There may still be a fall-through edge from BBI1 or BBI2 to TailBB; 2003 // check if there are any other predecessors besides those. 2004 unsigned NumPreds = TailBB->pred_size(); 2005 if (NumPreds > 1) 2006 CanMergeTail = false; 2007 else if (NumPreds == 1 && CanMergeTail) { 2008 MachineBasicBlock::pred_iterator PI = TailBB->pred_begin(); 2009 if (*PI != TrueBBI.BB && *PI != FalseBBI.BB) 2010 CanMergeTail = false; 2011 } 2012 if (CanMergeTail) { 2013 MergeBlocks(BBI, TailBBI); 2014 TailBBI.IsDone = true; 2015 } else { 2016 BBI.BB->addSuccessor(TailBB, BranchProbability::getOne()); 2017 InsertUncondBranch(*BBI.BB, *TailBB, TII); 2018 BBI.HasFallThrough = false; 2019 } 2020 } 2021 2022 // RemoveExtraEdges won't work if the block has an unanalyzable branch, 2023 // which can happen here if TailBB is unanalyzable and is merged, so 2024 // explicitly remove BBI1 and BBI2 as successors. 2025 BBI.BB->removeSuccessor(TrueBBI.BB); 2026 BBI.BB->removeSuccessor(FalseBBI.BB, /* NormalizeSuccessProbs */ true); 2027 RemoveExtraEdges(BBI); 2028 2029 // Update block info. 2030 BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true; 2031 InvalidatePreds(*BBI.BB); 2032 2033 // FIXME: Must maintain LiveIns. 2034 return true; 2035} 2036 2037static bool MaySpeculate(const MachineInstr &MI, 2038 SmallSet<unsigned, 4> &LaterRedefs) { 2039 bool SawStore = true; 2040 if (!MI.isSafeToMove(nullptr, SawStore)) 2041 return false; 2042 2043 for (const MachineOperand &MO : MI.operands()) { 2044 if (!MO.isReg()) 2045 continue; 2046 unsigned Reg = MO.getReg(); 2047 if (!Reg) 2048 continue; 2049 if (MO.isDef() && !LaterRedefs.count(Reg)) 2050 return false; 2051 } 2052 2053 return true; 2054} 2055 2056/// Predicate instructions from the start of the block to the specified end with 2057/// the specified condition. 2058void IfConverter::PredicateBlock(BBInfo &BBI, 2059 MachineBasicBlock::iterator E, 2060 SmallVectorImpl<MachineOperand> &Cond, 2061 SmallSet<unsigned, 4> *LaterRedefs) { 2062 bool AnyUnpred = false; 2063 bool MaySpec = LaterRedefs != nullptr; 2064 for (MachineInstr &I : make_range(BBI.BB->begin(), E)) { 2065 if (I.isDebugValue() || TII->isPredicated(I)) 2066 continue; 2067 // It may be possible not to predicate an instruction if it's the 'true' 2068 // side of a diamond and the 'false' side may re-define the instruction's 2069 // defs. 2070 if (MaySpec && MaySpeculate(I, *LaterRedefs)) { 2071 AnyUnpred = true; 2072 continue; 2073 } 2074 // If any instruction is predicated, then every instruction after it must 2075 // be predicated. 2076 MaySpec = false; 2077 if (!TII->PredicateInstruction(I, Cond)) { 2078#ifndef NDEBUG 2079 dbgs() << "Unable to predicate " << I << "!\n"; 2080#endif 2081 llvm_unreachable(nullptr); 2082 } 2083 2084 // If the predicated instruction now redefines a register as the result of 2085 // if-conversion, add an implicit kill. 2086 UpdatePredRedefs(I, Redefs); 2087 } 2088 2089 BBI.Predicate.append(Cond.begin(), Cond.end()); 2090 2091 BBI.IsAnalyzed = false; 2092 BBI.NonPredSize = 0; 2093 2094 ++NumIfConvBBs; 2095 if (AnyUnpred) 2096 ++NumUnpred; 2097} 2098 2099/// Copy and predicate instructions from source BB to the destination block. 2100/// Skip end of block branches if IgnoreBr is true. 2101void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, 2102 SmallVectorImpl<MachineOperand> &Cond, 2103 bool IgnoreBr) { 2104 MachineFunction &MF = *ToBBI.BB->getParent(); 2105 2106 MachineBasicBlock &FromMBB = *FromBBI.BB; 2107 for (MachineInstr &I : FromMBB) { 2108 // Do not copy the end of the block branches. 2109 if (IgnoreBr && I.isBranch()) 2110 break; 2111 2112 MachineInstr *MI = MF.CloneMachineInstr(&I); 2113 ToBBI.BB->insert(ToBBI.BB->end(), MI); 2114 ToBBI.NonPredSize++; 2115 unsigned ExtraPredCost = TII->getPredicationCost(I); 2116 unsigned NumCycles = SchedModel.computeInstrLatency(&I, false); 2117 if (NumCycles > 1) 2118 ToBBI.ExtraCost += NumCycles-1; 2119 ToBBI.ExtraCost2 += ExtraPredCost; 2120 2121 if (!TII->isPredicated(I) && !MI->isDebugValue()) { 2122 if (!TII->PredicateInstruction(*MI, Cond)) { 2123#ifndef NDEBUG 2124 dbgs() << "Unable to predicate " << I << "!\n"; 2125#endif 2126 llvm_unreachable(nullptr); 2127 } 2128 } 2129 2130 // If the predicated instruction now redefines a register as the result of 2131 // if-conversion, add an implicit kill. 2132 UpdatePredRedefs(*MI, Redefs); 2133 2134 // Some kill flags may not be correct anymore. 2135 if (!DontKill.empty()) 2136 RemoveKills(*MI, DontKill); 2137 } 2138 2139 if (!IgnoreBr) { 2140 std::vector<MachineBasicBlock *> Succs(FromMBB.succ_begin(), 2141 FromMBB.succ_end()); 2142 MachineBasicBlock *NBB = getNextBlock(FromMBB); 2143 MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr; 2144 2145 for (MachineBasicBlock *Succ : Succs) { 2146 // Fallthrough edge can't be transferred. 2147 if (Succ == FallThrough) 2148 continue; 2149 ToBBI.BB->addSuccessor(Succ); 2150 } 2151 } 2152 2153 ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end()); 2154 ToBBI.Predicate.append(Cond.begin(), Cond.end()); 2155 2156 ToBBI.ClobbersPred |= FromBBI.ClobbersPred; 2157 ToBBI.IsAnalyzed = false; 2158 2159 ++NumDupBBs; 2160} 2161 2162/// Move all instructions from FromBB to the end of ToBB. This will leave 2163/// FromBB as an empty block, so remove all of its successor edges except for 2164/// the fall-through edge. If AddEdges is true, i.e., when FromBBI's branch is 2165/// being moved, add those successor edges to ToBBI. 2166void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) { 2167 MachineBasicBlock &FromMBB = *FromBBI.BB; 2168 assert(!FromMBB.hasAddressTaken() && 2169 "Removing a BB whose address is taken!"); 2170 2171 // In case FromMBB contains terminators (e.g. return instruction), 2172 // first move the non-terminator instructions, then the terminators. 2173 MachineBasicBlock::iterator FromTI = FromMBB.getFirstTerminator(); 2174 MachineBasicBlock::iterator ToTI = ToBBI.BB->getFirstTerminator(); 2175 ToBBI.BB->splice(ToTI, &FromMBB, FromMBB.begin(), FromTI); 2176 2177 // If FromBB has non-predicated terminator we should copy it at the end. 2178 if (FromTI != FromMBB.end() && !TII->isPredicated(*FromTI)) 2179 ToTI = ToBBI.BB->end(); 2180 ToBBI.BB->splice(ToTI, &FromMBB, FromTI, FromMBB.end()); 2181 2182 // Force normalizing the successors' probabilities of ToBBI.BB to convert all 2183 // unknown probabilities into known ones. 2184 // FIXME: This usage is too tricky and in the future we would like to 2185 // eliminate all unknown probabilities in MBB. 2186 ToBBI.BB->normalizeSuccProbs(); 2187 2188 SmallVector<MachineBasicBlock *, 4> FromSuccs(FromMBB.succ_begin(), 2189 FromMBB.succ_end()); 2190 MachineBasicBlock *NBB = getNextBlock(FromMBB); 2191 MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr; 2192 // The edge probability from ToBBI.BB to FromMBB, which is only needed when 2193 // AddEdges is true and FromMBB is a successor of ToBBI.BB. 2194 auto To2FromProb = BranchProbability::getZero(); 2195 if (AddEdges && ToBBI.BB->isSuccessor(&FromMBB)) { 2196 To2FromProb = MBPI->getEdgeProbability(ToBBI.BB, &FromMBB); 2197 // Set the edge probability from ToBBI.BB to FromMBB to zero to avoid the 2198 // edge probability being merged to other edges when this edge is removed 2199 // later. 2200 ToBBI.BB->setSuccProbability(find(ToBBI.BB->successors(), &FromMBB), 2201 BranchProbability::getZero()); 2202 } 2203 2204 for (MachineBasicBlock *Succ : FromSuccs) { 2205 // Fallthrough edge can't be transferred. 2206 if (Succ == FallThrough) 2207 continue; 2208 2209 auto NewProb = BranchProbability::getZero(); 2210 if (AddEdges) { 2211 // Calculate the edge probability for the edge from ToBBI.BB to Succ, 2212 // which is a portion of the edge probability from FromMBB to Succ. The 2213 // portion ratio is the edge probability from ToBBI.BB to FromMBB (if 2214 // FromBBI is a successor of ToBBI.BB. See comment below for excepion). 2215 NewProb = MBPI->getEdgeProbability(&FromMBB, Succ); 2216 2217 // To2FromProb is 0 when FromMBB is not a successor of ToBBI.BB. This 2218 // only happens when if-converting a diamond CFG and FromMBB is the 2219 // tail BB. In this case FromMBB post-dominates ToBBI.BB and hence we 2220 // could just use the probabilities on FromMBB's out-edges when adding 2221 // new successors. 2222 if (!To2FromProb.isZero()) 2223 NewProb *= To2FromProb; 2224 } 2225 2226 FromMBB.removeSuccessor(Succ); 2227 2228 if (AddEdges) { 2229 // If the edge from ToBBI.BB to Succ already exists, update the 2230 // probability of this edge by adding NewProb to it. An example is shown 2231 // below, in which A is ToBBI.BB and B is FromMBB. In this case we 2232 // don't have to set C as A's successor as it already is. We only need to 2233 // update the edge probability on A->C. Note that B will not be 2234 // immediately removed from A's successors. It is possible that B->D is 2235 // not removed either if D is a fallthrough of B. Later the edge A->D 2236 // (generated here) and B->D will be combined into one edge. To maintain 2237 // correct edge probability of this combined edge, we need to set the edge 2238 // probability of A->B to zero, which is already done above. The edge 2239 // probability on A->D is calculated by scaling the original probability 2240 // on A->B by the probability of B->D. 2241 // 2242 // Before ifcvt: After ifcvt (assume B->D is kept): 2243 // 2244 // A A 2245 // /| /|\ 2246 // / B / B| 2247 // | /| | || 2248 // |/ | | |/ 2249 // C D C D 2250 // 2251 if (ToBBI.BB->isSuccessor(Succ)) 2252 ToBBI.BB->setSuccProbability( 2253 find(ToBBI.BB->successors(), Succ), 2254 MBPI->getEdgeProbability(ToBBI.BB, Succ) + NewProb); 2255 else 2256 ToBBI.BB->addSuccessor(Succ, NewProb); 2257 } 2258 } 2259 2260 // Now FromBBI always falls through to the next block! 2261 if (NBB && !FromMBB.isSuccessor(NBB)) 2262 FromMBB.addSuccessor(NBB); 2263 2264 // Normalize the probabilities of ToBBI.BB's successors with all adjustment 2265 // we've done above. 2266 ToBBI.BB->normalizeSuccProbs(); 2267 2268 ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end()); 2269 FromBBI.Predicate.clear(); 2270 2271 ToBBI.NonPredSize += FromBBI.NonPredSize; 2272 ToBBI.ExtraCost += FromBBI.ExtraCost; 2273 ToBBI.ExtraCost2 += FromBBI.ExtraCost2; 2274 FromBBI.NonPredSize = 0; 2275 FromBBI.ExtraCost = 0; 2276 FromBBI.ExtraCost2 = 0; 2277 2278 ToBBI.ClobbersPred |= FromBBI.ClobbersPred; 2279 ToBBI.HasFallThrough = FromBBI.HasFallThrough; 2280 ToBBI.IsAnalyzed = false; 2281 FromBBI.IsAnalyzed = false; 2282} 2283 2284FunctionPass * 2285llvm::createIfConverter(std::function<bool(const MachineFunction &)> Ftor) { 2286 return new IfConverter(std::move(Ftor)); 2287} 2288