1234285Sdim//===-- HexagonHardwareLoops.cpp - Identify and generate hardware loops ---===// 2234285Sdim// 3234285Sdim// The LLVM Compiler Infrastructure 4234285Sdim// 5234285Sdim// This file is distributed under the University of Illinois Open Source 6234285Sdim// License. See LICENSE.TXT for details. 7234285Sdim// 8234285Sdim//===----------------------------------------------------------------------===// 9234285Sdim// 10234285Sdim// This pass identifies loops where we can generate the Hexagon hardware 11234285Sdim// loop instruction. The hardware loop can perform loop branches with a 12234285Sdim// zero-cycle overhead. 13234285Sdim// 14234285Sdim// The pattern that defines the induction variable can changed depending on 15234285Sdim// prior optimizations. For example, the IndVarSimplify phase run by 'opt' 16234285Sdim// normalizes induction variables, and the Loop Strength Reduction pass 17234285Sdim// run by 'llc' may also make changes to the induction variable. 18234285Sdim// The pattern detected by this phase is due to running Strength Reduction. 19234285Sdim// 20234285Sdim// Criteria for hardware loops: 21234285Sdim// - Countable loops (w/ ind. var for a trip count) 22234285Sdim// - Assumes loops are normalized by IndVarSimplify 23234285Sdim// - Try inner-most loops first 24234285Sdim// - No nested hardware loops. 25234285Sdim// - No function calls in loops. 26234285Sdim// 27234285Sdim//===----------------------------------------------------------------------===// 28234285Sdim 29234285Sdim#define DEBUG_TYPE "hwloops" 30252723Sdim#include "llvm/ADT/SmallSet.h" 31234285Sdim#include "llvm/ADT/Statistic.h" 32234285Sdim#include "llvm/CodeGen/MachineDominators.h" 33234285Sdim#include "llvm/CodeGen/MachineFunction.h" 34234285Sdim#include "llvm/CodeGen/MachineFunctionPass.h" 35234285Sdim#include "llvm/CodeGen/MachineInstrBuilder.h" 36234285Sdim#include "llvm/CodeGen/MachineLoopInfo.h" 37234285Sdim#include "llvm/CodeGen/MachineRegisterInfo.h" 38252723Sdim#include "llvm/PassSupport.h" 39252723Sdim#include "llvm/Support/CommandLine.h" 40234285Sdim#include "llvm/Support/Debug.h" 41234285Sdim#include "llvm/Support/raw_ostream.h" 42234285Sdim#include "llvm/Target/TargetInstrInfo.h" 43252723Sdim#include "Hexagon.h" 44252723Sdim#include "HexagonTargetMachine.h" 45252723Sdim 46234285Sdim#include <algorithm> 47252723Sdim#include <vector> 48234285Sdim 49234285Sdimusing namespace llvm; 50234285Sdim 51252723Sdim#ifndef NDEBUG 52252723Sdimstatic cl::opt<int> HWLoopLimit("max-hwloop", cl::Hidden, cl::init(-1)); 53252723Sdim#endif 54252723Sdim 55234285SdimSTATISTIC(NumHWLoops, "Number of loops converted to hardware loops"); 56234285Sdim 57252723Sdimnamespace llvm { 58252723Sdim void initializeHexagonHardwareLoopsPass(PassRegistry&); 59252723Sdim} 60252723Sdim 61234285Sdimnamespace { 62234285Sdim class CountValue; 63234285Sdim struct HexagonHardwareLoops : public MachineFunctionPass { 64252723Sdim MachineLoopInfo *MLI; 65252723Sdim MachineRegisterInfo *MRI; 66252723Sdim MachineDominatorTree *MDT; 67252723Sdim const HexagonTargetMachine *TM; 68252723Sdim const HexagonInstrInfo *TII; 69252723Sdim const HexagonRegisterInfo *TRI; 70252723Sdim#ifndef NDEBUG 71252723Sdim static int Counter; 72252723Sdim#endif 73234285Sdim 74234285Sdim public: 75252723Sdim static char ID; 76234285Sdim 77252723Sdim HexagonHardwareLoops() : MachineFunctionPass(ID) { 78252723Sdim initializeHexagonHardwareLoopsPass(*PassRegistry::getPassRegistry()); 79252723Sdim } 80234285Sdim 81234285Sdim virtual bool runOnMachineFunction(MachineFunction &MF); 82234285Sdim 83234285Sdim const char *getPassName() const { return "Hexagon Hardware Loops"; } 84234285Sdim 85234285Sdim virtual void getAnalysisUsage(AnalysisUsage &AU) const { 86234285Sdim AU.addRequired<MachineDominatorTree>(); 87234285Sdim AU.addRequired<MachineLoopInfo>(); 88234285Sdim MachineFunctionPass::getAnalysisUsage(AU); 89234285Sdim } 90234285Sdim 91234285Sdim private: 92252723Sdim /// Kinds of comparisons in the compare instructions. 93252723Sdim struct Comparison { 94252723Sdim enum Kind { 95252723Sdim EQ = 0x01, 96252723Sdim NE = 0x02, 97252723Sdim L = 0x04, // Less-than property. 98252723Sdim G = 0x08, // Greater-than property. 99252723Sdim U = 0x40, // Unsigned property. 100252723Sdim LTs = L, 101252723Sdim LEs = L | EQ, 102252723Sdim GTs = G, 103252723Sdim GEs = G | EQ, 104252723Sdim LTu = L | U, 105252723Sdim LEu = L | EQ | U, 106252723Sdim GTu = G | U, 107252723Sdim GEu = G | EQ | U 108252723Sdim }; 109252723Sdim 110252723Sdim static Kind getSwappedComparison(Kind Cmp) { 111252723Sdim assert ((!((Cmp & L) && (Cmp & G))) && "Malformed comparison operator"); 112252723Sdim if ((Cmp & L) || (Cmp & G)) 113252723Sdim return (Kind)(Cmp ^ (L|G)); 114252723Sdim return Cmp; 115252723Sdim } 116252723Sdim }; 117252723Sdim 118252723Sdim /// \brief Find the register that contains the loop controlling 119234285Sdim /// induction variable. 120252723Sdim /// If successful, it will return true and set the \p Reg, \p IVBump 121252723Sdim /// and \p IVOp arguments. Otherwise it will return false. 122252723Sdim /// The returned induction register is the register R that follows the 123252723Sdim /// following induction pattern: 124252723Sdim /// loop: 125252723Sdim /// R = phi ..., [ R.next, LatchBlock ] 126252723Sdim /// R.next = R + #bump 127252723Sdim /// if (R.next < #N) goto loop 128252723Sdim /// IVBump is the immediate value added to R, and IVOp is the instruction 129252723Sdim /// "R.next = R + #bump". 130252723Sdim bool findInductionRegister(MachineLoop *L, unsigned &Reg, 131252723Sdim int64_t &IVBump, MachineInstr *&IVOp) const; 132234285Sdim 133252723Sdim /// \brief Analyze the statements in a loop to determine if the loop 134252723Sdim /// has a computable trip count and, if so, return a value that represents 135252723Sdim /// the trip count expression. 136252723Sdim CountValue *getLoopTripCount(MachineLoop *L, 137263509Sdim SmallVectorImpl<MachineInstr *> &OldInsts); 138234285Sdim 139252723Sdim /// \brief Return the expression that represents the number of times 140252723Sdim /// a loop iterates. The function takes the operands that represent the 141252723Sdim /// loop start value, loop end value, and induction value. Based upon 142252723Sdim /// these operands, the function attempts to compute the trip count. 143252723Sdim /// If the trip count is not directly available (as an immediate value, 144252723Sdim /// or a register), the function will attempt to insert computation of it 145252723Sdim /// to the loop's preheader. 146252723Sdim CountValue *computeCount(MachineLoop *Loop, 147252723Sdim const MachineOperand *Start, 148252723Sdim const MachineOperand *End, 149252723Sdim unsigned IVReg, 150252723Sdim int64_t IVBump, 151252723Sdim Comparison::Kind Cmp) const; 152234285Sdim 153252723Sdim /// \brief Return true if the instruction is not valid within a hardware 154252723Sdim /// loop. 155234285Sdim bool isInvalidLoopOperation(const MachineInstr *MI) const; 156234285Sdim 157252723Sdim /// \brief Return true if the loop contains an instruction that inhibits 158252723Sdim /// using the hardware loop. 159234285Sdim bool containsInvalidInstruction(MachineLoop *L) const; 160234285Sdim 161252723Sdim /// \brief Given a loop, check if we can convert it to a hardware loop. 162252723Sdim /// If so, then perform the conversion and return true. 163234285Sdim bool convertToHardwareLoop(MachineLoop *L); 164234285Sdim 165252723Sdim /// \brief Return true if the instruction is now dead. 166252723Sdim bool isDead(const MachineInstr *MI, 167263509Sdim SmallVectorImpl<MachineInstr *> &DeadPhis) const; 168252723Sdim 169252723Sdim /// \brief Remove the instruction if it is now dead. 170252723Sdim void removeIfDead(MachineInstr *MI); 171252723Sdim 172252723Sdim /// \brief Make sure that the "bump" instruction executes before the 173252723Sdim /// compare. We need that for the IV fixup, so that the compare 174252723Sdim /// instruction would not use a bumped value that has not yet been 175252723Sdim /// defined. If the instructions are out of order, try to reorder them. 176252723Sdim bool orderBumpCompare(MachineInstr *BumpI, MachineInstr *CmpI); 177252723Sdim 178252723Sdim /// \brief Get the instruction that loads an immediate value into \p R, 179252723Sdim /// or 0 if such an instruction does not exist. 180252723Sdim MachineInstr *defWithImmediate(unsigned R); 181252723Sdim 182252723Sdim /// \brief Get the immediate value referenced to by \p MO, either for 183252723Sdim /// immediate operands, or for register operands, where the register 184252723Sdim /// was defined with an immediate value. 185252723Sdim int64_t getImmediate(MachineOperand &MO); 186252723Sdim 187252723Sdim /// \brief Reset the given machine operand to now refer to a new immediate 188252723Sdim /// value. Assumes that the operand was already referencing an immediate 189252723Sdim /// value, either directly, or via a register. 190252723Sdim void setImmediate(MachineOperand &MO, int64_t Val); 191252723Sdim 192252723Sdim /// \brief Fix the data flow of the induction varible. 193252723Sdim /// The desired flow is: phi ---> bump -+-> comparison-in-latch. 194252723Sdim /// | 195252723Sdim /// +-> back to phi 196252723Sdim /// where "bump" is the increment of the induction variable: 197252723Sdim /// iv = iv + #const. 198252723Sdim /// Due to some prior code transformations, the actual flow may look 199252723Sdim /// like this: 200252723Sdim /// phi -+-> bump ---> back to phi 201252723Sdim /// | 202252723Sdim /// +-> comparison-in-latch (against upper_bound-bump), 203252723Sdim /// i.e. the comparison that controls the loop execution may be using 204252723Sdim /// the value of the induction variable from before the increment. 205252723Sdim /// 206252723Sdim /// Return true if the loop's flow is the desired one (i.e. it's 207252723Sdim /// either been fixed, or no fixing was necessary). 208252723Sdim /// Otherwise, return false. This can happen if the induction variable 209252723Sdim /// couldn't be identified, or if the value in the latch's comparison 210252723Sdim /// cannot be adjusted to reflect the post-bump value. 211252723Sdim bool fixupInductionVariable(MachineLoop *L); 212252723Sdim 213252723Sdim /// \brief Given a loop, if it does not have a preheader, create one. 214252723Sdim /// Return the block that is the preheader. 215252723Sdim MachineBasicBlock *createPreheaderForLoop(MachineLoop *L); 216234285Sdim }; 217234285Sdim 218234285Sdim char HexagonHardwareLoops::ID = 0; 219252723Sdim#ifndef NDEBUG 220252723Sdim int HexagonHardwareLoops::Counter = 0; 221252723Sdim#endif 222234285Sdim 223252723Sdim /// \brief Abstraction for a trip count of a loop. A smaller vesrsion 224252723Sdim /// of the MachineOperand class without the concerns of changing the 225252723Sdim /// operand representation. 226234285Sdim class CountValue { 227234285Sdim public: 228234285Sdim enum CountValueType { 229234285Sdim CV_Register, 230234285Sdim CV_Immediate 231234285Sdim }; 232234285Sdim private: 233234285Sdim CountValueType Kind; 234234285Sdim union Values { 235252723Sdim struct { 236252723Sdim unsigned Reg; 237252723Sdim unsigned Sub; 238252723Sdim } R; 239252723Sdim unsigned ImmVal; 240234285Sdim } Contents; 241234285Sdim 242234285Sdim public: 243252723Sdim explicit CountValue(CountValueType t, unsigned v, unsigned u = 0) { 244252723Sdim Kind = t; 245252723Sdim if (Kind == CV_Register) { 246252723Sdim Contents.R.Reg = v; 247252723Sdim Contents.R.Sub = u; 248252723Sdim } else { 249252723Sdim Contents.ImmVal = v; 250252723Sdim } 251252723Sdim } 252234285Sdim bool isReg() const { return Kind == CV_Register; } 253234285Sdim bool isImm() const { return Kind == CV_Immediate; } 254234285Sdim 255234285Sdim unsigned getReg() const { 256234285Sdim assert(isReg() && "Wrong CountValue accessor"); 257252723Sdim return Contents.R.Reg; 258234285Sdim } 259252723Sdim unsigned getSubReg() const { 260252723Sdim assert(isReg() && "Wrong CountValue accessor"); 261252723Sdim return Contents.R.Sub; 262234285Sdim } 263252723Sdim unsigned getImm() const { 264234285Sdim assert(isImm() && "Wrong CountValue accessor"); 265234285Sdim return Contents.ImmVal; 266234285Sdim } 267234285Sdim 268234285Sdim void print(raw_ostream &OS, const TargetMachine *TM = 0) const { 269252723Sdim const TargetRegisterInfo *TRI = TM ? TM->getRegisterInfo() : 0; 270252723Sdim if (isReg()) { OS << PrintReg(Contents.R.Reg, TRI, Contents.R.Sub); } 271252723Sdim if (isImm()) { OS << Contents.ImmVal; } 272234285Sdim } 273234285Sdim }; 274252723Sdim} // end anonymous namespace 275234285Sdim 276234285Sdim 277252723SdimINITIALIZE_PASS_BEGIN(HexagonHardwareLoops, "hwloops", 278252723Sdim "Hexagon Hardware Loops", false, false) 279252723SdimINITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) 280252723SdimINITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) 281252723SdimINITIALIZE_PASS_END(HexagonHardwareLoops, "hwloops", 282252723Sdim "Hexagon Hardware Loops", false, false) 283234285Sdim 284234285Sdim 285252723Sdim/// \brief Returns true if the instruction is a hardware loop instruction. 286234285Sdimstatic bool isHardwareLoop(const MachineInstr *MI) { 287234285Sdim return MI->getOpcode() == Hexagon::LOOP0_r || 288234285Sdim MI->getOpcode() == Hexagon::LOOP0_i; 289234285Sdim} 290234285Sdim 291234285SdimFunctionPass *llvm::createHexagonHardwareLoops() { 292234285Sdim return new HexagonHardwareLoops(); 293234285Sdim} 294234285Sdim 295234285Sdim 296234285Sdimbool HexagonHardwareLoops::runOnMachineFunction(MachineFunction &MF) { 297234285Sdim DEBUG(dbgs() << "********* Hexagon Hardware Loops *********\n"); 298234285Sdim 299234285Sdim bool Changed = false; 300234285Sdim 301234285Sdim MLI = &getAnalysis<MachineLoopInfo>(); 302234285Sdim MRI = &MF.getRegInfo(); 303252723Sdim MDT = &getAnalysis<MachineDominatorTree>(); 304252723Sdim TM = static_cast<const HexagonTargetMachine*>(&MF.getTarget()); 305252723Sdim TII = static_cast<const HexagonInstrInfo*>(TM->getInstrInfo()); 306252723Sdim TRI = static_cast<const HexagonRegisterInfo*>(TM->getRegisterInfo()); 307234285Sdim 308234285Sdim for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end(); 309234285Sdim I != E; ++I) { 310234285Sdim MachineLoop *L = *I; 311252723Sdim if (!L->getParentLoop()) 312234285Sdim Changed |= convertToHardwareLoop(L); 313234285Sdim } 314234285Sdim 315234285Sdim return Changed; 316234285Sdim} 317234285Sdim 318252723Sdim 319252723Sdimbool HexagonHardwareLoops::findInductionRegister(MachineLoop *L, 320252723Sdim unsigned &Reg, 321252723Sdim int64_t &IVBump, 322252723Sdim MachineInstr *&IVOp 323252723Sdim ) const { 324252723Sdim MachineBasicBlock *Header = L->getHeader(); 325252723Sdim MachineBasicBlock *Preheader = L->getLoopPreheader(); 326252723Sdim MachineBasicBlock *Latch = L->getLoopLatch(); 327252723Sdim if (!Header || !Preheader || !Latch) 328252723Sdim return false; 329252723Sdim 330252723Sdim // This pair represents an induction register together with an immediate 331252723Sdim // value that will be added to it in each loop iteration. 332252723Sdim typedef std::pair<unsigned,int64_t> RegisterBump; 333252723Sdim 334252723Sdim // Mapping: R.next -> (R, bump), where R, R.next and bump are derived 335252723Sdim // from an induction operation 336252723Sdim // R.next = R + bump 337252723Sdim // where bump is an immediate value. 338252723Sdim typedef std::map<unsigned,RegisterBump> InductionMap; 339252723Sdim 340252723Sdim InductionMap IndMap; 341252723Sdim 342252723Sdim typedef MachineBasicBlock::instr_iterator instr_iterator; 343252723Sdim for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); 344252723Sdim I != E && I->isPHI(); ++I) { 345252723Sdim MachineInstr *Phi = &*I; 346252723Sdim 347252723Sdim // Have a PHI instruction. Get the operand that corresponds to the 348252723Sdim // latch block, and see if is a result of an addition of form "reg+imm", 349252723Sdim // where the "reg" is defined by the PHI node we are looking at. 350252723Sdim for (unsigned i = 1, n = Phi->getNumOperands(); i < n; i += 2) { 351252723Sdim if (Phi->getOperand(i+1).getMBB() != Latch) 352252723Sdim continue; 353252723Sdim 354252723Sdim unsigned PhiOpReg = Phi->getOperand(i).getReg(); 355252723Sdim MachineInstr *DI = MRI->getVRegDef(PhiOpReg); 356252723Sdim unsigned UpdOpc = DI->getOpcode(); 357252723Sdim bool isAdd = (UpdOpc == Hexagon::ADD_ri); 358252723Sdim 359252723Sdim if (isAdd) { 360252723Sdim // If the register operand to the add is the PHI we're 361252723Sdim // looking at, this meets the induction pattern. 362252723Sdim unsigned IndReg = DI->getOperand(1).getReg(); 363252723Sdim if (MRI->getVRegDef(IndReg) == Phi) { 364252723Sdim unsigned UpdReg = DI->getOperand(0).getReg(); 365252723Sdim int64_t V = DI->getOperand(2).getImm(); 366252723Sdim IndMap.insert(std::make_pair(UpdReg, std::make_pair(IndReg, V))); 367252723Sdim } 368252723Sdim } 369252723Sdim } // for (i) 370252723Sdim } // for (instr) 371252723Sdim 372252723Sdim SmallVector<MachineOperand,2> Cond; 373252723Sdim MachineBasicBlock *TB = 0, *FB = 0; 374252723Sdim bool NotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Cond, false); 375252723Sdim if (NotAnalyzed) 376252723Sdim return false; 377252723Sdim 378252723Sdim unsigned CSz = Cond.size(); 379252723Sdim assert (CSz == 1 || CSz == 2); 380252723Sdim unsigned PredR = Cond[CSz-1].getReg(); 381252723Sdim 382252723Sdim MachineInstr *PredI = MRI->getVRegDef(PredR); 383252723Sdim if (!PredI->isCompare()) 384252723Sdim return false; 385252723Sdim 386252723Sdim unsigned CmpReg1 = 0, CmpReg2 = 0; 387252723Sdim int CmpImm = 0, CmpMask = 0; 388252723Sdim bool CmpAnalyzed = TII->analyzeCompare(PredI, CmpReg1, CmpReg2, 389252723Sdim CmpMask, CmpImm); 390252723Sdim // Fail if the compare was not analyzed, or it's not comparing a register 391252723Sdim // with an immediate value. Not checking the mask here, since we handle 392252723Sdim // the individual compare opcodes (including CMPb) later on. 393252723Sdim if (!CmpAnalyzed) 394252723Sdim return false; 395252723Sdim 396252723Sdim // Exactly one of the input registers to the comparison should be among 397252723Sdim // the induction registers. 398252723Sdim InductionMap::iterator IndMapEnd = IndMap.end(); 399252723Sdim InductionMap::iterator F = IndMapEnd; 400252723Sdim if (CmpReg1 != 0) { 401252723Sdim InductionMap::iterator F1 = IndMap.find(CmpReg1); 402252723Sdim if (F1 != IndMapEnd) 403252723Sdim F = F1; 404252723Sdim } 405252723Sdim if (CmpReg2 != 0) { 406252723Sdim InductionMap::iterator F2 = IndMap.find(CmpReg2); 407252723Sdim if (F2 != IndMapEnd) { 408252723Sdim if (F != IndMapEnd) 409252723Sdim return false; 410252723Sdim F = F2; 411252723Sdim } 412252723Sdim } 413252723Sdim if (F == IndMapEnd) 414252723Sdim return false; 415252723Sdim 416252723Sdim Reg = F->second.first; 417252723Sdim IVBump = F->second.second; 418252723Sdim IVOp = MRI->getVRegDef(F->first); 419252723Sdim return true; 420252723Sdim} 421252723Sdim 422252723Sdim 423252723Sdim/// \brief Analyze the statements in a loop to determine if the loop has 424252723Sdim/// a computable trip count and, if so, return a value that represents 425252723Sdim/// the trip count expression. 426234285Sdim/// 427252723Sdim/// This function iterates over the phi nodes in the loop to check for 428252723Sdim/// induction variable patterns that are used in the calculation for 429252723Sdim/// the number of time the loop is executed. 430252723SdimCountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L, 431263509Sdim SmallVectorImpl<MachineInstr *> &OldInsts) { 432234285Sdim MachineBasicBlock *TopMBB = L->getTopBlock(); 433234285Sdim MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(); 434234285Sdim assert(PI != TopMBB->pred_end() && 435234285Sdim "Loop must have more than one incoming edge!"); 436234285Sdim MachineBasicBlock *Backedge = *PI++; 437252723Sdim if (PI == TopMBB->pred_end()) // dead loop? 438252723Sdim return 0; 439234285Sdim MachineBasicBlock *Incoming = *PI++; 440252723Sdim if (PI != TopMBB->pred_end()) // multiple backedges? 441252723Sdim return 0; 442234285Sdim 443252723Sdim // Make sure there is one incoming and one backedge and determine which 444234285Sdim // is which. 445234285Sdim if (L->contains(Incoming)) { 446234285Sdim if (L->contains(Backedge)) 447234285Sdim return 0; 448234285Sdim std::swap(Incoming, Backedge); 449234285Sdim } else if (!L->contains(Backedge)) 450234285Sdim return 0; 451234285Sdim 452252723Sdim // Look for the cmp instruction to determine if we can get a useful trip 453252723Sdim // count. The trip count can be either a register or an immediate. The 454252723Sdim // location of the value depends upon the type (reg or imm). 455252723Sdim MachineBasicBlock *Latch = L->getLoopLatch(); 456252723Sdim if (!Latch) 457252723Sdim return 0; 458252723Sdim 459252723Sdim unsigned IVReg = 0; 460252723Sdim int64_t IVBump = 0; 461252723Sdim MachineInstr *IVOp; 462252723Sdim bool FoundIV = findInductionRegister(L, IVReg, IVBump, IVOp); 463252723Sdim if (!FoundIV) 464252723Sdim return 0; 465252723Sdim 466252723Sdim MachineBasicBlock *Preheader = L->getLoopPreheader(); 467252723Sdim 468252723Sdim MachineOperand *InitialValue = 0; 469252723Sdim MachineInstr *IV_Phi = MRI->getVRegDef(IVReg); 470252723Sdim for (unsigned i = 1, n = IV_Phi->getNumOperands(); i < n; i += 2) { 471252723Sdim MachineBasicBlock *MBB = IV_Phi->getOperand(i+1).getMBB(); 472252723Sdim if (MBB == Preheader) 473252723Sdim InitialValue = &IV_Phi->getOperand(i); 474252723Sdim else if (MBB == Latch) 475252723Sdim IVReg = IV_Phi->getOperand(i).getReg(); // Want IV reg after bump. 476234285Sdim } 477252723Sdim if (!InitialValue) 478252723Sdim return 0; 479234285Sdim 480252723Sdim SmallVector<MachineOperand,2> Cond; 481252723Sdim MachineBasicBlock *TB = 0, *FB = 0; 482252723Sdim bool NotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Cond, false); 483252723Sdim if (NotAnalyzed) 484252723Sdim return 0; 485234285Sdim 486252723Sdim MachineBasicBlock *Header = L->getHeader(); 487252723Sdim // TB must be non-null. If FB is also non-null, one of them must be 488252723Sdim // the header. Otherwise, branch to TB could be exiting the loop, and 489252723Sdim // the fall through can go to the header. 490252723Sdim assert (TB && "Latch block without a branch?"); 491252723Sdim assert ((!FB || TB == Header || FB == Header) && "Branches not to header?"); 492252723Sdim if (!TB || (FB && TB != Header && FB != Header)) 493252723Sdim return 0; 494252723Sdim 495252723Sdim // Branches of form "if (!P) ..." cause HexagonInstrInfo::AnalyzeBranch 496252723Sdim // to put imm(0), followed by P in the vector Cond. 497252723Sdim // If TB is not the header, it means that the "not-taken" path must lead 498252723Sdim // to the header. 499252723Sdim bool Negated = (Cond.size() > 1) ^ (TB != Header); 500252723Sdim unsigned PredReg = Cond[Cond.size()-1].getReg(); 501252723Sdim MachineInstr *CondI = MRI->getVRegDef(PredReg); 502252723Sdim unsigned CondOpc = CondI->getOpcode(); 503252723Sdim 504252723Sdim unsigned CmpReg1 = 0, CmpReg2 = 0; 505252723Sdim int Mask = 0, ImmValue = 0; 506252723Sdim bool AnalyzedCmp = TII->analyzeCompare(CondI, CmpReg1, CmpReg2, 507252723Sdim Mask, ImmValue); 508252723Sdim if (!AnalyzedCmp) 509252723Sdim return 0; 510252723Sdim 511252723Sdim // The comparison operator type determines how we compute the loop 512252723Sdim // trip count. 513252723Sdim OldInsts.push_back(CondI); 514252723Sdim OldInsts.push_back(IVOp); 515252723Sdim 516252723Sdim // Sadly, the following code gets information based on the position 517252723Sdim // of the operands in the compare instruction. This has to be done 518252723Sdim // this way, because the comparisons check for a specific relationship 519252723Sdim // between the operands (e.g. is-less-than), rather than to find out 520252723Sdim // what relationship the operands are in (as on PPC). 521252723Sdim Comparison::Kind Cmp; 522252723Sdim bool isSwapped = false; 523252723Sdim const MachineOperand &Op1 = CondI->getOperand(1); 524252723Sdim const MachineOperand &Op2 = CondI->getOperand(2); 525252723Sdim const MachineOperand *EndValue = 0; 526252723Sdim 527252723Sdim if (Op1.isReg()) { 528252723Sdim if (Op2.isImm() || Op1.getReg() == IVReg) 529252723Sdim EndValue = &Op2; 530252723Sdim else { 531252723Sdim EndValue = &Op1; 532252723Sdim isSwapped = true; 533252723Sdim } 534234285Sdim } 535234285Sdim 536252723Sdim if (!EndValue) 537252723Sdim return 0; 538234285Sdim 539252723Sdim switch (CondOpc) { 540252723Sdim case Hexagon::CMPEQri: 541252723Sdim case Hexagon::CMPEQrr: 542252723Sdim Cmp = !Negated ? Comparison::EQ : Comparison::NE; 543252723Sdim break; 544252723Sdim case Hexagon::CMPGTUri: 545252723Sdim case Hexagon::CMPGTUrr: 546252723Sdim Cmp = !Negated ? Comparison::GTu : Comparison::LEu; 547252723Sdim break; 548252723Sdim case Hexagon::CMPGTri: 549252723Sdim case Hexagon::CMPGTrr: 550252723Sdim Cmp = !Negated ? Comparison::GTs : Comparison::LEs; 551252723Sdim break; 552252723Sdim // Very limited support for byte/halfword compares. 553252723Sdim case Hexagon::CMPbEQri_V4: 554252723Sdim case Hexagon::CMPhEQri_V4: { 555252723Sdim if (IVBump != 1) 556252723Sdim return 0; 557234285Sdim 558252723Sdim int64_t InitV, EndV; 559252723Sdim // Since the comparisons are "ri", the EndValue should be an 560252723Sdim // immediate. Check it just in case. 561252723Sdim assert(EndValue->isImm() && "Unrecognized latch comparison"); 562252723Sdim EndV = EndValue->getImm(); 563252723Sdim // Allow InitialValue to be a register defined with an immediate. 564252723Sdim if (InitialValue->isReg()) { 565252723Sdim if (!defWithImmediate(InitialValue->getReg())) 566234285Sdim return 0; 567252723Sdim InitV = getImmediate(*InitialValue); 568234285Sdim } else { 569252723Sdim assert(InitialValue->isImm()); 570252723Sdim InitV = InitialValue->getImm(); 571234285Sdim } 572252723Sdim if (InitV >= EndV) 573252723Sdim return 0; 574252723Sdim if (CondOpc == Hexagon::CMPbEQri_V4) { 575252723Sdim if (!isInt<8>(InitV) || !isInt<8>(EndV)) 576252723Sdim return 0; 577252723Sdim } else { // Hexagon::CMPhEQri_V4 578252723Sdim if (!isInt<16>(InitV) || !isInt<16>(EndV)) 579252723Sdim return 0; 580252723Sdim } 581252723Sdim Cmp = !Negated ? Comparison::EQ : Comparison::NE; 582252723Sdim break; 583234285Sdim } 584252723Sdim default: 585252723Sdim return 0; 586234285Sdim } 587252723Sdim 588252723Sdim if (isSwapped) 589252723Sdim Cmp = Comparison::getSwappedComparison(Cmp); 590252723Sdim 591252723Sdim if (InitialValue->isReg()) { 592252723Sdim unsigned R = InitialValue->getReg(); 593252723Sdim MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent(); 594252723Sdim if (!MDT->properlyDominates(DefBB, Header)) 595252723Sdim return 0; 596252723Sdim OldInsts.push_back(MRI->getVRegDef(R)); 597252723Sdim } 598252723Sdim if (EndValue->isReg()) { 599252723Sdim unsigned R = EndValue->getReg(); 600252723Sdim MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent(); 601252723Sdim if (!MDT->properlyDominates(DefBB, Header)) 602252723Sdim return 0; 603252723Sdim } 604252723Sdim 605252723Sdim return computeCount(L, InitialValue, EndValue, IVReg, IVBump, Cmp); 606234285Sdim} 607234285Sdim 608252723Sdim/// \brief Helper function that returns the expression that represents the 609252723Sdim/// number of times a loop iterates. The function takes the operands that 610252723Sdim/// represent the loop start value, loop end value, and induction value. 611252723Sdim/// Based upon these operands, the function attempts to compute the trip count. 612252723SdimCountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop, 613252723Sdim const MachineOperand *Start, 614252723Sdim const MachineOperand *End, 615252723Sdim unsigned IVReg, 616252723Sdim int64_t IVBump, 617252723Sdim Comparison::Kind Cmp) const { 618252723Sdim // Cannot handle comparison EQ, i.e. while (A == B). 619252723Sdim if (Cmp == Comparison::EQ) 620252723Sdim return 0; 621252723Sdim 622252723Sdim // Check if either the start or end values are an assignment of an immediate. 623252723Sdim // If so, use the immediate value rather than the register. 624252723Sdim if (Start->isReg()) { 625252723Sdim const MachineInstr *StartValInstr = MRI->getVRegDef(Start->getReg()); 626252723Sdim if (StartValInstr && StartValInstr->getOpcode() == Hexagon::TFRI) 627252723Sdim Start = &StartValInstr->getOperand(1); 628252723Sdim } 629252723Sdim if (End->isReg()) { 630252723Sdim const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg()); 631252723Sdim if (EndValInstr && EndValInstr->getOpcode() == Hexagon::TFRI) 632252723Sdim End = &EndValInstr->getOperand(1); 633252723Sdim } 634252723Sdim 635252723Sdim assert (Start->isReg() || Start->isImm()); 636252723Sdim assert (End->isReg() || End->isImm()); 637252723Sdim 638252723Sdim bool CmpLess = Cmp & Comparison::L; 639252723Sdim bool CmpGreater = Cmp & Comparison::G; 640252723Sdim bool CmpHasEqual = Cmp & Comparison::EQ; 641252723Sdim 642252723Sdim // Avoid certain wrap-arounds. This doesn't detect all wrap-arounds. 643252723Sdim // If loop executes while iv is "less" with the iv value going down, then 644252723Sdim // the iv must wrap. 645252723Sdim if (CmpLess && IVBump < 0) 646252723Sdim return 0; 647252723Sdim // If loop executes while iv is "greater" with the iv value going up, then 648252723Sdim // the iv must wrap. 649252723Sdim if (CmpGreater && IVBump > 0) 650252723Sdim return 0; 651252723Sdim 652252723Sdim if (Start->isImm() && End->isImm()) { 653252723Sdim // Both, start and end are immediates. 654252723Sdim int64_t StartV = Start->getImm(); 655252723Sdim int64_t EndV = End->getImm(); 656252723Sdim int64_t Dist = EndV - StartV; 657252723Sdim if (Dist == 0) 658252723Sdim return 0; 659252723Sdim 660252723Sdim bool Exact = (Dist % IVBump) == 0; 661252723Sdim 662252723Sdim if (Cmp == Comparison::NE) { 663252723Sdim if (!Exact) 664252723Sdim return 0; 665252723Sdim if ((Dist < 0) ^ (IVBump < 0)) 666252723Sdim return 0; 667252723Sdim } 668252723Sdim 669252723Sdim // For comparisons that include the final value (i.e. include equality 670252723Sdim // with the final value), we need to increase the distance by 1. 671252723Sdim if (CmpHasEqual) 672252723Sdim Dist = Dist > 0 ? Dist+1 : Dist-1; 673252723Sdim 674252723Sdim // assert (CmpLess => Dist > 0); 675252723Sdim assert ((!CmpLess || Dist > 0) && "Loop should never iterate!"); 676252723Sdim // assert (CmpGreater => Dist < 0); 677252723Sdim assert ((!CmpGreater || Dist < 0) && "Loop should never iterate!"); 678252723Sdim 679252723Sdim // "Normalized" distance, i.e. with the bump set to +-1. 680252723Sdim int64_t Dist1 = (IVBump > 0) ? (Dist + (IVBump-1)) / IVBump 681252723Sdim : (-Dist + (-IVBump-1)) / (-IVBump); 682252723Sdim assert (Dist1 > 0 && "Fishy thing. Both operands have the same sign."); 683252723Sdim 684252723Sdim uint64_t Count = Dist1; 685252723Sdim 686252723Sdim if (Count > 0xFFFFFFFFULL) 687252723Sdim return 0; 688252723Sdim 689252723Sdim return new CountValue(CountValue::CV_Immediate, Count); 690252723Sdim } 691252723Sdim 692252723Sdim // A general case: Start and End are some values, but the actual 693252723Sdim // iteration count may not be available. If it is not, insert 694252723Sdim // a computation of it into the preheader. 695252723Sdim 696252723Sdim // If the induction variable bump is not a power of 2, quit. 697252723Sdim // Othwerise we'd need a general integer division. 698252723Sdim if (!isPowerOf2_64(abs64(IVBump))) 699252723Sdim return 0; 700252723Sdim 701252723Sdim MachineBasicBlock *PH = Loop->getLoopPreheader(); 702252723Sdim assert (PH && "Should have a preheader by now"); 703252723Sdim MachineBasicBlock::iterator InsertPos = PH->getFirstTerminator(); 704252723Sdim DebugLoc DL = (InsertPos != PH->end()) ? InsertPos->getDebugLoc() 705252723Sdim : DebugLoc(); 706252723Sdim 707252723Sdim // If Start is an immediate and End is a register, the trip count 708252723Sdim // will be "reg - imm". Hexagon's "subtract immediate" instruction 709252723Sdim // is actually "reg + -imm". 710252723Sdim 711252723Sdim // If the loop IV is going downwards, i.e. if the bump is negative, 712252723Sdim // then the iteration count (computed as End-Start) will need to be 713252723Sdim // negated. To avoid the negation, just swap Start and End. 714252723Sdim if (IVBump < 0) { 715252723Sdim std::swap(Start, End); 716252723Sdim IVBump = -IVBump; 717252723Sdim } 718252723Sdim // Cmp may now have a wrong direction, e.g. LEs may now be GEs. 719252723Sdim // Signedness, and "including equality" are preserved. 720252723Sdim 721252723Sdim bool RegToImm = Start->isReg() && End->isImm(); // for (reg..imm) 722252723Sdim bool RegToReg = Start->isReg() && End->isReg(); // for (reg..reg) 723252723Sdim 724252723Sdim int64_t StartV = 0, EndV = 0; 725252723Sdim if (Start->isImm()) 726252723Sdim StartV = Start->getImm(); 727252723Sdim if (End->isImm()) 728252723Sdim EndV = End->getImm(); 729252723Sdim 730252723Sdim int64_t AdjV = 0; 731252723Sdim // To compute the iteration count, we would need this computation: 732252723Sdim // Count = (End - Start + (IVBump-1)) / IVBump 733252723Sdim // or, when CmpHasEqual: 734252723Sdim // Count = (End - Start + (IVBump-1)+1) / IVBump 735252723Sdim // The "IVBump-1" part is the adjustment (AdjV). We can avoid 736252723Sdim // generating an instruction specifically to add it if we can adjust 737252723Sdim // the immediate values for Start or End. 738252723Sdim 739252723Sdim if (CmpHasEqual) { 740252723Sdim // Need to add 1 to the total iteration count. 741252723Sdim if (Start->isImm()) 742252723Sdim StartV--; 743252723Sdim else if (End->isImm()) 744252723Sdim EndV++; 745252723Sdim else 746252723Sdim AdjV += 1; 747252723Sdim } 748252723Sdim 749252723Sdim if (Cmp != Comparison::NE) { 750252723Sdim if (Start->isImm()) 751252723Sdim StartV -= (IVBump-1); 752252723Sdim else if (End->isImm()) 753252723Sdim EndV += (IVBump-1); 754252723Sdim else 755252723Sdim AdjV += (IVBump-1); 756252723Sdim } 757252723Sdim 758252723Sdim unsigned R = 0, SR = 0; 759252723Sdim if (Start->isReg()) { 760252723Sdim R = Start->getReg(); 761252723Sdim SR = Start->getSubReg(); 762252723Sdim } else { 763252723Sdim R = End->getReg(); 764252723Sdim SR = End->getSubReg(); 765252723Sdim } 766252723Sdim const TargetRegisterClass *RC = MRI->getRegClass(R); 767252723Sdim // Hardware loops cannot handle 64-bit registers. If it's a double 768252723Sdim // register, it has to have a subregister. 769252723Sdim if (!SR && RC == &Hexagon::DoubleRegsRegClass) 770252723Sdim return 0; 771252723Sdim const TargetRegisterClass *IntRC = &Hexagon::IntRegsRegClass; 772252723Sdim 773252723Sdim // Compute DistR (register with the distance between Start and End). 774252723Sdim unsigned DistR, DistSR; 775252723Sdim 776252723Sdim // Avoid special case, where the start value is an imm(0). 777252723Sdim if (Start->isImm() && StartV == 0) { 778252723Sdim DistR = End->getReg(); 779252723Sdim DistSR = End->getSubReg(); 780252723Sdim } else { 781252723Sdim const MCInstrDesc &SubD = RegToReg ? TII->get(Hexagon::SUB_rr) : 782252723Sdim (RegToImm ? TII->get(Hexagon::SUB_ri) : 783252723Sdim TII->get(Hexagon::ADD_ri)); 784252723Sdim unsigned SubR = MRI->createVirtualRegister(IntRC); 785252723Sdim MachineInstrBuilder SubIB = 786252723Sdim BuildMI(*PH, InsertPos, DL, SubD, SubR); 787252723Sdim 788252723Sdim if (RegToReg) { 789252723Sdim SubIB.addReg(End->getReg(), 0, End->getSubReg()) 790252723Sdim .addReg(Start->getReg(), 0, Start->getSubReg()); 791252723Sdim } else if (RegToImm) { 792252723Sdim SubIB.addImm(EndV) 793252723Sdim .addReg(Start->getReg(), 0, Start->getSubReg()); 794252723Sdim } else { // ImmToReg 795252723Sdim SubIB.addReg(End->getReg(), 0, End->getSubReg()) 796252723Sdim .addImm(-StartV); 797252723Sdim } 798252723Sdim DistR = SubR; 799252723Sdim DistSR = 0; 800252723Sdim } 801252723Sdim 802252723Sdim // From DistR, compute AdjR (register with the adjusted distance). 803252723Sdim unsigned AdjR, AdjSR; 804252723Sdim 805252723Sdim if (AdjV == 0) { 806252723Sdim AdjR = DistR; 807252723Sdim AdjSR = DistSR; 808252723Sdim } else { 809252723Sdim // Generate CountR = ADD DistR, AdjVal 810252723Sdim unsigned AddR = MRI->createVirtualRegister(IntRC); 811252723Sdim const MCInstrDesc &AddD = TII->get(Hexagon::ADD_ri); 812252723Sdim BuildMI(*PH, InsertPos, DL, AddD, AddR) 813252723Sdim .addReg(DistR, 0, DistSR) 814252723Sdim .addImm(AdjV); 815252723Sdim 816252723Sdim AdjR = AddR; 817252723Sdim AdjSR = 0; 818252723Sdim } 819252723Sdim 820252723Sdim // From AdjR, compute CountR (register with the final count). 821252723Sdim unsigned CountR, CountSR; 822252723Sdim 823252723Sdim if (IVBump == 1) { 824252723Sdim CountR = AdjR; 825252723Sdim CountSR = AdjSR; 826252723Sdim } else { 827252723Sdim // The IV bump is a power of two. Log_2(IV bump) is the shift amount. 828252723Sdim unsigned Shift = Log2_32(IVBump); 829252723Sdim 830252723Sdim // Generate NormR = LSR DistR, Shift. 831252723Sdim unsigned LsrR = MRI->createVirtualRegister(IntRC); 832252723Sdim const MCInstrDesc &LsrD = TII->get(Hexagon::LSR_ri); 833252723Sdim BuildMI(*PH, InsertPos, DL, LsrD, LsrR) 834252723Sdim .addReg(AdjR, 0, AdjSR) 835252723Sdim .addImm(Shift); 836252723Sdim 837252723Sdim CountR = LsrR; 838252723Sdim CountSR = 0; 839252723Sdim } 840252723Sdim 841252723Sdim return new CountValue(CountValue::CV_Register, CountR, CountSR); 842234285Sdim} 843234285Sdim 844234285Sdim 845252723Sdim/// \brief Return true if the operation is invalid within hardware loop. 846252723Sdimbool HexagonHardwareLoops::isInvalidLoopOperation( 847252723Sdim const MachineInstr *MI) const { 848252723Sdim 849234285Sdim // call is not allowed because the callee may use a hardware loop 850252723Sdim if (MI->getDesc().isCall()) 851234285Sdim return true; 852252723Sdim 853234285Sdim // do not allow nested hardware loops 854252723Sdim if (isHardwareLoop(MI)) 855234285Sdim return true; 856252723Sdim 857234285Sdim // check if the instruction defines a hardware loop register 858234285Sdim for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 859234285Sdim const MachineOperand &MO = MI->getOperand(i); 860252723Sdim if (!MO.isReg() || !MO.isDef()) 861252723Sdim continue; 862252723Sdim unsigned R = MO.getReg(); 863252723Sdim if (R == Hexagon::LC0 || R == Hexagon::LC1 || 864252723Sdim R == Hexagon::SA0 || R == Hexagon::SA1) 865234285Sdim return true; 866234285Sdim } 867234285Sdim return false; 868234285Sdim} 869234285Sdim 870252723Sdim 871252723Sdim/// \brief - Return true if the loop contains an instruction that inhibits 872252723Sdim/// the use of the hardware loop function. 873234285Sdimbool HexagonHardwareLoops::containsInvalidInstruction(MachineLoop *L) const { 874263509Sdim const std::vector<MachineBasicBlock *> &Blocks = L->getBlocks(); 875234285Sdim for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { 876234285Sdim MachineBasicBlock *MBB = Blocks[i]; 877234285Sdim for (MachineBasicBlock::iterator 878234285Sdim MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) { 879234285Sdim const MachineInstr *MI = &*MII; 880252723Sdim if (isInvalidLoopOperation(MI)) 881234285Sdim return true; 882234285Sdim } 883234285Sdim } 884234285Sdim return false; 885234285Sdim} 886234285Sdim 887252723Sdim 888252723Sdim/// \brief Returns true if the instruction is dead. This was essentially 889252723Sdim/// copied from DeadMachineInstructionElim::isDead, but with special cases 890252723Sdim/// for inline asm, physical registers and instructions with side effects 891252723Sdim/// removed. 892252723Sdimbool HexagonHardwareLoops::isDead(const MachineInstr *MI, 893263509Sdim SmallVectorImpl<MachineInstr *> &DeadPhis) const { 894252723Sdim // Examine each operand. 895252723Sdim for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 896252723Sdim const MachineOperand &MO = MI->getOperand(i); 897252723Sdim if (!MO.isReg() || !MO.isDef()) 898252723Sdim continue; 899252723Sdim 900252723Sdim unsigned Reg = MO.getReg(); 901252723Sdim if (MRI->use_nodbg_empty(Reg)) 902252723Sdim continue; 903252723Sdim 904252723Sdim typedef MachineRegisterInfo::use_nodbg_iterator use_nodbg_iterator; 905252723Sdim 906252723Sdim // This instruction has users, but if the only user is the phi node for the 907252723Sdim // parent block, and the only use of that phi node is this instruction, then 908252723Sdim // this instruction is dead: both it (and the phi node) can be removed. 909252723Sdim use_nodbg_iterator I = MRI->use_nodbg_begin(Reg); 910252723Sdim use_nodbg_iterator End = MRI->use_nodbg_end(); 911252723Sdim if (llvm::next(I) != End || !I.getOperand().getParent()->isPHI()) 912252723Sdim return false; 913252723Sdim 914252723Sdim MachineInstr *OnePhi = I.getOperand().getParent(); 915252723Sdim for (unsigned j = 0, f = OnePhi->getNumOperands(); j != f; ++j) { 916252723Sdim const MachineOperand &OPO = OnePhi->getOperand(j); 917252723Sdim if (!OPO.isReg() || !OPO.isDef()) 918252723Sdim continue; 919252723Sdim 920252723Sdim unsigned OPReg = OPO.getReg(); 921252723Sdim use_nodbg_iterator nextJ; 922252723Sdim for (use_nodbg_iterator J = MRI->use_nodbg_begin(OPReg); 923252723Sdim J != End; J = nextJ) { 924252723Sdim nextJ = llvm::next(J); 925252723Sdim MachineOperand &Use = J.getOperand(); 926252723Sdim MachineInstr *UseMI = Use.getParent(); 927252723Sdim 928252723Sdim // If the phi node has a user that is not MI, bail... 929252723Sdim if (MI != UseMI) 930252723Sdim return false; 931252723Sdim } 932252723Sdim } 933252723Sdim DeadPhis.push_back(OnePhi); 934252723Sdim } 935252723Sdim 936252723Sdim // If there are no defs with uses, the instruction is dead. 937252723Sdim return true; 938252723Sdim} 939252723Sdim 940252723Sdimvoid HexagonHardwareLoops::removeIfDead(MachineInstr *MI) { 941252723Sdim // This procedure was essentially copied from DeadMachineInstructionElim. 942252723Sdim 943252723Sdim SmallVector<MachineInstr*, 1> DeadPhis; 944252723Sdim if (isDead(MI, DeadPhis)) { 945252723Sdim DEBUG(dbgs() << "HW looping will remove: " << *MI); 946252723Sdim 947252723Sdim // It is possible that some DBG_VALUE instructions refer to this 948252723Sdim // instruction. Examine each def operand for such references; 949252723Sdim // if found, mark the DBG_VALUE as undef (but don't delete it). 950252723Sdim for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 951252723Sdim const MachineOperand &MO = MI->getOperand(i); 952252723Sdim if (!MO.isReg() || !MO.isDef()) 953252723Sdim continue; 954252723Sdim unsigned Reg = MO.getReg(); 955252723Sdim MachineRegisterInfo::use_iterator nextI; 956252723Sdim for (MachineRegisterInfo::use_iterator I = MRI->use_begin(Reg), 957252723Sdim E = MRI->use_end(); I != E; I = nextI) { 958252723Sdim nextI = llvm::next(I); // I is invalidated by the setReg 959252723Sdim MachineOperand &Use = I.getOperand(); 960252723Sdim MachineInstr *UseMI = Use.getParent(); 961252723Sdim if (UseMI == MI) 962252723Sdim continue; 963252723Sdim if (Use.isDebug()) 964252723Sdim UseMI->getOperand(0).setReg(0U); 965252723Sdim // This may also be a "instr -> phi -> instr" case which can 966252723Sdim // be removed too. 967252723Sdim } 968252723Sdim } 969252723Sdim 970252723Sdim MI->eraseFromParent(); 971252723Sdim for (unsigned i = 0; i < DeadPhis.size(); ++i) 972252723Sdim DeadPhis[i]->eraseFromParent(); 973252723Sdim } 974252723Sdim} 975252723Sdim 976252723Sdim/// \brief Check if the loop is a candidate for converting to a hardware 977252723Sdim/// loop. If so, then perform the transformation. 978234285Sdim/// 979252723Sdim/// This function works on innermost loops first. A loop can be converted 980252723Sdim/// if it is a counting loop; either a register value or an immediate. 981234285Sdim/// 982252723Sdim/// The code makes several assumptions about the representation of the loop 983252723Sdim/// in llvm. 984234285Sdimbool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) { 985252723Sdim // This is just for sanity. 986252723Sdim assert(L->getHeader() && "Loop without a header?"); 987252723Sdim 988234285Sdim bool Changed = false; 989234285Sdim // Process nested loops first. 990252723Sdim for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) 991234285Sdim Changed |= convertToHardwareLoop(*I); 992252723Sdim 993234285Sdim // If a nested loop has been converted, then we can't convert this loop. 994252723Sdim if (Changed) 995234285Sdim return Changed; 996252723Sdim 997252723Sdim#ifndef NDEBUG 998252723Sdim // Stop trying after reaching the limit (if any). 999252723Sdim int Limit = HWLoopLimit; 1000252723Sdim if (Limit >= 0) { 1001252723Sdim if (Counter >= HWLoopLimit) 1002252723Sdim return false; 1003252723Sdim Counter++; 1004234285Sdim } 1005252723Sdim#endif 1006252723Sdim 1007234285Sdim // Does the loop contain any invalid instructions? 1008252723Sdim if (containsInvalidInstruction(L)) 1009234285Sdim return false; 1010252723Sdim 1011252723Sdim // Is the induction variable bump feeding the latch condition? 1012252723Sdim if (!fixupInductionVariable(L)) 1013234285Sdim return false; 1014234285Sdim 1015234285Sdim MachineBasicBlock *LastMBB = L->getExitingBlock(); 1016234285Sdim // Don't generate hw loop if the loop has more than one exit. 1017252723Sdim if (LastMBB == 0) 1018234285Sdim return false; 1019252723Sdim 1020234285Sdim MachineBasicBlock::iterator LastI = LastMBB->getFirstTerminator(); 1021252723Sdim if (LastI == LastMBB->end()) 1022252723Sdim return false; 1023234285Sdim 1024252723Sdim // Ensure the loop has a preheader: the loop instruction will be 1025252723Sdim // placed there. 1026252723Sdim bool NewPreheader = false; 1027252723Sdim MachineBasicBlock *Preheader = L->getLoopPreheader(); 1028252723Sdim if (!Preheader) { 1029252723Sdim Preheader = createPreheaderForLoop(L); 1030252723Sdim if (!Preheader) 1031252723Sdim return false; 1032252723Sdim NewPreheader = true; 1033252723Sdim } 1034252723Sdim MachineBasicBlock::iterator InsertPos = Preheader->getFirstTerminator(); 1035252723Sdim 1036252723Sdim SmallVector<MachineInstr*, 2> OldInsts; 1037252723Sdim // Are we able to determine the trip count for the loop? 1038252723Sdim CountValue *TripCount = getLoopTripCount(L, OldInsts); 1039252723Sdim if (TripCount == 0) 1040252723Sdim return false; 1041252723Sdim 1042252723Sdim // Is the trip count available in the preheader? 1043252723Sdim if (TripCount->isReg()) { 1044252723Sdim // There will be a use of the register inserted into the preheader, 1045252723Sdim // so make sure that the register is actually defined at that point. 1046252723Sdim MachineInstr *TCDef = MRI->getVRegDef(TripCount->getReg()); 1047252723Sdim MachineBasicBlock *BBDef = TCDef->getParent(); 1048252723Sdim if (!NewPreheader) { 1049252723Sdim if (!MDT->dominates(BBDef, Preheader)) 1050252723Sdim return false; 1051252723Sdim } else { 1052252723Sdim // If we have just created a preheader, the dominator tree won't be 1053252723Sdim // aware of it. Check if the definition of the register dominates 1054252723Sdim // the header, but is not the header itself. 1055252723Sdim if (!MDT->properlyDominates(BBDef, L->getHeader())) 1056252723Sdim return false; 1057252723Sdim } 1058252723Sdim } 1059252723Sdim 1060234285Sdim // Determine the loop start. 1061234285Sdim MachineBasicBlock *LoopStart = L->getTopBlock(); 1062234285Sdim if (L->getLoopLatch() != LastMBB) { 1063234285Sdim // When the exit and latch are not the same, use the latch block as the 1064234285Sdim // start. 1065252723Sdim // The loop start address is used only after the 1st iteration, and the 1066252723Sdim // loop latch may contains instrs. that need to be executed after the 1067252723Sdim // first iteration. 1068234285Sdim LoopStart = L->getLoopLatch(); 1069234285Sdim // Make sure the latch is a successor of the exit, otherwise it won't work. 1070252723Sdim if (!LastMBB->isSuccessor(LoopStart)) 1071234285Sdim return false; 1072234285Sdim } 1073234285Sdim 1074252723Sdim // Convert the loop to a hardware loop. 1075234285Sdim DEBUG(dbgs() << "Change to hardware loop at "; L->dump()); 1076252723Sdim DebugLoc DL; 1077252723Sdim if (InsertPos != Preheader->end()) 1078252723Sdim DL = InsertPos->getDebugLoc(); 1079234285Sdim 1080234285Sdim if (TripCount->isReg()) { 1081234285Sdim // Create a copy of the loop count register. 1082252723Sdim unsigned CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass); 1083252723Sdim BuildMI(*Preheader, InsertPos, DL, TII->get(TargetOpcode::COPY), CountReg) 1084252723Sdim .addReg(TripCount->getReg(), 0, TripCount->getSubReg()); 1085245431Sdim // Add the Loop instruction to the beginning of the loop. 1086252723Sdim BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::LOOP0_r)) 1087252723Sdim .addMBB(LoopStart) 1088252723Sdim .addReg(CountReg); 1089234285Sdim } else { 1090252723Sdim assert(TripCount->isImm() && "Expecting immediate value for trip count"); 1091252723Sdim // Add the Loop immediate instruction to the beginning of the loop, 1092252723Sdim // if the immediate fits in the instructions. Otherwise, we need to 1093252723Sdim // create a new virtual register. 1094234285Sdim int64_t CountImm = TripCount->getImm(); 1095252723Sdim if (!TII->isValidOffset(Hexagon::LOOP0_i, CountImm)) { 1096252723Sdim unsigned CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass); 1097252723Sdim BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::TFRI), CountReg) 1098252723Sdim .addImm(CountImm); 1099252723Sdim BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::LOOP0_r)) 1100252723Sdim .addMBB(LoopStart).addReg(CountReg); 1101252723Sdim } else 1102252723Sdim BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::LOOP0_i)) 1103252723Sdim .addMBB(LoopStart).addImm(CountImm); 1104234285Sdim } 1105234285Sdim 1106252723Sdim // Make sure the loop start always has a reference in the CFG. We need 1107252723Sdim // to create a BlockAddress operand to get this mechanism to work both the 1108234285Sdim // MachineBasicBlock and BasicBlock objects need the flag set. 1109234285Sdim LoopStart->setHasAddressTaken(); 1110234285Sdim // This line is needed to set the hasAddressTaken flag on the BasicBlock 1111252723Sdim // object. 1112234285Sdim BlockAddress::get(const_cast<BasicBlock *>(LoopStart->getBasicBlock())); 1113234285Sdim 1114234285Sdim // Replace the loop branch with an endloop instruction. 1115252723Sdim DebugLoc LastIDL = LastI->getDebugLoc(); 1116252723Sdim BuildMI(*LastMBB, LastI, LastIDL, 1117252723Sdim TII->get(Hexagon::ENDLOOP0)).addMBB(LoopStart); 1118234285Sdim 1119234285Sdim // The loop ends with either: 1120234285Sdim // - a conditional branch followed by an unconditional branch, or 1121234285Sdim // - a conditional branch to the loop start. 1122252723Sdim if (LastI->getOpcode() == Hexagon::JMP_t || 1123252723Sdim LastI->getOpcode() == Hexagon::JMP_f) { 1124252723Sdim // Delete one and change/add an uncond. branch to out of the loop. 1125234285Sdim MachineBasicBlock *BranchTarget = LastI->getOperand(1).getMBB(); 1126234285Sdim LastI = LastMBB->erase(LastI); 1127234285Sdim if (!L->contains(BranchTarget)) { 1128252723Sdim if (LastI != LastMBB->end()) 1129252723Sdim LastI = LastMBB->erase(LastI); 1130234285Sdim SmallVector<MachineOperand, 0> Cond; 1131252723Sdim TII->InsertBranch(*LastMBB, BranchTarget, 0, Cond, LastIDL); 1132234285Sdim } 1133234285Sdim } else { 1134234285Sdim // Conditional branch to loop start; just delete it. 1135234285Sdim LastMBB->erase(LastI); 1136234285Sdim } 1137234285Sdim delete TripCount; 1138234285Sdim 1139252723Sdim // The induction operation and the comparison may now be 1140252723Sdim // unneeded. If these are unneeded, then remove them. 1141252723Sdim for (unsigned i = 0; i < OldInsts.size(); ++i) 1142252723Sdim removeIfDead(OldInsts[i]); 1143252723Sdim 1144234285Sdim ++NumHWLoops; 1145234285Sdim return true; 1146234285Sdim} 1147234285Sdim 1148252723Sdim 1149252723Sdimbool HexagonHardwareLoops::orderBumpCompare(MachineInstr *BumpI, 1150252723Sdim MachineInstr *CmpI) { 1151252723Sdim assert (BumpI != CmpI && "Bump and compare in the same instruction?"); 1152252723Sdim 1153252723Sdim MachineBasicBlock *BB = BumpI->getParent(); 1154252723Sdim if (CmpI->getParent() != BB) 1155252723Sdim return false; 1156252723Sdim 1157252723Sdim typedef MachineBasicBlock::instr_iterator instr_iterator; 1158252723Sdim // Check if things are in order to begin with. 1159252723Sdim for (instr_iterator I = BumpI, E = BB->instr_end(); I != E; ++I) 1160252723Sdim if (&*I == CmpI) 1161252723Sdim return true; 1162252723Sdim 1163252723Sdim // Out of order. 1164252723Sdim unsigned PredR = CmpI->getOperand(0).getReg(); 1165252723Sdim bool FoundBump = false; 1166252723Sdim instr_iterator CmpIt = CmpI, NextIt = llvm::next(CmpIt); 1167252723Sdim for (instr_iterator I = NextIt, E = BB->instr_end(); I != E; ++I) { 1168252723Sdim MachineInstr *In = &*I; 1169252723Sdim for (unsigned i = 0, n = In->getNumOperands(); i < n; ++i) { 1170252723Sdim MachineOperand &MO = In->getOperand(i); 1171252723Sdim if (MO.isReg() && MO.isUse()) { 1172252723Sdim if (MO.getReg() == PredR) // Found an intervening use of PredR. 1173252723Sdim return false; 1174252723Sdim } 1175252723Sdim } 1176252723Sdim 1177252723Sdim if (In == BumpI) { 1178252723Sdim instr_iterator After = BumpI; 1179252723Sdim instr_iterator From = CmpI; 1180252723Sdim BB->splice(llvm::next(After), BB, From); 1181252723Sdim FoundBump = true; 1182252723Sdim break; 1183252723Sdim } 1184252723Sdim } 1185252723Sdim assert (FoundBump && "Cannot determine instruction order"); 1186252723Sdim return FoundBump; 1187234285Sdim} 1188234285Sdim 1189234285Sdim 1190252723SdimMachineInstr *HexagonHardwareLoops::defWithImmediate(unsigned R) { 1191252723Sdim MachineInstr *DI = MRI->getVRegDef(R); 1192252723Sdim unsigned DOpc = DI->getOpcode(); 1193252723Sdim switch (DOpc) { 1194252723Sdim case Hexagon::TFRI: 1195252723Sdim case Hexagon::TFRI64: 1196252723Sdim case Hexagon::CONST32_Int_Real: 1197252723Sdim case Hexagon::CONST64_Int_Real: 1198252723Sdim return DI; 1199252723Sdim } 1200252723Sdim return 0; 1201234285Sdim} 1202234285Sdim 1203234285Sdim 1204252723Sdimint64_t HexagonHardwareLoops::getImmediate(MachineOperand &MO) { 1205252723Sdim if (MO.isImm()) 1206252723Sdim return MO.getImm(); 1207252723Sdim assert(MO.isReg()); 1208252723Sdim unsigned R = MO.getReg(); 1209252723Sdim MachineInstr *DI = defWithImmediate(R); 1210252723Sdim assert(DI && "Need an immediate operand"); 1211252723Sdim // All currently supported "define-with-immediate" instructions have the 1212252723Sdim // actual immediate value in the operand(1). 1213252723Sdim int64_t v = DI->getOperand(1).getImm(); 1214252723Sdim return v; 1215252723Sdim} 1216234285Sdim 1217252723Sdim 1218252723Sdimvoid HexagonHardwareLoops::setImmediate(MachineOperand &MO, int64_t Val) { 1219252723Sdim if (MO.isImm()) { 1220252723Sdim MO.setImm(Val); 1221252723Sdim return; 1222234285Sdim } 1223234285Sdim 1224252723Sdim assert(MO.isReg()); 1225252723Sdim unsigned R = MO.getReg(); 1226252723Sdim MachineInstr *DI = defWithImmediate(R); 1227252723Sdim if (MRI->hasOneNonDBGUse(R)) { 1228252723Sdim // If R has only one use, then just change its defining instruction to 1229252723Sdim // the new immediate value. 1230252723Sdim DI->getOperand(1).setImm(Val); 1231252723Sdim return; 1232252723Sdim } 1233234285Sdim 1234252723Sdim const TargetRegisterClass *RC = MRI->getRegClass(R); 1235252723Sdim unsigned NewR = MRI->createVirtualRegister(RC); 1236252723Sdim MachineBasicBlock &B = *DI->getParent(); 1237252723Sdim DebugLoc DL = DI->getDebugLoc(); 1238252723Sdim BuildMI(B, DI, DL, TII->get(DI->getOpcode()), NewR) 1239252723Sdim .addImm(Val); 1240252723Sdim MO.setReg(NewR); 1241252723Sdim} 1242234285Sdim 1243252723Sdim 1244252723Sdimbool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) { 1245252723Sdim MachineBasicBlock *Header = L->getHeader(); 1246252723Sdim MachineBasicBlock *Preheader = L->getLoopPreheader(); 1247252723Sdim MachineBasicBlock *Latch = L->getLoopLatch(); 1248252723Sdim 1249252723Sdim if (!Header || !Preheader || !Latch) 1250252723Sdim return false; 1251252723Sdim 1252252723Sdim // These data structures follow the same concept as the corresponding 1253252723Sdim // ones in findInductionRegister (where some comments are). 1254252723Sdim typedef std::pair<unsigned,int64_t> RegisterBump; 1255252723Sdim typedef std::pair<unsigned,RegisterBump> RegisterInduction; 1256252723Sdim typedef std::set<RegisterInduction> RegisterInductionSet; 1257252723Sdim 1258252723Sdim // Register candidates for induction variables, with their associated bumps. 1259252723Sdim RegisterInductionSet IndRegs; 1260252723Sdim 1261252723Sdim // Look for induction patterns: 1262252723Sdim // vreg1 = PHI ..., [ latch, vreg2 ] 1263252723Sdim // vreg2 = ADD vreg1, imm 1264252723Sdim typedef MachineBasicBlock::instr_iterator instr_iterator; 1265252723Sdim for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); 1266252723Sdim I != E && I->isPHI(); ++I) { 1267252723Sdim MachineInstr *Phi = &*I; 1268252723Sdim 1269252723Sdim // Have a PHI instruction. 1270252723Sdim for (unsigned i = 1, n = Phi->getNumOperands(); i < n; i += 2) { 1271252723Sdim if (Phi->getOperand(i+1).getMBB() != Latch) 1272252723Sdim continue; 1273252723Sdim 1274252723Sdim unsigned PhiReg = Phi->getOperand(i).getReg(); 1275252723Sdim MachineInstr *DI = MRI->getVRegDef(PhiReg); 1276252723Sdim unsigned UpdOpc = DI->getOpcode(); 1277252723Sdim bool isAdd = (UpdOpc == Hexagon::ADD_ri); 1278252723Sdim 1279252723Sdim if (isAdd) { 1280252723Sdim // If the register operand to the add/sub is the PHI we are looking 1281252723Sdim // at, this meets the induction pattern. 1282252723Sdim unsigned IndReg = DI->getOperand(1).getReg(); 1283252723Sdim if (MRI->getVRegDef(IndReg) == Phi) { 1284252723Sdim unsigned UpdReg = DI->getOperand(0).getReg(); 1285252723Sdim int64_t V = DI->getOperand(2).getImm(); 1286252723Sdim IndRegs.insert(std::make_pair(UpdReg, std::make_pair(IndReg, V))); 1287234285Sdim } 1288234285Sdim } 1289252723Sdim } // for (i) 1290252723Sdim } // for (instr) 1291252723Sdim 1292252723Sdim if (IndRegs.empty()) 1293252723Sdim return false; 1294252723Sdim 1295252723Sdim MachineBasicBlock *TB = 0, *FB = 0; 1296252723Sdim SmallVector<MachineOperand,2> Cond; 1297252723Sdim // AnalyzeBranch returns true if it fails to analyze branch. 1298252723Sdim bool NotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Cond, false); 1299252723Sdim if (NotAnalyzed) 1300252723Sdim return false; 1301252723Sdim 1302252723Sdim // Check if the latch branch is unconditional. 1303252723Sdim if (Cond.empty()) 1304252723Sdim return false; 1305252723Sdim 1306252723Sdim if (TB != Header && FB != Header) 1307252723Sdim // The latch does not go back to the header. Not a latch we know and love. 1308252723Sdim return false; 1309252723Sdim 1310252723Sdim // Expecting a predicate register as a condition. It won't be a hardware 1311252723Sdim // predicate register at this point yet, just a vreg. 1312252723Sdim // HexagonInstrInfo::AnalyzeBranch for negated branches inserts imm(0) 1313252723Sdim // into Cond, followed by the predicate register. For non-negated branches 1314252723Sdim // it's just the register. 1315252723Sdim unsigned CSz = Cond.size(); 1316252723Sdim if (CSz != 1 && CSz != 2) 1317252723Sdim return false; 1318252723Sdim 1319252723Sdim unsigned P = Cond[CSz-1].getReg(); 1320252723Sdim MachineInstr *PredDef = MRI->getVRegDef(P); 1321252723Sdim 1322252723Sdim if (!PredDef->isCompare()) 1323252723Sdim return false; 1324252723Sdim 1325252723Sdim SmallSet<unsigned,2> CmpRegs; 1326252723Sdim MachineOperand *CmpImmOp = 0; 1327252723Sdim 1328252723Sdim // Go over all operands to the compare and look for immediate and register 1329252723Sdim // operands. Assume that if the compare has a single register use and a 1330252723Sdim // single immediate operand, then the register is being compared with the 1331252723Sdim // immediate value. 1332252723Sdim for (unsigned i = 0, n = PredDef->getNumOperands(); i < n; ++i) { 1333252723Sdim MachineOperand &MO = PredDef->getOperand(i); 1334252723Sdim if (MO.isReg()) { 1335252723Sdim // Skip all implicit references. In one case there was: 1336252723Sdim // %vreg140<def> = FCMPUGT32_rr %vreg138, %vreg139, %USR<imp-use> 1337252723Sdim if (MO.isImplicit()) 1338252723Sdim continue; 1339252723Sdim if (MO.isUse()) { 1340252723Sdim unsigned R = MO.getReg(); 1341252723Sdim if (!defWithImmediate(R)) { 1342252723Sdim CmpRegs.insert(MO.getReg()); 1343252723Sdim continue; 1344252723Sdim } 1345252723Sdim // Consider the register to be the "immediate" operand. 1346252723Sdim if (CmpImmOp) 1347252723Sdim return false; 1348252723Sdim CmpImmOp = &MO; 1349252723Sdim } 1350252723Sdim } else if (MO.isImm()) { 1351252723Sdim if (CmpImmOp) // A second immediate argument? Confusing. Bail out. 1352252723Sdim return false; 1353252723Sdim CmpImmOp = &MO; 1354234285Sdim } 1355234285Sdim } 1356234285Sdim 1357252723Sdim if (CmpRegs.empty()) 1358252723Sdim return false; 1359234285Sdim 1360252723Sdim // Check if the compared register follows the order we want. Fix if needed. 1361252723Sdim for (RegisterInductionSet::iterator I = IndRegs.begin(), E = IndRegs.end(); 1362252723Sdim I != E; ++I) { 1363252723Sdim // This is a success. If the register used in the comparison is one that 1364252723Sdim // we have identified as a bumped (updated) induction register, there is 1365252723Sdim // nothing to do. 1366252723Sdim if (CmpRegs.count(I->first)) 1367252723Sdim return true; 1368252723Sdim 1369252723Sdim // Otherwise, if the register being compared comes out of a PHI node, 1370252723Sdim // and has been recognized as following the induction pattern, and is 1371252723Sdim // compared against an immediate, we can fix it. 1372252723Sdim const RegisterBump &RB = I->second; 1373252723Sdim if (CmpRegs.count(RB.first)) { 1374252723Sdim if (!CmpImmOp) 1375252723Sdim return false; 1376252723Sdim 1377252723Sdim int64_t CmpImm = getImmediate(*CmpImmOp); 1378252723Sdim int64_t V = RB.second; 1379252723Sdim if (V > 0 && CmpImm+V < CmpImm) // Overflow (64-bit). 1380252723Sdim return false; 1381252723Sdim if (V < 0 && CmpImm+V > CmpImm) // Overflow (64-bit). 1382252723Sdim return false; 1383252723Sdim CmpImm += V; 1384252723Sdim // Some forms of cmp-immediate allow u9 and s10. Assume the worst case 1385252723Sdim // scenario, i.e. an 8-bit value. 1386252723Sdim if (CmpImmOp->isImm() && !isInt<8>(CmpImm)) 1387252723Sdim return false; 1388252723Sdim 1389252723Sdim // Make sure that the compare happens after the bump. Otherwise, 1390252723Sdim // after the fixup, the compare would use a yet-undefined register. 1391252723Sdim MachineInstr *BumpI = MRI->getVRegDef(I->first); 1392252723Sdim bool Order = orderBumpCompare(BumpI, PredDef); 1393252723Sdim if (!Order) 1394252723Sdim return false; 1395252723Sdim 1396252723Sdim // Finally, fix the compare instruction. 1397252723Sdim setImmediate(*CmpImmOp, CmpImm); 1398252723Sdim for (unsigned i = 0, n = PredDef->getNumOperands(); i < n; ++i) { 1399252723Sdim MachineOperand &MO = PredDef->getOperand(i); 1400252723Sdim if (MO.isReg() && MO.getReg() == RB.first) { 1401252723Sdim MO.setReg(I->first); 1402252723Sdim return true; 1403252723Sdim } 1404252723Sdim } 1405252723Sdim } 1406252723Sdim } 1407252723Sdim 1408252723Sdim return false; 1409234285Sdim} 1410234285Sdim 1411234285Sdim 1412252723Sdim/// \brief Create a preheader for a given loop. 1413252723SdimMachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop( 1414252723Sdim MachineLoop *L) { 1415252723Sdim if (MachineBasicBlock *TmpPH = L->getLoopPreheader()) 1416252723Sdim return TmpPH; 1417252723Sdim 1418252723Sdim MachineBasicBlock *Header = L->getHeader(); 1419252723Sdim MachineBasicBlock *Latch = L->getLoopLatch(); 1420252723Sdim MachineFunction *MF = Header->getParent(); 1421252723Sdim DebugLoc DL; 1422252723Sdim 1423252723Sdim if (!Latch || Header->hasAddressTaken()) 1424252723Sdim return 0; 1425252723Sdim 1426252723Sdim typedef MachineBasicBlock::instr_iterator instr_iterator; 1427252723Sdim 1428252723Sdim // Verify that all existing predecessors have analyzable branches 1429252723Sdim // (or no branches at all). 1430252723Sdim typedef std::vector<MachineBasicBlock*> MBBVector; 1431252723Sdim MBBVector Preds(Header->pred_begin(), Header->pred_end()); 1432252723Sdim SmallVector<MachineOperand,2> Tmp1; 1433252723Sdim MachineBasicBlock *TB = 0, *FB = 0; 1434252723Sdim 1435252723Sdim if (TII->AnalyzeBranch(*Latch, TB, FB, Tmp1, false)) 1436252723Sdim return 0; 1437252723Sdim 1438252723Sdim for (MBBVector::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) { 1439252723Sdim MachineBasicBlock *PB = *I; 1440252723Sdim if (PB != Latch) { 1441252723Sdim bool NotAnalyzed = TII->AnalyzeBranch(*PB, TB, FB, Tmp1, false); 1442252723Sdim if (NotAnalyzed) 1443252723Sdim return 0; 1444252723Sdim } 1445252723Sdim } 1446252723Sdim 1447252723Sdim MachineBasicBlock *NewPH = MF->CreateMachineBasicBlock(); 1448252723Sdim MF->insert(Header, NewPH); 1449252723Sdim 1450252723Sdim if (Header->pred_size() > 2) { 1451252723Sdim // Ensure that the header has only two predecessors: the preheader and 1452252723Sdim // the loop latch. Any additional predecessors of the header should 1453252723Sdim // join at the newly created preheader. Inspect all PHI nodes from the 1454252723Sdim // header and create appropriate corresponding PHI nodes in the preheader. 1455252723Sdim 1456252723Sdim for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); 1457252723Sdim I != E && I->isPHI(); ++I) { 1458252723Sdim MachineInstr *PN = &*I; 1459252723Sdim 1460252723Sdim const MCInstrDesc &PD = TII->get(TargetOpcode::PHI); 1461252723Sdim MachineInstr *NewPN = MF->CreateMachineInstr(PD, DL); 1462252723Sdim NewPH->insert(NewPH->end(), NewPN); 1463252723Sdim 1464252723Sdim unsigned PR = PN->getOperand(0).getReg(); 1465252723Sdim const TargetRegisterClass *RC = MRI->getRegClass(PR); 1466252723Sdim unsigned NewPR = MRI->createVirtualRegister(RC); 1467252723Sdim NewPN->addOperand(MachineOperand::CreateReg(NewPR, true)); 1468252723Sdim 1469252723Sdim // Copy all non-latch operands of a header's PHI node to the newly 1470252723Sdim // created PHI node in the preheader. 1471252723Sdim for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) { 1472252723Sdim unsigned PredR = PN->getOperand(i).getReg(); 1473252723Sdim MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB(); 1474252723Sdim if (PredB == Latch) 1475252723Sdim continue; 1476252723Sdim 1477252723Sdim NewPN->addOperand(MachineOperand::CreateReg(PredR, false)); 1478252723Sdim NewPN->addOperand(MachineOperand::CreateMBB(PredB)); 1479252723Sdim } 1480252723Sdim 1481252723Sdim // Remove copied operands from the old PHI node and add the value 1482252723Sdim // coming from the preheader's PHI. 1483252723Sdim for (int i = PN->getNumOperands()-2; i > 0; i -= 2) { 1484252723Sdim MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB(); 1485252723Sdim if (PredB != Latch) { 1486252723Sdim PN->RemoveOperand(i+1); 1487252723Sdim PN->RemoveOperand(i); 1488252723Sdim } 1489252723Sdim } 1490252723Sdim PN->addOperand(MachineOperand::CreateReg(NewPR, false)); 1491252723Sdim PN->addOperand(MachineOperand::CreateMBB(NewPH)); 1492252723Sdim } 1493252723Sdim 1494234285Sdim } else { 1495252723Sdim assert(Header->pred_size() == 2); 1496252723Sdim 1497252723Sdim // The header has only two predecessors, but the non-latch predecessor 1498252723Sdim // is not a preheader (e.g. it has other successors, etc.) 1499252723Sdim // In such a case we don't need any extra PHI nodes in the new preheader, 1500252723Sdim // all we need is to adjust existing PHIs in the header to now refer to 1501252723Sdim // the new preheader. 1502252723Sdim for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); 1503252723Sdim I != E && I->isPHI(); ++I) { 1504252723Sdim MachineInstr *PN = &*I; 1505252723Sdim for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) { 1506252723Sdim MachineOperand &MO = PN->getOperand(i+1); 1507252723Sdim if (MO.getMBB() != Latch) 1508252723Sdim MO.setMBB(NewPH); 1509252723Sdim } 1510252723Sdim } 1511234285Sdim } 1512252723Sdim 1513252723Sdim // "Reroute" the CFG edges to link in the new preheader. 1514252723Sdim // If any of the predecessors falls through to the header, insert a branch 1515252723Sdim // to the new preheader in that place. 1516252723Sdim SmallVector<MachineOperand,1> Tmp2; 1517252723Sdim SmallVector<MachineOperand,1> EmptyCond; 1518252723Sdim 1519252723Sdim TB = FB = 0; 1520252723Sdim 1521252723Sdim for (MBBVector::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) { 1522252723Sdim MachineBasicBlock *PB = *I; 1523252723Sdim if (PB != Latch) { 1524252723Sdim Tmp2.clear(); 1525252723Sdim bool NotAnalyzed = TII->AnalyzeBranch(*PB, TB, FB, Tmp2, false); 1526252723Sdim (void)NotAnalyzed; // supress compiler warning 1527252723Sdim assert (!NotAnalyzed && "Should be analyzable!"); 1528252723Sdim if (TB != Header && (Tmp2.empty() || FB != Header)) 1529252723Sdim TII->InsertBranch(*PB, NewPH, 0, EmptyCond, DL); 1530252723Sdim PB->ReplaceUsesOfBlockWith(Header, NewPH); 1531252723Sdim } 1532252723Sdim } 1533252723Sdim 1534252723Sdim // It can happen that the latch block will fall through into the header. 1535252723Sdim // Insert an unconditional branch to the header. 1536252723Sdim TB = FB = 0; 1537252723Sdim bool LatchNotAnalyzed = TII->AnalyzeBranch(*Latch, TB, FB, Tmp2, false); 1538252723Sdim (void)LatchNotAnalyzed; // supress compiler warning 1539252723Sdim assert (!LatchNotAnalyzed && "Should be analyzable!"); 1540252723Sdim if (!TB && !FB) 1541252723Sdim TII->InsertBranch(*Latch, Header, 0, EmptyCond, DL); 1542252723Sdim 1543252723Sdim // Finally, the branch from the preheader to the header. 1544252723Sdim TII->InsertBranch(*NewPH, Header, 0, EmptyCond, DL); 1545252723Sdim NewPH->addSuccessor(Header); 1546252723Sdim 1547252723Sdim return NewPH; 1548234285Sdim} 1549