MLxExpansionPass.cpp revision 243830
1139749Simp//===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===// 2158078Smarcel// 3158078Smarcel// The LLVM Compiler Infrastructure 4158078Smarcel// 5119815Smarcel// This file is distributed under the University of Illinois Open Source 6119815Smarcel// License. See LICENSE.TXT for details. 7119815Smarcel// 8119815Smarcel//===----------------------------------------------------------------------===// 9119815Smarcel// 10119815Smarcel// Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of 11119815Smarcel// multiple and add / sub instructions) when special VMLx hazards are detected. 12119815Smarcel// 13119815Smarcel//===----------------------------------------------------------------------===// 14119815Smarcel 15119815Smarcel#define DEBUG_TYPE "mlx-expansion" 16119815Smarcel#include "ARM.h" 17119815Smarcel#include "ARMBaseInstrInfo.h" 18119815Smarcel#include "ARMSubtarget.h" 19119815Smarcel#include "llvm/CodeGen/MachineInstr.h" 20119815Smarcel#include "llvm/CodeGen/MachineInstrBuilder.h" 21119815Smarcel#include "llvm/CodeGen/MachineFunctionPass.h" 22119815Smarcel#include "llvm/CodeGen/MachineRegisterInfo.h" 23119815Smarcel#include "llvm/Target/TargetRegisterInfo.h" 24119815Smarcel#include "llvm/ADT/SmallPtrSet.h" 25119815Smarcel#include "llvm/ADT/Statistic.h" 26119815Smarcel#include "llvm/Support/CommandLine.h" 27119815Smarcel#include "llvm/Support/Debug.h" 28119815Smarcel#include "llvm/Support/raw_ostream.h" 29119815Smarcelusing namespace llvm; 30119815Smarcel 31119815Smarcelstatic cl::opt<bool> 32119815SmarcelForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden); 33119815Smarcelstatic cl::opt<unsigned> 34119815SmarcelExpandLimit("expand-limit", cl::init(~0U), cl::Hidden); 35119815Smarcel 36119815SmarcelSTATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded"); 37119815Smarcel 38119815Smarcelnamespace { 39119815Smarcel struct MLxExpansion : public MachineFunctionPass { 40119815Smarcel static char ID; 41119815Smarcel MLxExpansion() : MachineFunctionPass(ID) {} 42119815Smarcel 43119815Smarcel virtual bool runOnMachineFunction(MachineFunction &Fn); 44119815Smarcel 45158058Smarcel virtual const char *getPassName() const { 46158058Smarcel return "ARM MLA / MLS expansion pass"; 47119815Smarcel } 48119815Smarcel 49119815Smarcel private: 50119815Smarcel const ARMBaseInstrInfo *TII; 51119815Smarcel const TargetRegisterInfo *TRI; 52119815Smarcel MachineRegisterInfo *MRI; 53119815Smarcel 54119815Smarcel bool isLikeA9; 55119815Smarcel bool isSwift; 56119815Smarcel unsigned MIIdx; 57119815Smarcel MachineInstr* LastMIs[4]; 58119815Smarcel SmallPtrSet<MachineInstr*, 4> IgnoreStall; 59119815Smarcel 60119815Smarcel void clearStack(); 61119815Smarcel void pushStack(MachineInstr *MI); 62119815Smarcel MachineInstr *getAccDefMI(MachineInstr *MI) const; 63119815Smarcel unsigned getDefReg(MachineInstr *MI) const; 64158058Smarcel bool hasLoopHazard(MachineInstr *MI) const; 65158058Smarcel bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const; 66158058Smarcel bool FindMLxHazard(MachineInstr *MI); 67158058Smarcel void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI, 68119815Smarcel unsigned MulOpc, unsigned AddSubOpc, 69119815Smarcel bool NegAcc, bool HasLane); 70158058Smarcel bool ExpandFPMLxInstructions(MachineBasicBlock &MBB); 71119815Smarcel }; 72119815Smarcel char MLxExpansion::ID = 0; 73119815Smarcel} 74158078Smarcel 75158078Smarcelvoid MLxExpansion::clearStack() { 76158078Smarcel std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0); 77158078Smarcel MIIdx = 0; 78158058Smarcel} 79158058Smarcel 80158078Smarcelvoid MLxExpansion::pushStack(MachineInstr *MI) { 81158078Smarcel LastMIs[MIIdx] = MI; 82169646Smarcel if (++MIIdx == 4) 83158078Smarcel MIIdx = 0; 84158058Smarcel} 85158058Smarcel 86158058SmarcelMachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const { 87158058Smarcel // Look past COPY and INSERT_SUBREG instructions to find the 88158058Smarcel // real definition MI. This is important for _sfp instructions. 89158058Smarcel unsigned Reg = MI->getOperand(1).getReg(); 90158058Smarcel if (TargetRegisterInfo::isPhysicalRegister(Reg)) 91158058Smarcel return 0; 92158058Smarcel 93158058Smarcel MachineBasicBlock *MBB = MI->getParent(); 94158058Smarcel MachineInstr *DefMI = MRI->getVRegDef(Reg); 95158078Smarcel while (true) { 96158078Smarcel if (DefMI->getParent() != MBB) 97158058Smarcel break; 98158058Smarcel if (DefMI->isCopyLike()) { 99188472Skaiw Reg = DefMI->getOperand(1).getReg(); 100158078Smarcel if (TargetRegisterInfo::isVirtualRegister(Reg)) { 101158078Smarcel DefMI = MRI->getVRegDef(Reg); 102158078Smarcel continue; 103158078Smarcel } 104158078Smarcel } else if (DefMI->isInsertSubreg()) { 105158078Smarcel Reg = DefMI->getOperand(2).getReg(); 106158078Smarcel if (TargetRegisterInfo::isVirtualRegister(Reg)) { 107158078Smarcel DefMI = MRI->getVRegDef(Reg); 108158078Smarcel continue; 109158078Smarcel } 110158078Smarcel } 111158078Smarcel break; 112158078Smarcel } 113200257Smav return DefMI; 114189407Sjhb} 115200230Smarcel 116223672Shselaskyunsigned MLxExpansion::getDefReg(MachineInstr *MI) const { 117223672Shselasky unsigned Reg = MI->getOperand(0).getReg(); 118204533Sdelphij if (TargetRegisterInfo::isPhysicalRegister(Reg) || 119204533Sdelphij !MRI->hasOneNonDBGUse(Reg)) 120158058Smarcel return Reg; 121158058Smarcel 122119815Smarcel MachineBasicBlock *MBB = MI->getParent(); 123119815Smarcel MachineInstr *UseMI = &*MRI->use_nodbg_begin(Reg); 124119815Smarcel if (UseMI->getParent() != MBB) 125158058Smarcel return Reg; 126119815Smarcel 127158058Smarcel while (UseMI->isCopy() || UseMI->isInsertSubreg()) { 128119815Smarcel Reg = UseMI->getOperand(0).getReg(); 129158058Smarcel if (TargetRegisterInfo::isPhysicalRegister(Reg) || 130158058Smarcel !MRI->hasOneNonDBGUse(Reg)) 131158058Smarcel return Reg; 132158058Smarcel UseMI = &*MRI->use_nodbg_begin(Reg); 133119815Smarcel if (UseMI->getParent() != MBB) 134158058Smarcel return Reg; 135158058Smarcel } 136158058Smarcel 137158058Smarcel return Reg; 138158058Smarcel} 139158058Smarcel 140158058Smarcel/// hasLoopHazard - Check whether an MLx instruction is chained to itself across 141158058Smarcel/// a single-MBB loop. 142158058Smarcelbool MLxExpansion::hasLoopHazard(MachineInstr *MI) const { 143158058Smarcel unsigned Reg = MI->getOperand(1).getReg(); 144119815Smarcel if (TargetRegisterInfo::isPhysicalRegister(Reg)) 145119815Smarcel return false; 146119815Smarcel 147119815Smarcel MachineBasicBlock *MBB = MI->getParent(); 148119815Smarcel MachineInstr *DefMI = MRI->getVRegDef(Reg); 149119815Smarcel while (true) { 150119815Smarcelouter_continue: 151119815Smarcel if (DefMI->getParent() != MBB) 152119815Smarcel break; 153119815Smarcel 154158058Smarcel if (DefMI->isPHI()) { 155119815Smarcel for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) { 156119815Smarcel if (DefMI->getOperand(i + 1).getMBB() == MBB) { 157119815Smarcel unsigned SrcReg = DefMI->getOperand(i).getReg(); 158119815Smarcel if (TargetRegisterInfo::isVirtualRegister(SrcReg)) { 159119815Smarcel DefMI = MRI->getVRegDef(SrcReg); 160119815Smarcel goto outer_continue; 161119815Smarcel } 162119815Smarcel } 163119815Smarcel } 164119815Smarcel } else if (DefMI->isCopyLike()) { 165158058Smarcel Reg = DefMI->getOperand(1).getReg(); 166119815Smarcel if (TargetRegisterInfo::isVirtualRegister(Reg)) { 167119815Smarcel DefMI = MRI->getVRegDef(Reg); 168119815Smarcel continue; 169 } 170 } else if (DefMI->isInsertSubreg()) { 171 Reg = DefMI->getOperand(2).getReg(); 172 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 173 DefMI = MRI->getVRegDef(Reg); 174 continue; 175 } 176 } 177 178 break; 179 } 180 181 return DefMI == MI; 182} 183 184bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const { 185 // FIXME: Detect integer instructions properly. 186 const MCInstrDesc &MCID = MI->getDesc(); 187 unsigned Domain = MCID.TSFlags & ARMII::DomainMask; 188 if (MI->mayStore()) 189 return false; 190 unsigned Opcode = MCID.getOpcode(); 191 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 192 return false; 193 if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON)) 194 return MI->readsRegister(Reg, TRI); 195 return false; 196} 197 198static bool isFpMulInstruction(unsigned Opcode) { 199 switch (Opcode) { 200 case ARM::VMULS: 201 case ARM::VMULfd: 202 case ARM::VMULfq: 203 case ARM::VMULD: 204 case ARM::VMULslfd: 205 case ARM::VMULslfq: 206 return true; 207 default: 208 return false; 209 } 210} 211 212bool MLxExpansion::FindMLxHazard(MachineInstr *MI) { 213 if (NumExpand >= ExpandLimit) 214 return false; 215 216 if (ForceExapnd) 217 return true; 218 219 MachineInstr *DefMI = getAccDefMI(MI); 220 if (TII->isFpMLxInstruction(DefMI->getOpcode())) { 221 // r0 = vmla 222 // r3 = vmla r0, r1, r2 223 // takes 16 - 17 cycles 224 // 225 // r0 = vmla 226 // r4 = vmul r1, r2 227 // r3 = vadd r0, r4 228 // takes about 14 - 15 cycles even with vmul stalling for 4 cycles. 229 IgnoreStall.insert(DefMI); 230 return true; 231 } 232 233 // On Swift, we mostly care about hazards from multiplication instructions 234 // writing the accumulator and the pipelining of loop iterations by out-of- 235 // order execution. 236 if (isSwift) 237 return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI); 238 239 if (IgnoreStall.count(MI)) 240 return false; 241 242 // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the 243 // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall 244 // preserves the in-order retirement of the instructions. 245 // Look at the next few instructions, if *most* of them can cause hazards, 246 // then the scheduler can't *fix* this, we'd better break up the VMLA. 247 unsigned Limit1 = isLikeA9 ? 1 : 4; 248 unsigned Limit2 = isLikeA9 ? 1 : 4; 249 for (unsigned i = 1; i <= 4; ++i) { 250 int Idx = ((int)MIIdx - i + 4) % 4; 251 MachineInstr *NextMI = LastMIs[Idx]; 252 if (!NextMI) 253 continue; 254 255 if (TII->canCauseFpMLxStall(NextMI->getOpcode())) { 256 if (i <= Limit1) 257 return true; 258 } 259 260 // Look for VMLx RAW hazard. 261 if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI)) 262 return true; 263 } 264 265 return false; 266} 267 268/// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair 269/// of MUL + ADD / SUB instructions. 270void 271MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI, 272 unsigned MulOpc, unsigned AddSubOpc, 273 bool NegAcc, bool HasLane) { 274 unsigned DstReg = MI->getOperand(0).getReg(); 275 bool DstDead = MI->getOperand(0).isDead(); 276 unsigned AccReg = MI->getOperand(1).getReg(); 277 unsigned Src1Reg = MI->getOperand(2).getReg(); 278 unsigned Src2Reg = MI->getOperand(3).getReg(); 279 bool Src1Kill = MI->getOperand(2).isKill(); 280 bool Src2Kill = MI->getOperand(3).isKill(); 281 unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0; 282 unsigned NextOp = HasLane ? 5 : 4; 283 ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm(); 284 unsigned PredReg = MI->getOperand(++NextOp).getReg(); 285 286 const MCInstrDesc &MCID1 = TII->get(MulOpc); 287 const MCInstrDesc &MCID2 = TII->get(AddSubOpc); 288 const MachineFunction &MF = *MI->getParent()->getParent(); 289 unsigned TmpReg = MRI->createVirtualRegister( 290 TII->getRegClass(MCID1, 0, TRI, MF)); 291 292 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg) 293 .addReg(Src1Reg, getKillRegState(Src1Kill)) 294 .addReg(Src2Reg, getKillRegState(Src2Kill)); 295 if (HasLane) 296 MIB.addImm(LaneImm); 297 MIB.addImm(Pred).addReg(PredReg); 298 299 MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2) 300 .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead)); 301 302 if (NegAcc) { 303 bool AccKill = MRI->hasOneNonDBGUse(AccReg); 304 MIB.addReg(TmpReg, getKillRegState(true)) 305 .addReg(AccReg, getKillRegState(AccKill)); 306 } else { 307 MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true)); 308 } 309 MIB.addImm(Pred).addReg(PredReg); 310 311 DEBUG({ 312 dbgs() << "Expanding: " << *MI; 313 dbgs() << " to:\n"; 314 MachineBasicBlock::iterator MII = MI; 315 MII = llvm::prior(MII); 316 MachineInstr &MI2 = *MII; 317 MII = llvm::prior(MII); 318 MachineInstr &MI1 = *MII; 319 dbgs() << " " << MI1; 320 dbgs() << " " << MI2; 321 }); 322 323 MI->eraseFromParent(); 324 ++NumExpand; 325} 326 327bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) { 328 bool Changed = false; 329 330 clearStack(); 331 IgnoreStall.clear(); 332 333 unsigned Skip = 0; 334 MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend(); 335 while (MII != E) { 336 MachineInstr *MI = &*MII; 337 338 if (MI->isLabel() || MI->isImplicitDef() || MI->isCopy()) { 339 ++MII; 340 continue; 341 } 342 343 const MCInstrDesc &MCID = MI->getDesc(); 344 if (MI->isBarrier()) { 345 clearStack(); 346 Skip = 0; 347 ++MII; 348 continue; 349 } 350 351 unsigned Domain = MCID.TSFlags & ARMII::DomainMask; 352 if (Domain == ARMII::DomainGeneral) { 353 if (++Skip == 2) 354 // Assume dual issues of non-VFP / NEON instructions. 355 pushStack(0); 356 } else { 357 Skip = 0; 358 359 unsigned MulOpc, AddSubOpc; 360 bool NegAcc, HasLane; 361 if (!TII->isFpMLxInstruction(MCID.getOpcode(), 362 MulOpc, AddSubOpc, NegAcc, HasLane) || 363 !FindMLxHazard(MI)) 364 pushStack(MI); 365 else { 366 ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane); 367 E = MBB.rend(); // May have changed if MI was the 1st instruction. 368 Changed = true; 369 continue; 370 } 371 } 372 373 ++MII; 374 } 375 376 return Changed; 377} 378 379bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) { 380 TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo()); 381 TRI = Fn.getTarget().getRegisterInfo(); 382 MRI = &Fn.getRegInfo(); 383 const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>(); 384 isLikeA9 = STI->isLikeA9() || STI->isSwift(); 385 isSwift = STI->isSwift(); 386 387 bool Modified = false; 388 for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E; 389 ++MFI) { 390 MachineBasicBlock &MBB = *MFI; 391 Modified |= ExpandFPMLxInstructions(MBB); 392 } 393 394 return Modified; 395} 396 397FunctionPass *llvm::createMLxExpansionPass() { 398 return new MLxExpansion(); 399} 400