MLxExpansionPass.cpp revision 221345 
1//===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ----------=//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
11// multiple and add / sub instructions) when special VMLx hazards are detected.
12//
13//===----------------------------------------------------------------------===//
14
15#define DEBUG_TYPE "mlx-expansion"
16#include "ARM.h"
17#include "ARMBaseInstrInfo.h"
18#include "ARMSubtarget.h"
19#include "llvm/CodeGen/MachineInstr.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/Target/TargetRegisterInfo.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/Support/CommandLine.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/raw_ostream.h"
29using namespace llvm;
30
31static cl::opt<bool>
32ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
33static cl::opt<unsigned>
34ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
35
36STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
37
38namespace {
39  struct MLxExpansion : public MachineFunctionPass {
40    static char ID;
41    MLxExpansion() : MachineFunctionPass(ID) {}
42
43    virtual bool runOnMachineFunction(MachineFunction &Fn);
44
45    virtual const char *getPassName() const {
46      return "ARM MLA / MLS expansion pass";
47    }
48
49  private:
50    const ARMBaseInstrInfo *TII;
51    const TargetRegisterInfo *TRI;
52    MachineRegisterInfo *MRI;
53
54    bool isA9;
55    unsigned MIIdx;
56    MachineInstr* LastMIs[4];
57    SmallPtrSet<MachineInstr*, 4> IgnoreStall;
58
59    void clearStack();
60    void pushStack(MachineInstr *MI);
61    MachineInstr *getAccDefMI(MachineInstr *MI) const;
62    unsigned getDefReg(MachineInstr *MI) const;
63    bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
64    bool FindMLxHazard(MachineInstr *MI);
65    void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
66                                unsigned MulOpc, unsigned AddSubOpc,
67                                bool NegAcc, bool HasLane);
68    bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
69  };
70  char MLxExpansion::ID = 0;
71}
72
73void MLxExpansion::clearStack() {
74  std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0);
75  MIIdx = 0;
76}
77
78void MLxExpansion::pushStack(MachineInstr *MI) {
79  LastMIs[MIIdx] = MI;
80  if (++MIIdx == 4)
81    MIIdx = 0;
82}
83
84MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
85  // Look past COPY and INSERT_SUBREG instructions to find the
86  // real definition MI. This is important for _sfp instructions.
87  unsigned Reg = MI->getOperand(1).getReg();
88  if (TargetRegisterInfo::isPhysicalRegister(Reg))
89    return 0;
90
91  MachineBasicBlock *MBB = MI->getParent();
92  MachineInstr *DefMI = MRI->getVRegDef(Reg);
93  while (true) {
94    if (DefMI->getParent() != MBB)
95      break;
96    if (DefMI->isCopyLike()) {
97      Reg = DefMI->getOperand(1).getReg();
98      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
99        DefMI = MRI->getVRegDef(Reg);
100        continue;
101      }
102    } else if (DefMI->isInsertSubreg()) {
103      Reg = DefMI->getOperand(2).getReg();
104      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
105        DefMI = MRI->getVRegDef(Reg);
106        continue;
107      }
108    }
109    break;
110  }
111  return DefMI;
112}
113
114unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
115  unsigned Reg = MI->getOperand(0).getReg();
116  if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
117      !MRI->hasOneNonDBGUse(Reg))
118    return Reg;
119
120  MachineBasicBlock *MBB = MI->getParent();
121  MachineInstr *UseMI = &*MRI->use_nodbg_begin(Reg);
122  if (UseMI->getParent() != MBB)
123    return Reg;
124
125  while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
126    Reg = UseMI->getOperand(0).getReg();
127    if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
128        !MRI->hasOneNonDBGUse(Reg))
129      return Reg;
130    UseMI = &*MRI->use_nodbg_begin(Reg);
131    if (UseMI->getParent() != MBB)
132      return Reg;
133  }
134
135  return Reg;
136}
137
138bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
139  // FIXME: Detect integer instructions properly.
140  const TargetInstrDesc &TID = MI->getDesc();
141  unsigned Domain = TID.TSFlags & ARMII::DomainMask;
142  if (TID.mayStore())
143    return false;
144  unsigned Opcode = TID.getOpcode();
145  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
146    return false;
147  if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
148    return MI->readsRegister(Reg, TRI);
149  return false;
150}
151
152
153bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
154  if (NumExpand >= ExpandLimit)
155    return false;
156
157  if (ForceExapnd)
158    return true;
159
160  MachineInstr *DefMI = getAccDefMI(MI);
161  if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
162    // r0 = vmla
163    // r3 = vmla r0, r1, r2
164    // takes 16 - 17 cycles
165    //
166    // r0 = vmla
167    // r4 = vmul r1, r2
168    // r3 = vadd r0, r4
169    // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
170    IgnoreStall.insert(DefMI);
171    return true;
172  }
173
174  if (IgnoreStall.count(MI))
175    return false;
176
177  // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
178  // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
179  // preserves the in-order retirement of the instructions.
180  // Look at the next few instructions, if *most* of them can cause hazards,
181  // then the scheduler can't *fix* this, we'd better break up the VMLA.
182  unsigned Limit1 = isA9 ? 1 : 4;
183  unsigned Limit2 = isA9 ? 1 : 4;
184  for (unsigned i = 1; i <= 4; ++i) {
185    int Idx = ((int)MIIdx - i + 4) % 4;
186    MachineInstr *NextMI = LastMIs[Idx];
187    if (!NextMI)
188      continue;
189
190    if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
191      if (i <= Limit1)
192        return true;
193    }
194
195    // Look for VMLx RAW hazard.
196    if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
197      return true;
198  }
199
200  return false;
201}
202
203/// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
204/// of MUL + ADD / SUB instructions.
205void
206MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
207                                     unsigned MulOpc, unsigned AddSubOpc,
208                                     bool NegAcc, bool HasLane) {
209  unsigned DstReg = MI->getOperand(0).getReg();
210  bool DstDead = MI->getOperand(0).isDead();
211  unsigned AccReg = MI->getOperand(1).getReg();
212  unsigned Src1Reg = MI->getOperand(2).getReg();
213  unsigned Src2Reg = MI->getOperand(3).getReg();
214  bool Src1Kill = MI->getOperand(2).isKill();
215  bool Src2Kill = MI->getOperand(3).isKill();
216  unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
217  unsigned NextOp = HasLane ? 5 : 4;
218  ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
219  unsigned PredReg = MI->getOperand(++NextOp).getReg();
220
221  const TargetInstrDesc &TID1 = TII->get(MulOpc);
222  const TargetInstrDesc &TID2 = TII->get(AddSubOpc);
223  unsigned TmpReg = MRI->createVirtualRegister(TID1.getRegClass(0, TRI));
224
225  MachineInstrBuilder MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TID1, TmpReg)
226    .addReg(Src1Reg, getKillRegState(Src1Kill))
227    .addReg(Src2Reg, getKillRegState(Src2Kill));
228  if (HasLane)
229    MIB.addImm(LaneImm);
230  MIB.addImm(Pred).addReg(PredReg);
231
232  MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TID2)
233    .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
234
235  if (NegAcc) {
236    bool AccKill = MRI->hasOneNonDBGUse(AccReg);
237    MIB.addReg(TmpReg, getKillRegState(true))
238       .addReg(AccReg, getKillRegState(AccKill));
239  } else {
240    MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
241  }
242  MIB.addImm(Pred).addReg(PredReg);
243
244  DEBUG({
245      dbgs() << "Expanding: " << *MI;
246      dbgs() << "  to:\n";
247      MachineBasicBlock::iterator MII = MI;
248      MII = llvm::prior(MII);
249      MachineInstr &MI2 = *MII;
250      MII = llvm::prior(MII);
251      MachineInstr &MI1 = *MII;
252      dbgs() << "    " << MI1;
253      dbgs() << "    " << MI2;
254   });
255
256  MI->eraseFromParent();
257  ++NumExpand;
258}
259
260bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
261  bool Changed = false;
262
263  clearStack();
264  IgnoreStall.clear();
265
266  unsigned Skip = 0;
267  MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
268  while (MII != E) {
269    MachineInstr *MI = &*MII;
270
271    if (MI->isLabel() || MI->isImplicitDef() || MI->isCopy()) {
272      ++MII;
273      continue;
274    }
275
276    const TargetInstrDesc &TID = MI->getDesc();
277    if (TID.isBarrier()) {
278      clearStack();
279      Skip = 0;
280      ++MII;
281      continue;
282    }
283
284    unsigned Domain = TID.TSFlags & ARMII::DomainMask;
285    if (Domain == ARMII::DomainGeneral) {
286      if (++Skip == 2)
287        // Assume dual issues of non-VFP / NEON instructions.
288        pushStack(0);
289    } else {
290      Skip = 0;
291
292      unsigned MulOpc, AddSubOpc;
293      bool NegAcc, HasLane;
294      if (!TII->isFpMLxInstruction(TID.getOpcode(),
295                                   MulOpc, AddSubOpc, NegAcc, HasLane) ||
296          !FindMLxHazard(MI))
297        pushStack(MI);
298      else {
299        ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
300        E = MBB.rend(); // May have changed if MI was the 1st instruction.
301        Changed = true;
302        continue;
303      }
304    }
305
306    ++MII;
307  }
308
309  return Changed;
310}
311
312bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
313  TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
314  TRI = Fn.getTarget().getRegisterInfo();
315  MRI = &Fn.getRegInfo();
316  const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
317  isA9 = STI->isCortexA9();
318
319  bool Modified = false;
320  for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
321       ++MFI) {
322    MachineBasicBlock &MBB = *MFI;
323    Modified |= ExpandFPMLxInstructions(MBB);
324  }
325
326  return Modified;
327}
328
329FunctionPass *llvm::createMLxExpansionPass() {
330  return new MLxExpansion();
331}
332