1//===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9//
10
11#include "AMDGPU.h"
12#include "AMDGPUSubtarget.h"
13#include "SIInstrInfo.h"
14#include "SIMachineFunctionInfo.h"
15#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16#include "llvm/ADT/DepthFirstIterator.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/CodeGen/MachineFunctionPass.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/Support/Debug.h"
22#include "llvm/Support/raw_ostream.h"
23#include "llvm/Target/TargetMachine.h"
24
25#define DEBUG_TYPE "si-fold-operands"
26using namespace llvm;
27
28namespace {
29
30struct FoldCandidate {
31  MachineInstr *UseMI;
32  union {
33    MachineOperand *OpToFold;
34    uint64_t ImmToFold;
35    int FrameIndexToFold;
36  };
37  int ShrinkOpcode;
38  unsigned char UseOpNo;
39  MachineOperand::MachineOperandType Kind;
40  bool Commuted;
41
42  FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp,
43                bool Commuted_ = false,
44                int ShrinkOp = -1) :
45    UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
46    Kind(FoldOp->getType()),
47    Commuted(Commuted_) {
48    if (FoldOp->isImm()) {
49      ImmToFold = FoldOp->getImm();
50    } else if (FoldOp->isFI()) {
51      FrameIndexToFold = FoldOp->getIndex();
52    } else {
53      assert(FoldOp->isReg() || FoldOp->isGlobal());
54      OpToFold = FoldOp;
55    }
56  }
57
58  bool isFI() const {
59    return Kind == MachineOperand::MO_FrameIndex;
60  }
61
62  bool isImm() const {
63    return Kind == MachineOperand::MO_Immediate;
64  }
65
66  bool isReg() const {
67    return Kind == MachineOperand::MO_Register;
68  }
69
70  bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
71
72  bool isCommuted() const {
73    return Commuted;
74  }
75
76  bool needsShrink() const {
77    return ShrinkOpcode != -1;
78  }
79
80  int getShrinkOpcode() const {
81    return ShrinkOpcode;
82  }
83};
84
85class SIFoldOperands : public MachineFunctionPass {
86public:
87  static char ID;
88  MachineRegisterInfo *MRI;
89  const SIInstrInfo *TII;
90  const SIRegisterInfo *TRI;
91  const GCNSubtarget *ST;
92  const SIMachineFunctionInfo *MFI;
93
94  void foldOperand(MachineOperand &OpToFold,
95                   MachineInstr *UseMI,
96                   int UseOpIdx,
97                   SmallVectorImpl<FoldCandidate> &FoldList,
98                   SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
99
100  void foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;
101
102  const MachineOperand *isClamp(const MachineInstr &MI) const;
103  bool tryFoldClamp(MachineInstr &MI);
104
105  std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
106  bool tryFoldOMod(MachineInstr &MI);
107
108public:
109  SIFoldOperands() : MachineFunctionPass(ID) {
110    initializeSIFoldOperandsPass(*PassRegistry::getPassRegistry());
111  }
112
113  bool runOnMachineFunction(MachineFunction &MF) override;
114
115  StringRef getPassName() const override { return "SI Fold Operands"; }
116
117  void getAnalysisUsage(AnalysisUsage &AU) const override {
118    AU.setPreservesCFG();
119    MachineFunctionPass::getAnalysisUsage(AU);
120  }
121};
122
123} // End anonymous namespace.
124
125INITIALIZE_PASS(SIFoldOperands, DEBUG_TYPE,
126                "SI Fold Operands", false, false)
127
128char SIFoldOperands::ID = 0;
129
130char &llvm::SIFoldOperandsID = SIFoldOperands::ID;
131
132// Wrapper around isInlineConstant that understands special cases when
133// instruction types are replaced during operand folding.
134static bool isInlineConstantIfFolded(const SIInstrInfo *TII,
135                                     const MachineInstr &UseMI,
136                                     unsigned OpNo,
137                                     const MachineOperand &OpToFold) {
138  if (TII->isInlineConstant(UseMI, OpNo, OpToFold))
139    return true;
140
141  unsigned Opc = UseMI.getOpcode();
142  switch (Opc) {
143  case AMDGPU::V_MAC_F32_e64:
144  case AMDGPU::V_MAC_F16_e64:
145  case AMDGPU::V_FMAC_F32_e64:
146  case AMDGPU::V_FMAC_F16_e64: {
147    // Special case for mac. Since this is replaced with mad when folded into
148    // src2, we need to check the legality for the final instruction.
149    int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
150    if (static_cast<int>(OpNo) == Src2Idx) {
151      bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e64 ||
152                   Opc == AMDGPU::V_FMAC_F16_e64;
153      bool IsF32 = Opc == AMDGPU::V_MAC_F32_e64 ||
154                   Opc == AMDGPU::V_FMAC_F32_e64;
155
156      unsigned Opc = IsFMA ?
157        (IsF32 ? AMDGPU::V_FMA_F32 : AMDGPU::V_FMA_F16_gfx9) :
158        (IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16);
159      const MCInstrDesc &MadDesc = TII->get(Opc);
160      return TII->isInlineConstant(OpToFold, MadDesc.OpInfo[OpNo].OperandType);
161    }
162    return false;
163  }
164  default:
165    return false;
166  }
167}
168
169// TODO: Add heuristic that the frame index might not fit in the addressing mode
170// immediate offset to avoid materializing in loops.
171static bool frameIndexMayFold(const SIInstrInfo *TII,
172                              const MachineInstr &UseMI,
173                              int OpNo,
174                              const MachineOperand &OpToFold) {
175  return OpToFold.isFI() &&
176    (TII->isMUBUF(UseMI) || TII->isFLATScratch(UseMI)) &&
177    OpNo == AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), AMDGPU::OpName::vaddr);
178}
179
180FunctionPass *llvm::createSIFoldOperandsPass() {
181  return new SIFoldOperands();
182}
183
184static bool updateOperand(FoldCandidate &Fold,
185                          const SIInstrInfo &TII,
186                          const TargetRegisterInfo &TRI,
187                          const GCNSubtarget &ST) {
188  MachineInstr *MI = Fold.UseMI;
189  MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
190  assert(Old.isReg());
191
192  if (Fold.isImm()) {
193    if (MI->getDesc().TSFlags & SIInstrFlags::IsPacked &&
194        !(MI->getDesc().TSFlags & SIInstrFlags::IsMAI) &&
195        AMDGPU::isInlinableLiteralV216(static_cast<uint16_t>(Fold.ImmToFold),
196                                       ST.hasInv2PiInlineImm())) {
197      // Set op_sel/op_sel_hi on this operand or bail out if op_sel is
198      // already set.
199      unsigned Opcode = MI->getOpcode();
200      int OpNo = MI->getOperandNo(&Old);
201      int ModIdx = -1;
202      if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0))
203        ModIdx = AMDGPU::OpName::src0_modifiers;
204      else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1))
205        ModIdx = AMDGPU::OpName::src1_modifiers;
206      else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2))
207        ModIdx = AMDGPU::OpName::src2_modifiers;
208      assert(ModIdx != -1);
209      ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx);
210      MachineOperand &Mod = MI->getOperand(ModIdx);
211      unsigned Val = Mod.getImm();
212      if ((Val & SISrcMods::OP_SEL_0) || !(Val & SISrcMods::OP_SEL_1))
213        return false;
214      // Only apply the following transformation if that operand requries
215      // a packed immediate.
216      switch (TII.get(Opcode).OpInfo[OpNo].OperandType) {
217      case AMDGPU::OPERAND_REG_IMM_V2FP16:
218      case AMDGPU::OPERAND_REG_IMM_V2INT16:
219      case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
220      case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
221        // If upper part is all zero we do not need op_sel_hi.
222        if (!isUInt<16>(Fold.ImmToFold)) {
223          if (!(Fold.ImmToFold & 0xffff)) {
224            Mod.setImm(Mod.getImm() | SISrcMods::OP_SEL_0);
225            Mod.setImm(Mod.getImm() & ~SISrcMods::OP_SEL_1);
226            Old.ChangeToImmediate((Fold.ImmToFold >> 16) & 0xffff);
227            return true;
228          }
229          Mod.setImm(Mod.getImm() & ~SISrcMods::OP_SEL_1);
230          Old.ChangeToImmediate(Fold.ImmToFold & 0xffff);
231          return true;
232        }
233        break;
234      default:
235        break;
236      }
237    }
238  }
239
240  if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
241    MachineBasicBlock *MBB = MI->getParent();
242    auto Liveness = MBB->computeRegisterLiveness(&TRI, AMDGPU::VCC, MI, 16);
243    if (Liveness != MachineBasicBlock::LQR_Dead) {
244      LLVM_DEBUG(dbgs() << "Not shrinking " << MI << " due to vcc liveness\n");
245      return false;
246    }
247
248    MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
249    int Op32 = Fold.getShrinkOpcode();
250    MachineOperand &Dst0 = MI->getOperand(0);
251    MachineOperand &Dst1 = MI->getOperand(1);
252    assert(Dst0.isDef() && Dst1.isDef());
253
254    bool HaveNonDbgCarryUse = !MRI.use_nodbg_empty(Dst1.getReg());
255
256    const TargetRegisterClass *Dst0RC = MRI.getRegClass(Dst0.getReg());
257    Register NewReg0 = MRI.createVirtualRegister(Dst0RC);
258
259    MachineInstr *Inst32 = TII.buildShrunkInst(*MI, Op32);
260
261    if (HaveNonDbgCarryUse) {
262      BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), Dst1.getReg())
263        .addReg(AMDGPU::VCC, RegState::Kill);
264    }
265
266    // Keep the old instruction around to avoid breaking iterators, but
267    // replace it with a dummy instruction to remove uses.
268    //
269    // FIXME: We should not invert how this pass looks at operands to avoid
270    // this. Should track set of foldable movs instead of looking for uses
271    // when looking at a use.
272    Dst0.setReg(NewReg0);
273    for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
274      MI->RemoveOperand(I);
275    MI->setDesc(TII.get(AMDGPU::IMPLICIT_DEF));
276
277    if (Fold.isCommuted())
278      TII.commuteInstruction(*Inst32, false);
279    return true;
280  }
281
282  assert(!Fold.needsShrink() && "not handled");
283
284  if (Fold.isImm()) {
285    Old.ChangeToImmediate(Fold.ImmToFold);
286    return true;
287  }
288
289  if (Fold.isGlobal()) {
290    Old.ChangeToGA(Fold.OpToFold->getGlobal(), Fold.OpToFold->getOffset(),
291                   Fold.OpToFold->getTargetFlags());
292    return true;
293  }
294
295  if (Fold.isFI()) {
296    Old.ChangeToFrameIndex(Fold.FrameIndexToFold);
297    return true;
298  }
299
300  MachineOperand *New = Fold.OpToFold;
301  Old.substVirtReg(New->getReg(), New->getSubReg(), TRI);
302  Old.setIsUndef(New->isUndef());
303  return true;
304}
305
306static bool isUseMIInFoldList(ArrayRef<FoldCandidate> FoldList,
307                              const MachineInstr *MI) {
308  for (auto Candidate : FoldList) {
309    if (Candidate.UseMI == MI)
310      return true;
311  }
312  return false;
313}
314
315static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
316                                MachineInstr *MI, unsigned OpNo,
317                                MachineOperand *FoldOp, bool Commuted = false,
318                                int ShrinkOp = -1) {
319  // Skip additional folding on the same operand.
320  for (FoldCandidate &Fold : FoldList)
321    if (Fold.UseMI == MI && Fold.UseOpNo == OpNo)
322      return;
323  LLVM_DEBUG(dbgs() << "Append " << (Commuted ? "commuted" : "normal")
324                    << " operand " << OpNo << "\n  " << *MI << '\n');
325  FoldList.push_back(FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
326}
327
328static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
329                             MachineInstr *MI, unsigned OpNo,
330                             MachineOperand *OpToFold,
331                             const SIInstrInfo *TII) {
332  if (!TII->isOperandLegal(*MI, OpNo, OpToFold)) {
333    // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
334    unsigned Opc = MI->getOpcode();
335    if ((Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
336         Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
337        (int)OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)) {
338      bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e64 ||
339                   Opc == AMDGPU::V_FMAC_F16_e64;
340      bool IsF32 = Opc == AMDGPU::V_MAC_F32_e64 ||
341                   Opc == AMDGPU::V_FMAC_F32_e64;
342      unsigned NewOpc = IsFMA ?
343        (IsF32 ? AMDGPU::V_FMA_F32 : AMDGPU::V_FMA_F16_gfx9) :
344        (IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16);
345
346      // Check if changing this to a v_mad_{f16, f32} instruction will allow us
347      // to fold the operand.
348      MI->setDesc(TII->get(NewOpc));
349      bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold, TII);
350      if (FoldAsMAD) {
351        MI->untieRegOperand(OpNo);
352        return true;
353      }
354      MI->setDesc(TII->get(Opc));
355    }
356
357    // Special case for s_setreg_b32
358    if (Opc == AMDGPU::S_SETREG_B32 && OpToFold->isImm()) {
359      MI->setDesc(TII->get(AMDGPU::S_SETREG_IMM32_B32));
360      appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
361      return true;
362    }
363
364    // If we are already folding into another operand of MI, then
365    // we can't commute the instruction, otherwise we risk making the
366    // other fold illegal.
367    if (isUseMIInFoldList(FoldList, MI))
368      return false;
369
370    unsigned CommuteOpNo = OpNo;
371
372    // Operand is not legal, so try to commute the instruction to
373    // see if this makes it possible to fold.
374    unsigned CommuteIdx0 = TargetInstrInfo::CommuteAnyOperandIndex;
375    unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
376    bool CanCommute = TII->findCommutedOpIndices(*MI, CommuteIdx0, CommuteIdx1);
377
378    if (CanCommute) {
379      if (CommuteIdx0 == OpNo)
380        CommuteOpNo = CommuteIdx1;
381      else if (CommuteIdx1 == OpNo)
382        CommuteOpNo = CommuteIdx0;
383    }
384
385
386    // One of operands might be an Imm operand, and OpNo may refer to it after
387    // the call of commuteInstruction() below. Such situations are avoided
388    // here explicitly as OpNo must be a register operand to be a candidate
389    // for memory folding.
390    if (CanCommute && (!MI->getOperand(CommuteIdx0).isReg() ||
391                       !MI->getOperand(CommuteIdx1).isReg()))
392      return false;
393
394    if (!CanCommute ||
395        !TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1))
396      return false;
397
398    if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) {
399      if ((Opc == AMDGPU::V_ADD_I32_e64 ||
400           Opc == AMDGPU::V_SUB_I32_e64 ||
401           Opc == AMDGPU::V_SUBREV_I32_e64) && // FIXME
402          (OpToFold->isImm() || OpToFold->isFI() || OpToFold->isGlobal())) {
403        MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo();
404
405        // Verify the other operand is a VGPR, otherwise we would violate the
406        // constant bus restriction.
407        unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0;
408        MachineOperand &OtherOp = MI->getOperand(OtherIdx);
409        if (!OtherOp.isReg() ||
410            !TII->getRegisterInfo().isVGPR(MRI, OtherOp.getReg()))
411          return false;
412
413        assert(MI->getOperand(1).isDef());
414
415        // Make sure to get the 32-bit version of the commuted opcode.
416        unsigned MaybeCommutedOpc = MI->getOpcode();
417        int Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc);
418
419        appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true, Op32);
420        return true;
421      }
422
423      TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1);
424      return false;
425    }
426
427    appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true);
428    return true;
429  }
430
431  // Check the case where we might introduce a second constant operand to a
432  // scalar instruction
433  if (TII->isSALU(MI->getOpcode())) {
434    const MCInstrDesc &InstDesc = MI->getDesc();
435    const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
436    const SIRegisterInfo &SRI = TII->getRegisterInfo();
437
438    // Fine if the operand can be encoded as an inline constant
439    if (OpToFold->isImm()) {
440      if (!SRI.opCanUseInlineConstant(OpInfo.OperandType) ||
441          !TII->isInlineConstant(*OpToFold, OpInfo)) {
442        // Otherwise check for another constant
443        for (unsigned i = 0, e = InstDesc.getNumOperands(); i != e; ++i) {
444          auto &Op = MI->getOperand(i);
445          if (OpNo != i &&
446              TII->isLiteralConstantLike(Op, OpInfo)) {
447            return false;
448          }
449        }
450      }
451    }
452  }
453
454  appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
455  return true;
456}
457
458// If the use operand doesn't care about the value, this may be an operand only
459// used for register indexing, in which case it is unsafe to fold.
460static bool isUseSafeToFold(const SIInstrInfo *TII,
461                            const MachineInstr &MI,
462                            const MachineOperand &UseMO) {
463  return !UseMO.isUndef() && !TII->isSDWA(MI);
464  //return !MI.hasRegisterImplicitUseOperand(UseMO.getReg());
465}
466
467// Find a def of the UseReg, check if it is a reg_seqence and find initializers
468// for each subreg, tracking it to foldable inline immediate if possible.
469// Returns true on success.
470static bool getRegSeqInit(
471    SmallVectorImpl<std::pair<MachineOperand*, unsigned>> &Defs,
472    Register UseReg, uint8_t OpTy,
473    const SIInstrInfo *TII, const MachineRegisterInfo &MRI) {
474  MachineInstr *Def = MRI.getUniqueVRegDef(UseReg);
475  if (!Def || !Def->isRegSequence())
476    return false;
477
478  for (unsigned I = 1, E = Def->getNumExplicitOperands(); I < E; I += 2) {
479    MachineOperand *Sub = &Def->getOperand(I);
480    assert (Sub->isReg());
481
482    for (MachineInstr *SubDef = MRI.getUniqueVRegDef(Sub->getReg());
483         SubDef && Sub->isReg() && !Sub->getSubReg() &&
484         TII->isFoldableCopy(*SubDef);
485         SubDef = MRI.getUniqueVRegDef(Sub->getReg())) {
486      MachineOperand *Op = &SubDef->getOperand(1);
487      if (Op->isImm()) {
488        if (TII->isInlineConstant(*Op, OpTy))
489          Sub = Op;
490        break;
491      }
492      if (!Op->isReg())
493        break;
494      Sub = Op;
495    }
496
497    Defs.push_back(std::make_pair(Sub, Def->getOperand(I + 1).getImm()));
498  }
499
500  return true;
501}
502
503static bool tryToFoldACImm(const SIInstrInfo *TII,
504                           const MachineOperand &OpToFold,
505                           MachineInstr *UseMI,
506                           unsigned UseOpIdx,
507                           SmallVectorImpl<FoldCandidate> &FoldList) {
508  const MCInstrDesc &Desc = UseMI->getDesc();
509  const MCOperandInfo *OpInfo = Desc.OpInfo;
510  if (!OpInfo || UseOpIdx >= Desc.getNumOperands())
511    return false;
512
513  uint8_t OpTy = OpInfo[UseOpIdx].OperandType;
514  if (OpTy < AMDGPU::OPERAND_REG_INLINE_AC_FIRST ||
515      OpTy > AMDGPU::OPERAND_REG_INLINE_AC_LAST)
516    return false;
517
518  if (OpToFold.isImm() && TII->isInlineConstant(OpToFold, OpTy) &&
519      TII->isOperandLegal(*UseMI, UseOpIdx, &OpToFold)) {
520    UseMI->getOperand(UseOpIdx).ChangeToImmediate(OpToFold.getImm());
521    return true;
522  }
523
524  if (!OpToFold.isReg())
525    return false;
526
527  Register UseReg = OpToFold.getReg();
528  if (!Register::isVirtualRegister(UseReg))
529    return false;
530
531  if (llvm::find_if(FoldList, [UseMI](const FoldCandidate &FC) {
532        return FC.UseMI == UseMI; }) != FoldList.end())
533    return false;
534
535  MachineRegisterInfo &MRI = UseMI->getParent()->getParent()->getRegInfo();
536  SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
537  if (!getRegSeqInit(Defs, UseReg, OpTy, TII, MRI))
538    return false;
539
540  int32_t Imm;
541  for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
542    const MachineOperand *Op = Defs[I].first;
543    if (!Op->isImm())
544      return false;
545
546    auto SubImm = Op->getImm();
547    if (!I) {
548      Imm = SubImm;
549      if (!TII->isInlineConstant(*Op, OpTy) ||
550          !TII->isOperandLegal(*UseMI, UseOpIdx, Op))
551        return false;
552
553      continue;
554    }
555    if (Imm != SubImm)
556      return false; // Can only fold splat constants
557  }
558
559  appendFoldCandidate(FoldList, UseMI, UseOpIdx, Defs[0].first);
560  return true;
561}
562
563void SIFoldOperands::foldOperand(
564  MachineOperand &OpToFold,
565  MachineInstr *UseMI,
566  int UseOpIdx,
567  SmallVectorImpl<FoldCandidate> &FoldList,
568  SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
569  const MachineOperand &UseOp = UseMI->getOperand(UseOpIdx);
570
571  if (!isUseSafeToFold(TII, *UseMI, UseOp))
572    return;
573
574  // FIXME: Fold operands with subregs.
575  if (UseOp.isReg() && OpToFold.isReg()) {
576    if (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister)
577      return;
578  }
579
580  // Special case for REG_SEQUENCE: We can't fold literals into
581  // REG_SEQUENCE instructions, so we have to fold them into the
582  // uses of REG_SEQUENCE.
583  if (UseMI->isRegSequence()) {
584    Register RegSeqDstReg = UseMI->getOperand(0).getReg();
585    unsigned RegSeqDstSubReg = UseMI->getOperand(UseOpIdx + 1).getImm();
586
587    MachineRegisterInfo::use_iterator Next;
588    for (MachineRegisterInfo::use_iterator
589           RSUse = MRI->use_begin(RegSeqDstReg), RSE = MRI->use_end();
590         RSUse != RSE; RSUse = Next) {
591      Next = std::next(RSUse);
592
593      MachineInstr *RSUseMI = RSUse->getParent();
594
595      if (tryToFoldACImm(TII, UseMI->getOperand(0), RSUseMI,
596                         RSUse.getOperandNo(), FoldList))
597        continue;
598
599      if (RSUse->getSubReg() != RegSeqDstSubReg)
600        continue;
601
602      foldOperand(OpToFold, RSUseMI, RSUse.getOperandNo(), FoldList,
603                  CopiesToReplace);
604    }
605
606    return;
607  }
608
609  if (tryToFoldACImm(TII, OpToFold, UseMI, UseOpIdx, FoldList))
610    return;
611
612  if (frameIndexMayFold(TII, *UseMI, UseOpIdx, OpToFold)) {
613    // Sanity check that this is a stack access.
614    // FIXME: Should probably use stack pseudos before frame lowering.
615    MachineOperand *SOff = TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset);
616    if (!SOff->isReg() || (SOff->getReg() != MFI->getScratchWaveOffsetReg() &&
617                           SOff->getReg() != MFI->getStackPtrOffsetReg()))
618      return;
619
620    if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() !=
621        MFI->getScratchRSrcReg())
622      return;
623
624    // A frame index will resolve to a positive constant, so it should always be
625    // safe to fold the addressing mode, even pre-GFX9.
626    UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex());
627    SOff->setReg(MFI->getStackPtrOffsetReg());
628    return;
629  }
630
631  bool FoldingImmLike =
632      OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
633
634  if (FoldingImmLike && UseMI->isCopy()) {
635    Register DestReg = UseMI->getOperand(0).getReg();
636
637    // Don't fold into a copy to a physical register. Doing so would interfere
638    // with the register coalescer's logic which would avoid redundant
639    // initalizations.
640    if (DestReg.isPhysical())
641      return;
642
643    const TargetRegisterClass *DestRC =  MRI->getRegClass(DestReg);
644
645    Register SrcReg = UseMI->getOperand(1).getReg();
646    if (SrcReg.isVirtual()) { // XXX - This can be an assert?
647      const TargetRegisterClass * SrcRC = MRI->getRegClass(SrcReg);
648      if (TRI->isSGPRClass(SrcRC) && TRI->hasVectorRegisters(DestRC)) {
649        MachineRegisterInfo::use_iterator NextUse;
650        SmallVector<FoldCandidate, 4> CopyUses;
651        for (MachineRegisterInfo::use_iterator
652          Use = MRI->use_begin(DestReg), E = MRI->use_end();
653          Use != E; Use = NextUse) {
654          NextUse = std::next(Use);
655          FoldCandidate FC = FoldCandidate(Use->getParent(),
656           Use.getOperandNo(), &UseMI->getOperand(1));
657          CopyUses.push_back(FC);
658       }
659        for (auto & F : CopyUses) {
660          foldOperand(*F.OpToFold, F.UseMI, F.UseOpNo,
661           FoldList, CopiesToReplace);
662        }
663      }
664    }
665
666    if (DestRC == &AMDGPU::AGPR_32RegClass &&
667        TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
668      UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32));
669      UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
670      CopiesToReplace.push_back(UseMI);
671      return;
672    }
673
674    // In order to fold immediates into copies, we need to change the
675    // copy to a MOV.
676
677    unsigned MovOp = TII->getMovOpcode(DestRC);
678    if (MovOp == AMDGPU::COPY)
679      return;
680
681    UseMI->setDesc(TII->get(MovOp));
682    MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
683    MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
684    while (ImpOpI != ImpOpE) {
685      MachineInstr::mop_iterator Tmp = ImpOpI;
686      ImpOpI++;
687      UseMI->RemoveOperand(UseMI->getOperandNo(Tmp));
688    }
689    CopiesToReplace.push_back(UseMI);
690  } else {
691    if (UseMI->isCopy() && OpToFold.isReg() &&
692        UseMI->getOperand(0).getReg().isVirtual() &&
693        !UseMI->getOperand(1).getSubReg()) {
694      LLVM_DEBUG(dbgs() << "Folding " << OpToFold
695                        << "\n into " << *UseMI << '\n');
696      unsigned Size = TII->getOpSize(*UseMI, 1);
697      Register UseReg = OpToFold.getReg();
698      UseMI->getOperand(1).setReg(UseReg);
699      UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
700      UseMI->getOperand(1).setIsKill(false);
701      CopiesToReplace.push_back(UseMI);
702      OpToFold.setIsKill(false);
703
704      // That is very tricky to store a value into an AGPR. v_accvgpr_write_b32
705      // can only accept VGPR or inline immediate. Recreate a reg_sequence with
706      // its initializers right here, so we will rematerialize immediates and
707      // avoid copies via different reg classes.
708      SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
709      if (Size > 4 && TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) &&
710          getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32, TII,
711                        *MRI)) {
712        const DebugLoc &DL = UseMI->getDebugLoc();
713        MachineBasicBlock &MBB = *UseMI->getParent();
714
715        UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE));
716        for (unsigned I = UseMI->getNumOperands() - 1; I > 0; --I)
717          UseMI->RemoveOperand(I);
718
719        MachineInstrBuilder B(*MBB.getParent(), UseMI);
720        DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
721        SmallSetVector<TargetInstrInfo::RegSubRegPair, 32> SeenAGPRs;
722        for (unsigned I = 0; I < Size / 4; ++I) {
723          MachineOperand *Def = Defs[I].first;
724          TargetInstrInfo::RegSubRegPair CopyToVGPR;
725          if (Def->isImm() &&
726              TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
727            int64_t Imm = Def->getImm();
728
729            auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
730            BuildMI(MBB, UseMI, DL,
731                    TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), Tmp).addImm(Imm);
732            B.addReg(Tmp);
733          } else if (Def->isReg() && TRI->isAGPR(*MRI, Def->getReg())) {
734            auto Src = getRegSubRegPair(*Def);
735            Def->setIsKill(false);
736            if (!SeenAGPRs.insert(Src)) {
737              // We cannot build a reg_sequence out of the same registers, they
738              // must be copied. Better do it here before copyPhysReg() created
739              // several reads to do the AGPR->VGPR->AGPR copy.
740              CopyToVGPR = Src;
741            } else {
742              B.addReg(Src.Reg, Def->isUndef() ? RegState::Undef : 0,
743                       Src.SubReg);
744            }
745          } else {
746            assert(Def->isReg());
747            Def->setIsKill(false);
748            auto Src = getRegSubRegPair(*Def);
749
750            // Direct copy from SGPR to AGPR is not possible. To avoid creation
751            // of exploded copies SGPR->VGPR->AGPR in the copyPhysReg() later,
752            // create a copy here and track if we already have such a copy.
753            if (TRI->isSGPRReg(*MRI, Src.Reg)) {
754              CopyToVGPR = Src;
755            } else {
756              auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
757              BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def);
758              B.addReg(Tmp);
759            }
760          }
761
762          if (CopyToVGPR.Reg) {
763            Register Vgpr;
764            if (VGPRCopies.count(CopyToVGPR)) {
765              Vgpr = VGPRCopies[CopyToVGPR];
766            } else {
767              Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
768              BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def);
769              VGPRCopies[CopyToVGPR] = Vgpr;
770            }
771            auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
772            BuildMI(MBB, UseMI, DL,
773                    TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), Tmp).addReg(Vgpr);
774            B.addReg(Tmp);
775          }
776
777          B.addImm(Defs[I].second);
778        }
779        LLVM_DEBUG(dbgs() << "Folded " << *UseMI << '\n');
780        return;
781      }
782
783      if (Size != 4)
784        return;
785      if (TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) &&
786          TRI->isVGPR(*MRI, UseMI->getOperand(1).getReg()))
787        UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32));
788      else if (TRI->isVGPR(*MRI, UseMI->getOperand(0).getReg()) &&
789               TRI->isAGPR(*MRI, UseMI->getOperand(1).getReg()))
790        UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32));
791      return;
792    }
793
794    unsigned UseOpc = UseMI->getOpcode();
795    if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
796        (UseOpc == AMDGPU::V_READLANE_B32 &&
797         (int)UseOpIdx ==
798         AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) {
799      // %vgpr = V_MOV_B32 imm
800      // %sgpr = V_READFIRSTLANE_B32 %vgpr
801      // =>
802      // %sgpr = S_MOV_B32 imm
803      if (FoldingImmLike) {
804        if (execMayBeModifiedBeforeUse(*MRI,
805                                       UseMI->getOperand(UseOpIdx).getReg(),
806                                       *OpToFold.getParent(),
807                                       *UseMI))
808          return;
809
810        UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32));
811
812        // FIXME: ChangeToImmediate should clear subreg
813        UseMI->getOperand(1).setSubReg(0);
814        if (OpToFold.isImm())
815          UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
816        else
817          UseMI->getOperand(1).ChangeToFrameIndex(OpToFold.getIndex());
818        UseMI->RemoveOperand(2); // Remove exec read (or src1 for readlane)
819        return;
820      }
821
822      if (OpToFold.isReg() && TRI->isSGPRReg(*MRI, OpToFold.getReg())) {
823        if (execMayBeModifiedBeforeUse(*MRI,
824                                       UseMI->getOperand(UseOpIdx).getReg(),
825                                       *OpToFold.getParent(),
826                                       *UseMI))
827          return;
828
829        // %vgpr = COPY %sgpr0
830        // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
831        // =>
832        // %sgpr1 = COPY %sgpr0
833        UseMI->setDesc(TII->get(AMDGPU::COPY));
834        UseMI->getOperand(1).setReg(OpToFold.getReg());
835        UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
836        UseMI->getOperand(1).setIsKill(false);
837        UseMI->RemoveOperand(2); // Remove exec read (or src1 for readlane)
838        return;
839      }
840    }
841
842    const MCInstrDesc &UseDesc = UseMI->getDesc();
843
844    // Don't fold into target independent nodes.  Target independent opcodes
845    // don't have defined register classes.
846    if (UseDesc.isVariadic() ||
847        UseOp.isImplicit() ||
848        UseDesc.OpInfo[UseOpIdx].RegClass == -1)
849      return;
850  }
851
852  if (!FoldingImmLike) {
853    tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold, TII);
854
855    // FIXME: We could try to change the instruction from 64-bit to 32-bit
856    // to enable more folding opportunites.  The shrink operands pass
857    // already does this.
858    return;
859  }
860
861
862  const MCInstrDesc &FoldDesc = OpToFold.getParent()->getDesc();
863  const TargetRegisterClass *FoldRC =
864    TRI->getRegClass(FoldDesc.OpInfo[0].RegClass);
865
866  // Split 64-bit constants into 32-bits for folding.
867  if (UseOp.getSubReg() && AMDGPU::getRegBitWidth(FoldRC->getID()) == 64) {
868    Register UseReg = UseOp.getReg();
869    const TargetRegisterClass *UseRC = MRI->getRegClass(UseReg);
870
871    if (AMDGPU::getRegBitWidth(UseRC->getID()) != 64)
872      return;
873
874    APInt Imm(64, OpToFold.getImm());
875    if (UseOp.getSubReg() == AMDGPU::sub0) {
876      Imm = Imm.getLoBits(32);
877    } else {
878      assert(UseOp.getSubReg() == AMDGPU::sub1);
879      Imm = Imm.getHiBits(32);
880    }
881
882    MachineOperand ImmOp = MachineOperand::CreateImm(Imm.getSExtValue());
883    tryAddToFoldList(FoldList, UseMI, UseOpIdx, &ImmOp, TII);
884    return;
885  }
886
887
888
889  tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold, TII);
890}
891
892static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
893                                  uint32_t LHS, uint32_t RHS) {
894  switch (Opcode) {
895  case AMDGPU::V_AND_B32_e64:
896  case AMDGPU::V_AND_B32_e32:
897  case AMDGPU::S_AND_B32:
898    Result = LHS & RHS;
899    return true;
900  case AMDGPU::V_OR_B32_e64:
901  case AMDGPU::V_OR_B32_e32:
902  case AMDGPU::S_OR_B32:
903    Result = LHS | RHS;
904    return true;
905  case AMDGPU::V_XOR_B32_e64:
906  case AMDGPU::V_XOR_B32_e32:
907  case AMDGPU::S_XOR_B32:
908    Result = LHS ^ RHS;
909    return true;
910  case AMDGPU::V_LSHL_B32_e64:
911  case AMDGPU::V_LSHL_B32_e32:
912  case AMDGPU::S_LSHL_B32:
913    // The instruction ignores the high bits for out of bounds shifts.
914    Result = LHS << (RHS & 31);
915    return true;
916  case AMDGPU::V_LSHLREV_B32_e64:
917  case AMDGPU::V_LSHLREV_B32_e32:
918    Result = RHS << (LHS & 31);
919    return true;
920  case AMDGPU::V_LSHR_B32_e64:
921  case AMDGPU::V_LSHR_B32_e32:
922  case AMDGPU::S_LSHR_B32:
923    Result = LHS >> (RHS & 31);
924    return true;
925  case AMDGPU::V_LSHRREV_B32_e64:
926  case AMDGPU::V_LSHRREV_B32_e32:
927    Result = RHS >> (LHS & 31);
928    return true;
929  case AMDGPU::V_ASHR_I32_e64:
930  case AMDGPU::V_ASHR_I32_e32:
931  case AMDGPU::S_ASHR_I32:
932    Result = static_cast<int32_t>(LHS) >> (RHS & 31);
933    return true;
934  case AMDGPU::V_ASHRREV_I32_e64:
935  case AMDGPU::V_ASHRREV_I32_e32:
936    Result = static_cast<int32_t>(RHS) >> (LHS & 31);
937    return true;
938  default:
939    return false;
940  }
941}
942
943static unsigned getMovOpc(bool IsScalar) {
944  return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
945}
946
947/// Remove any leftover implicit operands from mutating the instruction. e.g.
948/// if we replace an s_and_b32 with a copy, we don't need the implicit scc def
949/// anymore.
950static void stripExtraCopyOperands(MachineInstr &MI) {
951  const MCInstrDesc &Desc = MI.getDesc();
952  unsigned NumOps = Desc.getNumOperands() +
953                    Desc.getNumImplicitUses() +
954                    Desc.getNumImplicitDefs();
955
956  for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
957    MI.RemoveOperand(I);
958}
959
960static void mutateCopyOp(MachineInstr &MI, const MCInstrDesc &NewDesc) {
961  MI.setDesc(NewDesc);
962  stripExtraCopyOperands(MI);
963}
964
965static MachineOperand *getImmOrMaterializedImm(MachineRegisterInfo &MRI,
966                                               MachineOperand &Op) {
967  if (Op.isReg()) {
968    // If this has a subregister, it obviously is a register source.
969    if (Op.getSubReg() != AMDGPU::NoSubRegister ||
970        !Register::isVirtualRegister(Op.getReg()))
971      return &Op;
972
973    MachineInstr *Def = MRI.getVRegDef(Op.getReg());
974    if (Def && Def->isMoveImmediate()) {
975      MachineOperand &ImmSrc = Def->getOperand(1);
976      if (ImmSrc.isImm())
977        return &ImmSrc;
978    }
979  }
980
981  return &Op;
982}
983
984// Try to simplify operations with a constant that may appear after instruction
985// selection.
986// TODO: See if a frame index with a fixed offset can fold.
987static bool tryConstantFoldOp(MachineRegisterInfo &MRI,
988                              const SIInstrInfo *TII,
989                              MachineInstr *MI,
990                              MachineOperand *ImmOp) {
991  unsigned Opc = MI->getOpcode();
992  if (Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
993      Opc == AMDGPU::S_NOT_B32) {
994    MI->getOperand(1).ChangeToImmediate(~ImmOp->getImm());
995    mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32)));
996    return true;
997  }
998
999  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1000  if (Src1Idx == -1)
1001    return false;
1002
1003  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1004  MachineOperand *Src0 = getImmOrMaterializedImm(MRI, MI->getOperand(Src0Idx));
1005  MachineOperand *Src1 = getImmOrMaterializedImm(MRI, MI->getOperand(Src1Idx));
1006
1007  if (!Src0->isImm() && !Src1->isImm())
1008    return false;
1009
1010  if (MI->getOpcode() == AMDGPU::V_LSHL_OR_B32) {
1011    if (Src0->isImm() && Src0->getImm() == 0) {
1012      // v_lshl_or_b32 0, X, Y -> copy Y
1013      // v_lshl_or_b32 0, X, K -> v_mov_b32 K
1014      bool UseCopy = TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->isReg();
1015      MI->RemoveOperand(Src1Idx);
1016      MI->RemoveOperand(Src0Idx);
1017
1018      MI->setDesc(TII->get(UseCopy ? AMDGPU::COPY : AMDGPU::V_MOV_B32_e32));
1019      return true;
1020    }
1021  }
1022
1023  // and k0, k1 -> v_mov_b32 (k0 & k1)
1024  // or k0, k1 -> v_mov_b32 (k0 | k1)
1025  // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1026  if (Src0->isImm() && Src1->isImm()) {
1027    int32_t NewImm;
1028    if (!evalBinaryInstruction(Opc, NewImm, Src0->getImm(), Src1->getImm()))
1029      return false;
1030
1031    const SIRegisterInfo &TRI = TII->getRegisterInfo();
1032    bool IsSGPR = TRI.isSGPRReg(MRI, MI->getOperand(0).getReg());
1033
1034    // Be careful to change the right operand, src0 may belong to a different
1035    // instruction.
1036    MI->getOperand(Src0Idx).ChangeToImmediate(NewImm);
1037    MI->RemoveOperand(Src1Idx);
1038    mutateCopyOp(*MI, TII->get(getMovOpc(IsSGPR)));
1039    return true;
1040  }
1041
1042  if (!MI->isCommutable())
1043    return false;
1044
1045  if (Src0->isImm() && !Src1->isImm()) {
1046    std::swap(Src0, Src1);
1047    std::swap(Src0Idx, Src1Idx);
1048  }
1049
1050  int32_t Src1Val = static_cast<int32_t>(Src1->getImm());
1051  if (Opc == AMDGPU::V_OR_B32_e64 ||
1052      Opc == AMDGPU::V_OR_B32_e32 ||
1053      Opc == AMDGPU::S_OR_B32) {
1054    if (Src1Val == 0) {
1055      // y = or x, 0 => y = copy x
1056      MI->RemoveOperand(Src1Idx);
1057      mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1058    } else if (Src1Val == -1) {
1059      // y = or x, -1 => y = v_mov_b32 -1
1060      MI->RemoveOperand(Src1Idx);
1061      mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32)));
1062    } else
1063      return false;
1064
1065    return true;
1066  }
1067
1068  if (MI->getOpcode() == AMDGPU::V_AND_B32_e64 ||
1069      MI->getOpcode() == AMDGPU::V_AND_B32_e32 ||
1070      MI->getOpcode() == AMDGPU::S_AND_B32) {
1071    if (Src1Val == 0) {
1072      // y = and x, 0 => y = v_mov_b32 0
1073      MI->RemoveOperand(Src0Idx);
1074      mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32)));
1075    } else if (Src1Val == -1) {
1076      // y = and x, -1 => y = copy x
1077      MI->RemoveOperand(Src1Idx);
1078      mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1079      stripExtraCopyOperands(*MI);
1080    } else
1081      return false;
1082
1083    return true;
1084  }
1085
1086  if (MI->getOpcode() == AMDGPU::V_XOR_B32_e64 ||
1087      MI->getOpcode() == AMDGPU::V_XOR_B32_e32 ||
1088      MI->getOpcode() == AMDGPU::S_XOR_B32) {
1089    if (Src1Val == 0) {
1090      // y = xor x, 0 => y = copy x
1091      MI->RemoveOperand(Src1Idx);
1092      mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1093      return true;
1094    }
1095  }
1096
1097  return false;
1098}
1099
1100// Try to fold an instruction into a simpler one
1101static bool tryFoldInst(const SIInstrInfo *TII,
1102                        MachineInstr *MI) {
1103  unsigned Opc = MI->getOpcode();
1104
1105  if (Opc == AMDGPU::V_CNDMASK_B32_e32    ||
1106      Opc == AMDGPU::V_CNDMASK_B32_e64    ||
1107      Opc == AMDGPU::V_CNDMASK_B64_PSEUDO) {
1108    const MachineOperand *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
1109    const MachineOperand *Src1 = TII->getNamedOperand(*MI, AMDGPU::OpName::src1);
1110    int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers);
1111    int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
1112    if (Src1->isIdenticalTo(*Src0) &&
1113        (Src1ModIdx == -1 || !MI->getOperand(Src1ModIdx).getImm()) &&
1114        (Src0ModIdx == -1 || !MI->getOperand(Src0ModIdx).getImm())) {
1115      LLVM_DEBUG(dbgs() << "Folded " << *MI << " into ");
1116      auto &NewDesc =
1117          TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false));
1118      int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1119      if (Src2Idx != -1)
1120        MI->RemoveOperand(Src2Idx);
1121      MI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1));
1122      if (Src1ModIdx != -1)
1123        MI->RemoveOperand(Src1ModIdx);
1124      if (Src0ModIdx != -1)
1125        MI->RemoveOperand(Src0ModIdx);
1126      mutateCopyOp(*MI, NewDesc);
1127      LLVM_DEBUG(dbgs() << *MI << '\n');
1128      return true;
1129    }
1130  }
1131
1132  return false;
1133}
1134
1135void SIFoldOperands::foldInstOperand(MachineInstr &MI,
1136                                     MachineOperand &OpToFold) const {
1137  // We need mutate the operands of new mov instructions to add implicit
1138  // uses of EXEC, but adding them invalidates the use_iterator, so defer
1139  // this.
1140  SmallVector<MachineInstr *, 4> CopiesToReplace;
1141  SmallVector<FoldCandidate, 4> FoldList;
1142  MachineOperand &Dst = MI.getOperand(0);
1143
1144  bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1145  if (FoldingImm) {
1146    unsigned NumLiteralUses = 0;
1147    MachineOperand *NonInlineUse = nullptr;
1148    int NonInlineUseOpNo = -1;
1149
1150    MachineRegisterInfo::use_iterator NextUse;
1151    for (MachineRegisterInfo::use_iterator
1152           Use = MRI->use_begin(Dst.getReg()), E = MRI->use_end();
1153         Use != E; Use = NextUse) {
1154      NextUse = std::next(Use);
1155      MachineInstr *UseMI = Use->getParent();
1156      unsigned OpNo = Use.getOperandNo();
1157
1158      // Folding the immediate may reveal operations that can be constant
1159      // folded or replaced with a copy. This can happen for example after
1160      // frame indices are lowered to constants or from splitting 64-bit
1161      // constants.
1162      //
1163      // We may also encounter cases where one or both operands are
1164      // immediates materialized into a register, which would ordinarily not
1165      // be folded due to multiple uses or operand constraints.
1166
1167      if (OpToFold.isImm() && tryConstantFoldOp(*MRI, TII, UseMI, &OpToFold)) {
1168        LLVM_DEBUG(dbgs() << "Constant folded " << *UseMI << '\n');
1169
1170        // Some constant folding cases change the same immediate's use to a new
1171        // instruction, e.g. and x, 0 -> 0. Make sure we re-visit the user
1172        // again. The same constant folded instruction could also have a second
1173        // use operand.
1174        NextUse = MRI->use_begin(Dst.getReg());
1175        FoldList.clear();
1176        continue;
1177      }
1178
1179      // Try to fold any inline immediate uses, and then only fold other
1180      // constants if they have one use.
1181      //
1182      // The legality of the inline immediate must be checked based on the use
1183      // operand, not the defining instruction, because 32-bit instructions
1184      // with 32-bit inline immediate sources may be used to materialize
1185      // constants used in 16-bit operands.
1186      //
1187      // e.g. it is unsafe to fold:
1188      //  s_mov_b32 s0, 1.0    // materializes 0x3f800000
1189      //  v_add_f16 v0, v1, s0 // 1.0 f16 inline immediate sees 0x00003c00
1190
1191      // Folding immediates with more than one use will increase program size.
1192      // FIXME: This will also reduce register usage, which may be better
1193      // in some cases. A better heuristic is needed.
1194      if (isInlineConstantIfFolded(TII, *UseMI, OpNo, OpToFold)) {
1195        foldOperand(OpToFold, UseMI, OpNo, FoldList, CopiesToReplace);
1196      } else if (frameIndexMayFold(TII, *UseMI, OpNo, OpToFold)) {
1197        foldOperand(OpToFold, UseMI, OpNo, FoldList,
1198                    CopiesToReplace);
1199      } else {
1200        if (++NumLiteralUses == 1) {
1201          NonInlineUse = &*Use;
1202          NonInlineUseOpNo = OpNo;
1203        }
1204      }
1205    }
1206
1207    if (NumLiteralUses == 1) {
1208      MachineInstr *UseMI = NonInlineUse->getParent();
1209      foldOperand(OpToFold, UseMI, NonInlineUseOpNo, FoldList, CopiesToReplace);
1210    }
1211  } else {
1212    // Folding register.
1213    SmallVector <MachineRegisterInfo::use_iterator, 4> UsesToProcess;
1214    for (MachineRegisterInfo::use_iterator
1215           Use = MRI->use_begin(Dst.getReg()), E = MRI->use_end();
1216         Use != E; ++Use) {
1217      UsesToProcess.push_back(Use);
1218    }
1219    for (auto U : UsesToProcess) {
1220      MachineInstr *UseMI = U->getParent();
1221
1222      foldOperand(OpToFold, UseMI, U.getOperandNo(),
1223        FoldList, CopiesToReplace);
1224    }
1225  }
1226
1227  MachineFunction *MF = MI.getParent()->getParent();
1228  // Make sure we add EXEC uses to any new v_mov instructions created.
1229  for (MachineInstr *Copy : CopiesToReplace)
1230    Copy->addImplicitDefUseOperands(*MF);
1231
1232  for (FoldCandidate &Fold : FoldList) {
1233    assert(!Fold.isReg() || Fold.OpToFold);
1234    if (Fold.isReg() && Register::isVirtualRegister(Fold.OpToFold->getReg())) {
1235      Register Reg = Fold.OpToFold->getReg();
1236      MachineInstr *DefMI = Fold.OpToFold->getParent();
1237      if (DefMI->readsRegister(AMDGPU::EXEC, TRI) &&
1238          execMayBeModifiedBeforeUse(*MRI, Reg, *DefMI, *Fold.UseMI))
1239        continue;
1240    }
1241    if (updateOperand(Fold, *TII, *TRI, *ST)) {
1242      // Clear kill flags.
1243      if (Fold.isReg()) {
1244        assert(Fold.OpToFold && Fold.OpToFold->isReg());
1245        // FIXME: Probably shouldn't bother trying to fold if not an
1246        // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
1247        // copies.
1248        MRI->clearKillFlags(Fold.OpToFold->getReg());
1249      }
1250      LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
1251                        << static_cast<int>(Fold.UseOpNo) << " of "
1252                        << *Fold.UseMI << '\n');
1253      tryFoldInst(TII, Fold.UseMI);
1254    } else if (Fold.isCommuted()) {
1255      // Restoring instruction's original operand order if fold has failed.
1256      TII->commuteInstruction(*Fold.UseMI, false);
1257    }
1258  }
1259}
1260
1261// Clamp patterns are canonically selected to v_max_* instructions, so only
1262// handle them.
1263const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const {
1264  unsigned Op = MI.getOpcode();
1265  switch (Op) {
1266  case AMDGPU::V_MAX_F32_e64:
1267  case AMDGPU::V_MAX_F16_e64:
1268  case AMDGPU::V_MAX_F64:
1269  case AMDGPU::V_PK_MAX_F16: {
1270    if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
1271      return nullptr;
1272
1273    // Make sure sources are identical.
1274    const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1275    const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1276    if (!Src0->isReg() || !Src1->isReg() ||
1277        Src0->getReg() != Src1->getReg() ||
1278        Src0->getSubReg() != Src1->getSubReg() ||
1279        Src0->getSubReg() != AMDGPU::NoSubRegister)
1280      return nullptr;
1281
1282    // Can't fold up if we have modifiers.
1283    if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
1284      return nullptr;
1285
1286    unsigned Src0Mods
1287      = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm();
1288    unsigned Src1Mods
1289      = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
1290
1291    // Having a 0 op_sel_hi would require swizzling the output in the source
1292    // instruction, which we can't do.
1293    unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1
1294                                                      : 0u;
1295    if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
1296      return nullptr;
1297    return Src0;
1298  }
1299  default:
1300    return nullptr;
1301  }
1302}
1303
1304// We obviously have multiple uses in a clamp since the register is used twice
1305// in the same instruction.
1306static bool hasOneNonDBGUseInst(const MachineRegisterInfo &MRI, unsigned Reg) {
1307  int Count = 0;
1308  for (auto I = MRI.use_instr_nodbg_begin(Reg), E = MRI.use_instr_nodbg_end();
1309       I != E; ++I) {
1310    if (++Count > 1)
1311      return false;
1312  }
1313
1314  return true;
1315}
1316
1317// FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
1318bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) {
1319  const MachineOperand *ClampSrc = isClamp(MI);
1320  if (!ClampSrc || !hasOneNonDBGUseInst(*MRI, ClampSrc->getReg()))
1321    return false;
1322
1323  MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg());
1324
1325  // The type of clamp must be compatible.
1326  if (TII->getClampMask(*Def) != TII->getClampMask(MI))
1327    return false;
1328
1329  MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp);
1330  if (!DefClamp)
1331    return false;
1332
1333  LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def
1334                    << '\n');
1335
1336  // Clamp is applied after omod, so it is OK if omod is set.
1337  DefClamp->setImm(1);
1338  MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
1339  MI.eraseFromParent();
1340  return true;
1341}
1342
1343static int getOModValue(unsigned Opc, int64_t Val) {
1344  switch (Opc) {
1345  case AMDGPU::V_MUL_F32_e64: {
1346    switch (static_cast<uint32_t>(Val)) {
1347    case 0x3f000000: // 0.5
1348      return SIOutMods::DIV2;
1349    case 0x40000000: // 2.0
1350      return SIOutMods::MUL2;
1351    case 0x40800000: // 4.0
1352      return SIOutMods::MUL4;
1353    default:
1354      return SIOutMods::NONE;
1355    }
1356  }
1357  case AMDGPU::V_MUL_F16_e64: {
1358    switch (static_cast<uint16_t>(Val)) {
1359    case 0x3800: // 0.5
1360      return SIOutMods::DIV2;
1361    case 0x4000: // 2.0
1362      return SIOutMods::MUL2;
1363    case 0x4400: // 4.0
1364      return SIOutMods::MUL4;
1365    default:
1366      return SIOutMods::NONE;
1367    }
1368  }
1369  default:
1370    llvm_unreachable("invalid mul opcode");
1371  }
1372}
1373
1374// FIXME: Does this really not support denormals with f16?
1375// FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
1376// handled, so will anything other than that break?
1377std::pair<const MachineOperand *, int>
1378SIFoldOperands::isOMod(const MachineInstr &MI) const {
1379  unsigned Op = MI.getOpcode();
1380  switch (Op) {
1381  case AMDGPU::V_MUL_F32_e64:
1382  case AMDGPU::V_MUL_F16_e64: {
1383    // If output denormals are enabled, omod is ignored.
1384    if ((Op == AMDGPU::V_MUL_F32_e64 && MFI->getMode().FP32Denormals) ||
1385        (Op == AMDGPU::V_MUL_F16_e64 && MFI->getMode().FP64FP16Denormals))
1386      return std::make_pair(nullptr, SIOutMods::NONE);
1387
1388    const MachineOperand *RegOp = nullptr;
1389    const MachineOperand *ImmOp = nullptr;
1390    const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1391    const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1392    if (Src0->isImm()) {
1393      ImmOp = Src0;
1394      RegOp = Src1;
1395    } else if (Src1->isImm()) {
1396      ImmOp = Src1;
1397      RegOp = Src0;
1398    } else
1399      return std::make_pair(nullptr, SIOutMods::NONE);
1400
1401    int OMod = getOModValue(Op, ImmOp->getImm());
1402    if (OMod == SIOutMods::NONE ||
1403        TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
1404        TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
1405        TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
1406        TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
1407      return std::make_pair(nullptr, SIOutMods::NONE);
1408
1409    return std::make_pair(RegOp, OMod);
1410  }
1411  case AMDGPU::V_ADD_F32_e64:
1412  case AMDGPU::V_ADD_F16_e64: {
1413    // If output denormals are enabled, omod is ignored.
1414    if ((Op == AMDGPU::V_ADD_F32_e64 && MFI->getMode().FP32Denormals) ||
1415        (Op == AMDGPU::V_ADD_F16_e64 && MFI->getMode().FP64FP16Denormals))
1416      return std::make_pair(nullptr, SIOutMods::NONE);
1417
1418    // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
1419    const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1420    const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1421
1422    if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
1423        Src0->getSubReg() == Src1->getSubReg() &&
1424        !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) &&
1425        !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) &&
1426        !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) &&
1427        !TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
1428      return std::make_pair(Src0, SIOutMods::MUL2);
1429
1430    return std::make_pair(nullptr, SIOutMods::NONE);
1431  }
1432  default:
1433    return std::make_pair(nullptr, SIOutMods::NONE);
1434  }
1435}
1436
1437// FIXME: Does this need to check IEEE bit on function?
1438bool SIFoldOperands::tryFoldOMod(MachineInstr &MI) {
1439  const MachineOperand *RegOp;
1440  int OMod;
1441  std::tie(RegOp, OMod) = isOMod(MI);
1442  if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
1443      RegOp->getSubReg() != AMDGPU::NoSubRegister ||
1444      !hasOneNonDBGUseInst(*MRI, RegOp->getReg()))
1445    return false;
1446
1447  MachineInstr *Def = MRI->getVRegDef(RegOp->getReg());
1448  MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod);
1449  if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
1450    return false;
1451
1452  // Clamp is applied after omod. If the source already has clamp set, don't
1453  // fold it.
1454  if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp))
1455    return false;
1456
1457  LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def << '\n');
1458
1459  DefOMod->setImm(OMod);
1460  MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
1461  MI.eraseFromParent();
1462  return true;
1463}
1464
1465bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
1466  if (skipFunction(MF.getFunction()))
1467    return false;
1468
1469  MRI = &MF.getRegInfo();
1470  ST = &MF.getSubtarget<GCNSubtarget>();
1471  TII = ST->getInstrInfo();
1472  TRI = &TII->getRegisterInfo();
1473  MFI = MF.getInfo<SIMachineFunctionInfo>();
1474
1475  // omod is ignored by hardware if IEEE bit is enabled. omod also does not
1476  // correctly handle signed zeros.
1477  //
1478  // FIXME: Also need to check strictfp
1479  bool IsIEEEMode = MFI->getMode().IEEE;
1480  bool HasNSZ = MFI->hasNoSignedZerosFPMath();
1481
1482  for (MachineBasicBlock *MBB : depth_first(&MF)) {
1483    MachineBasicBlock::iterator I, Next;
1484
1485    MachineOperand *CurrentKnownM0Val = nullptr;
1486    for (I = MBB->begin(); I != MBB->end(); I = Next) {
1487      Next = std::next(I);
1488      MachineInstr &MI = *I;
1489
1490      tryFoldInst(TII, &MI);
1491
1492      if (!TII->isFoldableCopy(MI)) {
1493        // Saw an unknown clobber of m0, so we no longer know what it is.
1494        if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI))
1495          CurrentKnownM0Val = nullptr;
1496
1497        // TODO: Omod might be OK if there is NSZ only on the source
1498        // instruction, and not the omod multiply.
1499        if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) ||
1500            !tryFoldOMod(MI))
1501          tryFoldClamp(MI);
1502
1503        continue;
1504      }
1505
1506      // Specially track simple redefs of m0 to the same value in a block, so we
1507      // can erase the later ones.
1508      if (MI.getOperand(0).getReg() == AMDGPU::M0) {
1509        MachineOperand &NewM0Val = MI.getOperand(1);
1510        if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(NewM0Val)) {
1511          MI.eraseFromParent();
1512          continue;
1513        }
1514
1515        // We aren't tracking other physical registers
1516        CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical()) ?
1517          nullptr : &NewM0Val;
1518        continue;
1519      }
1520
1521      MachineOperand &OpToFold = MI.getOperand(1);
1522      bool FoldingImm =
1523          OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1524
1525      // FIXME: We could also be folding things like TargetIndexes.
1526      if (!FoldingImm && !OpToFold.isReg())
1527        continue;
1528
1529      if (OpToFold.isReg() && !Register::isVirtualRegister(OpToFold.getReg()))
1530        continue;
1531
1532      // Prevent folding operands backwards in the function. For example,
1533      // the COPY opcode must not be replaced by 1 in this example:
1534      //
1535      //    %3 = COPY %vgpr0; VGPR_32:%3
1536      //    ...
1537      //    %vgpr0 = V_MOV_B32_e32 1, implicit %exec
1538      MachineOperand &Dst = MI.getOperand(0);
1539      if (Dst.isReg() && !Register::isVirtualRegister(Dst.getReg()))
1540        continue;
1541
1542      foldInstOperand(MI, OpToFold);
1543    }
1544  }
1545  return true;
1546}
1547