1//===--- AArch64StorePairSuppress.cpp --- Suppress store pair formation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass identifies floating point stores that should not be combined into
10// store pairs. Later we may do the same for floating point loads.
11// ===---------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "llvm/CodeGen/MachineFunction.h"
15#include "llvm/CodeGen/MachineFunctionPass.h"
16#include "llvm/CodeGen/MachineInstr.h"
17#include "llvm/CodeGen/MachineTraceMetrics.h"
18#include "llvm/CodeGen/TargetInstrInfo.h"
19#include "llvm/CodeGen/TargetSchedule.h"
20#include "llvm/Support/Debug.h"
21#include "llvm/Support/raw_ostream.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "aarch64-stp-suppress"
26
27#define STPSUPPRESS_PASS_NAME "AArch64 Store Pair Suppression"
28
29namespace {
30class AArch64StorePairSuppress : public MachineFunctionPass {
31  const AArch64InstrInfo *TII;
32  const TargetRegisterInfo *TRI;
33  const MachineRegisterInfo *MRI;
34  TargetSchedModel SchedModel;
35  MachineTraceMetrics *Traces;
36  MachineTraceMetrics::Ensemble *MinInstr;
37
38public:
39  static char ID;
40  AArch64StorePairSuppress() : MachineFunctionPass(ID) {
41    initializeAArch64StorePairSuppressPass(*PassRegistry::getPassRegistry());
42  }
43
44  StringRef getPassName() const override { return STPSUPPRESS_PASS_NAME; }
45
46  bool runOnMachineFunction(MachineFunction &F) override;
47
48private:
49  bool shouldAddSTPToBlock(const MachineBasicBlock *BB);
50
51  bool isNarrowFPStore(const MachineInstr &MI);
52
53  void getAnalysisUsage(AnalysisUsage &AU) const override {
54    AU.setPreservesCFG();
55    AU.addRequired<MachineTraceMetrics>();
56    AU.addPreserved<MachineTraceMetrics>();
57    MachineFunctionPass::getAnalysisUsage(AU);
58  }
59};
60char AArch64StorePairSuppress::ID = 0;
61} // anonymous
62
63INITIALIZE_PASS(AArch64StorePairSuppress, "aarch64-stp-suppress",
64                STPSUPPRESS_PASS_NAME, false, false)
65
66FunctionPass *llvm::createAArch64StorePairSuppressPass() {
67  return new AArch64StorePairSuppress();
68}
69
70/// Return true if an STP can be added to this block without increasing the
71/// critical resource height. STP is good to form in Ld/St limited blocks and
72/// bad to form in float-point limited blocks. This is true independent of the
73/// critical path. If the critical path is longer than the resource height, the
74/// extra vector ops can limit physreg renaming. Otherwise, it could simply
75/// oversaturate the vector units.
76bool AArch64StorePairSuppress::shouldAddSTPToBlock(const MachineBasicBlock *BB) {
77  if (!MinInstr)
78    MinInstr = Traces->getEnsemble(MachineTraceMetrics::TS_MinInstrCount);
79
80  MachineTraceMetrics::Trace BBTrace = MinInstr->getTrace(BB);
81  unsigned ResLength = BBTrace.getResourceLength();
82
83  // Get the machine model's scheduling class for STPQi.
84  // Bypass TargetSchedule's SchedClass resolution since we only have an opcode.
85  unsigned SCIdx = TII->get(AArch64::STPDi).getSchedClass();
86  const MCSchedClassDesc *SCDesc =
87      SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);
88
89  // If a subtarget does not define resources for STPQi, bail here.
90  if (SCDesc->isValid() && !SCDesc->isVariant()) {
91    unsigned ResLenWithSTP = BBTrace.getResourceLength(None, SCDesc);
92    if (ResLenWithSTP > ResLength) {
93      LLVM_DEBUG(dbgs() << "  Suppress STP in BB: " << BB->getNumber()
94                        << " resources " << ResLength << " -> " << ResLenWithSTP
95                        << "\n");
96      return false;
97    }
98  }
99  return true;
100}
101
102/// Return true if this is a floating-point store smaller than the V reg. On
103/// cyclone, these require a vector shuffle before storing a pair.
104/// Ideally we would call getMatchingPairOpcode() and have the machine model
105/// tell us if it's profitable with no cpu knowledge here.
106///
107/// FIXME: We plan to develop a decent Target abstraction for simple loads and
108/// stores. Until then use a nasty switch similar to AArch64LoadStoreOptimizer.
109bool AArch64StorePairSuppress::isNarrowFPStore(const MachineInstr &MI) {
110  switch (MI.getOpcode()) {
111  default:
112    return false;
113  case AArch64::STRSui:
114  case AArch64::STRDui:
115  case AArch64::STURSi:
116  case AArch64::STURDi:
117    return true;
118  }
119}
120
121bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
122  if (skipFunction(MF.getFunction()))
123    return false;
124
125  const TargetSubtargetInfo &ST = MF.getSubtarget();
126  TII = static_cast<const AArch64InstrInfo *>(ST.getInstrInfo());
127  TRI = ST.getRegisterInfo();
128  MRI = &MF.getRegInfo();
129  SchedModel.init(&ST);
130  Traces = &getAnalysis<MachineTraceMetrics>();
131  MinInstr = nullptr;
132
133  LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << MF.getName() << '\n');
134
135  if (!SchedModel.hasInstrSchedModel()) {
136    LLVM_DEBUG(dbgs() << "  Skipping pass: no machine model present.\n");
137    return false;
138  }
139
140  // Check for a sequence of stores to the same base address. We don't need to
141  // precisely determine whether a store pair can be formed. But we do want to
142  // filter out most situations where we can't form store pairs to avoid
143  // computing trace metrics in those cases.
144  for (auto &MBB : MF) {
145    bool SuppressSTP = false;
146    unsigned PrevBaseReg = 0;
147    for (auto &MI : MBB) {
148      if (!isNarrowFPStore(MI))
149        continue;
150      const MachineOperand *BaseOp;
151      int64_t Offset;
152      if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI) &&
153          BaseOp->isReg()) {
154        Register BaseReg = BaseOp->getReg();
155        if (PrevBaseReg == BaseReg) {
156          // If this block can take STPs, skip ahead to the next block.
157          if (!SuppressSTP && shouldAddSTPToBlock(MI.getParent()))
158            break;
159          // Otherwise, continue unpairing the stores in this block.
160          LLVM_DEBUG(dbgs() << "Unpairing store " << MI << "\n");
161          SuppressSTP = true;
162          TII->suppressLdStPair(MI);
163        }
164        PrevBaseReg = BaseReg;
165      } else
166        PrevBaseReg = 0;
167    }
168  }
169  // This pass just sets some internal MachineMemOperand flags. It can't really
170  // invalidate anything.
171  return false;
172}
173