1//===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9// This pass changes code to work around Cortex-A53 erratum 835769.
10// It works around it by inserting a nop instruction in code sequences that
11// in some circumstances may trigger the erratum.
12// It inserts a nop instruction between a sequence of the following 2 classes
13// of instructions:
14// instr 1: mem-instr (including loads, stores and prefetches).
15// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
16//===----------------------------------------------------------------------===//
17
18#include "AArch64.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/raw_ostream.h"
28#include "llvm/Target/TargetInstrInfo.h"
29
30using namespace llvm;
31
32#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
33
34STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
35
36//===----------------------------------------------------------------------===//
37// Helper functions
38
39// Is the instruction a match for the instruction that comes first in the
40// sequence of instructions that can trigger the erratum?
41static bool isFirstInstructionInSequence(MachineInstr *MI) {
42  // Must return true if this instruction is a load, a store or a prefetch.
43  switch (MI->getOpcode()) {
44  case AArch64::PRFMl:
45  case AArch64::PRFMroW:
46  case AArch64::PRFMroX:
47  case AArch64::PRFMui:
48  case AArch64::PRFUMi:
49    return true;
50  default:
51    return MI->mayLoadOrStore();
52  }
53}
54
55// Is the instruction a match for the instruction that comes second in the
56// sequence that can trigger the erratum?
57static bool isSecondInstructionInSequence(MachineInstr *MI) {
58  // Must return true for non-SIMD integer multiply-accumulates, writing
59  // to a 64-bit register.
60  switch (MI->getOpcode()) {
61  // Erratum cannot be triggered when the destination register is 32 bits,
62  // therefore only include the following.
63  case AArch64::MSUBXrrr:
64  case AArch64::MADDXrrr:
65  case AArch64::SMADDLrrr:
66  case AArch64::SMSUBLrrr:
67  case AArch64::UMADDLrrr:
68  case AArch64::UMSUBLrrr:
69    // Erratum can only be triggered by multiply-adds, not by regular
70    // non-accumulating multiplies, i.e. when Ra=XZR='11111'
71    return MI->getOperand(3).getReg() != AArch64::XZR;
72  default:
73    return false;
74  }
75}
76
77
78//===----------------------------------------------------------------------===//
79
80namespace {
81class AArch64A53Fix835769 : public MachineFunctionPass {
82  const TargetInstrInfo *TII;
83
84public:
85  static char ID;
86  explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {}
87
88  bool runOnMachineFunction(MachineFunction &F) override;
89
90  const char *getPassName() const override {
91    return "Workaround A53 erratum 835769 pass";
92  }
93
94  void getAnalysisUsage(AnalysisUsage &AU) const override {
95    AU.setPreservesCFG();
96    MachineFunctionPass::getAnalysisUsage(AU);
97  }
98
99private:
100  bool runOnBasicBlock(MachineBasicBlock &MBB);
101};
102char AArch64A53Fix835769::ID = 0;
103
104} // end anonymous namespace
105
106//===----------------------------------------------------------------------===//
107
108bool
109AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
110  DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
111  bool Changed = false;
112  TII = F.getSubtarget().getInstrInfo();
113
114  for (auto &MBB : F) {
115    Changed |= runOnBasicBlock(MBB);
116  }
117  return Changed;
118}
119
120// Return the block that was fallen through to get to MBB, if any,
121// otherwise nullptr.
122static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
123                                             const TargetInstrInfo *TII) {
124  // Get the previous machine basic block in the function.
125  MachineFunction::iterator MBBI(MBB);
126
127  // Can't go off top of function.
128  if (MBBI == MBB->getParent()->begin())
129    return nullptr;
130
131  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
132  SmallVector<MachineOperand, 2> Cond;
133
134  MachineBasicBlock *PrevBB = &*std::prev(MBBI);
135  for (MachineBasicBlock *S : MBB->predecessors())
136    if (S == PrevBB && !TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond) &&
137        !TBB && !FBB)
138      return S;
139
140  return nullptr;
141}
142
143// Iterate through fallen through blocks trying to find a previous non-pseudo if
144// there is one, otherwise return nullptr. Only look for instructions in
145// previous blocks, not the current block, since we only use this to look at
146// previous blocks.
147static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
148                                      const TargetInstrInfo *TII) {
149  MachineBasicBlock *FMBB = &MBB;
150
151  // If there is no non-pseudo in the current block, loop back around and try
152  // the previous block (if there is one).
153  while ((FMBB = getBBFallenThrough(FMBB, TII))) {
154    for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
155      if (!I.isPseudo())
156        return &I;
157  }
158
159  // There was no previous non-pseudo in the fallen through blocks
160  return nullptr;
161}
162
163static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
164                                       const TargetInstrInfo *TII) {
165  // If we are the first instruction of the block, put the NOP at the end of
166  // the previous fallthrough block
167  if (MI == &MBB.front()) {
168    MachineInstr *I = getLastNonPseudo(MBB, TII);
169    assert(I && "Expected instruction");
170    DebugLoc DL = I->getDebugLoc();
171    BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
172  }
173  else {
174    DebugLoc DL = MI->getDebugLoc();
175    BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
176  }
177
178  ++NumNopsAdded;
179}
180
181bool
182AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
183  bool Changed = false;
184  DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
185
186  // First, scan the basic block, looking for a sequence of 2 instructions
187  // that match the conditions under which the erratum may trigger.
188
189  // List of terminating instructions in matching sequences
190  std::vector<MachineInstr*> Sequences;
191  unsigned Idx = 0;
192  MachineInstr *PrevInstr = nullptr;
193
194  // Try and find the last non-pseudo instruction in any fallen through blocks,
195  // if there isn't one, then we use nullptr to represent that.
196  PrevInstr = getLastNonPseudo(MBB, TII);
197
198  for (auto &MI : MBB) {
199    MachineInstr *CurrInstr = &MI;
200    DEBUG(dbgs() << "  Examining: " << MI);
201    if (PrevInstr) {
202      DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
203                   << "    CurrInstr: " << *CurrInstr
204                   << "    isFirstInstructionInSequence(PrevInstr): "
205                   << isFirstInstructionInSequence(PrevInstr) << "\n"
206                   << "    isSecondInstructionInSequence(CurrInstr): "
207                   << isSecondInstructionInSequence(CurrInstr) << "\n");
208      if (isFirstInstructionInSequence(PrevInstr) &&
209          isSecondInstructionInSequence(CurrInstr)) {
210        DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
211        Sequences.push_back(CurrInstr);
212      }
213    }
214    if (!CurrInstr->isPseudo())
215      PrevInstr = CurrInstr;
216    ++Idx;
217  }
218
219  DEBUG(dbgs() << "Scan complete, " << Sequences.size()
220               << " occurrences of pattern found.\n");
221
222  // Then update the basic block, inserting nops between the detected sequences.
223  for (auto &MI : Sequences) {
224    Changed = true;
225    insertNopBeforeInstruction(MBB, MI, TII);
226  }
227
228  return Changed;
229}
230
231// Factory function used by AArch64TargetMachine to add the pass to
232// the passmanager.
233FunctionPass *llvm::createAArch64A53Fix835769() {
234  return new AArch64A53Fix835769();
235}
236