1//===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass changes code to work around Cortex-A53 erratum 835769.
9// It works around it by inserting a nop instruction in code sequences that
10// in some circumstances may trigger the erratum.
11// It inserts a nop instruction between a sequence of the following 2 classes
12// of instructions:
13// instr 1: mem-instr (including loads, stores and prefetches).
14// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
15//===----------------------------------------------------------------------===//
16
17#include "AArch64.h"
18#include "AArch64Subtarget.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/TargetInstrInfo.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/raw_ostream.h"
28
29using namespace llvm;
30
31#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
32
33STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
34
35//===----------------------------------------------------------------------===//
36// Helper functions
37
38// Is the instruction a match for the instruction that comes first in the
39// sequence of instructions that can trigger the erratum?
40static bool isFirstInstructionInSequence(MachineInstr *MI) {
41  // Must return true if this instruction is a load, a store or a prefetch.
42  switch (MI->getOpcode()) {
43  case AArch64::PRFMl:
44  case AArch64::PRFMroW:
45  case AArch64::PRFMroX:
46  case AArch64::PRFMui:
47  case AArch64::PRFUMi:
48    return true;
49  default:
50    return MI->mayLoadOrStore();
51  }
52}
53
54// Is the instruction a match for the instruction that comes second in the
55// sequence that can trigger the erratum?
56static bool isSecondInstructionInSequence(MachineInstr *MI) {
57  // Must return true for non-SIMD integer multiply-accumulates, writing
58  // to a 64-bit register.
59  switch (MI->getOpcode()) {
60  // Erratum cannot be triggered when the destination register is 32 bits,
61  // therefore only include the following.
62  case AArch64::MSUBXrrr:
63  case AArch64::MADDXrrr:
64  case AArch64::SMADDLrrr:
65  case AArch64::SMSUBLrrr:
66  case AArch64::UMADDLrrr:
67  case AArch64::UMSUBLrrr:
68    // Erratum can only be triggered by multiply-adds, not by regular
69    // non-accumulating multiplies, i.e. when Ra=XZR='11111'
70    return MI->getOperand(3).getReg() != AArch64::XZR;
71  default:
72    return false;
73  }
74}
75
76
77//===----------------------------------------------------------------------===//
78
79namespace {
80class AArch64A53Fix835769 : public MachineFunctionPass {
81  const TargetInstrInfo *TII;
82
83public:
84  static char ID;
85  explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
86    initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
87  }
88
89  bool runOnMachineFunction(MachineFunction &F) override;
90
91  MachineFunctionProperties getRequiredProperties() const override {
92    return MachineFunctionProperties().set(
93        MachineFunctionProperties::Property::NoVRegs);
94  }
95
96  StringRef getPassName() const override {
97    return "Workaround A53 erratum 835769 pass";
98  }
99
100  void getAnalysisUsage(AnalysisUsage &AU) const override {
101    AU.setPreservesCFG();
102    MachineFunctionPass::getAnalysisUsage(AU);
103  }
104
105private:
106  bool runOnBasicBlock(MachineBasicBlock &MBB);
107};
108char AArch64A53Fix835769::ID = 0;
109
110} // end anonymous namespace
111
112INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
113                "AArch64 fix for A53 erratum 835769", false, false)
114
115//===----------------------------------------------------------------------===//
116
117bool
118AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
119  LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
120  auto &STI = F.getSubtarget<AArch64Subtarget>();
121  // Fix not requested, skip pass.
122  if (!STI.fixCortexA53_835769())
123    return false;
124
125  bool Changed = false;
126  TII = STI.getInstrInfo();
127
128  for (auto &MBB : F) {
129    Changed |= runOnBasicBlock(MBB);
130  }
131  return Changed;
132}
133
134// Return the block that was fallen through to get to MBB, if any,
135// otherwise nullptr.
136static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
137                                             const TargetInstrInfo *TII) {
138  // Get the previous machine basic block in the function.
139  MachineFunction::iterator MBBI(MBB);
140
141  // Can't go off top of function.
142  if (MBBI == MBB->getParent()->begin())
143    return nullptr;
144
145  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
146  SmallVector<MachineOperand, 2> Cond;
147
148  MachineBasicBlock *PrevBB = &*std::prev(MBBI);
149  for (MachineBasicBlock *S : MBB->predecessors())
150    if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
151        !FBB)
152      return S;
153
154  return nullptr;
155}
156
157// Iterate through fallen through blocks trying to find a previous non-pseudo if
158// there is one, otherwise return nullptr. Only look for instructions in
159// previous blocks, not the current block, since we only use this to look at
160// previous blocks.
161static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
162                                      const TargetInstrInfo *TII) {
163  MachineBasicBlock *FMBB = &MBB;
164
165  // If there is no non-pseudo in the current block, loop back around and try
166  // the previous block (if there is one).
167  while ((FMBB = getBBFallenThrough(FMBB, TII))) {
168    for (MachineInstr &I : llvm::reverse(*FMBB))
169      if (!I.isPseudo())
170        return &I;
171  }
172
173  // There was no previous non-pseudo in the fallen through blocks
174  return nullptr;
175}
176
177static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
178                                       const TargetInstrInfo *TII) {
179  // If we are the first instruction of the block, put the NOP at the end of
180  // the previous fallthrough block
181  if (MI == &MBB.front()) {
182    MachineInstr *I = getLastNonPseudo(MBB, TII);
183    assert(I && "Expected instruction");
184    DebugLoc DL = I->getDebugLoc();
185    BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
186  }
187  else {
188    DebugLoc DL = MI->getDebugLoc();
189    BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
190  }
191
192  ++NumNopsAdded;
193}
194
195bool
196AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
197  bool Changed = false;
198  LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
199                    << " - scanning instructions...\n");
200
201  // First, scan the basic block, looking for a sequence of 2 instructions
202  // that match the conditions under which the erratum may trigger.
203
204  // List of terminating instructions in matching sequences
205  std::vector<MachineInstr*> Sequences;
206  unsigned Idx = 0;
207  MachineInstr *PrevInstr = nullptr;
208
209  // Try and find the last non-pseudo instruction in any fallen through blocks,
210  // if there isn't one, then we use nullptr to represent that.
211  PrevInstr = getLastNonPseudo(MBB, TII);
212
213  for (auto &MI : MBB) {
214    MachineInstr *CurrInstr = &MI;
215    LLVM_DEBUG(dbgs() << "  Examining: " << MI);
216    if (PrevInstr) {
217      LLVM_DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
218                        << "    CurrInstr: " << *CurrInstr
219                        << "    isFirstInstructionInSequence(PrevInstr): "
220                        << isFirstInstructionInSequence(PrevInstr) << "\n"
221                        << "    isSecondInstructionInSequence(CurrInstr): "
222                        << isSecondInstructionInSequence(CurrInstr) << "\n");
223      if (isFirstInstructionInSequence(PrevInstr) &&
224          isSecondInstructionInSequence(CurrInstr)) {
225        LLVM_DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
226        (void) Idx;
227        Sequences.push_back(CurrInstr);
228      }
229    }
230    if (!CurrInstr->isPseudo())
231      PrevInstr = CurrInstr;
232    ++Idx;
233  }
234
235  LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
236                    << " occurrences of pattern found.\n");
237
238  // Then update the basic block, inserting nops between the detected sequences.
239  for (auto &MI : Sequences) {
240    Changed = true;
241    insertNopBeforeInstruction(MBB, MI, TII);
242  }
243
244  return Changed;
245}
246
247// Factory function used by AArch64TargetMachine to add the pass to
248// the passmanager.
249FunctionPass *llvm::createAArch64A53Fix835769() {
250  return new AArch64A53Fix835769();
251}
252