1//===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass changes code to work around Cortex-A53 erratum 835769.
9// It works around it by inserting a nop instruction in code sequences that
10// in some circumstances may trigger the erratum.
11// It inserts a nop instruction between a sequence of the following 2 classes
12// of instructions:
13// instr 1: mem-instr (including loads, stores and prefetches).
14// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
15//===----------------------------------------------------------------------===//
16
17#include "AArch64.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/TargetInstrInfo.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/raw_ostream.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
31
32STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
33
34//===----------------------------------------------------------------------===//
35// Helper functions
36
37// Is the instruction a match for the instruction that comes first in the
38// sequence of instructions that can trigger the erratum?
39static bool isFirstInstructionInSequence(MachineInstr *MI) {
40  // Must return true if this instruction is a load, a store or a prefetch.
41  switch (MI->getOpcode()) {
42  case AArch64::PRFMl:
43  case AArch64::PRFMroW:
44  case AArch64::PRFMroX:
45  case AArch64::PRFMui:
46  case AArch64::PRFUMi:
47    return true;
48  default:
49    return MI->mayLoadOrStore();
50  }
51}
52
53// Is the instruction a match for the instruction that comes second in the
54// sequence that can trigger the erratum?
55static bool isSecondInstructionInSequence(MachineInstr *MI) {
56  // Must return true for non-SIMD integer multiply-accumulates, writing
57  // to a 64-bit register.
58  switch (MI->getOpcode()) {
59  // Erratum cannot be triggered when the destination register is 32 bits,
60  // therefore only include the following.
61  case AArch64::MSUBXrrr:
62  case AArch64::MADDXrrr:
63  case AArch64::SMADDLrrr:
64  case AArch64::SMSUBLrrr:
65  case AArch64::UMADDLrrr:
66  case AArch64::UMSUBLrrr:
67    // Erratum can only be triggered by multiply-adds, not by regular
68    // non-accumulating multiplies, i.e. when Ra=XZR='11111'
69    return MI->getOperand(3).getReg() != AArch64::XZR;
70  default:
71    return false;
72  }
73}
74
75
76//===----------------------------------------------------------------------===//
77
78namespace {
79class AArch64A53Fix835769 : public MachineFunctionPass {
80  const TargetInstrInfo *TII;
81
82public:
83  static char ID;
84  explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
85    initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
86  }
87
88  bool runOnMachineFunction(MachineFunction &F) override;
89
90  MachineFunctionProperties getRequiredProperties() const override {
91    return MachineFunctionProperties().set(
92        MachineFunctionProperties::Property::NoVRegs);
93  }
94
95  StringRef getPassName() const override {
96    return "Workaround A53 erratum 835769 pass";
97  }
98
99  void getAnalysisUsage(AnalysisUsage &AU) const override {
100    AU.setPreservesCFG();
101    MachineFunctionPass::getAnalysisUsage(AU);
102  }
103
104private:
105  bool runOnBasicBlock(MachineBasicBlock &MBB);
106};
107char AArch64A53Fix835769::ID = 0;
108
109} // end anonymous namespace
110
111INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
112                "AArch64 fix for A53 erratum 835769", false, false)
113
114//===----------------------------------------------------------------------===//
115
116bool
117AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
118  LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
119  bool Changed = false;
120  TII = F.getSubtarget().getInstrInfo();
121
122  for (auto &MBB : F) {
123    Changed |= runOnBasicBlock(MBB);
124  }
125  return Changed;
126}
127
128// Return the block that was fallen through to get to MBB, if any,
129// otherwise nullptr.
130static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
131                                             const TargetInstrInfo *TII) {
132  // Get the previous machine basic block in the function.
133  MachineFunction::iterator MBBI(MBB);
134
135  // Can't go off top of function.
136  if (MBBI == MBB->getParent()->begin())
137    return nullptr;
138
139  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
140  SmallVector<MachineOperand, 2> Cond;
141
142  MachineBasicBlock *PrevBB = &*std::prev(MBBI);
143  for (MachineBasicBlock *S : MBB->predecessors())
144    if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
145        !FBB)
146      return S;
147
148  return nullptr;
149}
150
151// Iterate through fallen through blocks trying to find a previous non-pseudo if
152// there is one, otherwise return nullptr. Only look for instructions in
153// previous blocks, not the current block, since we only use this to look at
154// previous blocks.
155static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
156                                      const TargetInstrInfo *TII) {
157  MachineBasicBlock *FMBB = &MBB;
158
159  // If there is no non-pseudo in the current block, loop back around and try
160  // the previous block (if there is one).
161  while ((FMBB = getBBFallenThrough(FMBB, TII))) {
162    for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
163      if (!I.isPseudo())
164        return &I;
165  }
166
167  // There was no previous non-pseudo in the fallen through blocks
168  return nullptr;
169}
170
171static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
172                                       const TargetInstrInfo *TII) {
173  // If we are the first instruction of the block, put the NOP at the end of
174  // the previous fallthrough block
175  if (MI == &MBB.front()) {
176    MachineInstr *I = getLastNonPseudo(MBB, TII);
177    assert(I && "Expected instruction");
178    DebugLoc DL = I->getDebugLoc();
179    BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
180  }
181  else {
182    DebugLoc DL = MI->getDebugLoc();
183    BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
184  }
185
186  ++NumNopsAdded;
187}
188
189bool
190AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
191  bool Changed = false;
192  LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
193                    << " - scanning instructions...\n");
194
195  // First, scan the basic block, looking for a sequence of 2 instructions
196  // that match the conditions under which the erratum may trigger.
197
198  // List of terminating instructions in matching sequences
199  std::vector<MachineInstr*> Sequences;
200  unsigned Idx = 0;
201  MachineInstr *PrevInstr = nullptr;
202
203  // Try and find the last non-pseudo instruction in any fallen through blocks,
204  // if there isn't one, then we use nullptr to represent that.
205  PrevInstr = getLastNonPseudo(MBB, TII);
206
207  for (auto &MI : MBB) {
208    MachineInstr *CurrInstr = &MI;
209    LLVM_DEBUG(dbgs() << "  Examining: " << MI);
210    if (PrevInstr) {
211      LLVM_DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
212                        << "    CurrInstr: " << *CurrInstr
213                        << "    isFirstInstructionInSequence(PrevInstr): "
214                        << isFirstInstructionInSequence(PrevInstr) << "\n"
215                        << "    isSecondInstructionInSequence(CurrInstr): "
216                        << isSecondInstructionInSequence(CurrInstr) << "\n");
217      if (isFirstInstructionInSequence(PrevInstr) &&
218          isSecondInstructionInSequence(CurrInstr)) {
219        LLVM_DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
220        Sequences.push_back(CurrInstr);
221      }
222    }
223    if (!CurrInstr->isPseudo())
224      PrevInstr = CurrInstr;
225    ++Idx;
226  }
227
228  LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
229                    << " occurrences of pattern found.\n");
230
231  // Then update the basic block, inserting nops between the detected sequences.
232  for (auto &MI : Sequences) {
233    Changed = true;
234    insertNopBeforeInstruction(MBB, MI, TII);
235  }
236
237  return Changed;
238}
239
240// Factory function used by AArch64TargetMachine to add the pass to
241// the passmanager.
242FunctionPass *llvm::createAArch64A53Fix835769() {
243  return new AArch64A53Fix835769();
244}
245