1//===-- AArch64A53Fix835769.cpp -------------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// This pass changes code to work around Cortex-A53 erratum 835769. 9// It works around it by inserting a nop instruction in code sequences that 10// in some circumstances may trigger the erratum. 11// It inserts a nop instruction between a sequence of the following 2 classes 12// of instructions: 13// instr 1: mem-instr (including loads, stores and prefetches). 14// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers. 15//===----------------------------------------------------------------------===// 16 17#include "AArch64.h" 18#include "AArch64Subtarget.h" 19#include "llvm/ADT/Statistic.h" 20#include "llvm/CodeGen/MachineFunction.h" 21#include "llvm/CodeGen/MachineFunctionPass.h" 22#include "llvm/CodeGen/MachineInstr.h" 23#include "llvm/CodeGen/MachineInstrBuilder.h" 24#include "llvm/CodeGen/MachineRegisterInfo.h" 25#include "llvm/CodeGen/TargetInstrInfo.h" 26#include "llvm/Support/Debug.h" 27#include "llvm/Support/raw_ostream.h" 28 29using namespace llvm; 30 31#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769" 32 33STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769"); 34 35//===----------------------------------------------------------------------===// 36// Helper functions 37 38// Is the instruction a match for the instruction that comes first in the 39// sequence of instructions that can trigger the erratum? 40static bool isFirstInstructionInSequence(MachineInstr *MI) { 41 // Must return true if this instruction is a load, a store or a prefetch. 42 switch (MI->getOpcode()) { 43 case AArch64::PRFMl: 44 case AArch64::PRFMroW: 45 case AArch64::PRFMroX: 46 case AArch64::PRFMui: 47 case AArch64::PRFUMi: 48 return true; 49 default: 50 return MI->mayLoadOrStore(); 51 } 52} 53 54// Is the instruction a match for the instruction that comes second in the 55// sequence that can trigger the erratum? 56static bool isSecondInstructionInSequence(MachineInstr *MI) { 57 // Must return true for non-SIMD integer multiply-accumulates, writing 58 // to a 64-bit register. 59 switch (MI->getOpcode()) { 60 // Erratum cannot be triggered when the destination register is 32 bits, 61 // therefore only include the following. 62 case AArch64::MSUBXrrr: 63 case AArch64::MADDXrrr: 64 case AArch64::SMADDLrrr: 65 case AArch64::SMSUBLrrr: 66 case AArch64::UMADDLrrr: 67 case AArch64::UMSUBLrrr: 68 // Erratum can only be triggered by multiply-adds, not by regular 69 // non-accumulating multiplies, i.e. when Ra=XZR='11111' 70 return MI->getOperand(3).getReg() != AArch64::XZR; 71 default: 72 return false; 73 } 74} 75 76 77//===----------------------------------------------------------------------===// 78 79namespace { 80class AArch64A53Fix835769 : public MachineFunctionPass { 81 const TargetInstrInfo *TII; 82 83public: 84 static char ID; 85 explicit AArch64A53Fix835769() : MachineFunctionPass(ID) { 86 initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry()); 87 } 88 89 bool runOnMachineFunction(MachineFunction &F) override; 90 91 MachineFunctionProperties getRequiredProperties() const override { 92 return MachineFunctionProperties().set( 93 MachineFunctionProperties::Property::NoVRegs); 94 } 95 96 StringRef getPassName() const override { 97 return "Workaround A53 erratum 835769 pass"; 98 } 99 100 void getAnalysisUsage(AnalysisUsage &AU) const override { 101 AU.setPreservesCFG(); 102 MachineFunctionPass::getAnalysisUsage(AU); 103 } 104 105private: 106 bool runOnBasicBlock(MachineBasicBlock &MBB); 107}; 108char AArch64A53Fix835769::ID = 0; 109 110} // end anonymous namespace 111 112INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", 113 "AArch64 fix for A53 erratum 835769", false, false) 114 115//===----------------------------------------------------------------------===// 116 117bool 118AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { 119 LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); 120 auto &STI = F.getSubtarget<AArch64Subtarget>(); 121 // Fix not requested, skip pass. 122 if (!STI.fixCortexA53_835769()) 123 return false; 124 125 bool Changed = false; 126 TII = STI.getInstrInfo(); 127 128 for (auto &MBB : F) { 129 Changed |= runOnBasicBlock(MBB); 130 } 131 return Changed; 132} 133 134// Return the block that was fallen through to get to MBB, if any, 135// otherwise nullptr. 136static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB, 137 const TargetInstrInfo *TII) { 138 // Get the previous machine basic block in the function. 139 MachineFunction::iterator MBBI(MBB); 140 141 // Can't go off top of function. 142 if (MBBI == MBB->getParent()->begin()) 143 return nullptr; 144 145 MachineBasicBlock *TBB = nullptr, *FBB = nullptr; 146 SmallVector<MachineOperand, 2> Cond; 147 148 MachineBasicBlock *PrevBB = &*std::prev(MBBI); 149 for (MachineBasicBlock *S : MBB->predecessors()) 150 if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB && 151 !FBB) 152 return S; 153 154 return nullptr; 155} 156 157// Iterate through fallen through blocks trying to find a previous non-pseudo if 158// there is one, otherwise return nullptr. Only look for instructions in 159// previous blocks, not the current block, since we only use this to look at 160// previous blocks. 161static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB, 162 const TargetInstrInfo *TII) { 163 MachineBasicBlock *FMBB = &MBB; 164 165 // If there is no non-pseudo in the current block, loop back around and try 166 // the previous block (if there is one). 167 while ((FMBB = getBBFallenThrough(FMBB, TII))) { 168 for (MachineInstr &I : llvm::reverse(*FMBB)) 169 if (!I.isPseudo()) 170 return &I; 171 } 172 173 // There was no previous non-pseudo in the fallen through blocks 174 return nullptr; 175} 176 177static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI, 178 const TargetInstrInfo *TII) { 179 // If we are the first instruction of the block, put the NOP at the end of 180 // the previous fallthrough block 181 if (MI == &MBB.front()) { 182 MachineInstr *I = getLastNonPseudo(MBB, TII); 183 assert(I && "Expected instruction"); 184 DebugLoc DL = I->getDebugLoc(); 185 BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0); 186 } 187 else { 188 DebugLoc DL = MI->getDebugLoc(); 189 BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0); 190 } 191 192 ++NumNopsAdded; 193} 194 195bool 196AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) { 197 bool Changed = false; 198 LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB 199 << " - scanning instructions...\n"); 200 201 // First, scan the basic block, looking for a sequence of 2 instructions 202 // that match the conditions under which the erratum may trigger. 203 204 // List of terminating instructions in matching sequences 205 std::vector<MachineInstr*> Sequences; 206 unsigned Idx = 0; 207 MachineInstr *PrevInstr = nullptr; 208 209 // Try and find the last non-pseudo instruction in any fallen through blocks, 210 // if there isn't one, then we use nullptr to represent that. 211 PrevInstr = getLastNonPseudo(MBB, TII); 212 213 for (auto &MI : MBB) { 214 MachineInstr *CurrInstr = &MI; 215 LLVM_DEBUG(dbgs() << " Examining: " << MI); 216 if (PrevInstr) { 217 LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr 218 << " CurrInstr: " << *CurrInstr 219 << " isFirstInstructionInSequence(PrevInstr): " 220 << isFirstInstructionInSequence(PrevInstr) << "\n" 221 << " isSecondInstructionInSequence(CurrInstr): " 222 << isSecondInstructionInSequence(CurrInstr) << "\n"); 223 if (isFirstInstructionInSequence(PrevInstr) && 224 isSecondInstructionInSequence(CurrInstr)) { 225 LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n"); 226 (void) Idx; 227 Sequences.push_back(CurrInstr); 228 } 229 } 230 if (!CurrInstr->isPseudo()) 231 PrevInstr = CurrInstr; 232 ++Idx; 233 } 234 235 LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size() 236 << " occurrences of pattern found.\n"); 237 238 // Then update the basic block, inserting nops between the detected sequences. 239 for (auto &MI : Sequences) { 240 Changed = true; 241 insertNopBeforeInstruction(MBB, MI, TII); 242 } 243 244 return Changed; 245} 246 247// Factory function used by AArch64TargetMachine to add the pass to 248// the passmanager. 249FunctionPass *llvm::createAArch64A53Fix835769() { 250 return new AArch64A53Fix835769(); 251} 252