12868Speytoia//===-- AArch64A53Fix835769.cpp -------------------------------------------===// 22868Speytoia// 32868Speytoia// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 42868Speytoia// See https://llvm.org/LICENSE.txt for license information. 52868Speytoia// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 62868Speytoia// 72868Speytoia//===----------------------------------------------------------------------===// 82868Speytoia// This pass changes code to work around Cortex-A53 erratum 835769. 92868Speytoia// It works around it by inserting a nop instruction in code sequences that 102868Speytoia// in some circumstances may trigger the erratum. 112868Speytoia// It inserts a nop instruction between a sequence of the following 2 classes 122868Speytoia// of instructions: 132868Speytoia// instr 1: mem-instr (including loads, stores and prefetches). 142868Speytoia// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers. 152868Speytoia//===----------------------------------------------------------------------===// 162868Speytoia 172868Speytoia#include "AArch64.h" 182868Speytoia#include "llvm/ADT/Statistic.h" 192868Speytoia#include "llvm/CodeGen/MachineFunction.h" 202868Speytoia#include "llvm/CodeGen/MachineFunctionPass.h" 212868Speytoia#include "llvm/CodeGen/MachineInstr.h" 222868Speytoia#include "llvm/CodeGen/MachineInstrBuilder.h" 232868Speytoia#include "llvm/CodeGen/MachineRegisterInfo.h" 242868Speytoia#include "llvm/CodeGen/TargetInstrInfo.h" 252868Speytoia#include "llvm/Support/Debug.h" 262868Speytoia#include "llvm/Support/raw_ostream.h" 272868Speytoia 282868Speytoiausing namespace llvm; 292868Speytoia 302868Speytoia#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769" 312868Speytoia 322868SpeytoiaSTATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769"); 332868Speytoia 342868Speytoia//===----------------------------------------------------------------------===// 352868Speytoia// Helper functions 362868Speytoia 372868Speytoia// Is the instruction a match for the instruction that comes first in the 382868Speytoia// sequence of instructions that can trigger the erratum? 392868Speytoiastatic bool isFirstInstructionInSequence(MachineInstr *MI) { 402868Speytoia // Must return true if this instruction is a load, a store or a prefetch. 412868Speytoia switch (MI->getOpcode()) { 422868Speytoia case AArch64::PRFMl: 432868Speytoia case AArch64::PRFMroW: 442868Speytoia case AArch64::PRFMroX: 452868Speytoia case AArch64::PRFMui: 462868Speytoia case AArch64::PRFUMi: 472868Speytoia return true; 482868Speytoia default: 492868Speytoia return MI->mayLoadOrStore(); 502868Speytoia } 512868Speytoia} 522868Speytoia 532868Speytoia// Is the instruction a match for the instruction that comes second in the 542868Speytoia// sequence that can trigger the erratum? 552868Speytoiastatic bool isSecondInstructionInSequence(MachineInstr *MI) { 562868Speytoia // Must return true for non-SIMD integer multiply-accumulates, writing 572868Speytoia // to a 64-bit register. 582868Speytoia switch (MI->getOpcode()) { 592868Speytoia // Erratum cannot be triggered when the destination register is 32 bits, 602868Speytoia // therefore only include the following. 612868Speytoia case AArch64::MSUBXrrr: 622868Speytoia case AArch64::MADDXrrr: 632868Speytoia case AArch64::SMADDLrrr: 642868Speytoia case AArch64::SMSUBLrrr: 652868Speytoia case AArch64::UMADDLrrr: 662868Speytoia case AArch64::UMSUBLrrr: 672868Speytoia // Erratum can only be triggered by multiply-adds, not by regular 682868Speytoia // non-accumulating multiplies, i.e. when Ra=XZR='11111' 692868Speytoia return MI->getOperand(3).getReg() != AArch64::XZR; 702868Speytoia default: 712868Speytoia return false; 722868Speytoia } 732868Speytoia} 742868Speytoia 752868Speytoia 762868Speytoia//===----------------------------------------------------------------------===// 772868Speytoia 782868Speytoianamespace { 79class AArch64A53Fix835769 : public MachineFunctionPass { 80 const TargetInstrInfo *TII; 81 82public: 83 static char ID; 84 explicit AArch64A53Fix835769() : MachineFunctionPass(ID) { 85 initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry()); 86 } 87 88 bool runOnMachineFunction(MachineFunction &F) override; 89 90 MachineFunctionProperties getRequiredProperties() const override { 91 return MachineFunctionProperties().set( 92 MachineFunctionProperties::Property::NoVRegs); 93 } 94 95 StringRef getPassName() const override { 96 return "Workaround A53 erratum 835769 pass"; 97 } 98 99 void getAnalysisUsage(AnalysisUsage &AU) const override { 100 AU.setPreservesCFG(); 101 MachineFunctionPass::getAnalysisUsage(AU); 102 } 103 104private: 105 bool runOnBasicBlock(MachineBasicBlock &MBB); 106}; 107char AArch64A53Fix835769::ID = 0; 108 109} // end anonymous namespace 110 111INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", 112 "AArch64 fix for A53 erratum 835769", false, false) 113 114//===----------------------------------------------------------------------===// 115 116bool 117AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { 118 LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); 119 bool Changed = false; 120 TII = F.getSubtarget().getInstrInfo(); 121 122 for (auto &MBB : F) { 123 Changed |= runOnBasicBlock(MBB); 124 } 125 return Changed; 126} 127 128// Return the block that was fallen through to get to MBB, if any, 129// otherwise nullptr. 130static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB, 131 const TargetInstrInfo *TII) { 132 // Get the previous machine basic block in the function. 133 MachineFunction::iterator MBBI(MBB); 134 135 // Can't go off top of function. 136 if (MBBI == MBB->getParent()->begin()) 137 return nullptr; 138 139 MachineBasicBlock *TBB = nullptr, *FBB = nullptr; 140 SmallVector<MachineOperand, 2> Cond; 141 142 MachineBasicBlock *PrevBB = &*std::prev(MBBI); 143 for (MachineBasicBlock *S : MBB->predecessors()) 144 if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB && 145 !FBB) 146 return S; 147 148 return nullptr; 149} 150 151// Iterate through fallen through blocks trying to find a previous non-pseudo if 152// there is one, otherwise return nullptr. Only look for instructions in 153// previous blocks, not the current block, since we only use this to look at 154// previous blocks. 155static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB, 156 const TargetInstrInfo *TII) { 157 MachineBasicBlock *FMBB = &MBB; 158 159 // If there is no non-pseudo in the current block, loop back around and try 160 // the previous block (if there is one). 161 while ((FMBB = getBBFallenThrough(FMBB, TII))) { 162 for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend())) 163 if (!I.isPseudo()) 164 return &I; 165 } 166 167 // There was no previous non-pseudo in the fallen through blocks 168 return nullptr; 169} 170 171static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI, 172 const TargetInstrInfo *TII) { 173 // If we are the first instruction of the block, put the NOP at the end of 174 // the previous fallthrough block 175 if (MI == &MBB.front()) { 176 MachineInstr *I = getLastNonPseudo(MBB, TII); 177 assert(I && "Expected instruction"); 178 DebugLoc DL = I->getDebugLoc(); 179 BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0); 180 } 181 else { 182 DebugLoc DL = MI->getDebugLoc(); 183 BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0); 184 } 185 186 ++NumNopsAdded; 187} 188 189bool 190AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) { 191 bool Changed = false; 192 LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB 193 << " - scanning instructions...\n"); 194 195 // First, scan the basic block, looking for a sequence of 2 instructions 196 // that match the conditions under which the erratum may trigger. 197 198 // List of terminating instructions in matching sequences 199 std::vector<MachineInstr*> Sequences; 200 unsigned Idx = 0; 201 MachineInstr *PrevInstr = nullptr; 202 203 // Try and find the last non-pseudo instruction in any fallen through blocks, 204 // if there isn't one, then we use nullptr to represent that. 205 PrevInstr = getLastNonPseudo(MBB, TII); 206 207 for (auto &MI : MBB) { 208 MachineInstr *CurrInstr = &MI; 209 LLVM_DEBUG(dbgs() << " Examining: " << MI); 210 if (PrevInstr) { 211 LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr 212 << " CurrInstr: " << *CurrInstr 213 << " isFirstInstructionInSequence(PrevInstr): " 214 << isFirstInstructionInSequence(PrevInstr) << "\n" 215 << " isSecondInstructionInSequence(CurrInstr): " 216 << isSecondInstructionInSequence(CurrInstr) << "\n"); 217 if (isFirstInstructionInSequence(PrevInstr) && 218 isSecondInstructionInSequence(CurrInstr)) { 219 LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n"); 220 Sequences.push_back(CurrInstr); 221 } 222 } 223 if (!CurrInstr->isPseudo()) 224 PrevInstr = CurrInstr; 225 ++Idx; 226 } 227 228 LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size() 229 << " occurrences of pattern found.\n"); 230 231 // Then update the basic block, inserting nops between the detected sequences. 232 for (auto &MI : Sequences) { 233 Changed = true; 234 insertNopBeforeInstruction(MBB, MI, TII); 235 } 236 237 return Changed; 238} 239 240// Factory function used by AArch64TargetMachine to add the pass to 241// the passmanager. 242FunctionPass *llvm::createAArch64A53Fix835769() { 243 return new AArch64A53Fix835769(); 244} 245