1343171Sdim//===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===// 2343171Sdim// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6343171Sdim// 7343171Sdim//===----------------------------------------------------------------------===// 8343171Sdim// 9343171Sdim// This pass applies cache prefetch instructions based on a profile. The pass 10343171Sdim// assumes DiscriminateMemOps ran immediately before, to ensure debug info 11343171Sdim// matches the one used at profile generation time. The profile is encoded in 12343171Sdim// afdo format (text or binary). It contains prefetch hints recommendations. 13343171Sdim// Each recommendation is made in terms of debug info locations, a type (i.e. 14343171Sdim// nta, t{0|1|2}) and a delta. The debug info identifies an instruction with a 15343171Sdim// memory operand (see X86DiscriminateMemOps). The prefetch will be made for 16343171Sdim// a location at that memory operand + the delta specified in the 17343171Sdim// recommendation. 18343171Sdim// 19343171Sdim//===----------------------------------------------------------------------===// 20343171Sdim 21343171Sdim#include "X86.h" 22343171Sdim#include "X86InstrBuilder.h" 23343171Sdim#include "X86InstrInfo.h" 24343171Sdim#include "X86MachineFunctionInfo.h" 25343171Sdim#include "X86Subtarget.h" 26343171Sdim#include "llvm/CodeGen/MachineModuleInfo.h" 27343171Sdim#include "llvm/IR/DebugInfoMetadata.h" 28343171Sdim#include "llvm/ProfileData/SampleProf.h" 29343171Sdim#include "llvm/ProfileData/SampleProfReader.h" 30343171Sdim#include "llvm/Transforms/IPO/SampleProfile.h" 31343171Sdimusing namespace llvm; 32343171Sdimusing namespace sampleprof; 33343171Sdim 34343171Sdimstatic cl::opt<std::string> 35343171Sdim PrefetchHintsFile("prefetch-hints-file", 36343806Sdim cl::desc("Path to the prefetch hints profile. See also " 37343806Sdim "-x86-discriminate-memops"), 38343171Sdim cl::Hidden); 39343171Sdimnamespace { 40343171Sdim 41343171Sdimclass X86InsertPrefetch : public MachineFunctionPass { 42343171Sdim void getAnalysisUsage(AnalysisUsage &AU) const override; 43343171Sdim bool doInitialization(Module &) override; 44343171Sdim 45343171Sdim bool runOnMachineFunction(MachineFunction &MF) override; 46343171Sdim struct PrefetchInfo { 47343171Sdim unsigned InstructionID; 48343171Sdim int64_t Delta; 49343171Sdim }; 50343171Sdim typedef SmallVectorImpl<PrefetchInfo> Prefetches; 51343171Sdim bool findPrefetchInfo(const FunctionSamples *Samples, const MachineInstr &MI, 52343171Sdim Prefetches &prefetches) const; 53343171Sdim 54343171Sdimpublic: 55343171Sdim static char ID; 56343171Sdim X86InsertPrefetch(const std::string &PrefetchHintsFilename); 57343171Sdim StringRef getPassName() const override { 58343171Sdim return "X86 Insert Cache Prefetches"; 59343171Sdim } 60343171Sdim 61343171Sdimprivate: 62343171Sdim std::string Filename; 63343171Sdim std::unique_ptr<SampleProfileReader> Reader; 64343171Sdim}; 65343171Sdim 66343171Sdimusing PrefetchHints = SampleRecord::CallTargetMap; 67343171Sdim 68343171Sdim// Return any prefetching hints for the specified MachineInstruction. The hints 69343171Sdim// are returned as pairs (name, delta). 70343171SdimErrorOr<PrefetchHints> getPrefetchHints(const FunctionSamples *TopSamples, 71343171Sdim const MachineInstr &MI) { 72343171Sdim if (const auto &Loc = MI.getDebugLoc()) 73343171Sdim if (const auto *Samples = TopSamples->findFunctionSamples(Loc)) 74343171Sdim return Samples->findCallTargetMapAt(FunctionSamples::getOffset(Loc), 75343171Sdim Loc->getBaseDiscriminator()); 76343171Sdim return std::error_code(); 77343171Sdim} 78343171Sdim 79343171Sdim// The prefetch instruction can't take memory operands involving vector 80343171Sdim// registers. 81343171Sdimbool IsMemOpCompatibleWithPrefetch(const MachineInstr &MI, int Op) { 82360784Sdim Register BaseReg = MI.getOperand(Op + X86::AddrBaseReg).getReg(); 83360784Sdim Register IndexReg = MI.getOperand(Op + X86::AddrIndexReg).getReg(); 84343171Sdim return (BaseReg == 0 || 85343171Sdim X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg) || 86343171Sdim X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg)) && 87343171Sdim (IndexReg == 0 || 88343171Sdim X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg) || 89343171Sdim X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg)); 90343171Sdim} 91343171Sdim 92343171Sdim} // end anonymous namespace 93343171Sdim 94343171Sdim//===----------------------------------------------------------------------===// 95343171Sdim// Implementation 96343171Sdim//===----------------------------------------------------------------------===// 97343171Sdim 98343171Sdimchar X86InsertPrefetch::ID = 0; 99343171Sdim 100343171SdimX86InsertPrefetch::X86InsertPrefetch(const std::string &PrefetchHintsFilename) 101343171Sdim : MachineFunctionPass(ID), Filename(PrefetchHintsFilename) {} 102343171Sdim 103343171Sdim/// Return true if the provided MachineInstruction has cache prefetch hints. In 104343171Sdim/// that case, the prefetch hints are stored, in order, in the Prefetches 105343171Sdim/// vector. 106343171Sdimbool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples, 107343171Sdim const MachineInstr &MI, 108343171Sdim Prefetches &Prefetches) const { 109343171Sdim assert(Prefetches.empty() && 110343171Sdim "Expected caller passed empty PrefetchInfo vector."); 111360784Sdim static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = { 112343171Sdim {"_nta_", X86::PREFETCHNTA}, 113343171Sdim {"_t0_", X86::PREFETCHT0}, 114343171Sdim {"_t1_", X86::PREFETCHT1}, 115343171Sdim {"_t2_", X86::PREFETCHT2}, 116343171Sdim }; 117343171Sdim static const char *SerializedPrefetchPrefix = "__prefetch"; 118343171Sdim 119343171Sdim const ErrorOr<PrefetchHints> T = getPrefetchHints(TopSamples, MI); 120343171Sdim if (!T) 121343171Sdim return false; 122343171Sdim int16_t max_index = -1; 123343171Sdim // Convert serialized prefetch hints into PrefetchInfo objects, and populate 124343171Sdim // the Prefetches vector. 125343171Sdim for (const auto &S_V : *T) { 126343171Sdim StringRef Name = S_V.getKey(); 127343171Sdim if (Name.consume_front(SerializedPrefetchPrefix)) { 128343171Sdim int64_t D = static_cast<int64_t>(S_V.second); 129343171Sdim unsigned IID = 0; 130343171Sdim for (const auto &HintType : HintTypes) { 131343171Sdim if (Name.startswith(HintType.first)) { 132343171Sdim Name = Name.drop_front(HintType.first.size()); 133343171Sdim IID = HintType.second; 134343171Sdim break; 135343171Sdim } 136343171Sdim } 137343171Sdim if (IID == 0) 138343171Sdim return false; 139343171Sdim uint8_t index = 0; 140343171Sdim Name.consumeInteger(10, index); 141343171Sdim 142343171Sdim if (index >= Prefetches.size()) 143343171Sdim Prefetches.resize(index + 1); 144343171Sdim Prefetches[index] = {IID, D}; 145343171Sdim max_index = std::max(max_index, static_cast<int16_t>(index)); 146343171Sdim } 147343171Sdim } 148343171Sdim assert(max_index + 1 >= 0 && 149343171Sdim "Possible overflow: max_index + 1 should be positive."); 150343171Sdim assert(static_cast<size_t>(max_index + 1) == Prefetches.size() && 151343171Sdim "The number of prefetch hints received should match the number of " 152343171Sdim "PrefetchInfo objects returned"); 153343171Sdim return !Prefetches.empty(); 154343171Sdim} 155343171Sdim 156343171Sdimbool X86InsertPrefetch::doInitialization(Module &M) { 157343171Sdim if (Filename.empty()) 158343171Sdim return false; 159343171Sdim 160343171Sdim LLVMContext &Ctx = M.getContext(); 161343171Sdim ErrorOr<std::unique_ptr<SampleProfileReader>> ReaderOrErr = 162343171Sdim SampleProfileReader::create(Filename, Ctx); 163343171Sdim if (std::error_code EC = ReaderOrErr.getError()) { 164343171Sdim std::string Msg = "Could not open profile: " + EC.message(); 165343171Sdim Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg, 166343171Sdim DiagnosticSeverity::DS_Warning)); 167343171Sdim return false; 168343171Sdim } 169343171Sdim Reader = std::move(ReaderOrErr.get()); 170343171Sdim Reader->read(); 171343171Sdim return true; 172343171Sdim} 173343171Sdim 174343171Sdimvoid X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const { 175343171Sdim AU.setPreservesAll(); 176360784Sdim AU.addRequired<MachineModuleInfoWrapperPass>(); 177343171Sdim} 178343171Sdim 179343171Sdimbool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) { 180343171Sdim if (!Reader) 181343171Sdim return false; 182343171Sdim const FunctionSamples *Samples = Reader->getSamplesFor(MF.getFunction()); 183343171Sdim if (!Samples) 184343171Sdim return false; 185343171Sdim 186343171Sdim bool Changed = false; 187343171Sdim 188343171Sdim const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 189343171Sdim SmallVector<PrefetchInfo, 4> Prefetches; 190343171Sdim for (auto &MBB : MF) { 191343171Sdim for (auto MI = MBB.instr_begin(); MI != MBB.instr_end();) { 192343171Sdim auto Current = MI; 193343171Sdim ++MI; 194343171Sdim 195343171Sdim int Offset = X86II::getMemoryOperandNo(Current->getDesc().TSFlags); 196343171Sdim if (Offset < 0) 197343171Sdim continue; 198343171Sdim unsigned Bias = X86II::getOperandBias(Current->getDesc()); 199343171Sdim int MemOpOffset = Offset + Bias; 200343171Sdim // FIXME(mtrofin): ORE message when the recommendation cannot be taken. 201343171Sdim if (!IsMemOpCompatibleWithPrefetch(*Current, MemOpOffset)) 202343171Sdim continue; 203343171Sdim Prefetches.clear(); 204343171Sdim if (!findPrefetchInfo(Samples, *Current, Prefetches)) 205343171Sdim continue; 206343171Sdim assert(!Prefetches.empty() && 207343171Sdim "The Prefetches vector should contain at least a value if " 208343171Sdim "findPrefetchInfo returned true."); 209343171Sdim for (auto &PrefInfo : Prefetches) { 210343171Sdim unsigned PFetchInstrID = PrefInfo.InstructionID; 211343171Sdim int64_t Delta = PrefInfo.Delta; 212343171Sdim const MCInstrDesc &Desc = TII->get(PFetchInstrID); 213343171Sdim MachineInstr *PFetch = 214343171Sdim MF.CreateMachineInstr(Desc, Current->getDebugLoc(), true); 215343171Sdim MachineInstrBuilder MIB(MF, PFetch); 216343171Sdim 217343171Sdim assert(X86::AddrBaseReg == 0 && X86::AddrScaleAmt == 1 && 218343171Sdim X86::AddrIndexReg == 2 && X86::AddrDisp == 3 && 219343171Sdim X86::AddrSegmentReg == 4 && 220343171Sdim "Unexpected change in X86 operand offset order."); 221343171Sdim 222343171Sdim // This assumes X86::AddBaseReg = 0, {...}ScaleAmt = 1, etc. 223343171Sdim // FIXME(mtrofin): consider adding a: 224343171Sdim // MachineInstrBuilder::set(unsigned offset, op). 225343171Sdim MIB.addReg(Current->getOperand(MemOpOffset + X86::AddrBaseReg).getReg()) 226343171Sdim .addImm( 227343171Sdim Current->getOperand(MemOpOffset + X86::AddrScaleAmt).getImm()) 228343171Sdim .addReg( 229343171Sdim Current->getOperand(MemOpOffset + X86::AddrIndexReg).getReg()) 230343171Sdim .addImm(Current->getOperand(MemOpOffset + X86::AddrDisp).getImm() + 231343171Sdim Delta) 232343171Sdim .addReg(Current->getOperand(MemOpOffset + X86::AddrSegmentReg) 233343171Sdim .getReg()); 234343171Sdim 235343171Sdim if (!Current->memoperands_empty()) { 236343171Sdim MachineMemOperand *CurrentOp = *(Current->memoperands_begin()); 237343171Sdim MIB.addMemOperand(MF.getMachineMemOperand( 238343171Sdim CurrentOp, CurrentOp->getOffset() + Delta, CurrentOp->getSize())); 239343171Sdim } 240343171Sdim 241343171Sdim // Insert before Current. This is because Current may clobber some of 242343171Sdim // the registers used to describe the input memory operand. 243343171Sdim MBB.insert(Current, PFetch); 244343171Sdim Changed = true; 245343171Sdim } 246343171Sdim } 247343171Sdim } 248343171Sdim return Changed; 249343171Sdim} 250343171Sdim 251343171SdimFunctionPass *llvm::createX86InsertPrefetchPass() { 252343171Sdim return new X86InsertPrefetch(PrefetchHintsFile); 253343171Sdim} 254