1//===- X86DiscriminateMemOps.cpp - Unique IDs for Mem Ops -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// This pass aids profile-driven cache prefetch insertion by ensuring all
10/// instructions that have a memory operand are distinguishible from each other.
11///
12//===----------------------------------------------------------------------===//
13
14#include "X86.h"
15#include "X86InstrBuilder.h"
16#include "X86InstrInfo.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86Subtarget.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21#include "llvm/ProfileData/SampleProf.h"
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Transforms/IPO/SampleProfile.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "x86-discriminate-memops"
28
29static cl::opt<bool> EnableDiscriminateMemops(
30    DEBUG_TYPE, cl::init(false),
31    cl::desc("Generate unique debug info for each instruction with a memory "
32             "operand. Should be enabled for profile-driven cache prefetching, "
33             "both in the build of the binary being profiled, as well as in "
34             "the build of the binary consuming the profile."),
35    cl::Hidden);
36
37static cl::opt<bool> BypassPrefetchInstructions(
38    "x86-bypass-prefetch-instructions", cl::init(true),
39    cl::desc("When discriminating instructions with memory operands, ignore "
40             "prefetch instructions. This ensures the other memory operand "
41             "instructions have the same identifiers after inserting "
42             "prefetches, allowing for successive insertions."),
43    cl::Hidden);
44
45namespace {
46
47using Location = std::pair<StringRef, unsigned>;
48
49Location diToLocation(const DILocation *Loc) {
50  return std::make_pair(Loc->getFilename(), Loc->getLine());
51}
52
53/// Ensure each instruction having a memory operand has a distinct <LineNumber,
54/// Discriminator> pair.
55void updateDebugInfo(MachineInstr *MI, const DILocation *Loc) {
56  DebugLoc DL(Loc);
57  MI->setDebugLoc(DL);
58}
59
60class X86DiscriminateMemOps : public MachineFunctionPass {
61  bool runOnMachineFunction(MachineFunction &MF) override;
62  StringRef getPassName() const override {
63    return "X86 Discriminate Memory Operands";
64  }
65
66public:
67  static char ID;
68
69  /// Default construct and initialize the pass.
70  X86DiscriminateMemOps();
71};
72
73bool IsPrefetchOpcode(unsigned Opcode) {
74  return Opcode == X86::PREFETCHNTA || Opcode == X86::PREFETCHT0 ||
75         Opcode == X86::PREFETCHT1 || Opcode == X86::PREFETCHT2;
76}
77} // end anonymous namespace
78
79//===----------------------------------------------------------------------===//
80//            Implementation
81//===----------------------------------------------------------------------===//
82
83char X86DiscriminateMemOps::ID = 0;
84
85/// Default construct and initialize the pass.
86X86DiscriminateMemOps::X86DiscriminateMemOps() : MachineFunctionPass(ID) {}
87
88bool X86DiscriminateMemOps::runOnMachineFunction(MachineFunction &MF) {
89  if (!EnableDiscriminateMemops)
90    return false;
91
92  DISubprogram *FDI = MF.getFunction().getSubprogram();
93  if (!FDI || !FDI->getUnit()->getDebugInfoForProfiling())
94    return false;
95
96  // Have a default DILocation, if we find instructions with memops that don't
97  // have any debug info.
98  const DILocation *ReferenceDI =
99      DILocation::get(FDI->getContext(), FDI->getLine(), 0, FDI);
100  assert(ReferenceDI && "ReferenceDI should not be nullptr");
101  DenseMap<Location, unsigned> MemOpDiscriminators;
102  MemOpDiscriminators[diToLocation(ReferenceDI)] = 0;
103
104  // Figure out the largest discriminator issued for each Location. When we
105  // issue new discriminators, we can thus avoid issuing discriminators
106  // belonging to instructions that don't have memops. This isn't a requirement
107  // for the goals of this pass, however, it avoids unnecessary ambiguity.
108  for (auto &MBB : MF) {
109    for (auto &MI : MBB) {
110      const auto &DI = MI.getDebugLoc();
111      if (!DI)
112        continue;
113      if (BypassPrefetchInstructions && IsPrefetchOpcode(MI.getDesc().Opcode))
114        continue;
115      Location Loc = diToLocation(DI);
116      MemOpDiscriminators[Loc] =
117          std::max(MemOpDiscriminators[Loc], DI->getBaseDiscriminator());
118    }
119  }
120
121  // Keep track of the discriminators seen at each Location. If an instruction's
122  // DebugInfo has a Location and discriminator we've already seen, replace its
123  // discriminator with a new one, to guarantee uniqueness.
124  DenseMap<Location, DenseSet<unsigned>> Seen;
125
126  bool Changed = false;
127  for (auto &MBB : MF) {
128    for (auto &MI : MBB) {
129      if (X86II::getMemoryOperandNo(MI.getDesc().TSFlags) < 0)
130        continue;
131      if (BypassPrefetchInstructions && IsPrefetchOpcode(MI.getDesc().Opcode))
132        continue;
133      const DILocation *DI = MI.getDebugLoc();
134      bool HasDebug = DI;
135      if (!HasDebug) {
136        DI = ReferenceDI;
137      }
138      Location L = diToLocation(DI);
139      DenseSet<unsigned> &Set = Seen[L];
140      const std::pair<DenseSet<unsigned>::iterator, bool> TryInsert =
141          Set.insert(DI->getBaseDiscriminator());
142      if (!TryInsert.second || !HasDebug) {
143        unsigned BF, DF, CI = 0;
144        DILocation::decodeDiscriminator(DI->getDiscriminator(), BF, DF, CI);
145        Optional<unsigned> EncodedDiscriminator = DILocation::encodeDiscriminator(
146            MemOpDiscriminators[L] + 1, DF, CI);
147
148        if (!EncodedDiscriminator) {
149          // FIXME(mtrofin): The assumption is that this scenario is infrequent/OK
150          // not to support. If evidence points otherwise, we can explore synthesizeing
151          // unique DIs by adding fake line numbers, or by constructing 64 bit
152          // discriminators.
153          LLVM_DEBUG(dbgs() << "Unable to create a unique discriminator "
154                     "for instruction with memory operand in: "
155                     << DI->getFilename() << " Line: " << DI->getLine()
156                     << " Column: " << DI->getColumn()
157                     << ". This is likely due to a large macro expansion. \n");
158          continue;
159        }
160        // Since we were able to encode, bump the MemOpDiscriminators.
161        ++MemOpDiscriminators[L];
162        DI = DI->cloneWithDiscriminator(EncodedDiscriminator.getValue());
163        assert(DI && "DI should not be nullptr");
164        updateDebugInfo(&MI, DI);
165        Changed = true;
166        std::pair<DenseSet<unsigned>::iterator, bool> MustInsert =
167            Set.insert(DI->getBaseDiscriminator());
168        (void)MustInsert; // Silence warning in release build.
169        assert(MustInsert.second && "New discriminator shouldn't be present in set");
170      }
171
172      // Bump the reference DI to avoid cramming discriminators on line 0.
173      // FIXME(mtrofin): pin ReferenceDI on blocks or first instruction with DI
174      // in a block. It's more consistent than just relying on the last memop
175      // instruction we happened to see.
176      ReferenceDI = DI;
177    }
178  }
179  return Changed;
180}
181
182FunctionPass *llvm::createX86DiscriminateMemOpsPass() {
183  return new X86DiscriminateMemOps();
184}
185