1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/Triple.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/BranchProbabilityInfo.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/Constant.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalValue.h"
33#include "llvm/IR/GlobalVariable.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/Instruction.h"
36#include "llvm/IR/Instructions.h"
37#include "llvm/IR/IntrinsicInst.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Type.h"
40#include "llvm/InitializePasses.h"
41#include "llvm/Pass.h"
42#include "llvm/ProfileData/InstrProf.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/CommandLine.h"
45#include "llvm/Support/Error.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Transforms/Utils/BasicBlockUtils.h"
48#include "llvm/Transforms/Utils/ModuleUtils.h"
49#include "llvm/Transforms/Utils/SSAUpdater.h"
50#include <algorithm>
51#include <cassert>
52#include <cstddef>
53#include <cstdint>
54#include <string>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "instrprof"
59
60// The start and end values of precise value profile range for memory
61// intrinsic sizes
62cl::opt<std::string> MemOPSizeRange(
63    "memop-size-range",
64    cl::desc("Set the range of size in memory intrinsic calls to be profiled "
65             "precisely, in a format of <start_val>:<end_val>"),
66    cl::init(""));
67
68// The value that considered to be large value in  memory intrinsic.
69cl::opt<unsigned> MemOPSizeLarge(
70    "memop-size-large",
71    cl::desc("Set large value thresthold in memory intrinsic size profiling. "
72             "Value of 0 disables the large value profiling."),
73    cl::init(8192));
74
75namespace {
76
77cl::opt<bool> DoHashBasedCounterSplit(
78    "hash-based-counter-split",
79    cl::desc("Rename counter variable of a comdat function based on cfg hash"),
80    cl::init(true));
81
82cl::opt<bool> RuntimeCounterRelocation(
83    "runtime-counter-relocation",
84    cl::desc("Enable relocating counters at runtime."),
85    cl::init(false));
86
87cl::opt<bool> ValueProfileStaticAlloc(
88    "vp-static-alloc",
89    cl::desc("Do static counter allocation for value profiler"),
90    cl::init(true));
91
92cl::opt<double> NumCountersPerValueSite(
93    "vp-counters-per-site",
94    cl::desc("The average number of profile counters allocated "
95             "per value profiling site."),
96    // This is set to a very small value because in real programs, only
97    // a very small percentage of value sites have non-zero targets, e.g, 1/30.
98    // For those sites with non-zero profile, the average number of targets
99    // is usually smaller than 2.
100    cl::init(1.0));
101
102cl::opt<bool> AtomicCounterUpdateAll(
103    "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
104    cl::desc("Make all profile counter updates atomic (for testing only)"),
105    cl::init(false));
106
107cl::opt<bool> AtomicCounterUpdatePromoted(
108    "atomic-counter-update-promoted", cl::ZeroOrMore,
109    cl::desc("Do counter update using atomic fetch add "
110             " for promoted counters only"),
111    cl::init(false));
112
113cl::opt<bool> AtomicFirstCounter(
114    "atomic-first-counter", cl::ZeroOrMore,
115    cl::desc("Use atomic fetch add for first counter in a function (usually "
116             "the entry counter)"),
117    cl::init(false));
118
119// If the option is not specified, the default behavior about whether
120// counter promotion is done depends on how instrumentaiton lowering
121// pipeline is setup, i.e., the default value of true of this option
122// does not mean the promotion will be done by default. Explicitly
123// setting this option can override the default behavior.
124cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
125                                 cl::desc("Do counter register promotion"),
126                                 cl::init(false));
127cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
128    cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
129    cl::desc("Max number counter promotions per loop to avoid"
130             " increasing register pressure too much"));
131
132// A debug option
133cl::opt<int>
134    MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
135                       cl::desc("Max number of allowed counter promotions"));
136
137cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
138    cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
139    cl::desc("The max number of exiting blocks of a loop to allow "
140             " speculative counter promotion"));
141
142cl::opt<bool> SpeculativeCounterPromotionToLoop(
143    cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
144    cl::desc("When the option is false, if the target block is in a loop, "
145             "the promotion will be disallowed unless the promoted counter "
146             " update can be further/iteratively promoted into an acyclic "
147             " region."));
148
149cl::opt<bool> IterativeCounterPromotion(
150    cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
151    cl::desc("Allow counter promotion across the whole loop nest."));
152
153class InstrProfilingLegacyPass : public ModulePass {
154  InstrProfiling InstrProf;
155
156public:
157  static char ID;
158
159  InstrProfilingLegacyPass() : ModulePass(ID) {}
160  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
161      : ModulePass(ID), InstrProf(Options, IsCS) {
162    initializeInstrProfilingLegacyPassPass(*PassRegistry::getPassRegistry());
163  }
164
165  StringRef getPassName() const override {
166    return "Frontend instrumentation-based coverage lowering";
167  }
168
169  bool runOnModule(Module &M) override {
170    auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
171      return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
172    };
173    return InstrProf.run(M, GetTLI);
174  }
175
176  void getAnalysisUsage(AnalysisUsage &AU) const override {
177    AU.setPreservesCFG();
178    AU.addRequired<TargetLibraryInfoWrapperPass>();
179  }
180};
181
182///
183/// A helper class to promote one counter RMW operation in the loop
184/// into register update.
185///
186/// RWM update for the counter will be sinked out of the loop after
187/// the transformation.
188///
189class PGOCounterPromoterHelper : public LoadAndStorePromoter {
190public:
191  PGOCounterPromoterHelper(
192      Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
193      BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
194      ArrayRef<Instruction *> InsertPts,
195      DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
196      LoopInfo &LI)
197      : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
198        InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
199    assert(isa<LoadInst>(L));
200    assert(isa<StoreInst>(S));
201    SSA.AddAvailableValue(PH, Init);
202  }
203
204  void doExtraRewritesBeforeFinalDeletion() override {
205    for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
206      BasicBlock *ExitBlock = ExitBlocks[i];
207      Instruction *InsertPos = InsertPts[i];
208      // Get LiveIn value into the ExitBlock. If there are multiple
209      // predecessors, the value is defined by a PHI node in this
210      // block.
211      Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
212      Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
213      Type *Ty = LiveInValue->getType();
214      IRBuilder<> Builder(InsertPos);
215      if (AtomicCounterUpdatePromoted)
216        // automic update currently can only be promoted across the current
217        // loop, not the whole loop nest.
218        Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
219                                AtomicOrdering::SequentiallyConsistent);
220      else {
221        LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
222        auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
223        auto *NewStore = Builder.CreateStore(NewVal, Addr);
224
225        // Now update the parent loop's candidate list:
226        if (IterativeCounterPromotion) {
227          auto *TargetLoop = LI.getLoopFor(ExitBlock);
228          if (TargetLoop)
229            LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
230        }
231      }
232    }
233  }
234
235private:
236  Instruction *Store;
237  ArrayRef<BasicBlock *> ExitBlocks;
238  ArrayRef<Instruction *> InsertPts;
239  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
240  LoopInfo &LI;
241};
242
243/// A helper class to do register promotion for all profile counter
244/// updates in a loop.
245///
246class PGOCounterPromoter {
247public:
248  PGOCounterPromoter(
249      DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
250      Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
251      : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
252        LI(LI), BFI(BFI) {
253
254    // Skip collection of ExitBlocks and InsertPts for loops that will not be
255    // able to have counters promoted.
256    SmallVector<BasicBlock *, 8> LoopExitBlocks;
257    SmallPtrSet<BasicBlock *, 8> BlockSet;
258
259    L.getExitBlocks(LoopExitBlocks);
260    if (!isPromotionPossible(&L, LoopExitBlocks))
261      return;
262
263    for (BasicBlock *ExitBlock : LoopExitBlocks) {
264      if (BlockSet.insert(ExitBlock).second) {
265        ExitBlocks.push_back(ExitBlock);
266        InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
267      }
268    }
269  }
270
271  bool run(int64_t *NumPromoted) {
272    // Skip 'infinite' loops:
273    if (ExitBlocks.size() == 0)
274      return false;
275    unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
276    if (MaxProm == 0)
277      return false;
278
279    unsigned Promoted = 0;
280    for (auto &Cand : LoopToCandidates[&L]) {
281
282      SmallVector<PHINode *, 4> NewPHIs;
283      SSAUpdater SSA(&NewPHIs);
284      Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
285
286      // If BFI is set, we will use it to guide the promotions.
287      if (BFI) {
288        auto *BB = Cand.first->getParent();
289        auto InstrCount = BFI->getBlockProfileCount(BB);
290        if (!InstrCount)
291          continue;
292        auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
293        // If the average loop trip count is not greater than 1.5, we skip
294        // promotion.
295        if (PreheaderCount &&
296            (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
297          continue;
298      }
299
300      PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
301                                        L.getLoopPreheader(), ExitBlocks,
302                                        InsertPts, LoopToCandidates, LI);
303      Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
304      Promoted++;
305      if (Promoted >= MaxProm)
306        break;
307
308      (*NumPromoted)++;
309      if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
310        break;
311    }
312
313    LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
314                      << L.getLoopDepth() << ")\n");
315    return Promoted != 0;
316  }
317
318private:
319  bool allowSpeculativeCounterPromotion(Loop *LP) {
320    SmallVector<BasicBlock *, 8> ExitingBlocks;
321    L.getExitingBlocks(ExitingBlocks);
322    // Not considierered speculative.
323    if (ExitingBlocks.size() == 1)
324      return true;
325    if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
326      return false;
327    return true;
328  }
329
330  // Check whether the loop satisfies the basic conditions needed to perform
331  // Counter Promotions.
332  bool isPromotionPossible(Loop *LP,
333                           const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
334    // We can't insert into a catchswitch.
335    if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
336          return isa<CatchSwitchInst>(Exit->getTerminator());
337        }))
338      return false;
339
340    if (!LP->hasDedicatedExits())
341      return false;
342
343    BasicBlock *PH = LP->getLoopPreheader();
344    if (!PH)
345      return false;
346
347    return true;
348  }
349
350  // Returns the max number of Counter Promotions for LP.
351  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
352    SmallVector<BasicBlock *, 8> LoopExitBlocks;
353    LP->getExitBlocks(LoopExitBlocks);
354    if (!isPromotionPossible(LP, LoopExitBlocks))
355      return 0;
356
357    SmallVector<BasicBlock *, 8> ExitingBlocks;
358    LP->getExitingBlocks(ExitingBlocks);
359
360    // If BFI is set, we do more aggressive promotions based on BFI.
361    if (BFI)
362      return (unsigned)-1;
363
364    // Not considierered speculative.
365    if (ExitingBlocks.size() == 1)
366      return MaxNumOfPromotionsPerLoop;
367
368    if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
369      return 0;
370
371    // Whether the target block is in a loop does not matter:
372    if (SpeculativeCounterPromotionToLoop)
373      return MaxNumOfPromotionsPerLoop;
374
375    // Now check the target block:
376    unsigned MaxProm = MaxNumOfPromotionsPerLoop;
377    for (auto *TargetBlock : LoopExitBlocks) {
378      auto *TargetLoop = LI.getLoopFor(TargetBlock);
379      if (!TargetLoop)
380        continue;
381      unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
382      unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
383      MaxProm =
384          std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
385                                PendingCandsInTarget);
386    }
387    return MaxProm;
388  }
389
390  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
391  SmallVector<BasicBlock *, 8> ExitBlocks;
392  SmallVector<Instruction *, 8> InsertPts;
393  Loop &L;
394  LoopInfo &LI;
395  BlockFrequencyInfo *BFI;
396};
397
398} // end anonymous namespace
399
400PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
401  FunctionAnalysisManager &FAM =
402      AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
403  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
404    return FAM.getResult<TargetLibraryAnalysis>(F);
405  };
406  if (!run(M, GetTLI))
407    return PreservedAnalyses::all();
408
409  return PreservedAnalyses::none();
410}
411
412char InstrProfilingLegacyPass::ID = 0;
413INITIALIZE_PASS_BEGIN(
414    InstrProfilingLegacyPass, "instrprof",
415    "Frontend instrumentation-based coverage lowering.", false, false)
416INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
417INITIALIZE_PASS_END(
418    InstrProfilingLegacyPass, "instrprof",
419    "Frontend instrumentation-based coverage lowering.", false, false)
420
421ModulePass *
422llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options,
423                                     bool IsCS) {
424  return new InstrProfilingLegacyPass(Options, IsCS);
425}
426
427static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
428  InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
429  if (Inc)
430    return Inc;
431  return dyn_cast<InstrProfIncrementInst>(Instr);
432}
433
434bool InstrProfiling::lowerIntrinsics(Function *F) {
435  bool MadeChange = false;
436  PromotionCandidates.clear();
437  for (BasicBlock &BB : *F) {
438    for (auto I = BB.begin(), E = BB.end(); I != E;) {
439      auto Instr = I++;
440      InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
441      if (Inc) {
442        lowerIncrement(Inc);
443        MadeChange = true;
444      } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
445        lowerValueProfileInst(Ind);
446        MadeChange = true;
447      }
448    }
449  }
450
451  if (!MadeChange)
452    return false;
453
454  promoteCounterLoadStores(F);
455  return true;
456}
457
458bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
459  if (RuntimeCounterRelocation.getNumOccurrences() > 0)
460    return RuntimeCounterRelocation;
461
462  return TT.isOSFuchsia();
463}
464
465bool InstrProfiling::isCounterPromotionEnabled() const {
466  if (DoCounterPromotion.getNumOccurrences() > 0)
467    return DoCounterPromotion;
468
469  return Options.DoCounterPromotion;
470}
471
472void InstrProfiling::promoteCounterLoadStores(Function *F) {
473  if (!isCounterPromotionEnabled())
474    return;
475
476  DominatorTree DT(*F);
477  LoopInfo LI(DT);
478  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
479
480  std::unique_ptr<BlockFrequencyInfo> BFI;
481  if (Options.UseBFIInPromotion) {
482    std::unique_ptr<BranchProbabilityInfo> BPI;
483    BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
484    BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
485  }
486
487  for (const auto &LoadStore : PromotionCandidates) {
488    auto *CounterLoad = LoadStore.first;
489    auto *CounterStore = LoadStore.second;
490    BasicBlock *BB = CounterLoad->getParent();
491    Loop *ParentLoop = LI.getLoopFor(BB);
492    if (!ParentLoop)
493      continue;
494    LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
495  }
496
497  SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
498
499  // Do a post-order traversal of the loops so that counter updates can be
500  // iteratively hoisted outside the loop nest.
501  for (auto *Loop : llvm::reverse(Loops)) {
502    PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
503    Promoter.run(&TotalCountersPromoted);
504  }
505}
506
507/// Check if the module contains uses of any profiling intrinsics.
508static bool containsProfilingIntrinsics(Module &M) {
509  if (auto *F = M.getFunction(
510          Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
511    if (!F->use_empty())
512      return true;
513  if (auto *F = M.getFunction(
514          Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
515    if (!F->use_empty())
516      return true;
517  if (auto *F = M.getFunction(
518          Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
519    if (!F->use_empty())
520      return true;
521  return false;
522}
523
524bool InstrProfiling::run(
525    Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
526  this->M = &M;
527  this->GetTLI = std::move(GetTLI);
528  NamesVar = nullptr;
529  NamesSize = 0;
530  ProfileDataMap.clear();
531  UsedVars.clear();
532  getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
533                              MemOPSizeRangeLast);
534  TT = Triple(M.getTargetTriple());
535
536  // Emit the runtime hook even if no counters are present.
537  bool MadeChange = emitRuntimeHook();
538
539  // Improve compile time by avoiding linear scans when there is no work.
540  GlobalVariable *CoverageNamesVar =
541      M.getNamedGlobal(getCoverageUnusedNamesVarName());
542  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
543    return MadeChange;
544
545  // We did not know how many value sites there would be inside
546  // the instrumented function. This is counting the number of instrumented
547  // target value sites to enter it as field in the profile data variable.
548  for (Function &F : M) {
549    InstrProfIncrementInst *FirstProfIncInst = nullptr;
550    for (BasicBlock &BB : F)
551      for (auto I = BB.begin(), E = BB.end(); I != E; I++)
552        if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
553          computeNumValueSiteCounts(Ind);
554        else if (FirstProfIncInst == nullptr)
555          FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
556
557    // Value profiling intrinsic lowering requires per-function profile data
558    // variable to be created first.
559    if (FirstProfIncInst != nullptr)
560      static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
561  }
562
563  for (Function &F : M)
564    MadeChange |= lowerIntrinsics(&F);
565
566  if (CoverageNamesVar) {
567    lowerCoverageData(CoverageNamesVar);
568    MadeChange = true;
569  }
570
571  if (!MadeChange)
572    return false;
573
574  emitVNodes();
575  emitNameData();
576  emitRegistration();
577  emitUses();
578  emitInitialization();
579  return true;
580}
581
582static FunctionCallee
583getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
584                              bool IsRange = false) {
585  LLVMContext &Ctx = M.getContext();
586  auto *ReturnTy = Type::getVoidTy(M.getContext());
587
588  AttributeList AL;
589  if (auto AK = TLI.getExtAttrForI32Param(false))
590    AL = AL.addParamAttribute(M.getContext(), 2, AK);
591
592  if (!IsRange) {
593    Type *ParamTypes[] = {
594#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
595#include "llvm/ProfileData/InstrProfData.inc"
596    };
597    auto *ValueProfilingCallTy =
598        FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
599    return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
600                                 ValueProfilingCallTy, AL);
601  } else {
602    Type *RangeParamTypes[] = {
603#define VALUE_RANGE_PROF 1
604#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
605#include "llvm/ProfileData/InstrProfData.inc"
606#undef VALUE_RANGE_PROF
607    };
608    auto *ValueRangeProfilingCallTy =
609        FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
610    return M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
611                                 ValueRangeProfilingCallTy, AL);
612  }
613}
614
615void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
616  GlobalVariable *Name = Ind->getName();
617  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
618  uint64_t Index = Ind->getIndex()->getZExtValue();
619  auto It = ProfileDataMap.find(Name);
620  if (It == ProfileDataMap.end()) {
621    PerFunctionProfileData PD;
622    PD.NumValueSites[ValueKind] = Index + 1;
623    ProfileDataMap[Name] = PD;
624  } else if (It->second.NumValueSites[ValueKind] <= Index)
625    It->second.NumValueSites[ValueKind] = Index + 1;
626}
627
628void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
629  GlobalVariable *Name = Ind->getName();
630  auto It = ProfileDataMap.find(Name);
631  assert(It != ProfileDataMap.end() && It->second.DataVar &&
632         "value profiling detected in function with no counter incerement");
633
634  GlobalVariable *DataVar = It->second.DataVar;
635  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
636  uint64_t Index = Ind->getIndex()->getZExtValue();
637  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
638    Index += It->second.NumValueSites[Kind];
639
640  IRBuilder<> Builder(Ind);
641  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
642                  llvm::InstrProfValueKind::IPVK_MemOPSize);
643  CallInst *Call = nullptr;
644  auto *TLI = &GetTLI(*Ind->getFunction());
645
646  // To support value profiling calls within Windows exception handlers, funclet
647  // information contained within operand bundles needs to be copied over to
648  // the library call. This is required for the IR to be processed by the
649  // WinEHPrepare pass.
650  SmallVector<OperandBundleDef, 1> OpBundles;
651  Ind->getOperandBundlesAsDefs(OpBundles);
652  if (!IsRange) {
653    Value *Args[3] = {Ind->getTargetValue(),
654                      Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
655                      Builder.getInt32(Index)};
656    Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
657                              OpBundles);
658  } else {
659    Value *Args[6] = {
660        Ind->getTargetValue(),
661        Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
662        Builder.getInt32(Index),
663        Builder.getInt64(MemOPSizeRangeStart),
664        Builder.getInt64(MemOPSizeRangeLast),
665        Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
666    Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true),
667                              Args, OpBundles);
668  }
669  if (auto AK = TLI->getExtAttrForI32Param(false))
670    Call->addParamAttr(2, AK);
671  Ind->replaceAllUsesWith(Call);
672  Ind->eraseFromParent();
673}
674
675void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
676  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
677
678  IRBuilder<> Builder(Inc);
679  uint64_t Index = Inc->getIndex()->getZExtValue();
680  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
681                                                   Counters, 0, Index);
682
683  if (isRuntimeCounterRelocationEnabled()) {
684    Type *Int64Ty = Type::getInt64Ty(M->getContext());
685    Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext());
686    Function *Fn = Inc->getParent()->getParent();
687    Instruction &I = Fn->getEntryBlock().front();
688    LoadInst *LI = dyn_cast<LoadInst>(&I);
689    if (!LI) {
690      IRBuilder<> Builder(&I);
691      Type *Int64Ty = Type::getInt64Ty(M->getContext());
692      GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
693      if (!Bias) {
694        Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
695                                  Constant::getNullValue(Int64Ty),
696                                  getInstrProfCounterBiasVarName());
697        Bias->setVisibility(GlobalVariable::HiddenVisibility);
698      }
699      LI = Builder.CreateLoad(Int64Ty, Bias);
700    }
701    auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI);
702    Addr = Builder.CreateIntToPtr(Add, Int64PtrTy);
703  }
704
705  if (Options.Atomic || AtomicCounterUpdateAll ||
706      (Index == 0 && AtomicFirstCounter)) {
707    Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
708                            AtomicOrdering::Monotonic);
709  } else {
710    Value *IncStep = Inc->getStep();
711    Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
712    auto *Count = Builder.CreateAdd(Load, Inc->getStep());
713    auto *Store = Builder.CreateStore(Count, Addr);
714    if (isCounterPromotionEnabled())
715      PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
716  }
717  Inc->eraseFromParent();
718}
719
720void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
721  ConstantArray *Names =
722      cast<ConstantArray>(CoverageNamesVar->getInitializer());
723  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
724    Constant *NC = Names->getOperand(I);
725    Value *V = NC->stripPointerCasts();
726    assert(isa<GlobalVariable>(V) && "Missing reference to function name");
727    GlobalVariable *Name = cast<GlobalVariable>(V);
728
729    Name->setLinkage(GlobalValue::PrivateLinkage);
730    ReferencedNames.push_back(Name);
731    NC->dropAllReferences();
732  }
733  CoverageNamesVar->eraseFromParent();
734}
735
736/// Get the name of a profiling variable for a particular function.
737static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
738  StringRef NamePrefix = getInstrProfNameVarPrefix();
739  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
740  Function *F = Inc->getParent()->getParent();
741  Module *M = F->getParent();
742  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
743      !canRenameComdatFunc(*F))
744    return (Prefix + Name).str();
745  uint64_t FuncHash = Inc->getHash()->getZExtValue();
746  SmallVector<char, 24> HashPostfix;
747  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
748    return (Prefix + Name).str();
749  return (Prefix + Name + "." + Twine(FuncHash)).str();
750}
751
752static inline bool shouldRecordFunctionAddr(Function *F) {
753  // Check the linkage
754  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
755  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
756      !HasAvailableExternallyLinkage)
757    return true;
758
759  // A function marked 'alwaysinline' with available_externally linkage can't
760  // have its address taken. Doing so would create an undefined external ref to
761  // the function, which would fail to link.
762  if (HasAvailableExternallyLinkage &&
763      F->hasFnAttribute(Attribute::AlwaysInline))
764    return false;
765
766  // Prohibit function address recording if the function is both internal and
767  // COMDAT. This avoids the profile data variable referencing internal symbols
768  // in COMDAT.
769  if (F->hasLocalLinkage() && F->hasComdat())
770    return false;
771
772  // Check uses of this function for other than direct calls or invokes to it.
773  // Inline virtual functions have linkeOnceODR linkage. When a key method
774  // exists, the vtable will only be emitted in the TU where the key method
775  // is defined. In a TU where vtable is not available, the function won't
776  // be 'addresstaken'. If its address is not recorded here, the profile data
777  // with missing address may be picked by the linker leading  to missing
778  // indirect call target info.
779  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
780}
781
782static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
783  // Don't do this for Darwin.  compiler-rt uses linker magic.
784  if (TT.isOSDarwin())
785    return false;
786  // Use linker script magic to get data/cnts/name start/end.
787  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
788      TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() ||
789      TT.isOSWindows())
790    return false;
791
792  return true;
793}
794
795GlobalVariable *
796InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
797  GlobalVariable *NamePtr = Inc->getName();
798  auto It = ProfileDataMap.find(NamePtr);
799  PerFunctionProfileData PD;
800  if (It != ProfileDataMap.end()) {
801    if (It->second.RegionCounters)
802      return It->second.RegionCounters;
803    PD = It->second;
804  }
805
806  // Match the linkage and visibility of the name global. COFF supports using
807  // comdats with internal symbols, so do that if we can.
808  Function *Fn = Inc->getParent()->getParent();
809  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
810  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
811  if (TT.isOSBinFormatCOFF()) {
812    Linkage = GlobalValue::InternalLinkage;
813    Visibility = GlobalValue::DefaultVisibility;
814  }
815
816  // Move the name variable to the right section. Place them in a COMDAT group
817  // if the associated function is a COMDAT. This will make sure that only one
818  // copy of counters of the COMDAT function will be emitted after linking. Keep
819  // in mind that this pass may run before the inliner, so we need to create a
820  // new comdat group for the counters and profiling data. If we use the comdat
821  // of the parent function, that will result in relocations against discarded
822  // sections.
823  bool NeedComdat = needsComdatForCounter(*Fn, *M);
824  if (NeedComdat) {
825    if (TT.isOSBinFormatCOFF()) {
826      // For COFF, put the counters, data, and values each into their own
827      // comdats. We can't use a group because the Visual C++ linker will
828      // report duplicate symbol errors if there are multiple external symbols
829      // with the same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
830      Linkage = GlobalValue::LinkOnceODRLinkage;
831      Visibility = GlobalValue::HiddenVisibility;
832    }
833  }
834  auto MaybeSetComdat = [=](GlobalVariable *GV) {
835    if (NeedComdat)
836      GV->setComdat(M->getOrInsertComdat(GV->getName()));
837  };
838
839  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
840  LLVMContext &Ctx = M->getContext();
841  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
842
843  // Create the counters variable.
844  auto *CounterPtr =
845      new GlobalVariable(*M, CounterTy, false, Linkage,
846                         Constant::getNullValue(CounterTy),
847                         getVarName(Inc, getInstrProfCountersVarPrefix()));
848  CounterPtr->setVisibility(Visibility);
849  CounterPtr->setSection(
850      getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
851  CounterPtr->setAlignment(Align(8));
852  MaybeSetComdat(CounterPtr);
853  CounterPtr->setLinkage(Linkage);
854
855  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
856  // Allocate statically the array of pointers to value profile nodes for
857  // the current function.
858  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
859  if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
860    uint64_t NS = 0;
861    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
862      NS += PD.NumValueSites[Kind];
863    if (NS) {
864      ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
865
866      auto *ValuesVar =
867          new GlobalVariable(*M, ValuesTy, false, Linkage,
868                             Constant::getNullValue(ValuesTy),
869                             getVarName(Inc, getInstrProfValuesVarPrefix()));
870      ValuesVar->setVisibility(Visibility);
871      ValuesVar->setSection(
872          getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
873      ValuesVar->setAlignment(Align(8));
874      MaybeSetComdat(ValuesVar);
875      ValuesPtrExpr =
876          ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
877    }
878  }
879
880  // Create data variable.
881  auto *Int16Ty = Type::getInt16Ty(Ctx);
882  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
883  Type *DataTypes[] = {
884#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
885#include "llvm/ProfileData/InstrProfData.inc"
886  };
887  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
888
889  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
890                               ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
891                               : ConstantPointerNull::get(Int8PtrTy);
892
893  Constant *Int16ArrayVals[IPVK_Last + 1];
894  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
895    Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
896
897  Constant *DataVals[] = {
898#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
899#include "llvm/ProfileData/InstrProfData.inc"
900  };
901  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
902                                  ConstantStruct::get(DataTy, DataVals),
903                                  getVarName(Inc, getInstrProfDataVarPrefix()));
904  Data->setVisibility(Visibility);
905  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
906  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
907  MaybeSetComdat(Data);
908  Data->setLinkage(Linkage);
909
910  PD.RegionCounters = CounterPtr;
911  PD.DataVar = Data;
912  ProfileDataMap[NamePtr] = PD;
913
914  // Mark the data variable as used so that it isn't stripped out.
915  UsedVars.push_back(Data);
916  // Now that the linkage set by the FE has been passed to the data and counter
917  // variables, reset Name variable's linkage and visibility to private so that
918  // it can be removed later by the compiler.
919  NamePtr->setLinkage(GlobalValue::PrivateLinkage);
920  // Collect the referenced names to be used by emitNameData.
921  ReferencedNames.push_back(NamePtr);
922
923  return CounterPtr;
924}
925
926void InstrProfiling::emitVNodes() {
927  if (!ValueProfileStaticAlloc)
928    return;
929
930  // For now only support this on platforms that do
931  // not require runtime registration to discover
932  // named section start/end.
933  if (needsRuntimeRegistrationOfSectionRange(TT))
934    return;
935
936  size_t TotalNS = 0;
937  for (auto &PD : ProfileDataMap) {
938    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
939      TotalNS += PD.second.NumValueSites[Kind];
940  }
941
942  if (!TotalNS)
943    return;
944
945  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
946// Heuristic for small programs with very few total value sites.
947// The default value of vp-counters-per-site is chosen based on
948// the observation that large apps usually have a low percentage
949// of value sites that actually have any profile data, and thus
950// the average number of counters per site is low. For small
951// apps with very few sites, this may not be true. Bump up the
952// number of counters in this case.
953#define INSTR_PROF_MIN_VAL_COUNTS 10
954  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
955    NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
956
957  auto &Ctx = M->getContext();
958  Type *VNodeTypes[] = {
959#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
960#include "llvm/ProfileData/InstrProfData.inc"
961  };
962  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
963
964  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
965  auto *VNodesVar = new GlobalVariable(
966      *M, VNodesTy, false, GlobalValue::PrivateLinkage,
967      Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
968  VNodesVar->setSection(
969      getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
970  UsedVars.push_back(VNodesVar);
971}
972
973void InstrProfiling::emitNameData() {
974  std::string UncompressedData;
975
976  if (ReferencedNames.empty())
977    return;
978
979  std::string CompressedNameStr;
980  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
981                                          DoInstrProfNameCompression)) {
982    report_fatal_error(toString(std::move(E)), false);
983  }
984
985  auto &Ctx = M->getContext();
986  auto *NamesVal = ConstantDataArray::getString(
987      Ctx, StringRef(CompressedNameStr), false);
988  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
989                                GlobalValue::PrivateLinkage, NamesVal,
990                                getInstrProfNamesVarName());
991  NamesSize = CompressedNameStr.size();
992  NamesVar->setSection(
993      getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
994  // On COFF, it's important to reduce the alignment down to 1 to prevent the
995  // linker from inserting padding before the start of the names section or
996  // between names entries.
997  NamesVar->setAlignment(Align(1));
998  UsedVars.push_back(NamesVar);
999
1000  for (auto *NamePtr : ReferencedNames)
1001    NamePtr->eraseFromParent();
1002}
1003
1004void InstrProfiling::emitRegistration() {
1005  if (!needsRuntimeRegistrationOfSectionRange(TT))
1006    return;
1007
1008  // Construct the function.
1009  auto *VoidTy = Type::getVoidTy(M->getContext());
1010  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1011  auto *Int64Ty = Type::getInt64Ty(M->getContext());
1012  auto *RegisterFTy = FunctionType::get(VoidTy, false);
1013  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1014                                     getInstrProfRegFuncsName(), M);
1015  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1016  if (Options.NoRedZone)
1017    RegisterF->addFnAttr(Attribute::NoRedZone);
1018
1019  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1020  auto *RuntimeRegisterF =
1021      Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
1022                       getInstrProfRegFuncName(), M);
1023
1024  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1025  for (Value *Data : UsedVars)
1026    if (Data != NamesVar && !isa<Function>(Data))
1027      IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1028
1029  if (NamesVar) {
1030    Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1031    auto *NamesRegisterTy =
1032        FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1033    auto *NamesRegisterF =
1034        Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
1035                         getInstrProfNamesRegFuncName(), M);
1036    IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1037                                    IRB.getInt64(NamesSize)});
1038  }
1039
1040  IRB.CreateRetVoid();
1041}
1042
1043bool InstrProfiling::emitRuntimeHook() {
1044  // We expect the linker to be invoked with -u<hook_var> flag for Linux or
1045  // Fuchsia, in which case there is no need to emit the user function.
1046  if (TT.isOSLinux() || TT.isOSFuchsia())
1047    return false;
1048
1049  // If the module's provided its own runtime, we don't need to do anything.
1050  if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1051    return false;
1052
1053  // Declare an external variable that will pull in the runtime initialization.
1054  auto *Int32Ty = Type::getInt32Ty(M->getContext());
1055  auto *Var =
1056      new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
1057                         nullptr, getInstrProfRuntimeHookVarName());
1058
1059  // Make a function that uses it.
1060  auto *User = Function::Create(FunctionType::get(Int32Ty, false),
1061                                GlobalValue::LinkOnceODRLinkage,
1062                                getInstrProfRuntimeHookVarUseFuncName(), M);
1063  User->addFnAttr(Attribute::NoInline);
1064  if (Options.NoRedZone)
1065    User->addFnAttr(Attribute::NoRedZone);
1066  User->setVisibility(GlobalValue::HiddenVisibility);
1067  if (TT.supportsCOMDAT())
1068    User->setComdat(M->getOrInsertComdat(User->getName()));
1069
1070  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1071  auto *Load = IRB.CreateLoad(Int32Ty, Var);
1072  IRB.CreateRet(Load);
1073
1074  // Mark the user variable as used so that it isn't stripped out.
1075  UsedVars.push_back(User);
1076  return true;
1077}
1078
1079void InstrProfiling::emitUses() {
1080  if (!UsedVars.empty())
1081    appendToUsed(*M, UsedVars);
1082}
1083
1084void InstrProfiling::emitInitialization() {
1085  // Create ProfileFileName variable. Don't don't this for the
1086  // context-sensitive instrumentation lowering: This lowering is after
1087  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1088  // have already create the variable before LTO/ThinLTO linking.
1089  if (!IsCS)
1090    createProfileFileNameVar(*M, Options.InstrProfileOutput);
1091  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1092  if (!RegisterF)
1093    return;
1094
1095  // Create the initialization function.
1096  auto *VoidTy = Type::getVoidTy(M->getContext());
1097  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1098                             GlobalValue::InternalLinkage,
1099                             getInstrProfInitFuncName(), M);
1100  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1101  F->addFnAttr(Attribute::NoInline);
1102  if (Options.NoRedZone)
1103    F->addFnAttr(Attribute::NoRedZone);
1104
1105  // Add the basic block and the necessary calls.
1106  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1107  IRB.CreateCall(RegisterF, {});
1108  IRB.CreateRetVoid();
1109
1110  appendToGlobalCtors(*M, F, 0);
1111}
1112