1//===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines interfaces to access the target independent code
10// generation passes provided by the LLVM backend.
11//
12//===---------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/TargetPassConfig.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Analysis/BasicAliasAnalysis.h"
19#include "llvm/Analysis/CallGraphSCCPass.h"
20#include "llvm/Analysis/ScopedNoAliasAA.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
23#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
24#include "llvm/CodeGen/CSEConfigBase.h"
25#include "llvm/CodeGen/MachineFunctionPass.h"
26#include "llvm/CodeGen/MachinePassRegistry.h"
27#include "llvm/CodeGen/Passes.h"
28#include "llvm/CodeGen/RegAllocRegistry.h"
29#include "llvm/IR/IRPrintingPasses.h"
30#include "llvm/IR/LegacyPassManager.h"
31#include "llvm/IR/PassInstrumentation.h"
32#include "llvm/IR/Verifier.h"
33#include "llvm/InitializePasses.h"
34#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCTargetOptions.h"
36#include "llvm/Pass.h"
37#include "llvm/Support/CodeGen.h"
38#include "llvm/Support/CommandLine.h"
39#include "llvm/Support/Compiler.h"
40#include "llvm/Support/Debug.h"
41#include "llvm/Support/Discriminator.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/SaveAndRestore.h"
44#include "llvm/Support/Threading.h"
45#include "llvm/Target/CGPassBuilderOption.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Transforms/Scalar.h"
48#include "llvm/Transforms/Utils.h"
49#include <cassert>
50#include <optional>
51#include <string>
52
53using namespace llvm;
54
55static cl::opt<bool>
56    EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,
57               cl::desc("Enable interprocedural register allocation "
58                        "to reduce load/store at procedure calls."));
59static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
60    cl::desc("Disable Post Regalloc Scheduler"));
61static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
62    cl::desc("Disable branch folding"));
63static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
64    cl::desc("Disable tail duplication"));
65static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
66    cl::desc("Disable pre-register allocation tail duplication"));
67static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
68    cl::Hidden, cl::desc("Disable probability-driven block placement"));
69static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
70    cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
71static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
72    cl::desc("Disable Stack Slot Coloring"));
73static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
74    cl::desc("Disable Machine Dead Code Elimination"));
75static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
76    cl::desc("Disable Early If-conversion"));
77static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
78    cl::desc("Disable Machine LICM"));
79static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
80    cl::desc("Disable Machine Common Subexpression Elimination"));
81static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
82    "optimize-regalloc", cl::Hidden,
83    cl::desc("Enable optimized register allocation compilation path."));
84static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
85    cl::Hidden,
86    cl::desc("Disable Machine LICM"));
87static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
88    cl::desc("Disable Machine Sinking"));
89static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
90    cl::Hidden,
91    cl::desc("Disable PostRA Machine Sinking"));
92static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
93    cl::desc("Disable Loop Strength Reduction Pass"));
94static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
95    cl::Hidden, cl::desc("Disable ConstantHoisting"));
96static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
97    cl::desc("Disable Codegen Prepare"));
98static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
99    cl::desc("Disable Copy Propagation pass"));
100static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
101    cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
102static cl::opt<bool> EnableImplicitNullChecks(
103    "enable-implicit-null-checks",
104    cl::desc("Fold null checks into faulting memory operations"),
105    cl::init(false), cl::Hidden);
106static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",
107    cl::desc("Disable MergeICmps Pass"),
108    cl::init(false), cl::Hidden);
109static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
110    cl::desc("Print LLVM IR produced by the loop-reduce pass"));
111static cl::opt<bool> PrintISelInput("print-isel-input", cl::Hidden,
112    cl::desc("Print LLVM IR input to isel pass"));
113static cl::opt<bool> PrintGCInfo("print-gc", cl::Hidden,
114    cl::desc("Dump garbage collector data"));
115static cl::opt<cl::boolOrDefault>
116    VerifyMachineCode("verify-machineinstrs", cl::Hidden,
117                      cl::desc("Verify generated machine code"));
118static cl::opt<cl::boolOrDefault>
119    DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,
120                        cl::desc("Debugify MIR before and Strip debug after "
121                                 "each pass except those known to be unsafe "
122                                 "when debug info is present"));
123static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(
124    "debugify-check-and-strip-all-safe", cl::Hidden,
125    cl::desc(
126        "Debugify MIR before, by checking and stripping the debug info after, "
127        "each pass except those known to be unsafe when debug info is "
128        "present"));
129// Enable or disable the MachineOutliner.
130static cl::opt<RunOutliner> EnableMachineOutliner(
131    "enable-machine-outliner", cl::desc("Enable the machine outliner"),
132    cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault),
133    cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always",
134                          "Run on all functions guaranteed to be beneficial"),
135               clEnumValN(RunOutliner::NeverOutline, "never",
136                          "Disable all outlining"),
137               // Sentinel value for unspecified option.
138               clEnumValN(RunOutliner::AlwaysOutline, "", "")));
139// Disable the pass to fix unwind information. Whether the pass is included in
140// the pipeline is controlled via the target options, this option serves as
141// manual override.
142static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,
143                                     cl::desc("Disable the CFI fixup pass"));
144// Enable or disable FastISel. Both options are needed, because
145// FastISel is enabled by default with -fast, and we wish to be
146// able to enable or disable fast-isel independently from -O0.
147static cl::opt<cl::boolOrDefault>
148EnableFastISelOption("fast-isel", cl::Hidden,
149  cl::desc("Enable the \"fast\" instruction selector"));
150
151static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
152    "global-isel", cl::Hidden,
153    cl::desc("Enable the \"global\" instruction selector"));
154
155// FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
156//        first...
157static cl::opt<bool>
158    PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden,
159                   cl::desc("Print machine instrs after ISel"));
160
161static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
162    "global-isel-abort", cl::Hidden,
163    cl::desc("Enable abort calls when \"global\" instruction selection "
164             "fails to lower/select an instruction"),
165    cl::values(
166        clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
167        clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
168        clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
169                   "Disable the abort but emit a diagnostic on failure")));
170
171// An option that disables inserting FS-AFDO discriminators before emit.
172// This is mainly for debugging and tuning purpose.
173static cl::opt<bool>
174    FSNoFinalDiscrim("fs-no-final-discrim", cl::init(false), cl::Hidden,
175                     cl::desc("Do not insert FS-AFDO discriminators before "
176                              "emit."));
177// Disable MIRProfileLoader before RegAlloc. This is for for debugging and
178// tuning purpose.
179static cl::opt<bool> DisableRAFSProfileLoader(
180    "disable-ra-fsprofile-loader", cl::init(false), cl::Hidden,
181    cl::desc("Disable MIRProfileLoader before RegAlloc"));
182// Disable MIRProfileLoader before BloackPlacement. This is for for debugging
183// and tuning purpose.
184static cl::opt<bool> DisableLayoutFSProfileLoader(
185    "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
186    cl::desc("Disable MIRProfileLoader before BlockPlacement"));
187// Specify FSProfile file name.
188static cl::opt<std::string>
189    FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
190                  cl::desc("Flow Sensitive profile file name."), cl::Hidden);
191// Specify Remapping file for FSProfile.
192static cl::opt<std::string> FSRemappingFile(
193    "fs-remapping-file", cl::init(""), cl::value_desc("filename"),
194    cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
195
196// Temporary option to allow experimenting with MachineScheduler as a post-RA
197// scheduler. Targets can "properly" enable this with
198// substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
199// Targets can return true in targetSchedulesPostRAScheduling() and
200// insert a PostRA scheduling pass wherever it wants.
201static cl::opt<bool> MISchedPostRA(
202    "misched-postra", cl::Hidden,
203    cl::desc(
204        "Run MachineScheduler post regalloc (independent of preRA sched)"));
205
206// Experimental option to run live interval analysis early.
207static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
208    cl::desc("Run live interval analysis earlier in the pipeline"));
209
210/// Option names for limiting the codegen pipeline.
211/// Those are used in error reporting and we didn't want
212/// to duplicate their names all over the place.
213static const char StartAfterOptName[] = "start-after";
214static const char StartBeforeOptName[] = "start-before";
215static const char StopAfterOptName[] = "stop-after";
216static const char StopBeforeOptName[] = "stop-before";
217
218static cl::opt<std::string>
219    StartAfterOpt(StringRef(StartAfterOptName),
220                  cl::desc("Resume compilation after a specific pass"),
221                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
222
223static cl::opt<std::string>
224    StartBeforeOpt(StringRef(StartBeforeOptName),
225                   cl::desc("Resume compilation before a specific pass"),
226                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
227
228static cl::opt<std::string>
229    StopAfterOpt(StringRef(StopAfterOptName),
230                 cl::desc("Stop compilation after a specific pass"),
231                 cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
232
233static cl::opt<std::string>
234    StopBeforeOpt(StringRef(StopBeforeOptName),
235                  cl::desc("Stop compilation before a specific pass"),
236                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
237
238/// Enable the machine function splitter pass.
239static cl::opt<bool> EnableMachineFunctionSplitter(
240    "enable-split-machine-functions", cl::Hidden,
241    cl::desc("Split out cold blocks from machine functions based on profile "
242             "information."));
243
244/// Disable the expand reductions pass for testing.
245static cl::opt<bool> DisableExpandReductions(
246    "disable-expand-reductions", cl::init(false), cl::Hidden,
247    cl::desc("Disable the expand reduction intrinsics pass from running"));
248
249/// Disable the select optimization pass.
250static cl::opt<bool> DisableSelectOptimize(
251    "disable-select-optimize", cl::init(true), cl::Hidden,
252    cl::desc("Disable the select-optimization pass from running"));
253
254/// Allow standard passes to be disabled by command line options. This supports
255/// simple binary flags that either suppress the pass or do nothing.
256/// i.e. -disable-mypass=false has no effect.
257/// These should be converted to boolOrDefault in order to use applyOverride.
258static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
259                                       bool Override) {
260  if (Override)
261    return IdentifyingPassPtr();
262  return PassID;
263}
264
265/// Allow standard passes to be disabled by the command line, regardless of who
266/// is adding the pass.
267///
268/// StandardID is the pass identified in the standard pass pipeline and provided
269/// to addPass(). It may be a target-specific ID in the case that the target
270/// directly adds its own pass, but in that case we harmlessly fall through.
271///
272/// TargetID is the pass that the target has configured to override StandardID.
273///
274/// StandardID may be a pseudo ID. In that case TargetID is the name of the real
275/// pass to run. This allows multiple options to control a single pass depending
276/// on where in the pipeline that pass is added.
277static IdentifyingPassPtr overridePass(AnalysisID StandardID,
278                                       IdentifyingPassPtr TargetID) {
279  if (StandardID == &PostRASchedulerID)
280    return applyDisable(TargetID, DisablePostRASched);
281
282  if (StandardID == &BranchFolderPassID)
283    return applyDisable(TargetID, DisableBranchFold);
284
285  if (StandardID == &TailDuplicateID)
286    return applyDisable(TargetID, DisableTailDuplicate);
287
288  if (StandardID == &EarlyTailDuplicateID)
289    return applyDisable(TargetID, DisableEarlyTailDup);
290
291  if (StandardID == &MachineBlockPlacementID)
292    return applyDisable(TargetID, DisableBlockPlacement);
293
294  if (StandardID == &StackSlotColoringID)
295    return applyDisable(TargetID, DisableSSC);
296
297  if (StandardID == &DeadMachineInstructionElimID)
298    return applyDisable(TargetID, DisableMachineDCE);
299
300  if (StandardID == &EarlyIfConverterID)
301    return applyDisable(TargetID, DisableEarlyIfConversion);
302
303  if (StandardID == &EarlyMachineLICMID)
304    return applyDisable(TargetID, DisableMachineLICM);
305
306  if (StandardID == &MachineCSEID)
307    return applyDisable(TargetID, DisableMachineCSE);
308
309  if (StandardID == &MachineLICMID)
310    return applyDisable(TargetID, DisablePostRAMachineLICM);
311
312  if (StandardID == &MachineSinkingID)
313    return applyDisable(TargetID, DisableMachineSink);
314
315  if (StandardID == &PostRAMachineSinkingID)
316    return applyDisable(TargetID, DisablePostRAMachineSink);
317
318  if (StandardID == &MachineCopyPropagationID)
319    return applyDisable(TargetID, DisableCopyProp);
320
321  return TargetID;
322}
323
324// Find the FSProfile file name. The internal option takes the precedence
325// before getting from TargetMachine.
326static std::string getFSProfileFile(const TargetMachine *TM) {
327  if (!FSProfileFile.empty())
328    return FSProfileFile.getValue();
329  const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
330  if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
331    return std::string();
332  return PGOOpt->ProfileFile;
333}
334
335// Find the Profile remapping file name. The internal option takes the
336// precedence before getting from TargetMachine.
337static std::string getFSRemappingFile(const TargetMachine *TM) {
338  if (!FSRemappingFile.empty())
339    return FSRemappingFile.getValue();
340  const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
341  if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
342    return std::string();
343  return PGOOpt->ProfileRemappingFile;
344}
345
346//===---------------------------------------------------------------------===//
347/// TargetPassConfig
348//===---------------------------------------------------------------------===//
349
350INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
351                "Target Pass Configuration", false, false)
352char TargetPassConfig::ID = 0;
353
354namespace {
355
356struct InsertedPass {
357  AnalysisID TargetPassID;
358  IdentifyingPassPtr InsertedPassID;
359
360  InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
361      : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
362
363  Pass *getInsertedPass() const {
364    assert(InsertedPassID.isValid() && "Illegal Pass ID!");
365    if (InsertedPassID.isInstance())
366      return InsertedPassID.getInstance();
367    Pass *NP = Pass::createPass(InsertedPassID.getID());
368    assert(NP && "Pass ID not registered");
369    return NP;
370  }
371};
372
373} // end anonymous namespace
374
375namespace llvm {
376
377extern cl::opt<bool> EnableFSDiscriminator;
378
379class PassConfigImpl {
380public:
381  // List of passes explicitly substituted by this target. Normally this is
382  // empty, but it is a convenient way to suppress or replace specific passes
383  // that are part of a standard pass pipeline without overridding the entire
384  // pipeline. This mechanism allows target options to inherit a standard pass's
385  // user interface. For example, a target may disable a standard pass by
386  // default by substituting a pass ID of zero, and the user may still enable
387  // that standard pass with an explicit command line option.
388  DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
389
390  /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
391  /// is inserted after each instance of the first one.
392  SmallVector<InsertedPass, 4> InsertedPasses;
393};
394
395} // end namespace llvm
396
397// Out of line virtual method.
398TargetPassConfig::~TargetPassConfig() {
399  delete Impl;
400}
401
402static const PassInfo *getPassInfo(StringRef PassName) {
403  if (PassName.empty())
404    return nullptr;
405
406  const PassRegistry &PR = *PassRegistry::getPassRegistry();
407  const PassInfo *PI = PR.getPassInfo(PassName);
408  if (!PI)
409    report_fatal_error(Twine('\"') + Twine(PassName) +
410                       Twine("\" pass is not registered."));
411  return PI;
412}
413
414static AnalysisID getPassIDFromName(StringRef PassName) {
415  const PassInfo *PI = getPassInfo(PassName);
416  return PI ? PI->getTypeInfo() : nullptr;
417}
418
419static std::pair<StringRef, unsigned>
420getPassNameAndInstanceNum(StringRef PassName) {
421  StringRef Name, InstanceNumStr;
422  std::tie(Name, InstanceNumStr) = PassName.split(',');
423
424  unsigned InstanceNum = 0;
425  if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum))
426    report_fatal_error("invalid pass instance specifier " + PassName);
427
428  return std::make_pair(Name, InstanceNum);
429}
430
431void TargetPassConfig::setStartStopPasses() {
432  StringRef StartBeforeName;
433  std::tie(StartBeforeName, StartBeforeInstanceNum) =
434    getPassNameAndInstanceNum(StartBeforeOpt);
435
436  StringRef StartAfterName;
437  std::tie(StartAfterName, StartAfterInstanceNum) =
438    getPassNameAndInstanceNum(StartAfterOpt);
439
440  StringRef StopBeforeName;
441  std::tie(StopBeforeName, StopBeforeInstanceNum)
442    = getPassNameAndInstanceNum(StopBeforeOpt);
443
444  StringRef StopAfterName;
445  std::tie(StopAfterName, StopAfterInstanceNum)
446    = getPassNameAndInstanceNum(StopAfterOpt);
447
448  StartBefore = getPassIDFromName(StartBeforeName);
449  StartAfter = getPassIDFromName(StartAfterName);
450  StopBefore = getPassIDFromName(StopBeforeName);
451  StopAfter = getPassIDFromName(StopAfterName);
452  if (StartBefore && StartAfter)
453    report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
454                       Twine(StartAfterOptName) + Twine(" specified!"));
455  if (StopBefore && StopAfter)
456    report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
457                       Twine(StopAfterOptName) + Twine(" specified!"));
458  Started = (StartAfter == nullptr) && (StartBefore == nullptr);
459}
460
461CGPassBuilderOption llvm::getCGPassBuilderOption() {
462  CGPassBuilderOption Opt;
463
464#define SET_OPTION(Option)                                                     \
465  if (Option.getNumOccurrences())                                              \
466    Opt.Option = Option;
467
468  SET_OPTION(EnableFastISelOption)
469  SET_OPTION(EnableGlobalISelAbort)
470  SET_OPTION(EnableGlobalISelOption)
471  SET_OPTION(EnableIPRA)
472  SET_OPTION(OptimizeRegAlloc)
473  SET_OPTION(VerifyMachineCode)
474
475#define SET_BOOLEAN_OPTION(Option) Opt.Option = Option;
476
477  SET_BOOLEAN_OPTION(EarlyLiveIntervals)
478  SET_BOOLEAN_OPTION(EnableBlockPlacementStats)
479  SET_BOOLEAN_OPTION(EnableImplicitNullChecks)
480  SET_BOOLEAN_OPTION(EnableMachineOutliner)
481  SET_BOOLEAN_OPTION(MISchedPostRA)
482  SET_BOOLEAN_OPTION(DisableMergeICmps)
483  SET_BOOLEAN_OPTION(DisableLSR)
484  SET_BOOLEAN_OPTION(DisableConstantHoisting)
485  SET_BOOLEAN_OPTION(DisableCGP)
486  SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
487  SET_BOOLEAN_OPTION(DisableSelectOptimize)
488  SET_BOOLEAN_OPTION(PrintLSR)
489  SET_BOOLEAN_OPTION(PrintISelInput)
490  SET_BOOLEAN_OPTION(PrintGCInfo)
491
492  return Opt;
493}
494
495static void registerPartialPipelineCallback(PassInstrumentationCallbacks &PIC,
496                                            LLVMTargetMachine &LLVMTM) {
497  StringRef StartBefore;
498  StringRef StartAfter;
499  StringRef StopBefore;
500  StringRef StopAfter;
501
502  unsigned StartBeforeInstanceNum = 0;
503  unsigned StartAfterInstanceNum = 0;
504  unsigned StopBeforeInstanceNum = 0;
505  unsigned StopAfterInstanceNum = 0;
506
507  std::tie(StartBefore, StartBeforeInstanceNum) =
508      getPassNameAndInstanceNum(StartBeforeOpt);
509  std::tie(StartAfter, StartAfterInstanceNum) =
510      getPassNameAndInstanceNum(StartAfterOpt);
511  std::tie(StopBefore, StopBeforeInstanceNum) =
512      getPassNameAndInstanceNum(StopBeforeOpt);
513  std::tie(StopAfter, StopAfterInstanceNum) =
514      getPassNameAndInstanceNum(StopAfterOpt);
515
516  if (StartBefore.empty() && StartAfter.empty() && StopBefore.empty() &&
517      StopAfter.empty())
518    return;
519
520  std::tie(StartBefore, std::ignore) =
521      LLVMTM.getPassNameFromLegacyName(StartBefore);
522  std::tie(StartAfter, std::ignore) =
523      LLVMTM.getPassNameFromLegacyName(StartAfter);
524  std::tie(StopBefore, std::ignore) =
525      LLVMTM.getPassNameFromLegacyName(StopBefore);
526  std::tie(StopAfter, std::ignore) =
527      LLVMTM.getPassNameFromLegacyName(StopAfter);
528  if (!StartBefore.empty() && !StartAfter.empty())
529    report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
530                       Twine(StartAfterOptName) + Twine(" specified!"));
531  if (!StopBefore.empty() && !StopAfter.empty())
532    report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
533                       Twine(StopAfterOptName) + Twine(" specified!"));
534
535  PIC.registerShouldRunOptionalPassCallback(
536      [=, EnableCurrent = StartBefore.empty() && StartAfter.empty(),
537       EnableNext = std::optional<bool>(), StartBeforeCount = 0u,
538       StartAfterCount = 0u, StopBeforeCount = 0u,
539       StopAfterCount = 0u](StringRef P, Any) mutable {
540        bool StartBeforePass = !StartBefore.empty() && P.contains(StartBefore);
541        bool StartAfterPass = !StartAfter.empty() && P.contains(StartAfter);
542        bool StopBeforePass = !StopBefore.empty() && P.contains(StopBefore);
543        bool StopAfterPass = !StopAfter.empty() && P.contains(StopAfter);
544
545        // Implement -start-after/-stop-after
546        if (EnableNext) {
547          EnableCurrent = *EnableNext;
548          EnableNext.reset();
549        }
550
551        // Using PIC.registerAfterPassCallback won't work because if this
552        // callback returns false, AfterPassCallback is also skipped.
553        if (StartAfterPass && StartAfterCount++ == StartAfterInstanceNum) {
554          assert(!EnableNext && "Error: assign to EnableNext more than once");
555          EnableNext = true;
556        }
557        if (StopAfterPass && StopAfterCount++ == StopAfterInstanceNum) {
558          assert(!EnableNext && "Error: assign to EnableNext more than once");
559          EnableNext = false;
560        }
561
562        if (StartBeforePass && StartBeforeCount++ == StartBeforeInstanceNum)
563          EnableCurrent = true;
564        if (StopBeforePass && StopBeforeCount++ == StopBeforeInstanceNum)
565          EnableCurrent = false;
566        return EnableCurrent;
567      });
568}
569
570void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
571                                   LLVMTargetMachine &LLVMTM) {
572
573  // Register a callback for disabling passes.
574  PIC.registerShouldRunOptionalPassCallback([](StringRef P, Any) {
575
576#define DISABLE_PASS(Option, Name)                                             \
577  if (Option && P.contains(#Name))                                             \
578    return false;
579    DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
580    DISABLE_PASS(DisableBranchFold, BranchFolderPass)
581    DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
582    DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterPass)
583    DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
584    DISABLE_PASS(DisableMachineCSE, MachineCSEPass)
585    DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
586    DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
587    DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
588    DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
589    DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
590    DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
591    DISABLE_PASS(DisableSSC, StackSlotColoringPass)
592    DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
593
594    return true;
595  });
596
597  registerPartialPipelineCallback(PIC, LLVMTM);
598}
599
600// Out of line constructor provides default values for pass options and
601// registers all common codegen passes.
602TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
603    : ImmutablePass(ID), PM(&pm), TM(&TM) {
604  Impl = new PassConfigImpl();
605
606  // Register all target independent codegen passes to activate their PassIDs,
607  // including this pass itself.
608  initializeCodeGen(*PassRegistry::getPassRegistry());
609
610  // Also register alias analysis passes required by codegen passes.
611  initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
612  initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
613
614  if (EnableIPRA.getNumOccurrences())
615    TM.Options.EnableIPRA = EnableIPRA;
616  else {
617    // If not explicitly specified, use target default.
618    TM.Options.EnableIPRA |= TM.useIPRA();
619  }
620
621  if (TM.Options.EnableIPRA)
622    setRequiresCodeGenSCCOrder();
623
624  if (EnableGlobalISelAbort.getNumOccurrences())
625    TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
626
627  setStartStopPasses();
628}
629
630CodeGenOpt::Level TargetPassConfig::getOptLevel() const {
631  return TM->getOptLevel();
632}
633
634/// Insert InsertedPassID pass after TargetPassID.
635void TargetPassConfig::insertPass(AnalysisID TargetPassID,
636                                  IdentifyingPassPtr InsertedPassID) {
637  assert(((!InsertedPassID.isInstance() &&
638           TargetPassID != InsertedPassID.getID()) ||
639          (InsertedPassID.isInstance() &&
640           TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
641         "Insert a pass after itself!");
642  Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID);
643}
644
645/// createPassConfig - Create a pass configuration object to be used by
646/// addPassToEmitX methods for generating a pipeline of CodeGen passes.
647///
648/// Targets may override this to extend TargetPassConfig.
649TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) {
650  return new TargetPassConfig(*this, PM);
651}
652
653TargetPassConfig::TargetPassConfig()
654  : ImmutablePass(ID) {
655  report_fatal_error("Trying to construct TargetPassConfig without a target "
656                     "machine. Scheduling a CodeGen pass without a target "
657                     "triple set?");
658}
659
660bool TargetPassConfig::willCompleteCodeGenPipeline() {
661  return StopBeforeOpt.empty() && StopAfterOpt.empty();
662}
663
664bool TargetPassConfig::hasLimitedCodeGenPipeline() {
665  return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
666         !willCompleteCodeGenPipeline();
667}
668
669std::string
670TargetPassConfig::getLimitedCodeGenPipelineReason(const char *Separator) {
671  if (!hasLimitedCodeGenPipeline())
672    return std::string();
673  std::string Res;
674  static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
675                                              &StopAfterOpt, &StopBeforeOpt};
676  static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
677                                   StopAfterOptName, StopBeforeOptName};
678  bool IsFirst = true;
679  for (int Idx = 0; Idx < 4; ++Idx)
680    if (!PassNames[Idx]->empty()) {
681      if (!IsFirst)
682        Res += Separator;
683      IsFirst = false;
684      Res += OptNames[Idx];
685    }
686  return Res;
687}
688
689// Helper to verify the analysis is really immutable.
690void TargetPassConfig::setOpt(bool &Opt, bool Val) {
691  assert(!Initialized && "PassConfig is immutable");
692  Opt = Val;
693}
694
695void TargetPassConfig::substitutePass(AnalysisID StandardID,
696                                      IdentifyingPassPtr TargetID) {
697  Impl->TargetPasses[StandardID] = TargetID;
698}
699
700IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
701  DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
702    I = Impl->TargetPasses.find(ID);
703  if (I == Impl->TargetPasses.end())
704    return ID;
705  return I->second;
706}
707
708bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
709  IdentifyingPassPtr TargetID = getPassSubstitution(ID);
710  IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);
711  return !FinalPtr.isValid() || FinalPtr.isInstance() ||
712      FinalPtr.getID() != ID;
713}
714
715/// Add a pass to the PassManager if that pass is supposed to be run.  If the
716/// Started/Stopped flags indicate either that the compilation should start at
717/// a later pass or that it should stop after an earlier pass, then do not add
718/// the pass.  Finally, compare the current pass against the StartAfter
719/// and StopAfter options and change the Started/Stopped flags accordingly.
720void TargetPassConfig::addPass(Pass *P) {
721  assert(!Initialized && "PassConfig is immutable");
722
723  // Cache the Pass ID here in case the pass manager finds this pass is
724  // redundant with ones already scheduled / available, and deletes it.
725  // Fundamentally, once we add the pass to the manager, we no longer own it
726  // and shouldn't reference it.
727  AnalysisID PassID = P->getPassID();
728
729  if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
730    Started = true;
731  if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
732    Stopped = true;
733  if (Started && !Stopped) {
734    if (AddingMachinePasses) {
735      // Construct banner message before PM->add() as that may delete the pass.
736      std::string Banner =
737          std::string("After ") + std::string(P->getPassName());
738      addMachinePrePasses();
739      PM->add(P);
740      addMachinePostPasses(Banner);
741    } else {
742      PM->add(P);
743    }
744
745    // Add the passes after the pass P if there is any.
746    for (const auto &IP : Impl->InsertedPasses)
747      if (IP.TargetPassID == PassID)
748        addPass(IP.getInsertedPass());
749  } else {
750    delete P;
751  }
752
753  if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
754    Stopped = true;
755
756  if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
757    Started = true;
758  if (Stopped && !Started)
759    report_fatal_error("Cannot stop compilation after pass that is not run");
760}
761
762/// Add a CodeGen pass at this point in the pipeline after checking for target
763/// and command line overrides.
764///
765/// addPass cannot return a pointer to the pass instance because is internal the
766/// PassManager and the instance we create here may already be freed.
767AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
768  IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
769  IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
770  if (!FinalPtr.isValid())
771    return nullptr;
772
773  Pass *P;
774  if (FinalPtr.isInstance())
775    P = FinalPtr.getInstance();
776  else {
777    P = Pass::createPass(FinalPtr.getID());
778    if (!P)
779      llvm_unreachable("Pass ID not registered");
780  }
781  AnalysisID FinalID = P->getPassID();
782  addPass(P); // Ends the lifetime of P.
783
784  return FinalID;
785}
786
787void TargetPassConfig::printAndVerify(const std::string &Banner) {
788  addPrintPass(Banner);
789  addVerifyPass(Banner);
790}
791
792void TargetPassConfig::addPrintPass(const std::string &Banner) {
793  if (PrintAfterISel)
794    PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
795}
796
797void TargetPassConfig::addVerifyPass(const std::string &Banner) {
798  bool Verify = VerifyMachineCode == cl::BOU_TRUE;
799#ifdef EXPENSIVE_CHECKS
800  if (VerifyMachineCode == cl::BOU_UNSET)
801    Verify = TM->isMachineVerifierClean();
802#endif
803  if (Verify)
804    PM->add(createMachineVerifierPass(Banner));
805}
806
807void TargetPassConfig::addDebugifyPass() {
808  PM->add(createDebugifyMachineModulePass());
809}
810
811void TargetPassConfig::addStripDebugPass() {
812  PM->add(createStripDebugMachineModulePass(/*OnlyDebugified=*/true));
813}
814
815void TargetPassConfig::addCheckDebugPass() {
816  PM->add(createCheckDebugMachineModulePass());
817}
818
819void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
820  if (AllowDebugify && DebugifyIsSafe &&
821      (DebugifyAndStripAll == cl::BOU_TRUE ||
822       DebugifyCheckAndStripAll == cl::BOU_TRUE))
823    addDebugifyPass();
824}
825
826void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
827  if (DebugifyIsSafe) {
828    if (DebugifyCheckAndStripAll == cl::BOU_TRUE) {
829      addCheckDebugPass();
830      addStripDebugPass();
831    } else if (DebugifyAndStripAll == cl::BOU_TRUE)
832      addStripDebugPass();
833  }
834  addVerifyPass(Banner);
835}
836
837/// Add common target configurable passes that perform LLVM IR to IR transforms
838/// following machine independent optimization.
839void TargetPassConfig::addIRPasses() {
840  // Before running any passes, run the verifier to determine if the input
841  // coming from the front-end and/or optimizer is valid.
842  if (!DisableVerify)
843    addPass(createVerifierPass());
844
845  if (getOptLevel() != CodeGenOpt::None) {
846    // Basic AliasAnalysis support.
847    // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
848    // BasicAliasAnalysis wins if they disagree. This is intended to help
849    // support "obvious" type-punning idioms.
850    addPass(createTypeBasedAAWrapperPass());
851    addPass(createScopedNoAliasAAWrapperPass());
852    addPass(createBasicAAWrapperPass());
853
854    // Run loop strength reduction before anything else.
855    if (!DisableLSR) {
856      addPass(createCanonicalizeFreezeInLoopsPass());
857      addPass(createLoopStrengthReducePass());
858      if (PrintLSR)
859        addPass(createPrintFunctionPass(dbgs(),
860                                        "\n\n*** Code after LSR ***\n"));
861    }
862
863    // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
864    // loads and compares. ExpandMemCmpPass then tries to expand those calls
865    // into optimally-sized loads and compares. The transforms are enabled by a
866    // target lowering hook.
867    if (!DisableMergeICmps)
868      addPass(createMergeICmpsLegacyPass());
869    addPass(createExpandMemCmpPass());
870  }
871
872  // Run GC lowering passes for builtin collectors
873  // TODO: add a pass insertion point here
874  addPass(&GCLoweringID);
875  addPass(&ShadowStackGCLoweringID);
876  addPass(createLowerConstantIntrinsicsPass());
877
878  // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
879  // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
880  if (TM->getTargetTriple().isOSBinFormatMachO() &&
881      TM->Options.LowerGlobalDtorsViaCxaAtExit)
882    addPass(createLowerGlobalDtorsLegacyPass());
883
884  // Make sure that no unreachable blocks are instruction selected.
885  addPass(createUnreachableBlockEliminationPass());
886
887  // Prepare expensive constants for SelectionDAG.
888  if (getOptLevel() != CodeGenOpt::None && !DisableConstantHoisting)
889    addPass(createConstantHoistingPass());
890
891  if (getOptLevel() != CodeGenOpt::None)
892    addPass(createReplaceWithVeclibLegacyPass());
893
894  if (getOptLevel() != CodeGenOpt::None && !DisablePartialLibcallInlining)
895    addPass(createPartiallyInlineLibCallsPass());
896
897  // Expand vector predication intrinsics into standard IR instructions.
898  // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
899  // passes since it emits those kinds of intrinsics.
900  addPass(createExpandVectorPredicationPass());
901
902  // Add scalarization of target's unsupported masked memory intrinsics pass.
903  // the unsupported intrinsic will be replaced with a chain of basic blocks,
904  // that stores/loads element one-by-one if the appropriate mask bit is set.
905  addPass(createScalarizeMaskedMemIntrinLegacyPass());
906
907  // Expand reduction intrinsics into shuffle sequences if the target wants to.
908  // Allow disabling it for testing purposes.
909  if (!DisableExpandReductions)
910    addPass(createExpandReductionsPass());
911
912  if (getOptLevel() != CodeGenOpt::None)
913    addPass(createTLSVariableHoistPass());
914
915  // Convert conditional moves to conditional jumps when profitable.
916  if (getOptLevel() != CodeGenOpt::None && !DisableSelectOptimize)
917    addPass(createSelectOptimizePass());
918}
919
920/// Turn exception handling constructs into something the code generators can
921/// handle.
922void TargetPassConfig::addPassesToHandleExceptions() {
923  const MCAsmInfo *MCAI = TM->getMCAsmInfo();
924  assert(MCAI && "No MCAsmInfo");
925  switch (MCAI->getExceptionHandlingType()) {
926  case ExceptionHandling::SjLj:
927    // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
928    // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
929    // catch info can get misplaced when a selector ends up more than one block
930    // removed from the parent invoke(s). This could happen when a landing
931    // pad is shared by multiple invokes and is also a target of a normal
932    // edge from elsewhere.
933    addPass(createSjLjEHPreparePass(TM));
934    [[fallthrough]];
935  case ExceptionHandling::DwarfCFI:
936  case ExceptionHandling::ARM:
937  case ExceptionHandling::AIX:
938    addPass(createDwarfEHPass(getOptLevel()));
939    break;
940  case ExceptionHandling::WinEH:
941    // We support using both GCC-style and MSVC-style exceptions on Windows, so
942    // add both preparation passes. Each pass will only actually run if it
943    // recognizes the personality function.
944    addPass(createWinEHPass());
945    addPass(createDwarfEHPass(getOptLevel()));
946    break;
947  case ExceptionHandling::Wasm:
948    // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
949    // on catchpads and cleanuppads because it does not outline them into
950    // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
951    // should remove PHIs there.
952    addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
953    addPass(createWasmEHPass());
954    break;
955  case ExceptionHandling::None:
956    addPass(createLowerInvokePass());
957
958    // The lower invoke pass may create unreachable code. Remove it.
959    addPass(createUnreachableBlockEliminationPass());
960    break;
961  }
962}
963
964/// Add pass to prepare the LLVM IR for code generation. This should be done
965/// before exception handling preparation passes.
966void TargetPassConfig::addCodeGenPrepare() {
967  if (getOptLevel() != CodeGenOpt::None && !DisableCGP)
968    addPass(createCodeGenPreparePass());
969}
970
971/// Add common passes that perform LLVM IR to IR transforms in preparation for
972/// instruction selection.
973void TargetPassConfig::addISelPrepare() {
974  addPreISel();
975
976  // Force codegen to run according to the callgraph.
977  if (requiresCodeGenSCCOrder())
978    addPass(new DummyCGSCCPass);
979
980  addPass(createReturnProtectorPass());
981
982  // Add both the safe stack and the stack protection passes: each of them will
983  // only protect functions that have corresponding attributes.
984  addPass(createSafeStackPass());
985  addPass(createStackProtectorPass());
986
987  if (PrintISelInput)
988    addPass(createPrintFunctionPass(
989        dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));
990
991  // All passes which modify the LLVM IR are now complete; run the verifier
992  // to ensure that the IR is valid.
993  if (!DisableVerify)
994    addPass(createVerifierPass());
995}
996
997bool TargetPassConfig::addCoreISelPasses() {
998  // Enable FastISel with -fast-isel, but allow that to be overridden.
999  TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
1000
1001  // Determine an instruction selector.
1002  enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
1003  SelectorType Selector;
1004
1005  if (EnableFastISelOption == cl::BOU_TRUE)
1006    Selector = SelectorType::FastISel;
1007  else if (EnableGlobalISelOption == cl::BOU_TRUE ||
1008           (TM->Options.EnableGlobalISel &&
1009            EnableGlobalISelOption != cl::BOU_FALSE))
1010    Selector = SelectorType::GlobalISel;
1011  else if (TM->getOptLevel() == CodeGenOpt::None && TM->getO0WantsFastISel())
1012    Selector = SelectorType::FastISel;
1013  else
1014    Selector = SelectorType::SelectionDAG;
1015
1016  // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
1017  if (Selector == SelectorType::FastISel) {
1018    TM->setFastISel(true);
1019    TM->setGlobalISel(false);
1020  } else if (Selector == SelectorType::GlobalISel) {
1021    TM->setFastISel(false);
1022    TM->setGlobalISel(true);
1023  }
1024
1025  // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
1026  //        analyses needing to be re-run. This can result in being unable to
1027  //        schedule passes (particularly with 'Function Alias Analysis
1028  //        Results'). It's not entirely clear why but AFAICT this seems to be
1029  //        due to one FunctionPassManager not being able to use analyses from a
1030  //        previous one. As we're injecting a ModulePass we break the usual
1031  //        pass manager into two. GlobalISel with the fallback path disabled
1032  //        and -run-pass seem to be unaffected. The majority of GlobalISel
1033  //        testing uses -run-pass so this probably isn't too bad.
1034  SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
1035  if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1036    DebugifyIsSafe = false;
1037
1038  // Add instruction selector passes.
1039  if (Selector == SelectorType::GlobalISel) {
1040    SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
1041    if (addIRTranslator())
1042      return true;
1043
1044    addPreLegalizeMachineIR();
1045
1046    if (addLegalizeMachineIR())
1047      return true;
1048
1049    // Before running the register bank selector, ask the target if it
1050    // wants to run some passes.
1051    addPreRegBankSelect();
1052
1053    if (addRegBankSelect())
1054      return true;
1055
1056    addPreGlobalInstructionSelect();
1057
1058    if (addGlobalInstructionSelect())
1059      return true;
1060
1061    // Pass to reset the MachineFunction if the ISel failed.
1062    addPass(createResetMachineFunctionPass(
1063        reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
1064
1065    // Provide a fallback path when we do not want to abort on
1066    // not-yet-supported input.
1067    if (!isGlobalISelAbortEnabled() && addInstSelector())
1068      return true;
1069
1070  } else if (addInstSelector())
1071    return true;
1072
1073  // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
1074  // FinalizeISel.
1075  addPass(&FinalizeISelID);
1076
1077  // Print the instruction selected machine code...
1078  printAndVerify("After Instruction Selection");
1079
1080  return false;
1081}
1082
1083bool TargetPassConfig::addISelPasses() {
1084  if (TM->useEmulatedTLS())
1085    addPass(createLowerEmuTLSPass());
1086
1087  addPass(createPreISelIntrinsicLoweringPass());
1088  PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
1089  addPass(createExpandLargeDivRemPass());
1090  addPass(createExpandLargeFpConvertPass());
1091  addIRPasses();
1092  addCodeGenPrepare();
1093  addPassesToHandleExceptions();
1094  addISelPrepare();
1095
1096  return addCoreISelPasses();
1097}
1098
1099/// -regalloc=... command line option.
1100static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
1101static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
1102               RegisterPassParser<RegisterRegAlloc>>
1103    RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
1104             cl::desc("Register allocator to use"));
1105
1106/// Add the complete set of target-independent postISel code generator passes.
1107///
1108/// This can be read as the standard order of major LLVM CodeGen stages. Stages
1109/// with nontrivial configuration or multiple passes are broken out below in
1110/// add%Stage routines.
1111///
1112/// Any TargetPassConfig::addXX routine may be overriden by the Target. The
1113/// addPre/Post methods with empty header implementations allow injecting
1114/// target-specific fixups just before or after major stages. Additionally,
1115/// targets have the flexibility to change pass order within a stage by
1116/// overriding default implementation of add%Stage routines below. Each
1117/// technique has maintainability tradeoffs because alternate pass orders are
1118/// not well supported. addPre/Post works better if the target pass is easily
1119/// tied to a common pass. But if it has subtle dependencies on multiple passes,
1120/// the target should override the stage instead.
1121///
1122/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
1123/// before/after any target-independent pass. But it's currently overkill.
1124void TargetPassConfig::addMachinePasses() {
1125  AddingMachinePasses = true;
1126
1127  // Add passes that optimize machine instructions in SSA form.
1128  if (getOptLevel() != CodeGenOpt::None) {
1129    addMachineSSAOptimization();
1130  } else {
1131    // If the target requests it, assign local variables to stack slots relative
1132    // to one another and simplify frame index references where possible.
1133    addPass(&LocalStackSlotAllocationID);
1134  }
1135
1136  if (TM->Options.EnableIPRA)
1137    addPass(createRegUsageInfoPropPass());
1138
1139  // Run pre-ra passes.
1140  addPreRegAlloc();
1141
1142  // Debugifying the register allocator passes seems to provoke some
1143  // non-determinism that affects CodeGen and there doesn't seem to be a point
1144  // where it becomes safe again so stop debugifying here.
1145  DebugifyIsSafe = false;
1146
1147  // Add a FSDiscriminator pass right before RA, so that we could get
1148  // more precise SampleFDO profile for RA.
1149  if (EnableFSDiscriminator) {
1150    addPass(createMIRAddFSDiscriminatorsPass(
1151        sampleprof::FSDiscriminatorPass::Pass1));
1152    const std::string ProfileFile = getFSProfileFile(TM);
1153    if (!ProfileFile.empty() && !DisableRAFSProfileLoader)
1154      addPass(
1155          createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
1156                                     sampleprof::FSDiscriminatorPass::Pass1));
1157  }
1158
1159  // Run register allocation and passes that are tightly coupled with it,
1160  // including phi elimination and scheduling.
1161  if (getOptimizeRegAlloc())
1162    addOptimizedRegAlloc();
1163  else
1164    addFastRegAlloc();
1165
1166  // Run post-ra passes.
1167  addPostRegAlloc();
1168
1169  addPass(&RemoveRedundantDebugValuesID);
1170
1171  addPass(&FixupStatepointCallerSavedID);
1172
1173  // Insert prolog/epilog code.  Eliminate abstract frame index references...
1174  if (getOptLevel() != CodeGenOpt::None) {
1175    addPass(&PostRAMachineSinkingID);
1176    addPass(&ShrinkWrapID);
1177  }
1178
1179  // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1180  // do so if it hasn't been disabled, substituted, or overridden.
1181  if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))
1182      addPass(createPrologEpilogInserterPass());
1183
1184  /// Add passes that optimize machine instructions after register allocation.
1185  if (getOptLevel() != CodeGenOpt::None)
1186    addMachineLateOptimization();
1187
1188  // Expand pseudo instructions before second scheduling pass.
1189  addPass(&ExpandPostRAPseudosID);
1190
1191  // Run pre-sched2 passes.
1192  addPreSched2();
1193
1194  if (EnableImplicitNullChecks)
1195    addPass(&ImplicitNullChecksID);
1196
1197  // Second pass scheduler.
1198  // Let Target optionally insert this pass by itself at some other
1199  // point.
1200  if (getOptLevel() != CodeGenOpt::None &&
1201      !TM->targetSchedulesPostRAScheduling()) {
1202    if (MISchedPostRA)
1203      addPass(&PostMachineSchedulerID);
1204    else
1205      addPass(&PostRASchedulerID);
1206  }
1207
1208  // GC
1209  if (addGCPasses()) {
1210    if (PrintGCInfo)
1211      addPass(createGCInfoPrinter(dbgs()));
1212  }
1213
1214  // Basic block placement.
1215  if (getOptLevel() != CodeGenOpt::None)
1216    addBlockPlacement();
1217
1218  // Insert before XRay Instrumentation.
1219  addPass(&FEntryInserterID);
1220
1221  addPass(&XRayInstrumentationID);
1222  addPass(&PatchableFunctionID);
1223
1224  if (EnableFSDiscriminator && !FSNoFinalDiscrim)
1225    // Add FS discriminators here so that all the instruction duplicates
1226    // in different BBs get their own discriminators. With this, we can "sum"
1227    // the SampleFDO counters instead of using MAX. This will improve the
1228    // SampleFDO profile quality.
1229    addPass(createMIRAddFSDiscriminatorsPass(
1230        sampleprof::FSDiscriminatorPass::PassLast));
1231
1232  addPreEmitPass();
1233
1234  if (TM->Options.EnableIPRA)
1235    // Collect register usage information and produce a register mask of
1236    // clobbered registers, to be used to optimize call sites.
1237    addPass(createRegUsageInfoCollector());
1238
1239  // FIXME: Some backends are incompatible with running the verifier after
1240  // addPreEmitPass.  Maybe only pass "false" here for those targets?
1241  addPass(&FuncletLayoutID);
1242
1243  addPass(&StackMapLivenessID);
1244  addPass(&LiveDebugValuesID);
1245  addPass(&MachineSanitizerBinaryMetadataID);
1246
1247  if (TM->Options.EnableMachineOutliner && getOptLevel() != CodeGenOpt::None &&
1248      EnableMachineOutliner != RunOutliner::NeverOutline) {
1249    bool RunOnAllFunctions =
1250        (EnableMachineOutliner == RunOutliner::AlwaysOutline);
1251    bool AddOutliner =
1252        RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
1253    if (AddOutliner)
1254      addPass(createMachineOutlinerPass(RunOnAllFunctions));
1255  }
1256
1257  // Machine function splitter uses the basic block sections feature. Both
1258  // cannot be enabled at the same time. Basic block sections takes precedence.
1259  // FIXME: In principle, BasicBlockSection::Labels and splitting can used
1260  // together. Update this check once we have addressed any issues.
1261  if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) {
1262    if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1263      addPass(llvm::createBasicBlockSectionsProfileReaderPass(
1264          TM->getBBSectionsFuncListBuf()));
1265    }
1266    addPass(llvm::createBasicBlockSectionsPass());
1267  } else if (TM->Options.EnableMachineFunctionSplitter ||
1268             EnableMachineFunctionSplitter) {
1269    addPass(createMachineFunctionSplitterPass());
1270  }
1271
1272  if (!DisableCFIFixup && TM->Options.EnableCFIFixup)
1273    addPass(createCFIFixup());
1274
1275  PM->add(createStackFrameLayoutAnalysisPass());
1276
1277  // Add passes that directly emit MI after all other MI passes.
1278  addPreEmitPass2();
1279
1280  AddingMachinePasses = false;
1281}
1282
1283/// Add passes that optimize machine instructions in SSA form.
1284void TargetPassConfig::addMachineSSAOptimization() {
1285  // Pre-ra tail duplication.
1286  addPass(&EarlyTailDuplicateID);
1287
1288  // Optimize PHIs before DCE: removing dead PHI cycles may make more
1289  // instructions dead.
1290  addPass(&OptimizePHIsID);
1291
1292  // This pass merges large allocas. StackSlotColoring is a different pass
1293  // which merges spill slots.
1294  addPass(&StackColoringID);
1295
1296  // If the target requests it, assign local variables to stack slots relative
1297  // to one another and simplify frame index references where possible.
1298  addPass(&LocalStackSlotAllocationID);
1299
1300  // With optimization, dead code should already be eliminated. However
1301  // there is one known exception: lowered code for arguments that are only
1302  // used by tail calls, where the tail calls reuse the incoming stack
1303  // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1304  addPass(&DeadMachineInstructionElimID);
1305
1306  // Allow targets to insert passes that improve instruction level parallelism,
1307  // like if-conversion. Such passes will typically need dominator trees and
1308  // loop info, just like LICM and CSE below.
1309  addILPOpts();
1310
1311  addPass(&EarlyMachineLICMID);
1312  addPass(&MachineCSEID);
1313
1314  addPass(&MachineSinkingID);
1315
1316  addPass(&PeepholeOptimizerID);
1317  // Clean-up the dead code that may have been generated by peephole
1318  // rewriting.
1319  addPass(&DeadMachineInstructionElimID);
1320}
1321
1322//===---------------------------------------------------------------------===//
1323/// Register Allocation Pass Configuration
1324//===---------------------------------------------------------------------===//
1325
1326bool TargetPassConfig::getOptimizeRegAlloc() const {
1327  switch (OptimizeRegAlloc) {
1328  case cl::BOU_UNSET: return getOptLevel() != CodeGenOpt::None;
1329  case cl::BOU_TRUE:  return true;
1330  case cl::BOU_FALSE: return false;
1331  }
1332  llvm_unreachable("Invalid optimize-regalloc state");
1333}
1334
1335/// A dummy default pass factory indicates whether the register allocator is
1336/// overridden on the command line.
1337static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1338
1339static RegisterRegAlloc
1340defaultRegAlloc("default",
1341                "pick register allocator based on -O option",
1342                useDefaultRegisterAllocator);
1343
1344static void initializeDefaultRegisterAllocatorOnce() {
1345  if (!RegisterRegAlloc::getDefault())
1346    RegisterRegAlloc::setDefault(RegAlloc);
1347}
1348
1349/// Instantiate the default register allocator pass for this target for either
1350/// the optimized or unoptimized allocation path. This will be added to the pass
1351/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1352/// in the optimized case.
1353///
1354/// A target that uses the standard regalloc pass order for fast or optimized
1355/// allocation may still override this for per-target regalloc
1356/// selection. But -regalloc=... always takes precedence.
1357FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1358  if (Optimized)
1359    return createGreedyRegisterAllocator();
1360  else
1361    return createFastRegisterAllocator();
1362}
1363
1364/// Find and instantiate the register allocation pass requested by this target
1365/// at the current optimization level.  Different register allocators are
1366/// defined as separate passes because they may require different analysis.
1367///
1368/// This helper ensures that the regalloc= option is always available,
1369/// even for targets that override the default allocator.
1370///
1371/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1372/// this can be folded into addPass.
1373FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1374  // Initialize the global default.
1375  llvm::call_once(InitializeDefaultRegisterAllocatorFlag,
1376                  initializeDefaultRegisterAllocatorOnce);
1377
1378  RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1379  if (Ctor != useDefaultRegisterAllocator)
1380    return Ctor();
1381
1382  // With no -regalloc= override, ask the target for a regalloc pass.
1383  return createTargetRegisterAllocator(Optimized);
1384}
1385
1386bool TargetPassConfig::isCustomizedRegAlloc() {
1387  return RegAlloc !=
1388         (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1389}
1390
1391bool TargetPassConfig::addRegAssignAndRewriteFast() {
1392  if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1393      RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1394    report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc.");
1395
1396  addPass(createRegAllocPass(false));
1397
1398  // Allow targets to change the register assignments after
1399  // fast register allocation.
1400  addPostFastRegAllocRewrite();
1401  return true;
1402}
1403
1404bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1405  // Add the selected register allocation pass.
1406  addPass(createRegAllocPass(true));
1407
1408  // Allow targets to change the register assignments before rewriting.
1409  addPreRewrite();
1410
1411  // Finally rewrite virtual registers.
1412  addPass(&VirtRegRewriterID);
1413
1414  // Regalloc scoring for ML-driven eviction - noop except when learning a new
1415  // eviction policy.
1416  addPass(createRegAllocScoringPass());
1417  return true;
1418}
1419
1420/// Return true if the default global register allocator is in use and
1421/// has not be overriden on the command line with '-regalloc=...'
1422bool TargetPassConfig::usingDefaultRegAlloc() const {
1423  return RegAlloc.getNumOccurrences() == 0;
1424}
1425
1426/// Add the minimum set of target-independent passes that are required for
1427/// register allocation. No coalescing or scheduling.
1428void TargetPassConfig::addFastRegAlloc() {
1429  addPass(&PHIEliminationID);
1430  addPass(&TwoAddressInstructionPassID);
1431
1432  addRegAssignAndRewriteFast();
1433}
1434
1435/// Add standard target-independent passes that are tightly coupled with
1436/// optimized register allocation, including coalescing, machine instruction
1437/// scheduling, and register allocation itself.
1438void TargetPassConfig::addOptimizedRegAlloc() {
1439  addPass(&DetectDeadLanesID);
1440
1441  addPass(&ProcessImplicitDefsID);
1442
1443  // LiveVariables currently requires pure SSA form.
1444  //
1445  // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1446  // LiveVariables can be removed completely, and LiveIntervals can be directly
1447  // computed. (We still either need to regenerate kill flags after regalloc, or
1448  // preferably fix the scavenger to not depend on them).
1449  // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1450  // When LiveVariables is removed this has to be removed/moved either.
1451  // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1452  // after it with -stop-before/-stop-after.
1453  addPass(&UnreachableMachineBlockElimID);
1454  addPass(&LiveVariablesID);
1455
1456  // Edge splitting is smarter with machine loop info.
1457  addPass(&MachineLoopInfoID);
1458  addPass(&PHIEliminationID);
1459
1460  // Eventually, we want to run LiveIntervals before PHI elimination.
1461  if (EarlyLiveIntervals)
1462    addPass(&LiveIntervalsID);
1463
1464  addPass(&TwoAddressInstructionPassID);
1465  addPass(&RegisterCoalescerID);
1466
1467  // The machine scheduler may accidentally create disconnected components
1468  // when moving subregister definitions around, avoid this by splitting them to
1469  // separate vregs before. Splitting can also improve reg. allocation quality.
1470  addPass(&RenameIndependentSubregsID);
1471
1472  // PreRA instruction scheduling.
1473  addPass(&MachineSchedulerID);
1474
1475  if (addRegAssignAndRewriteOptimized()) {
1476    // Perform stack slot coloring and post-ra machine LICM.
1477    addPass(&StackSlotColoringID);
1478
1479    // Allow targets to expand pseudo instructions depending on the choice of
1480    // registers before MachineCopyPropagation.
1481    addPostRewrite();
1482
1483    // Copy propagate to forward register uses and try to eliminate COPYs that
1484    // were not coalesced.
1485    addPass(&MachineCopyPropagationID);
1486
1487    // Run post-ra machine LICM to hoist reloads / remats.
1488    //
1489    // FIXME: can this move into MachineLateOptimization?
1490    addPass(&MachineLICMID);
1491  }
1492}
1493
1494//===---------------------------------------------------------------------===//
1495/// Post RegAlloc Pass Configuration
1496//===---------------------------------------------------------------------===//
1497
1498/// Add passes that optimize machine instructions after register allocation.
1499void TargetPassConfig::addMachineLateOptimization() {
1500  // Cleanup of redundant immediate/address loads.
1501  addPass(&MachineLateInstrsCleanupID);
1502
1503  // Branch folding must be run after regalloc and prolog/epilog insertion.
1504  addPass(&BranchFolderPassID);
1505
1506  // Tail duplication.
1507  // Note that duplicating tail just increases code size and degrades
1508  // performance for targets that require Structured Control Flow.
1509  // In addition it can also make CFG irreducible. Thus we disable it.
1510  if (!TM->requiresStructuredCFG())
1511    addPass(&TailDuplicateID);
1512
1513  // Copy propagation.
1514  addPass(&MachineCopyPropagationID);
1515}
1516
1517/// Add standard GC passes.
1518bool TargetPassConfig::addGCPasses() {
1519  addPass(&GCMachineCodeAnalysisID);
1520  return true;
1521}
1522
1523/// Add standard basic block placement passes.
1524void TargetPassConfig::addBlockPlacement() {
1525  if (EnableFSDiscriminator) {
1526    addPass(createMIRAddFSDiscriminatorsPass(
1527        sampleprof::FSDiscriminatorPass::Pass2));
1528    const std::string ProfileFile = getFSProfileFile(TM);
1529    if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)
1530      addPass(
1531          createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
1532                                     sampleprof::FSDiscriminatorPass::Pass2));
1533  }
1534  if (addPass(&MachineBlockPlacementID)) {
1535    // Run a separate pass to collect block placement statistics.
1536    if (EnableBlockPlacementStats)
1537      addPass(&MachineBlockPlacementStatsID);
1538  }
1539}
1540
1541//===---------------------------------------------------------------------===//
1542/// GlobalISel Configuration
1543//===---------------------------------------------------------------------===//
1544bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1545  return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1546}
1547
1548bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1549  return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1550}
1551
1552bool TargetPassConfig::isGISelCSEEnabled() const {
1553  return true;
1554}
1555
1556std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1557  return std::make_unique<CSEConfigBase>();
1558}
1559