1//===- InlineAdvisor.h - Inlining decision making abstraction -*- C++ ---*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9#ifndef LLVM_ANALYSIS_INLINEADVISOR_H 10#define LLVM_ANALYSIS_INLINEADVISOR_H 11 12#include "llvm/Analysis/InlineCost.h" 13#include "llvm/Config/llvm-config.h" 14#include "llvm/IR/PassManager.h" 15#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" 16#include <memory> 17#include <unordered_set> 18 19namespace llvm { 20class BasicBlock; 21class CallBase; 22class Function; 23class Module; 24class OptimizationRemarkEmitter; 25 26/// There are 3 scenarios we can use the InlineAdvisor: 27/// - Default - use manual heuristics. 28/// 29/// - Release mode, the expected mode for production, day to day deployments. 30/// In this mode, when building the compiler, we also compile a pre-trained ML 31/// model to native code, and link it as a static library. This mode has low 32/// overhead and no additional dependencies for the compiler runtime. 33/// 34/// - Development mode, for training new models. 35/// In this mode, we trade off runtime performance for flexibility. This mode 36/// requires the full C Tensorflow API library, and evaluates models 37/// dynamically. This mode also permits generating training logs, for offline 38/// training. 39enum class InliningAdvisorMode : int { 40 Default, 41 Release, 42 Development 43}; 44 45class InlineAdvisor; 46/// Capture state between an inlining decision having had been made, and 47/// its impact being observable. When collecting model training data, this 48/// allows recording features/decisions/partial reward data sets. 49/// 50/// Derivations of this type are expected to be tightly coupled with their 51/// InliningAdvisors. The base type implements the minimal contractual 52/// obligations. 53class InlineAdvice { 54public: 55 InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 56 OptimizationRemarkEmitter &ORE, bool IsInliningRecommended); 57 58 InlineAdvice(InlineAdvice &&) = delete; 59 InlineAdvice(const InlineAdvice &) = delete; 60 virtual ~InlineAdvice() { 61 assert(Recorded && "InlineAdvice should have been informed of the " 62 "inliner's decision in all cases"); 63 } 64 65 /// Exactly one of the record* APIs must be called. Implementers may extend 66 /// behavior by implementing the corresponding record*Impl. 67 /// 68 /// Call after inlining succeeded, and did not result in deleting the callee. 69 void recordInlining(); 70 71 /// Call after inlining succeeded, and resulted in deleting the callee. 72 void recordInliningWithCalleeDeleted(); 73 74 /// Call after the decision for a call site was to not inline. 75 void recordUnsuccessfulInlining(const InlineResult &Result) { 76 markRecorded(); 77 recordUnsuccessfulInliningImpl(Result); 78 } 79 80 /// Call to indicate inlining was not attempted. 81 void recordUnattemptedInlining() { 82 markRecorded(); 83 recordUnattemptedInliningImpl(); 84 } 85 86 /// Get the inlining recommendation. 87 bool isInliningRecommended() const { return IsInliningRecommended; } 88 const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; } 89 const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; } 90 91protected: 92 virtual void recordInliningImpl() {} 93 virtual void recordInliningWithCalleeDeletedImpl() {} 94 virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {} 95 virtual void recordUnattemptedInliningImpl() {} 96 97 InlineAdvisor *const Advisor; 98 /// Caller and Callee are pre-inlining. 99 Function *const Caller; 100 Function *const Callee; 101 102 // Capture the context of CB before inlining, as a successful inlining may 103 // change that context, and we want to report success or failure in the 104 // original context. 105 const DebugLoc DLoc; 106 const BasicBlock *const Block; 107 OptimizationRemarkEmitter &ORE; 108 const bool IsInliningRecommended; 109 110private: 111 void markRecorded() { 112 assert(!Recorded && "Recording should happen exactly once"); 113 Recorded = true; 114 } 115 void recordInlineStatsIfNeeded(); 116 117 bool Recorded = false; 118}; 119 120class DefaultInlineAdvice : public InlineAdvice { 121public: 122 DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 123 Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE, 124 bool EmitRemarks = true) 125 : InlineAdvice(Advisor, CB, ORE, OIC.hasValue()), OriginalCB(&CB), 126 OIC(OIC), EmitRemarks(EmitRemarks) {} 127 128private: 129 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override; 130 void recordInliningWithCalleeDeletedImpl() override; 131 void recordInliningImpl() override; 132 133private: 134 CallBase *const OriginalCB; 135 Optional<InlineCost> OIC; 136 bool EmitRemarks; 137}; 138 139/// Interface for deciding whether to inline a call site or not. 140class InlineAdvisor { 141public: 142 InlineAdvisor(InlineAdvisor &&) = delete; 143 virtual ~InlineAdvisor(); 144 145 /// Get an InlineAdvice containing a recommendation on whether to 146 /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to 147 /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates 148 /// only mandatory (always-inline) call sites should be recommended - this 149 /// allows the InlineAdvisor track such inlininings. 150 /// Returns an InlineAdvice with the inlining recommendation. 151 std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB, 152 bool MandatoryOnly = false); 153 154 /// This must be called when the Inliner pass is entered, to allow the 155 /// InlineAdvisor update internal state, as result of function passes run 156 /// between Inliner pass runs (for the same module). 157 virtual void onPassEntry() {} 158 159 /// This must be called when the Inliner pass is exited, as function passes 160 /// may be run subsequently. This allows an implementation of InlineAdvisor 161 /// to prepare for a partial update. 162 virtual void onPassExit() {} 163 164protected: 165 InlineAdvisor(Module &M, FunctionAnalysisManager &FAM); 166 virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0; 167 virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB, 168 bool Advice); 169 170 Module &M; 171 FunctionAnalysisManager &FAM; 172 std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; 173 174 /// We may want to defer deleting functions to after the inlining for a whole 175 /// module has finished. This allows us to reliably use function pointers as 176 /// unique identifiers, as an efficient implementation detail of the 177 /// InlineAdvisor. Otherwise, it is possible the memory allocator 178 /// re-allocate Function objects at the same address of a deleted Function; 179 /// and Functions are potentially created during the function passes called 180 /// after each SCC inlining (e.g. argument promotion does that). 181 void freeDeletedFunctions(); 182 183 bool isFunctionDeleted(const Function *F) const { 184 return DeletedFunctions.count(F); 185 } 186 187 enum class MandatoryInliningKind { NotMandatory, Always, Never }; 188 189 static MandatoryInliningKind getMandatoryKind(CallBase &CB, 190 FunctionAnalysisManager &FAM, 191 OptimizationRemarkEmitter &ORE); 192 193 OptimizationRemarkEmitter &getCallerORE(CallBase &CB); 194 195private: 196 friend class InlineAdvice; 197 void markFunctionAsDeleted(Function *F); 198 std::unordered_set<const Function *> DeletedFunctions; 199}; 200 201/// The default (manual heuristics) implementation of the InlineAdvisor. This 202/// implementation does not need to keep state between inliner pass runs, and is 203/// reusable as-is for inliner pass test scenarios, as well as for regular use. 204class DefaultInlineAdvisor : public InlineAdvisor { 205public: 206 DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 207 InlineParams Params) 208 : InlineAdvisor(M, FAM), Params(Params) {} 209 210private: 211 std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; 212 213 void onPassExit() override { freeDeletedFunctions(); } 214 215 InlineParams Params; 216}; 217 218/// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor 219/// needs to capture state right before inlining commences over a module. 220class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> { 221public: 222 static AnalysisKey Key; 223 InlineAdvisorAnalysis() = default; 224 struct Result { 225 Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} 226 bool invalidate(Module &, const PreservedAnalyses &, 227 ModuleAnalysisManager::Invalidator &) { 228 // InlineAdvisor must be preserved across analysis invalidations. 229 return false; 230 } 231 bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, 232 StringRef ReplayFile); 233 InlineAdvisor *getAdvisor() const { return Advisor.get(); } 234 void clear() { Advisor.reset(); } 235 236 private: 237 Module &M; 238 ModuleAnalysisManager &MAM; 239 std::unique_ptr<InlineAdvisor> Advisor; 240 }; 241 242 Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } 243}; 244 245#ifdef LLVM_HAVE_TF_AOT 246std::unique_ptr<InlineAdvisor> 247getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM); 248#endif 249 250#ifdef LLVM_HAVE_TF_API 251std::unique_ptr<InlineAdvisor> 252getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, 253 std::function<bool(CallBase &)> GetDefaultAdvice); 254#endif 255 256// Default (manual policy) decision making helper APIs. Shared with the legacy 257// pass manager inliner. 258 259/// Return the cost only if the inliner should attempt to inline at the given 260/// CallSite. If we return the cost, we will emit an optimisation remark later 261/// using that cost, so we won't do so from this function. Return None if 262/// inlining should not be attempted. 263Optional<InlineCost> 264shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost, 265 OptimizationRemarkEmitter &ORE, bool EnableDeferral = true); 266 267/// Emit ORE message. 268void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, 269 const BasicBlock *Block, const Function &Callee, 270 const Function &Caller, const InlineCost &IC, 271 bool ForProfileContext = false, 272 const char *PassName = nullptr); 273 274/// get call site location as string 275std::string getCallSiteLocation(DebugLoc DLoc); 276 277/// Add location info to ORE message. 278void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); 279 280/// Set the inline-remark attribute. 281void setInlineRemark(CallBase &CB, StringRef Message); 282 283/// Utility for extracting the inline cost message to a string. 284std::string inlineCostStr(const InlineCost &IC); 285} // namespace llvm 286#endif // LLVM_ANALYSIS_INLINEADVISOR_H 287