1//===- InlineAlways.cpp - Code to inline always_inline functions ----------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file implements a custom inliner that handles only functions that 10// are marked as "always inline". 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/Transforms/IPO/AlwaysInliner.h" 15#include "llvm/ADT/SetVector.h" 16#include "llvm/Analysis/AssumptionCache.h" 17#include "llvm/Analysis/InlineCost.h" 18#include "llvm/Analysis/TargetLibraryInfo.h" 19#include "llvm/IR/CallingConv.h" 20#include "llvm/IR/DataLayout.h" 21#include "llvm/IR/Instructions.h" 22#include "llvm/IR/Module.h" 23#include "llvm/IR/Type.h" 24#include "llvm/InitializePasses.h" 25#include "llvm/Transforms/IPO.h" 26#include "llvm/Transforms/IPO/Inliner.h" 27#include "llvm/Transforms/Utils/Cloning.h" 28#include "llvm/Transforms/Utils/ModuleUtils.h" 29 30using namespace llvm; 31 32#define DEBUG_TYPE "inline" 33 34PreservedAnalyses AlwaysInlinerPass::run(Module &M, 35 ModuleAnalysisManager &MAM) { 36 // Add inline assumptions during code generation. 37 FunctionAnalysisManager &FAM = 38 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 39 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 40 return FAM.getResult<AssumptionAnalysis>(F); 41 }; 42 InlineFunctionInfo IFI(/*cg=*/nullptr, GetAssumptionCache); 43 44 SmallSetVector<CallBase *, 16> Calls; 45 bool Changed = false; 46 SmallVector<Function *, 16> InlinedFunctions; 47 for (Function &F : M) 48 if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) && 49 isInlineViable(F).isSuccess()) { 50 Calls.clear(); 51 52 for (User *U : F.users()) 53 if (auto *CB = dyn_cast<CallBase>(U)) 54 if (CB->getCalledFunction() == &F) 55 Calls.insert(CB); 56 57 for (CallBase *CB : Calls) 58 // FIXME: We really shouldn't be able to fail to inline at this point! 59 // We should do something to log or check the inline failures here. 60 Changed |= 61 InlineFunction(*CB, IFI, /*CalleeAAR=*/nullptr, InsertLifetime) 62 .isSuccess(); 63 64 // Remember to try and delete this function afterward. This both avoids 65 // re-walking the rest of the module and avoids dealing with any iterator 66 // invalidation issues while deleting functions. 67 InlinedFunctions.push_back(&F); 68 } 69 70 // Remove any live functions. 71 erase_if(InlinedFunctions, [&](Function *F) { 72 F->removeDeadConstantUsers(); 73 return !F->isDefTriviallyDead(); 74 }); 75 76 // Delete the non-comdat ones from the module and also from our vector. 77 auto NonComdatBegin = partition( 78 InlinedFunctions, [&](Function *F) { return F->hasComdat(); }); 79 for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end())) 80 M.getFunctionList().erase(F); 81 InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end()); 82 83 if (!InlinedFunctions.empty()) { 84 // Now we just have the comdat functions. Filter out the ones whose comdats 85 // are not actually dead. 86 filterDeadComdatFunctions(M, InlinedFunctions); 87 // The remaining functions are actually dead. 88 for (Function *F : InlinedFunctions) 89 M.getFunctionList().erase(F); 90 } 91 92 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 93} 94 95namespace { 96 97/// Inliner pass which only handles "always inline" functions. 98/// 99/// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner 100/// base class to provide several facilities such as array alloca merging. 101class AlwaysInlinerLegacyPass : public LegacyInlinerBase { 102 103public: 104 AlwaysInlinerLegacyPass() : LegacyInlinerBase(ID, /*InsertLifetime*/ true) { 105 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 106 } 107 108 AlwaysInlinerLegacyPass(bool InsertLifetime) 109 : LegacyInlinerBase(ID, InsertLifetime) { 110 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 111 } 112 113 /// Main run interface method. We override here to avoid calling skipSCC(). 114 bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); } 115 116 static char ID; // Pass identification, replacement for typeid 117 118 InlineCost getInlineCost(CallBase &CB) override; 119 120 using llvm::Pass::doFinalization; 121 bool doFinalization(CallGraph &CG) override { 122 return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true); 123 } 124}; 125} 126 127char AlwaysInlinerLegacyPass::ID = 0; 128INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", 129 "Inliner for always_inline functions", false, false) 130INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 131INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass) 132INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 133INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 134INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", 135 "Inliner for always_inline functions", false, false) 136 137Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { 138 return new AlwaysInlinerLegacyPass(InsertLifetime); 139} 140 141/// Get the inline cost for the always-inliner. 142/// 143/// The always inliner *only* handles functions which are marked with the 144/// attribute to force inlining. As such, it is dramatically simpler and avoids 145/// using the powerful (but expensive) inline cost analysis. Instead it uses 146/// a very simple and boring direct walk of the instructions looking for 147/// impossible-to-inline constructs. 148/// 149/// Note, it would be possible to go to some lengths to cache the information 150/// computed here, but as we only expect to do this for relatively few and 151/// small functions which have the explicit attribute to force inlining, it is 152/// likely not worth it in practice. 153InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallBase &CB) { 154 Function *Callee = CB.getCalledFunction(); 155 156 // Only inline direct calls to functions with always-inline attributes 157 // that are viable for inlining. 158 if (!Callee) 159 return InlineCost::getNever("indirect call"); 160 161 // FIXME: We shouldn't even get here for declarations. 162 if (Callee->isDeclaration()) 163 return InlineCost::getNever("no definition"); 164 165 if (!CB.hasFnAttr(Attribute::AlwaysInline)) 166 return InlineCost::getNever("no alwaysinline attribute"); 167 168 auto IsViable = isInlineViable(*Callee); 169 if (!IsViable.isSuccess()) 170 return InlineCost::getNever(IsViable.getFailureReason()); 171 172 return InlineCost::getAlways("always inliner"); 173} 174