1356843Sdim//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// 2356843Sdim// 3356843Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4356843Sdim// See https://llvm.org/LICENSE.txt for license information. 5356843Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6356843Sdim// 7356843Sdim//===----------------------------------------------------------------------===// 8356843Sdim// 9356843Sdim// Populates the VFABI attribute with the scalar-to-vector mappings 10356843Sdim// from the TargetLibraryInfo. 11356843Sdim// 12356843Sdim//===----------------------------------------------------------------------===// 13356843Sdim 14356843Sdim#include "llvm/Transforms/Utils/InjectTLIMappings.h" 15356843Sdim#include "llvm/ADT/Statistic.h" 16356843Sdim#include "llvm/Analysis/VectorUtils.h" 17356843Sdim#include "llvm/IR/InstIterator.h" 18356843Sdim#include "llvm/Transforms/Utils.h" 19356843Sdim#include "llvm/Transforms/Utils/ModuleUtils.h" 20356843Sdim 21356843Sdimusing namespace llvm; 22356843Sdim 23356843Sdim#define DEBUG_TYPE "inject-tli-mappings" 24356843Sdim 25356843SdimSTATISTIC(NumCallInjected, 26356843Sdim "Number of calls in which the mappings have been injected."); 27356843Sdim 28356843SdimSTATISTIC(NumVFDeclAdded, 29356843Sdim "Number of function declarations that have been added."); 30356843SdimSTATISTIC(NumCompUsedAdded, 31356843Sdim "Number of `@llvm.compiler.used` operands that have been added."); 32356843Sdim 33356843Sdim/// Helper function to map the TLI name to a strings that holds 34356843Sdim/// scalar-to-vector mapping. 35356843Sdim/// 36356843Sdim/// _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>) 37356843Sdim/// 38356843Sdim/// where: 39356843Sdim/// 40356843Sdim/// <isa> = "_LLVM_" 41356843Sdim/// <mask> = "N". Note: TLI does not support masked interfaces. 42356843Sdim/// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor` 43356843Sdim/// field of the `VecDesc` struct. 44356843Sdim/// <vparams> = "v", as many as are the number of parameters of CI. 45356843Sdim/// <scalarname> = the name of the scalar function called by CI. 46356843Sdim/// <vectorname> = the name of the vector function mapped by the TLI. 47356843Sdimstatic std::string mangleTLIName(StringRef VectorName, const CallInst &CI, 48356843Sdim unsigned VF) { 49356843Sdim SmallString<256> Buffer; 50356843Sdim llvm::raw_svector_ostream Out(Buffer); 51356843Sdim Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF; 52356843Sdim for (unsigned I = 0; I < CI.getNumArgOperands(); ++I) 53356843Sdim Out << "v"; 54356843Sdim Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")"; 55356843Sdim return Out.str(); 56356843Sdim} 57356843Sdim 58356843Sdim/// A helper function for converting Scalar types to vector types. 59356843Sdim/// If the incoming type is void, we return void. If the VF is 1, we return 60356843Sdim/// the scalar type. 61356843Sdimstatic Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) { 62356843Sdim if (Scalar->isVoidTy() || VF == 1) 63356843Sdim return Scalar; 64356843Sdim return VectorType::get(Scalar, {VF, isScalable}); 65356843Sdim} 66356843Sdim 67356843Sdim/// A helper function that adds the vector function declaration that 68356843Sdim/// vectorizes the CallInst CI with a vectorization factor of VF 69356843Sdim/// lanes. The TLI assumes that all parameters and the return type of 70356843Sdim/// CI (other than void) need to be widened to a VectorType of VF 71356843Sdim/// lanes. 72356843Sdimstatic void addVariantDeclaration(CallInst &CI, const unsigned VF, 73356843Sdim const StringRef VFName) { 74356843Sdim Module *M = CI.getModule(); 75356843Sdim 76356843Sdim // Add function declaration. 77356843Sdim Type *RetTy = ToVectorTy(CI.getType(), VF); 78356843Sdim SmallVector<Type *, 4> Tys; 79356843Sdim for (Value *ArgOperand : CI.arg_operands()) 80356843Sdim Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); 81356843Sdim assert(!CI.getFunctionType()->isVarArg() && 82356843Sdim "VarArg functions are not supported."); 83356843Sdim FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); 84356843Sdim Function *VectorF = 85356843Sdim Function::Create(FTy, Function::ExternalLinkage, VFName, M); 86356843Sdim VectorF->copyAttributesFrom(CI.getCalledFunction()); 87356843Sdim ++NumVFDeclAdded; 88356843Sdim LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName 89356843Sdim << "` of type " << *(VectorF->getType()) << "\n"); 90356843Sdim 91356843Sdim // Make function declaration (without a body) "sticky" in the IR by 92356843Sdim // listing it in the @llvm.compiler.used intrinsic. 93356843Sdim assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " 94356843Sdim "only on declarations."); 95356843Sdim appendToCompilerUsed(*M, {VectorF}); 96356843Sdim LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName 97356843Sdim << "` to `@llvm.compiler.used`.\n"); 98356843Sdim ++NumCompUsedAdded; 99356843Sdim} 100356843Sdim 101356843Sdimstatic void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { 102356843Sdim // This is needed to make sure we don't query the TLI for calls to 103356843Sdim // bitcast of function pointers, like `%call = call i32 (i32*, ...) 104356843Sdim // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, 105356843Sdim // as such calls make the `isFunctionVectorizable` raise an 106356843Sdim // exception. 107356843Sdim if (CI.isNoBuiltin() || !CI.getCalledFunction()) 108356843Sdim return; 109356843Sdim 110356843Sdim const std::string ScalarName = CI.getCalledFunction()->getName(); 111356843Sdim // Nothing to be done if the TLI thinks the function is not 112356843Sdim // vectorizable. 113356843Sdim if (!TLI.isFunctionVectorizable(ScalarName)) 114356843Sdim return; 115356843Sdim SmallVector<std::string, 8> Mappings; 116356843Sdim VFABI::getVectorVariantNames(CI, Mappings); 117356843Sdim Module *M = CI.getModule(); 118356843Sdim const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(), 119356843Sdim Mappings.end()); 120356843Sdim // All VFs in the TLI are powers of 2. 121356843Sdim for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF; 122356843Sdim VF *= 2) { 123356843Sdim const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF); 124356843Sdim if (!TLIName.empty()) { 125356843Sdim std::string MangledName = mangleTLIName(TLIName, CI, VF); 126356843Sdim if (!OriginalSetOfMappings.count(MangledName)) { 127356843Sdim Mappings.push_back(MangledName); 128356843Sdim ++NumCallInjected; 129356843Sdim } 130356843Sdim Function *VariantF = M->getFunction(TLIName); 131356843Sdim if (!VariantF) 132356843Sdim addVariantDeclaration(CI, VF, TLIName); 133356843Sdim } 134356843Sdim } 135356843Sdim 136356843Sdim VFABI::setVectorVariantNames(&CI, Mappings); 137356843Sdim} 138356843Sdim 139356843Sdimstatic bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 140356843Sdim for (auto &I : instructions(F)) 141356843Sdim if (auto CI = dyn_cast<CallInst>(&I)) 142356843Sdim addMappingsFromTLI(TLI, *CI); 143356843Sdim // Even if the pass adds IR attributes, the analyses are preserved. 144356843Sdim return false; 145356843Sdim} 146356843Sdim 147356843Sdim//////////////////////////////////////////////////////////////////////////////// 148356843Sdim// New pass manager implementation. 149356843Sdim//////////////////////////////////////////////////////////////////////////////// 150356843SdimPreservedAnalyses InjectTLIMappings::run(Function &F, 151356843Sdim FunctionAnalysisManager &AM) { 152356843Sdim const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 153356843Sdim runImpl(TLI, F); 154356843Sdim // Even if the pass adds IR attributes, the analyses are preserved. 155356843Sdim return PreservedAnalyses::all(); 156356843Sdim} 157356843Sdim 158356843Sdim//////////////////////////////////////////////////////////////////////////////// 159356843Sdim// Legacy PM Implementation. 160356843Sdim//////////////////////////////////////////////////////////////////////////////// 161356843Sdimbool InjectTLIMappingsLegacy::runOnFunction(Function &F) { 162356843Sdim const TargetLibraryInfo &TLI = 163356843Sdim getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 164356843Sdim return runImpl(TLI, F); 165356843Sdim} 166356843Sdim 167356843Sdimvoid InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { 168356843Sdim AU.setPreservesCFG(); 169356843Sdim AU.addRequired<TargetLibraryInfoWrapperPass>(); 170356843Sdim AU.addPreserved<TargetLibraryInfoWrapperPass>(); 171356843Sdim} 172356843Sdim 173356843Sdim//////////////////////////////////////////////////////////////////////////////// 174356843Sdim// Legacy Pass manager initialization 175356843Sdim//////////////////////////////////////////////////////////////////////////////// 176356843Sdimchar InjectTLIMappingsLegacy::ID = 0; 177356843Sdim 178356843SdimINITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, 179356843Sdim "Inject TLI Mappings", false, false) 180356843SdimINITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 181356843SdimINITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", 182356843Sdim false, false) 183356843Sdim 184356843SdimFunctionPass *llvm::createInjectTLIMappingsLegacyPass() { 185356843Sdim return new InjectTLIMappingsLegacy(); 186356843Sdim} 187