InjectTLIMappings.cpp revision 360784 
1//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection  ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Populates the VFABI attribute with the scalar-to-vector mappings
10// from the TargetLibraryInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/InjectTLIMappings.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/IR/InstIterator.h"
18#include "llvm/Transforms/Utils.h"
19#include "llvm/Transforms/Utils/ModuleUtils.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "inject-tli-mappings"
24
25STATISTIC(NumCallInjected,
26          "Number of calls in which the mappings have been injected.");
27
28STATISTIC(NumVFDeclAdded,
29          "Number of function declarations that have been added.");
30STATISTIC(NumCompUsedAdded,
31          "Number of `@llvm.compiler.used` operands that have been added.");
32
33/// Helper function to map the TLI name to a strings that holds
34/// scalar-to-vector mapping.
35///
36///    _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>)
37///
38/// where:
39///
40/// <isa> = "_LLVM_"
41/// <mask> = "N". Note: TLI does not support masked interfaces.
42/// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor`
43///          field of the `VecDesc` struct.
44/// <vparams> = "v", as many as are the number of parameters of CI.
45/// <scalarname> = the name of the scalar function called by CI.
46/// <vectorname> = the name of the vector function mapped by the TLI.
47static std::string mangleTLIName(StringRef VectorName, const CallInst &CI,
48                                 unsigned VF) {
49  SmallString<256> Buffer;
50  llvm::raw_svector_ostream Out(Buffer);
51  Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF;
52  for (unsigned I = 0; I < CI.getNumArgOperands(); ++I)
53    Out << "v";
54  Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")";
55  return Out.str();
56}
57
58/// A helper function for converting Scalar types to vector types.
59/// If the incoming type is void, we return void. If the VF is 1, we return
60/// the scalar type.
61static Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) {
62  if (Scalar->isVoidTy() || VF == 1)
63    return Scalar;
64  return VectorType::get(Scalar, {VF, isScalable});
65}
66
67/// A helper function that adds the vector function declaration that
68/// vectorizes the CallInst CI with a vectorization factor of VF
69/// lanes. The TLI assumes that all parameters and the return type of
70/// CI (other than void) need to be widened to a VectorType of VF
71/// lanes.
72static void addVariantDeclaration(CallInst &CI, const unsigned VF,
73                                  const StringRef VFName) {
74  Module *M = CI.getModule();
75
76  // Add function declaration.
77  Type *RetTy = ToVectorTy(CI.getType(), VF);
78  SmallVector<Type *, 4> Tys;
79  for (Value *ArgOperand : CI.arg_operands())
80    Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
81  assert(!CI.getFunctionType()->isVarArg() &&
82         "VarArg functions are not supported.");
83  FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
84  Function *VectorF =
85      Function::Create(FTy, Function::ExternalLinkage, VFName, M);
86  VectorF->copyAttributesFrom(CI.getCalledFunction());
87  ++NumVFDeclAdded;
88  LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
89                    << "` of type " << *(VectorF->getType()) << "\n");
90
91  // Make function declaration (without a body) "sticky" in the IR by
92  // listing it in the @llvm.compiler.used intrinsic.
93  assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
94                             "only on declarations.");
95  appendToCompilerUsed(*M, {VectorF});
96  LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
97                    << "` to `@llvm.compiler.used`.\n");
98  ++NumCompUsedAdded;
99}
100
101static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
102  // This is needed to make sure we don't query the TLI for calls to
103  // bitcast of function pointers, like `%call = call i32 (i32*, ...)
104  // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
105  // as such calls make the `isFunctionVectorizable` raise an
106  // exception.
107  if (CI.isNoBuiltin() || !CI.getCalledFunction())
108    return;
109
110  const std::string ScalarName = CI.getCalledFunction()->getName();
111  // Nothing to be done if the TLI thinks the function is not
112  // vectorizable.
113  if (!TLI.isFunctionVectorizable(ScalarName))
114    return;
115  SmallVector<std::string, 8> Mappings;
116  VFABI::getVectorVariantNames(CI, Mappings);
117  Module *M = CI.getModule();
118  const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
119                                                   Mappings.end());
120  //  All VFs in the TLI are powers of 2.
121  for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
122       VF *= 2) {
123    const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF);
124    if (!TLIName.empty()) {
125      std::string MangledName = mangleTLIName(TLIName, CI, VF);
126      if (!OriginalSetOfMappings.count(MangledName)) {
127        Mappings.push_back(MangledName);
128        ++NumCallInjected;
129      }
130      Function *VariantF = M->getFunction(TLIName);
131      if (!VariantF)
132        addVariantDeclaration(CI, VF, TLIName);
133    }
134  }
135
136  VFABI::setVectorVariantNames(&CI, Mappings);
137}
138
139static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
140  for (auto &I : instructions(F))
141    if (auto CI = dyn_cast<CallInst>(&I))
142      addMappingsFromTLI(TLI, *CI);
143  // Even if the pass adds IR attributes, the analyses are preserved.
144  return false;
145}
146
147////////////////////////////////////////////////////////////////////////////////
148// New pass manager implementation.
149////////////////////////////////////////////////////////////////////////////////
150PreservedAnalyses InjectTLIMappings::run(Function &F,
151                                         FunctionAnalysisManager &AM) {
152  const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
153  runImpl(TLI, F);
154  // Even if the pass adds IR attributes, the analyses are preserved.
155  return PreservedAnalyses::all();
156}
157
158////////////////////////////////////////////////////////////////////////////////
159// Legacy PM Implementation.
160////////////////////////////////////////////////////////////////////////////////
161bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
162  const TargetLibraryInfo &TLI =
163      getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
164  return runImpl(TLI, F);
165}
166
167void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
168  AU.setPreservesCFG();
169  AU.addRequired<TargetLibraryInfoWrapperPass>();
170  AU.addPreserved<TargetLibraryInfoWrapperPass>();
171}
172
173////////////////////////////////////////////////////////////////////////////////
174// Legacy Pass manager initialization
175////////////////////////////////////////////////////////////////////////////////
176char InjectTLIMappingsLegacy::ID = 0;
177
178INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
179                      "Inject TLI Mappings", false, false)
180INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
181INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
182                    false, false)
183
184FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
185  return new InjectTLIMappingsLegacy();
186}
187