1//===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions
10//
11//===----------------------------------------------------------------------===//
12
13#include "NVPTXUtilities.h"
14#include "NVPTX.h"
15#include "NVPTXTargetMachine.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/GlobalVariable.h"
19#include "llvm/IR/InstIterator.h"
20#include "llvm/IR/Module.h"
21#include "llvm/IR/Operator.h"
22#include "llvm/Support/Mutex.h"
23#include <algorithm>
24#include <cstring>
25#include <map>
26#include <mutex>
27#include <string>
28#include <vector>
29
30namespace llvm {
31
32namespace {
33typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t;
34typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;
35
36struct AnnotationCache {
37  sys::Mutex Lock;
38  std::map<const Module *, global_val_annot_t> Cache;
39};
40
41AnnotationCache &getAnnotationCache() {
42  static AnnotationCache AC;
43  return AC;
44}
45} // anonymous namespace
46
47void clearAnnotationCache(const Module *Mod) {
48  auto &AC = getAnnotationCache();
49  std::lock_guard<sys::Mutex> Guard(AC.Lock);
50  AC.Cache.erase(Mod);
51}
52
53static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval) {
54  auto &AC = getAnnotationCache();
55  std::lock_guard<sys::Mutex> Guard(AC.Lock);
56  assert(md && "Invalid mdnode for annotation");
57  assert((md->getNumOperands() % 2) == 1 && "Invalid number of operands");
58  // start index = 1, to skip the global variable key
59  // increment = 2, to skip the value for each property-value pairs
60  for (unsigned i = 1, e = md->getNumOperands(); i != e; i += 2) {
61    // property
62    const MDString *prop = dyn_cast<MDString>(md->getOperand(i));
63    assert(prop && "Annotation property not a string");
64
65    // value
66    ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(md->getOperand(i + 1));
67    assert(Val && "Value operand not a constant int");
68
69    std::string keyname = prop->getString().str();
70    if (retval.find(keyname) != retval.end())
71      retval[keyname].push_back(Val->getZExtValue());
72    else {
73      std::vector<unsigned> tmp;
74      tmp.push_back(Val->getZExtValue());
75      retval[keyname] = tmp;
76    }
77  }
78}
79
80static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
81  auto &AC = getAnnotationCache();
82  std::lock_guard<sys::Mutex> Guard(AC.Lock);
83  NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations");
84  if (!NMD)
85    return;
86  key_val_pair_t tmp;
87  for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
88    const MDNode *elem = NMD->getOperand(i);
89
90    GlobalValue *entity =
91        mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0));
92    // entity may be null due to DCE
93    if (!entity)
94      continue;
95    if (entity != gv)
96      continue;
97
98    // accumulate annotations for entity in tmp
99    cacheAnnotationFromMD(elem, tmp);
100  }
101
102  if (tmp.empty()) // no annotations for this gv
103    return;
104
105  if (AC.Cache.find(m) != AC.Cache.end())
106    AC.Cache[m][gv] = std::move(tmp);
107  else {
108    global_val_annot_t tmp1;
109    tmp1[gv] = std::move(tmp);
110    AC.Cache[m] = std::move(tmp1);
111  }
112}
113
114bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
115                           unsigned &retval) {
116  auto &AC = getAnnotationCache();
117  std::lock_guard<sys::Mutex> Guard(AC.Lock);
118  const Module *m = gv->getParent();
119  if (AC.Cache.find(m) == AC.Cache.end())
120    cacheAnnotationFromMD(m, gv);
121  else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
122    cacheAnnotationFromMD(m, gv);
123  if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
124    return false;
125  retval = AC.Cache[m][gv][prop][0];
126  return true;
127}
128
129bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
130                           std::vector<unsigned> &retval) {
131  auto &AC = getAnnotationCache();
132  std::lock_guard<sys::Mutex> Guard(AC.Lock);
133  const Module *m = gv->getParent();
134  if (AC.Cache.find(m) == AC.Cache.end())
135    cacheAnnotationFromMD(m, gv);
136  else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
137    cacheAnnotationFromMD(m, gv);
138  if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
139    return false;
140  retval = AC.Cache[m][gv][prop];
141  return true;
142}
143
144bool isTexture(const Value &val) {
145  if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
146    unsigned annot;
147    if (findOneNVVMAnnotation(gv, "texture", annot)) {
148      assert((annot == 1) && "Unexpected annotation on a texture symbol");
149      return true;
150    }
151  }
152  return false;
153}
154
155bool isSurface(const Value &val) {
156  if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
157    unsigned annot;
158    if (findOneNVVMAnnotation(gv, "surface", annot)) {
159      assert((annot == 1) && "Unexpected annotation on a surface symbol");
160      return true;
161    }
162  }
163  return false;
164}
165
166bool isSampler(const Value &val) {
167  const char *AnnotationName = "sampler";
168
169  if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
170    unsigned annot;
171    if (findOneNVVMAnnotation(gv, AnnotationName, annot)) {
172      assert((annot == 1) && "Unexpected annotation on a sampler symbol");
173      return true;
174    }
175  }
176  if (const Argument *arg = dyn_cast<Argument>(&val)) {
177    const Function *func = arg->getParent();
178    std::vector<unsigned> annot;
179    if (findAllNVVMAnnotation(func, AnnotationName, annot)) {
180      if (is_contained(annot, arg->getArgNo()))
181        return true;
182    }
183  }
184  return false;
185}
186
187bool isImageReadOnly(const Value &val) {
188  if (const Argument *arg = dyn_cast<Argument>(&val)) {
189    const Function *func = arg->getParent();
190    std::vector<unsigned> annot;
191    if (findAllNVVMAnnotation(func, "rdoimage", annot)) {
192      if (is_contained(annot, arg->getArgNo()))
193        return true;
194    }
195  }
196  return false;
197}
198
199bool isImageWriteOnly(const Value &val) {
200  if (const Argument *arg = dyn_cast<Argument>(&val)) {
201    const Function *func = arg->getParent();
202    std::vector<unsigned> annot;
203    if (findAllNVVMAnnotation(func, "wroimage", annot)) {
204      if (is_contained(annot, arg->getArgNo()))
205        return true;
206    }
207  }
208  return false;
209}
210
211bool isImageReadWrite(const Value &val) {
212  if (const Argument *arg = dyn_cast<Argument>(&val)) {
213    const Function *func = arg->getParent();
214    std::vector<unsigned> annot;
215    if (findAllNVVMAnnotation(func, "rdwrimage", annot)) {
216      if (is_contained(annot, arg->getArgNo()))
217        return true;
218    }
219  }
220  return false;
221}
222
223bool isImage(const Value &val) {
224  return isImageReadOnly(val) || isImageWriteOnly(val) || isImageReadWrite(val);
225}
226
227bool isManaged(const Value &val) {
228  if(const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
229    unsigned annot;
230    if (findOneNVVMAnnotation(gv, "managed", annot)) {
231      assert((annot == 1) && "Unexpected annotation on a managed symbol");
232      return true;
233    }
234  }
235  return false;
236}
237
238std::string getTextureName(const Value &val) {
239  assert(val.hasName() && "Found texture variable with no name");
240  return std::string(val.getName());
241}
242
243std::string getSurfaceName(const Value &val) {
244  assert(val.hasName() && "Found surface variable with no name");
245  return std::string(val.getName());
246}
247
248std::string getSamplerName(const Value &val) {
249  assert(val.hasName() && "Found sampler variable with no name");
250  return std::string(val.getName());
251}
252
253bool getMaxNTIDx(const Function &F, unsigned &x) {
254  return findOneNVVMAnnotation(&F, "maxntidx", x);
255}
256
257bool getMaxNTIDy(const Function &F, unsigned &y) {
258  return findOneNVVMAnnotation(&F, "maxntidy", y);
259}
260
261bool getMaxNTIDz(const Function &F, unsigned &z) {
262  return findOneNVVMAnnotation(&F, "maxntidz", z);
263}
264
265bool getMaxClusterRank(const Function &F, unsigned &x) {
266  return findOneNVVMAnnotation(&F, "maxclusterrank", x);
267}
268
269bool getReqNTIDx(const Function &F, unsigned &x) {
270  return findOneNVVMAnnotation(&F, "reqntidx", x);
271}
272
273bool getReqNTIDy(const Function &F, unsigned &y) {
274  return findOneNVVMAnnotation(&F, "reqntidy", y);
275}
276
277bool getReqNTIDz(const Function &F, unsigned &z) {
278  return findOneNVVMAnnotation(&F, "reqntidz", z);
279}
280
281bool getMinCTASm(const Function &F, unsigned &x) {
282  return findOneNVVMAnnotation(&F, "minctasm", x);
283}
284
285bool getMaxNReg(const Function &F, unsigned &x) {
286  return findOneNVVMAnnotation(&F, "maxnreg", x);
287}
288
289bool isKernelFunction(const Function &F) {
290  unsigned x = 0;
291  bool retval = findOneNVVMAnnotation(&F, "kernel", x);
292  if (!retval) {
293    // There is no NVVM metadata, check the calling convention
294    return F.getCallingConv() == CallingConv::PTX_Kernel;
295  }
296  return (x == 1);
297}
298
299bool getAlign(const Function &F, unsigned index, unsigned &align) {
300  std::vector<unsigned> Vs;
301  bool retval = findAllNVVMAnnotation(&F, "align", Vs);
302  if (!retval)
303    return false;
304  for (unsigned v : Vs) {
305    if ((v >> 16) == index) {
306      align = v & 0xFFFF;
307      return true;
308    }
309  }
310  return false;
311}
312
313bool getAlign(const CallInst &I, unsigned index, unsigned &align) {
314  if (MDNode *alignNode = I.getMetadata("callalign")) {
315    for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
316      if (const ConstantInt *CI =
317              mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {
318        unsigned v = CI->getZExtValue();
319        if ((v >> 16) == index) {
320          align = v & 0xFFFF;
321          return true;
322        }
323        if ((v >> 16) > index) {
324          return false;
325        }
326      }
327    }
328  }
329  return false;
330}
331
332Function *getMaybeBitcastedCallee(const CallBase *CB) {
333  return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
334}
335
336bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) {
337  const auto &ST =
338      *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();
339  if (!ST.hasNoReturn())
340    return false;
341
342  assert((isa<Function>(V) || isa<CallInst>(V)) &&
343         "Expect either a call instruction or a function");
344
345  if (const CallInst *CallI = dyn_cast<CallInst>(V))
346    return CallI->doesNotReturn() &&
347           CallI->getFunctionType()->getReturnType()->isVoidTy();
348
349  const Function *F = cast<Function>(V);
350  return F->doesNotReturn() &&
351         F->getFunctionType()->getReturnType()->isVoidTy() &&
352         !isKernelFunction(*F);
353}
354
355bool Isv2x16VT(EVT VT) {
356  return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16);
357}
358
359} // namespace llvm
360