1//===- NVPTXUtilities.cpp - Utility Functions -----------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file contains miscellaneous utility functions 10// 11//===----------------------------------------------------------------------===// 12 13#include "NVPTXUtilities.h" 14#include "NVPTX.h" 15#include "NVPTXTargetMachine.h" 16#include "llvm/IR/Constants.h" 17#include "llvm/IR/Function.h" 18#include "llvm/IR/GlobalVariable.h" 19#include "llvm/IR/InstIterator.h" 20#include "llvm/IR/Module.h" 21#include "llvm/IR/Operator.h" 22#include "llvm/Support/Mutex.h" 23#include <algorithm> 24#include <cstring> 25#include <map> 26#include <mutex> 27#include <string> 28#include <vector> 29 30namespace llvm { 31 32namespace { 33typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t; 34typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t; 35 36struct AnnotationCache { 37 sys::Mutex Lock; 38 std::map<const Module *, global_val_annot_t> Cache; 39}; 40 41AnnotationCache &getAnnotationCache() { 42 static AnnotationCache AC; 43 return AC; 44} 45} // anonymous namespace 46 47void clearAnnotationCache(const Module *Mod) { 48 auto &AC = getAnnotationCache(); 49 std::lock_guard<sys::Mutex> Guard(AC.Lock); 50 AC.Cache.erase(Mod); 51} 52 53static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval) { 54 auto &AC = getAnnotationCache(); 55 std::lock_guard<sys::Mutex> Guard(AC.Lock); 56 assert(md && "Invalid mdnode for annotation"); 57 assert((md->getNumOperands() % 2) == 1 && "Invalid number of operands"); 58 // start index = 1, to skip the global variable key 59 // increment = 2, to skip the value for each property-value pairs 60 for (unsigned i = 1, e = md->getNumOperands(); i != e; i += 2) { 61 // property 62 const MDString *prop = dyn_cast<MDString>(md->getOperand(i)); 63 assert(prop && "Annotation property not a string"); 64 65 // value 66 ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(md->getOperand(i + 1)); 67 assert(Val && "Value operand not a constant int"); 68 69 std::string keyname = prop->getString().str(); 70 if (retval.find(keyname) != retval.end()) 71 retval[keyname].push_back(Val->getZExtValue()); 72 else { 73 std::vector<unsigned> tmp; 74 tmp.push_back(Val->getZExtValue()); 75 retval[keyname] = tmp; 76 } 77 } 78} 79 80static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) { 81 auto &AC = getAnnotationCache(); 82 std::lock_guard<sys::Mutex> Guard(AC.Lock); 83 NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations"); 84 if (!NMD) 85 return; 86 key_val_pair_t tmp; 87 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { 88 const MDNode *elem = NMD->getOperand(i); 89 90 GlobalValue *entity = 91 mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0)); 92 // entity may be null due to DCE 93 if (!entity) 94 continue; 95 if (entity != gv) 96 continue; 97 98 // accumulate annotations for entity in tmp 99 cacheAnnotationFromMD(elem, tmp); 100 } 101 102 if (tmp.empty()) // no annotations for this gv 103 return; 104 105 if (AC.Cache.find(m) != AC.Cache.end()) 106 AC.Cache[m][gv] = std::move(tmp); 107 else { 108 global_val_annot_t tmp1; 109 tmp1[gv] = std::move(tmp); 110 AC.Cache[m] = std::move(tmp1); 111 } 112} 113 114bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop, 115 unsigned &retval) { 116 auto &AC = getAnnotationCache(); 117 std::lock_guard<sys::Mutex> Guard(AC.Lock); 118 const Module *m = gv->getParent(); 119 if (AC.Cache.find(m) == AC.Cache.end()) 120 cacheAnnotationFromMD(m, gv); 121 else if (AC.Cache[m].find(gv) == AC.Cache[m].end()) 122 cacheAnnotationFromMD(m, gv); 123 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end()) 124 return false; 125 retval = AC.Cache[m][gv][prop][0]; 126 return true; 127} 128 129bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop, 130 std::vector<unsigned> &retval) { 131 auto &AC = getAnnotationCache(); 132 std::lock_guard<sys::Mutex> Guard(AC.Lock); 133 const Module *m = gv->getParent(); 134 if (AC.Cache.find(m) == AC.Cache.end()) 135 cacheAnnotationFromMD(m, gv); 136 else if (AC.Cache[m].find(gv) == AC.Cache[m].end()) 137 cacheAnnotationFromMD(m, gv); 138 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end()) 139 return false; 140 retval = AC.Cache[m][gv][prop]; 141 return true; 142} 143 144bool isTexture(const Value &val) { 145 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 146 unsigned annot; 147 if (findOneNVVMAnnotation(gv, "texture", annot)) { 148 assert((annot == 1) && "Unexpected annotation on a texture symbol"); 149 return true; 150 } 151 } 152 return false; 153} 154 155bool isSurface(const Value &val) { 156 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 157 unsigned annot; 158 if (findOneNVVMAnnotation(gv, "surface", annot)) { 159 assert((annot == 1) && "Unexpected annotation on a surface symbol"); 160 return true; 161 } 162 } 163 return false; 164} 165 166bool isSampler(const Value &val) { 167 const char *AnnotationName = "sampler"; 168 169 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 170 unsigned annot; 171 if (findOneNVVMAnnotation(gv, AnnotationName, annot)) { 172 assert((annot == 1) && "Unexpected annotation on a sampler symbol"); 173 return true; 174 } 175 } 176 if (const Argument *arg = dyn_cast<Argument>(&val)) { 177 const Function *func = arg->getParent(); 178 std::vector<unsigned> annot; 179 if (findAllNVVMAnnotation(func, AnnotationName, annot)) { 180 if (is_contained(annot, arg->getArgNo())) 181 return true; 182 } 183 } 184 return false; 185} 186 187bool isImageReadOnly(const Value &val) { 188 if (const Argument *arg = dyn_cast<Argument>(&val)) { 189 const Function *func = arg->getParent(); 190 std::vector<unsigned> annot; 191 if (findAllNVVMAnnotation(func, "rdoimage", annot)) { 192 if (is_contained(annot, arg->getArgNo())) 193 return true; 194 } 195 } 196 return false; 197} 198 199bool isImageWriteOnly(const Value &val) { 200 if (const Argument *arg = dyn_cast<Argument>(&val)) { 201 const Function *func = arg->getParent(); 202 std::vector<unsigned> annot; 203 if (findAllNVVMAnnotation(func, "wroimage", annot)) { 204 if (is_contained(annot, arg->getArgNo())) 205 return true; 206 } 207 } 208 return false; 209} 210 211bool isImageReadWrite(const Value &val) { 212 if (const Argument *arg = dyn_cast<Argument>(&val)) { 213 const Function *func = arg->getParent(); 214 std::vector<unsigned> annot; 215 if (findAllNVVMAnnotation(func, "rdwrimage", annot)) { 216 if (is_contained(annot, arg->getArgNo())) 217 return true; 218 } 219 } 220 return false; 221} 222 223bool isImage(const Value &val) { 224 return isImageReadOnly(val) || isImageWriteOnly(val) || isImageReadWrite(val); 225} 226 227bool isManaged(const Value &val) { 228 if(const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 229 unsigned annot; 230 if (findOneNVVMAnnotation(gv, "managed", annot)) { 231 assert((annot == 1) && "Unexpected annotation on a managed symbol"); 232 return true; 233 } 234 } 235 return false; 236} 237 238std::string getTextureName(const Value &val) { 239 assert(val.hasName() && "Found texture variable with no name"); 240 return std::string(val.getName()); 241} 242 243std::string getSurfaceName(const Value &val) { 244 assert(val.hasName() && "Found surface variable with no name"); 245 return std::string(val.getName()); 246} 247 248std::string getSamplerName(const Value &val) { 249 assert(val.hasName() && "Found sampler variable with no name"); 250 return std::string(val.getName()); 251} 252 253bool getMaxNTIDx(const Function &F, unsigned &x) { 254 return findOneNVVMAnnotation(&F, "maxntidx", x); 255} 256 257bool getMaxNTIDy(const Function &F, unsigned &y) { 258 return findOneNVVMAnnotation(&F, "maxntidy", y); 259} 260 261bool getMaxNTIDz(const Function &F, unsigned &z) { 262 return findOneNVVMAnnotation(&F, "maxntidz", z); 263} 264 265bool getMaxClusterRank(const Function &F, unsigned &x) { 266 return findOneNVVMAnnotation(&F, "maxclusterrank", x); 267} 268 269bool getReqNTIDx(const Function &F, unsigned &x) { 270 return findOneNVVMAnnotation(&F, "reqntidx", x); 271} 272 273bool getReqNTIDy(const Function &F, unsigned &y) { 274 return findOneNVVMAnnotation(&F, "reqntidy", y); 275} 276 277bool getReqNTIDz(const Function &F, unsigned &z) { 278 return findOneNVVMAnnotation(&F, "reqntidz", z); 279} 280 281bool getMinCTASm(const Function &F, unsigned &x) { 282 return findOneNVVMAnnotation(&F, "minctasm", x); 283} 284 285bool getMaxNReg(const Function &F, unsigned &x) { 286 return findOneNVVMAnnotation(&F, "maxnreg", x); 287} 288 289bool isKernelFunction(const Function &F) { 290 unsigned x = 0; 291 bool retval = findOneNVVMAnnotation(&F, "kernel", x); 292 if (!retval) { 293 // There is no NVVM metadata, check the calling convention 294 return F.getCallingConv() == CallingConv::PTX_Kernel; 295 } 296 return (x == 1); 297} 298 299bool getAlign(const Function &F, unsigned index, unsigned &align) { 300 std::vector<unsigned> Vs; 301 bool retval = findAllNVVMAnnotation(&F, "align", Vs); 302 if (!retval) 303 return false; 304 for (unsigned v : Vs) { 305 if ((v >> 16) == index) { 306 align = v & 0xFFFF; 307 return true; 308 } 309 } 310 return false; 311} 312 313bool getAlign(const CallInst &I, unsigned index, unsigned &align) { 314 if (MDNode *alignNode = I.getMetadata("callalign")) { 315 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) { 316 if (const ConstantInt *CI = 317 mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) { 318 unsigned v = CI->getZExtValue(); 319 if ((v >> 16) == index) { 320 align = v & 0xFFFF; 321 return true; 322 } 323 if ((v >> 16) > index) { 324 return false; 325 } 326 } 327 } 328 } 329 return false; 330} 331 332Function *getMaybeBitcastedCallee(const CallBase *CB) { 333 return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts()); 334} 335 336bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) { 337 const auto &ST = 338 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl(); 339 if (!ST.hasNoReturn()) 340 return false; 341 342 assert((isa<Function>(V) || isa<CallInst>(V)) && 343 "Expect either a call instruction or a function"); 344 345 if (const CallInst *CallI = dyn_cast<CallInst>(V)) 346 return CallI->doesNotReturn() && 347 CallI->getFunctionType()->getReturnType()->isVoidTy(); 348 349 const Function *F = cast<Function>(V); 350 return F->doesNotReturn() && 351 F->getFunctionType()->getReturnType()->isVoidTy() && 352 !isKernelFunction(*F); 353} 354 355bool Isv2x16VT(EVT VT) { 356 return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16); 357} 358 359} // namespace llvm 360