1249259Sdim//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 2249259Sdim// 3249259Sdim// The LLVM Compiler Infrastructure 4249259Sdim// 5249259Sdim// This file is distributed under the University of Illinois Open Source 6249259Sdim// License. See LICENSE.TXT for details. 7249259Sdim// 8249259Sdim//===----------------------------------------------------------------------===// 9249259Sdim// 10249259Sdim// This file implements the auto-upgrade helper functions 11249259Sdim// 12249259Sdim//===----------------------------------------------------------------------===// 13249259Sdim 14249259Sdim#include "llvm/AutoUpgrade.h" 15249259Sdim#include "llvm/IR/Constants.h" 16249259Sdim#include "llvm/IR/Function.h" 17249259Sdim#include "llvm/IR/IRBuilder.h" 18249259Sdim#include "llvm/IR/Instruction.h" 19249259Sdim#include "llvm/IR/IntrinsicInst.h" 20249259Sdim#include "llvm/IR/LLVMContext.h" 21249259Sdim#include "llvm/IR/Module.h" 22249259Sdim#include "llvm/Support/CFG.h" 23249259Sdim#include "llvm/Support/CallSite.h" 24249259Sdim#include "llvm/Support/ErrorHandling.h" 25249259Sdim#include <cstring> 26249259Sdimusing namespace llvm; 27249259Sdim 28249259Sdim// Upgrade the declarations of the SSE4.1 functions whose arguments have 29249259Sdim// changed their type from v4f32 to v2i64. 30249259Sdimstatic bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, 31249259Sdim Function *&NewFn) { 32249259Sdim // Check whether this is an old version of the function, which received 33249259Sdim // v4f32 arguments. 34249259Sdim Type *Arg0Type = F->getFunctionType()->getParamType(0); 35249259Sdim if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 36249259Sdim return false; 37249259Sdim 38249259Sdim // Yes, it's old, replace it with new version. 39249259Sdim F->setName(F->getName() + ".old"); 40249259Sdim NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 41249259Sdim return true; 42249259Sdim} 43249259Sdim 44249259Sdimstatic bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 45249259Sdim assert(F && "Illegal to upgrade a non-existent Function."); 46249259Sdim 47249259Sdim // Quickly eliminate it, if it's not a candidate. 48249259Sdim StringRef Name = F->getName(); 49249259Sdim if (Name.size() <= 8 || !Name.startswith("llvm.")) 50249259Sdim return false; 51249259Sdim Name = Name.substr(5); // Strip off "llvm." 52249259Sdim 53249259Sdim switch (Name[0]) { 54249259Sdim default: break; 55249259Sdim case 'a': { 56249259Sdim if (Name.startswith("arm.neon.vclz")) { 57249259Sdim Type* args[2] = { 58249259Sdim F->arg_begin()->getType(), 59249259Sdim Type::getInt1Ty(F->getContext()) 60249259Sdim }; 61249259Sdim // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 62249259Sdim // the end of the name. Change name from llvm.arm.neon.vclz.* to 63249259Sdim // llvm.ctlz.* 64249259Sdim FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 65249259Sdim NewFn = Function::Create(fType, F->getLinkage(), 66249259Sdim "llvm.ctlz." + Name.substr(14), F->getParent()); 67249259Sdim return true; 68249259Sdim } 69249259Sdim if (Name.startswith("arm.neon.vcnt")) { 70249259Sdim NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 71249259Sdim F->arg_begin()->getType()); 72249259Sdim return true; 73249259Sdim } 74249259Sdim break; 75249259Sdim } 76249259Sdim case 'c': { 77249259Sdim if (Name.startswith("ctlz.") && F->arg_size() == 1) { 78249259Sdim F->setName(Name + ".old"); 79249259Sdim NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 80249259Sdim F->arg_begin()->getType()); 81249259Sdim return true; 82249259Sdim } 83249259Sdim if (Name.startswith("cttz.") && F->arg_size() == 1) { 84249259Sdim F->setName(Name + ".old"); 85249259Sdim NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 86249259Sdim F->arg_begin()->getType()); 87249259Sdim return true; 88249259Sdim } 89249259Sdim break; 90249259Sdim } 91249259Sdim case 'x': { 92249259Sdim if (Name.startswith("x86.sse2.pcmpeq.") || 93249259Sdim Name.startswith("x86.sse2.pcmpgt.") || 94249259Sdim Name.startswith("x86.avx2.pcmpeq.") || 95249259Sdim Name.startswith("x86.avx2.pcmpgt.") || 96249259Sdim Name.startswith("x86.avx.vpermil.") || 97249259Sdim Name == "x86.avx.movnt.dq.256" || 98249259Sdim Name == "x86.avx.movnt.pd.256" || 99249259Sdim Name == "x86.avx.movnt.ps.256" || 100249259Sdim (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { 101249259Sdim NewFn = 0; 102249259Sdim return true; 103249259Sdim } 104249259Sdim // SSE4.1 ptest functions may have an old signature. 105249259Sdim if (Name.startswith("x86.sse41.ptest")) { 106249259Sdim if (Name == "x86.sse41.ptestc") 107249259Sdim return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); 108249259Sdim if (Name == "x86.sse41.ptestz") 109249259Sdim return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn); 110249259Sdim if (Name == "x86.sse41.ptestnzc") 111249259Sdim return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 112249259Sdim } 113249259Sdim // frcz.ss/sd may need to have an argument dropped 114249259Sdim if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) { 115249259Sdim F->setName(Name + ".old"); 116249259Sdim NewFn = Intrinsic::getDeclaration(F->getParent(), 117249259Sdim Intrinsic::x86_xop_vfrcz_ss); 118249259Sdim return true; 119249259Sdim } 120249259Sdim if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) { 121249259Sdim F->setName(Name + ".old"); 122249259Sdim NewFn = Intrinsic::getDeclaration(F->getParent(), 123249259Sdim Intrinsic::x86_xop_vfrcz_sd); 124249259Sdim return true; 125249259Sdim } 126249259Sdim // Fix the FMA4 intrinsics to remove the 4 127249259Sdim if (Name.startswith("x86.fma4.")) { 128249259Sdim F->setName("llvm.x86.fma" + Name.substr(8)); 129249259Sdim NewFn = F; 130249259Sdim return true; 131249259Sdim } 132249259Sdim break; 133249259Sdim } 134249259Sdim } 135249259Sdim 136249259Sdim // This may not belong here. This function is effectively being overloaded 137249259Sdim // to both detect an intrinsic which needs upgrading, and to provide the 138249259Sdim // upgraded form of the intrinsic. We should perhaps have two separate 139249259Sdim // functions for this. 140249259Sdim return false; 141249259Sdim} 142249259Sdim 143249259Sdimbool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 144249259Sdim NewFn = 0; 145249259Sdim bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 146249259Sdim 147249259Sdim // Upgrade intrinsic attributes. This does not change the function. 148249259Sdim if (NewFn) 149249259Sdim F = NewFn; 150249259Sdim if (unsigned id = F->getIntrinsicID()) 151249259Sdim F->setAttributes(Intrinsic::getAttributes(F->getContext(), 152249259Sdim (Intrinsic::ID)id)); 153249259Sdim return Upgraded; 154249259Sdim} 155249259Sdim 156249259Sdimbool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 157249259Sdim // Nothing to do yet. 158249259Sdim return false; 159249259Sdim} 160249259Sdim 161249259Sdim// UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the 162249259Sdim// upgraded intrinsic. All argument and return casting must be provided in 163249259Sdim// order to seamlessly integrate with existing context. 164249259Sdimvoid llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 165249259Sdim Function *F = CI->getCalledFunction(); 166249259Sdim LLVMContext &C = CI->getContext(); 167249259Sdim IRBuilder<> Builder(C); 168249259Sdim Builder.SetInsertPoint(CI->getParent(), CI); 169249259Sdim 170249259Sdim assert(F && "Intrinsic call is not direct?"); 171249259Sdim 172249259Sdim if (!NewFn) { 173249259Sdim // Get the Function's name. 174249259Sdim StringRef Name = F->getName(); 175249259Sdim 176249259Sdim Value *Rep; 177249259Sdim // Upgrade packed integer vector compares intrinsics to compare instructions 178249259Sdim if (Name.startswith("llvm.x86.sse2.pcmpeq.") || 179249259Sdim Name.startswith("llvm.x86.avx2.pcmpeq.")) { 180249259Sdim Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), 181249259Sdim "pcmpeq"); 182249259Sdim // need to sign extend since icmp returns vector of i1 183249259Sdim Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 184249259Sdim } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") || 185249259Sdim Name.startswith("llvm.x86.avx2.pcmpgt.")) { 186249259Sdim Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), 187249259Sdim "pcmpgt"); 188249259Sdim // need to sign extend since icmp returns vector of i1 189249259Sdim Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 190249259Sdim } else if (Name == "llvm.x86.avx.movnt.dq.256" || 191249259Sdim Name == "llvm.x86.avx.movnt.ps.256" || 192249259Sdim Name == "llvm.x86.avx.movnt.pd.256") { 193249259Sdim IRBuilder<> Builder(C); 194249259Sdim Builder.SetInsertPoint(CI->getParent(), CI); 195249259Sdim 196249259Sdim Module *M = F->getParent(); 197249259Sdim SmallVector<Value *, 1> Elts; 198249259Sdim Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1)); 199249259Sdim MDNode *Node = MDNode::get(C, Elts); 200249259Sdim 201249259Sdim Value *Arg0 = CI->getArgOperand(0); 202249259Sdim Value *Arg1 = CI->getArgOperand(1); 203249259Sdim 204249259Sdim // Convert the type of the pointer to a pointer to the stored type. 205249259Sdim Value *BC = Builder.CreateBitCast(Arg0, 206249259Sdim PointerType::getUnqual(Arg1->getType()), 207249259Sdim "cast"); 208249259Sdim StoreInst *SI = Builder.CreateStore(Arg1, BC); 209249259Sdim SI->setMetadata(M->getMDKindID("nontemporal"), Node); 210249259Sdim SI->setAlignment(16); 211249259Sdim 212249259Sdim // Remove intrinsic. 213249259Sdim CI->eraseFromParent(); 214249259Sdim return; 215249259Sdim } else if (Name.startswith("llvm.x86.xop.vpcom")) { 216249259Sdim Intrinsic::ID intID; 217249259Sdim if (Name.endswith("ub")) 218249259Sdim intID = Intrinsic::x86_xop_vpcomub; 219249259Sdim else if (Name.endswith("uw")) 220249259Sdim intID = Intrinsic::x86_xop_vpcomuw; 221249259Sdim else if (Name.endswith("ud")) 222249259Sdim intID = Intrinsic::x86_xop_vpcomud; 223249259Sdim else if (Name.endswith("uq")) 224249259Sdim intID = Intrinsic::x86_xop_vpcomuq; 225249259Sdim else if (Name.endswith("b")) 226249259Sdim intID = Intrinsic::x86_xop_vpcomb; 227249259Sdim else if (Name.endswith("w")) 228249259Sdim intID = Intrinsic::x86_xop_vpcomw; 229249259Sdim else if (Name.endswith("d")) 230249259Sdim intID = Intrinsic::x86_xop_vpcomd; 231249259Sdim else if (Name.endswith("q")) 232249259Sdim intID = Intrinsic::x86_xop_vpcomq; 233249259Sdim else 234249259Sdim llvm_unreachable("Unknown suffix"); 235249259Sdim 236249259Sdim Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom" 237249259Sdim unsigned Imm; 238249259Sdim if (Name.startswith("lt")) 239249259Sdim Imm = 0; 240249259Sdim else if (Name.startswith("le")) 241249259Sdim Imm = 1; 242249259Sdim else if (Name.startswith("gt")) 243249259Sdim Imm = 2; 244249259Sdim else if (Name.startswith("ge")) 245249259Sdim Imm = 3; 246249259Sdim else if (Name.startswith("eq")) 247249259Sdim Imm = 4; 248249259Sdim else if (Name.startswith("ne")) 249249259Sdim Imm = 5; 250249259Sdim else if (Name.startswith("true")) 251249259Sdim Imm = 6; 252249259Sdim else if (Name.startswith("false")) 253249259Sdim Imm = 7; 254249259Sdim else 255249259Sdim llvm_unreachable("Unknown condition"); 256249259Sdim 257249259Sdim Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID); 258249259Sdim Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0), 259249259Sdim CI->getArgOperand(1), Builder.getInt8(Imm)); 260249259Sdim } else { 261249259Sdim bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; 262249259Sdim if (Name == "llvm.x86.avx.vpermil.pd.256") 263249259Sdim PD256 = true; 264249259Sdim else if (Name == "llvm.x86.avx.vpermil.pd") 265249259Sdim PD128 = true; 266249259Sdim else if (Name == "llvm.x86.avx.vpermil.ps.256") 267249259Sdim PS256 = true; 268249259Sdim else if (Name == "llvm.x86.avx.vpermil.ps") 269249259Sdim PS128 = true; 270249259Sdim 271249259Sdim if (PD256 || PD128 || PS256 || PS128) { 272249259Sdim Value *Op0 = CI->getArgOperand(0); 273249259Sdim unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 274249259Sdim SmallVector<Constant*, 8> Idxs; 275249259Sdim 276249259Sdim if (PD128) 277249259Sdim for (unsigned i = 0; i != 2; ++i) 278249259Sdim Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1)); 279249259Sdim else if (PD256) 280249259Sdim for (unsigned l = 0; l != 4; l+=2) 281249259Sdim for (unsigned i = 0; i != 2; ++i) 282249259Sdim Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l)); 283249259Sdim else if (PS128) 284249259Sdim for (unsigned i = 0; i != 4; ++i) 285249259Sdim Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3)); 286249259Sdim else if (PS256) 287249259Sdim for (unsigned l = 0; l != 8; l+=4) 288249259Sdim for (unsigned i = 0; i != 4; ++i) 289249259Sdim Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l)); 290249259Sdim else 291249259Sdim llvm_unreachable("Unexpected function"); 292249259Sdim 293249259Sdim Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs)); 294249259Sdim } else { 295249259Sdim llvm_unreachable("Unknown function for CallInst upgrade."); 296249259Sdim } 297249259Sdim } 298249259Sdim 299249259Sdim CI->replaceAllUsesWith(Rep); 300249259Sdim CI->eraseFromParent(); 301249259Sdim return; 302249259Sdim } 303249259Sdim 304249259Sdim std::string Name = CI->getName().str(); 305249259Sdim CI->setName(Name + ".old"); 306249259Sdim 307249259Sdim switch (NewFn->getIntrinsicID()) { 308249259Sdim default: 309249259Sdim llvm_unreachable("Unknown function for CallInst upgrade."); 310249259Sdim 311249259Sdim case Intrinsic::ctlz: 312249259Sdim case Intrinsic::cttz: 313249259Sdim assert(CI->getNumArgOperands() == 1 && 314249259Sdim "Mismatch between function args and call args"); 315249259Sdim CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 316249259Sdim Builder.getFalse(), Name)); 317249259Sdim CI->eraseFromParent(); 318249259Sdim return; 319249259Sdim 320249259Sdim case Intrinsic::arm_neon_vclz: { 321249259Sdim // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.* 322249259Sdim CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 323249259Sdim Builder.getFalse(), 324249259Sdim "llvm.ctlz." + Name.substr(14))); 325249259Sdim CI->eraseFromParent(); 326249259Sdim return; 327249259Sdim } 328249259Sdim case Intrinsic::ctpop: { 329249259Sdim CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0))); 330249259Sdim CI->eraseFromParent(); 331249259Sdim return; 332249259Sdim } 333249259Sdim 334249259Sdim case Intrinsic::x86_xop_vfrcz_ss: 335249259Sdim case Intrinsic::x86_xop_vfrcz_sd: 336249259Sdim CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1), 337249259Sdim Name)); 338249259Sdim CI->eraseFromParent(); 339249259Sdim return; 340249259Sdim 341249259Sdim case Intrinsic::x86_sse41_ptestc: 342249259Sdim case Intrinsic::x86_sse41_ptestz: 343249259Sdim case Intrinsic::x86_sse41_ptestnzc: { 344249259Sdim // The arguments for these intrinsics used to be v4f32, and changed 345249259Sdim // to v2i64. This is purely a nop, since those are bitwise intrinsics. 346249259Sdim // So, the only thing required is a bitcast for both arguments. 347249259Sdim // First, check the arguments have the old type. 348249259Sdim Value *Arg0 = CI->getArgOperand(0); 349249259Sdim if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 350249259Sdim return; 351249259Sdim 352249259Sdim // Old intrinsic, add bitcasts 353249259Sdim Value *Arg1 = CI->getArgOperand(1); 354249259Sdim 355249259Sdim Value *BC0 = 356249259Sdim Builder.CreateBitCast(Arg0, 357249259Sdim VectorType::get(Type::getInt64Ty(C), 2), 358249259Sdim "cast"); 359249259Sdim Value *BC1 = 360249259Sdim Builder.CreateBitCast(Arg1, 361249259Sdim VectorType::get(Type::getInt64Ty(C), 2), 362249259Sdim "cast"); 363249259Sdim 364249259Sdim CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name); 365249259Sdim CI->replaceAllUsesWith(NewCall); 366249259Sdim CI->eraseFromParent(); 367249259Sdim return; 368249259Sdim } 369249259Sdim } 370249259Sdim} 371249259Sdim 372249259Sdim// This tests each Function to determine if it needs upgrading. When we find 373249259Sdim// one we are interested in, we then upgrade all calls to reflect the new 374249259Sdim// function. 375249259Sdimvoid llvm::UpgradeCallsToIntrinsic(Function* F) { 376249259Sdim assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 377249259Sdim 378249259Sdim // Upgrade the function and check if it is a totaly new function. 379249259Sdim Function *NewFn; 380249259Sdim if (UpgradeIntrinsicFunction(F, NewFn)) { 381249259Sdim if (NewFn != F) { 382249259Sdim // Replace all uses to the old function with the new one if necessary. 383249259Sdim for (Value::use_iterator UI = F->use_begin(), UE = F->use_end(); 384249259Sdim UI != UE; ) { 385249259Sdim if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 386249259Sdim UpgradeIntrinsicCall(CI, NewFn); 387249259Sdim } 388249259Sdim // Remove old function, no longer used, from the module. 389249259Sdim F->eraseFromParent(); 390249259Sdim } 391249259Sdim } 392249259Sdim} 393249259Sdim 394