LowerExpectIntrinsic.cpp revision 226633
1263508Sdim#define DEBUG_TYPE "lower-expect-intrinsic" 2249259Sdim#include "llvm/Constants.h" 3249259Sdim#include "llvm/Function.h" 4249259Sdim#include "llvm/BasicBlock.h" 5249259Sdim#include "llvm/LLVMContext.h" 6249259Sdim#include "llvm/Instructions.h" 7249259Sdim#include "llvm/Intrinsics.h" 8249259Sdim#include "llvm/Metadata.h" 9249259Sdim#include "llvm/Pass.h" 10249259Sdim#include "llvm/Transforms/Scalar.h" 11249259Sdim#include "llvm/Support/CommandLine.h" 12249259Sdim#include "llvm/Support/Debug.h" 13249259Sdim#include "llvm/ADT/Statistic.h" 14249259Sdim#include <vector> 15249259Sdim 16249259Sdimusing namespace llvm; 17249259Sdim 18249259SdimSTATISTIC(IfHandled, "Number of 'expect' intrinsic intructions handled"); 19249259Sdim 20249259Sdimstatic cl::opt<uint32_t> 21249259SdimLikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), 22249259Sdim cl::desc("Weight of the branch likely to be taken (default = 64)")); 23249259Sdimstatic cl::opt<uint32_t> 24249259SdimUnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), 25249259Sdim cl::desc("Weight of the branch unlikely to be taken (default = 4)")); 26249259Sdim 27249259Sdimnamespace { 28249259Sdim 29249259Sdim class LowerExpectIntrinsic : public FunctionPass { 30249259Sdim 31249259Sdim bool HandleSwitchExpect(SwitchInst *SI); 32249259Sdim 33249259Sdim bool HandleIfExpect(BranchInst *BI); 34249259Sdim 35249259Sdim public: 36249259Sdim static char ID; 37249259Sdim LowerExpectIntrinsic() : FunctionPass(ID) { 38249259Sdim initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); 39249259Sdim } 40249259Sdim 41249259Sdim bool runOnFunction(Function &F); 42249259Sdim }; 43249259Sdim} 44249259Sdim 45249259Sdim 46249259Sdimbool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) { 47249259Sdim CallInst *CI = dyn_cast<CallInst>(SI->getCondition()); 48249259Sdim if (!CI) 49249259Sdim return false; 50249259Sdim 51249259Sdim Function *Fn = CI->getCalledFunction(); 52249259Sdim if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 53249259Sdim return false; 54249259Sdim 55249259Sdim Value *ArgValue = CI->getArgOperand(0); 56249259Sdim ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 57249259Sdim if (!ExpectedValue) 58249259Sdim return false; 59249259Sdim 60249259Sdim LLVMContext &Context = CI->getContext(); 61249259Sdim Type *Int32Ty = Type::getInt32Ty(Context); 62249259Sdim 63249259Sdim unsigned caseNo = SI->findCaseValue(ExpectedValue); 64249259Sdim std::vector<Value *> Vec; 65249259Sdim unsigned n = SI->getNumCases(); 66249259Sdim Vec.resize(n + 1); // +1 for MDString 67249259Sdim 68249259Sdim Vec[0] = MDString::get(Context, "branch_weights"); 69249259Sdim for (unsigned i = 0; i < n; ++i) { 70249259Sdim Vec[i + 1] = ConstantInt::get(Int32Ty, i == caseNo ? LikelyBranchWeight : UnlikelyBranchWeight); 71249259Sdim } 72249259Sdim 73249259Sdim MDNode *WeightsNode = llvm::MDNode::get(Context, Vec); 74249259Sdim SI->setMetadata(LLVMContext::MD_prof, WeightsNode); 75249259Sdim 76249259Sdim SI->setCondition(ArgValue); 77249259Sdim return true; 78249259Sdim} 79249259Sdim 80249259Sdim 81249259Sdimbool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) { 82249259Sdim if (BI->isUnconditional()) 83249259Sdim return false; 84249259Sdim 85249259Sdim // Handle non-optimized IR code like: 86249259Sdim // %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1) 87249259Sdim // %tobool = icmp ne i64 %expval, 0 88249259Sdim // br i1 %tobool, label %if.then, label %if.end 89249259Sdim 90249259Sdim ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition()); 91249259Sdim if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE) 92249259Sdim return false; 93249259Sdim 94249259Sdim CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0)); 95249259Sdim if (!CI) 96249259Sdim return false; 97249259Sdim 98249259Sdim Function *Fn = CI->getCalledFunction(); 99249259Sdim if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 100249259Sdim return false; 101249259Sdim 102249259Sdim Value *ArgValue = CI->getArgOperand(0); 103249259Sdim ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 104249259Sdim if (!ExpectedValue) 105249259Sdim return false; 106249259Sdim 107249259Sdim LLVMContext &Context = CI->getContext(); 108249259Sdim Type *Int32Ty = Type::getInt32Ty(Context); 109249259Sdim bool Likely = ExpectedValue->isOne(); 110249259Sdim 111249259Sdim // If expect value is equal to 1 it means that we are more likely to take 112249259Sdim // branch 0, in other case more likely is branch 1. 113249259Sdim Value *Ops[] = { 114249259Sdim MDString::get(Context, "branch_weights"), 115249259Sdim ConstantInt::get(Int32Ty, Likely ? LikelyBranchWeight : UnlikelyBranchWeight), 116249259Sdim ConstantInt::get(Int32Ty, Likely ? UnlikelyBranchWeight : LikelyBranchWeight) 117249259Sdim }; 118249259Sdim 119249259Sdim MDNode *WeightsNode = MDNode::get(Context, Ops); 120249259Sdim BI->setMetadata(LLVMContext::MD_prof, WeightsNode); 121249259Sdim 122249259Sdim CmpI->setOperand(0, ArgValue); 123249259Sdim return true; 124249259Sdim} 125249259Sdim 126249259Sdim 127249259Sdimbool LowerExpectIntrinsic::runOnFunction(Function &F) { 128249259Sdim for (Function::iterator I = F.begin(), E = F.end(); I != E;) { 129249259Sdim BasicBlock *BB = I++; 130249259Sdim 131249259Sdim // Create "block_weights" metadata. 132249259Sdim if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) { 133249259Sdim if (HandleIfExpect(BI)) 134249259Sdim IfHandled++; 135249259Sdim } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) { 136249259Sdim if (HandleSwitchExpect(SI)) 137249259Sdim IfHandled++; 138249259Sdim } 139249259Sdim 140249259Sdim // remove llvm.expect intrinsics. 141249259Sdim for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); 142249259Sdim BI != BE; ) { 143249259Sdim CallInst *CI = dyn_cast<CallInst>(BI++); 144249259Sdim if (!CI) 145249259Sdim continue; 146249259Sdim 147249259Sdim Function *Fn = CI->getCalledFunction(); 148249259Sdim if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { 149249259Sdim Value *Exp = CI->getArgOperand(0); 150249259Sdim CI->replaceAllUsesWith(Exp); 151249259Sdim CI->eraseFromParent(); 152249259Sdim } 153249259Sdim } 154249259Sdim } 155249259Sdim 156249259Sdim return false; 157249259Sdim} 158249259Sdim 159249259Sdim 160249259Sdimchar LowerExpectIntrinsic::ID = 0; 161249259SdimINITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' " 162249259Sdim "Intrinsics", false, false) 163249259Sdim 164249259SdimFunctionPass *llvm::createLowerExpectIntrinsicPass() { 165249259Sdim return new LowerExpectIntrinsic(); 166249259Sdim} 167249259Sdim