LowerExpectIntrinsic.cpp revision 249423
1//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This pass lowers the 'expect' intrinsic to LLVM metadata. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "lower-expect-intrinsic" 15#include "llvm/Transforms/Scalar.h" 16#include "llvm/ADT/Statistic.h" 17#include "llvm/IR/BasicBlock.h" 18#include "llvm/IR/Constants.h" 19#include "llvm/IR/Function.h" 20#include "llvm/IR/Instructions.h" 21#include "llvm/IR/Intrinsics.h" 22#include "llvm/IR/LLVMContext.h" 23#include "llvm/IR/MDBuilder.h" 24#include "llvm/IR/Metadata.h" 25#include "llvm/Pass.h" 26#include "llvm/Support/CommandLine.h" 27#include "llvm/Support/Debug.h" 28#include <vector> 29 30using namespace llvm; 31 32STATISTIC(IfHandled, "Number of 'expect' intrinsic intructions handled"); 33 34static cl::opt<uint32_t> 35LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), 36 cl::desc("Weight of the branch likely to be taken (default = 64)")); 37static cl::opt<uint32_t> 38UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), 39 cl::desc("Weight of the branch unlikely to be taken (default = 4)")); 40 41namespace { 42 43 class LowerExpectIntrinsic : public FunctionPass { 44 45 bool HandleSwitchExpect(SwitchInst *SI); 46 47 bool HandleIfExpect(BranchInst *BI); 48 49 public: 50 static char ID; 51 LowerExpectIntrinsic() : FunctionPass(ID) { 52 initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); 53 } 54 55 bool runOnFunction(Function &F); 56 }; 57} 58 59 60bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) { 61 CallInst *CI = dyn_cast<CallInst>(SI->getCondition()); 62 if (!CI) 63 return false; 64 65 Function *Fn = CI->getCalledFunction(); 66 if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 67 return false; 68 69 Value *ArgValue = CI->getArgOperand(0); 70 ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 71 if (!ExpectedValue) 72 return false; 73 74 SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue); 75 unsigned n = SI->getNumCases(); // +1 for default case. 76 std::vector<uint32_t> Weights(n + 1); 77 78 Weights[0] = Case == SI->case_default() ? LikelyBranchWeight 79 : UnlikelyBranchWeight; 80 for (unsigned i = 0; i != n; ++i) 81 Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight 82 : UnlikelyBranchWeight; 83 84 SI->setMetadata(LLVMContext::MD_prof, 85 MDBuilder(CI->getContext()).createBranchWeights(Weights)); 86 87 SI->setCondition(ArgValue); 88 return true; 89} 90 91 92bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) { 93 if (BI->isUnconditional()) 94 return false; 95 96 // Handle non-optimized IR code like: 97 // %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1) 98 // %tobool = icmp ne i64 %expval, 0 99 // br i1 %tobool, label %if.then, label %if.end 100 101 ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition()); 102 if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE) 103 return false; 104 105 CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0)); 106 if (!CI) 107 return false; 108 109 Function *Fn = CI->getCalledFunction(); 110 if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 111 return false; 112 113 Value *ArgValue = CI->getArgOperand(0); 114 ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 115 if (!ExpectedValue) 116 return false; 117 118 MDBuilder MDB(CI->getContext()); 119 MDNode *Node; 120 121 // If expect value is equal to 1 it means that we are more likely to take 122 // branch 0, in other case more likely is branch 1. 123 if (ExpectedValue->isOne()) 124 Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); 125 else 126 Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); 127 128 BI->setMetadata(LLVMContext::MD_prof, Node); 129 130 CmpI->setOperand(0, ArgValue); 131 return true; 132} 133 134 135bool LowerExpectIntrinsic::runOnFunction(Function &F) { 136 for (Function::iterator I = F.begin(), E = F.end(); I != E;) { 137 BasicBlock *BB = I++; 138 139 // Create "block_weights" metadata. 140 if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) { 141 if (HandleIfExpect(BI)) 142 IfHandled++; 143 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) { 144 if (HandleSwitchExpect(SI)) 145 IfHandled++; 146 } 147 148 // remove llvm.expect intrinsics. 149 for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); 150 BI != BE; ) { 151 CallInst *CI = dyn_cast<CallInst>(BI++); 152 if (!CI) 153 continue; 154 155 Function *Fn = CI->getCalledFunction(); 156 if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { 157 Value *Exp = CI->getArgOperand(0); 158 CI->replaceAllUsesWith(Exp); 159 CI->eraseFromParent(); 160 } 161 } 162 } 163 164 return false; 165} 166 167 168char LowerExpectIntrinsic::ID = 0; 169INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' " 170 "Intrinsics", false, false) 171 172FunctionPass *llvm::createLowerExpectIntrinsicPass() { 173 return new LowerExpectIntrinsic(); 174} 175