1234353Sdim//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// 2234353Sdim// 3234353Sdim// The LLVM Compiler Infrastructure 4234353Sdim// 5234353Sdim// This file is distributed under the University of Illinois Open Source 6234353Sdim// License. See LICENSE.TXT for details. 7234353Sdim// 8234353Sdim//===----------------------------------------------------------------------===// 9234353Sdim// 10234353Sdim// This pass lowers the 'expect' intrinsic to LLVM metadata. 11234353Sdim// 12234353Sdim//===----------------------------------------------------------------------===// 13234353Sdim 14224133Sdim#define DEBUG_TYPE "lower-expect-intrinsic" 15249423Sdim#include "llvm/Transforms/Scalar.h" 16249423Sdim#include "llvm/ADT/Statistic.h" 17249423Sdim#include "llvm/IR/BasicBlock.h" 18249423Sdim#include "llvm/IR/Constants.h" 19249423Sdim#include "llvm/IR/Function.h" 20249423Sdim#include "llvm/IR/Instructions.h" 21249423Sdim#include "llvm/IR/Intrinsics.h" 22249423Sdim#include "llvm/IR/LLVMContext.h" 23249423Sdim#include "llvm/IR/MDBuilder.h" 24249423Sdim#include "llvm/IR/Metadata.h" 25224133Sdim#include "llvm/Pass.h" 26224133Sdim#include "llvm/Support/CommandLine.h" 27224133Sdim#include "llvm/Support/Debug.h" 28224133Sdim#include <vector> 29224133Sdim 30224133Sdimusing namespace llvm; 31224133Sdim 32263508SdimSTATISTIC(IfHandled, "Number of 'expect' intrinsic instructions handled"); 33224133Sdim 34224133Sdimstatic cl::opt<uint32_t> 35224133SdimLikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), 36224133Sdim cl::desc("Weight of the branch likely to be taken (default = 64)")); 37224133Sdimstatic cl::opt<uint32_t> 38224133SdimUnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), 39224133Sdim cl::desc("Weight of the branch unlikely to be taken (default = 4)")); 40224133Sdim 41224133Sdimnamespace { 42224133Sdim 43224133Sdim class LowerExpectIntrinsic : public FunctionPass { 44224133Sdim 45224133Sdim bool HandleSwitchExpect(SwitchInst *SI); 46224133Sdim 47224133Sdim bool HandleIfExpect(BranchInst *BI); 48224133Sdim 49224133Sdim public: 50224133Sdim static char ID; 51224133Sdim LowerExpectIntrinsic() : FunctionPass(ID) { 52224133Sdim initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); 53224133Sdim } 54224133Sdim 55224133Sdim bool runOnFunction(Function &F); 56224133Sdim }; 57224133Sdim} 58224133Sdim 59224133Sdim 60224133Sdimbool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) { 61224133Sdim CallInst *CI = dyn_cast<CallInst>(SI->getCondition()); 62224133Sdim if (!CI) 63224133Sdim return false; 64224133Sdim 65224133Sdim Function *Fn = CI->getCalledFunction(); 66224133Sdim if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 67224133Sdim return false; 68224133Sdim 69224133Sdim Value *ArgValue = CI->getArgOperand(0); 70224133Sdim ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 71224133Sdim if (!ExpectedValue) 72224133Sdim return false; 73224133Sdim 74234353Sdim SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue); 75239462Sdim unsigned n = SI->getNumCases(); // +1 for default case. 76239462Sdim std::vector<uint32_t> Weights(n + 1); 77224133Sdim 78239462Sdim Weights[0] = Case == SI->case_default() ? LikelyBranchWeight 79239462Sdim : UnlikelyBranchWeight; 80239462Sdim for (unsigned i = 0; i != n; ++i) 81239462Sdim Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight 82239462Sdim : UnlikelyBranchWeight; 83224133Sdim 84239462Sdim SI->setMetadata(LLVMContext::MD_prof, 85239462Sdim MDBuilder(CI->getContext()).createBranchWeights(Weights)); 86224133Sdim 87224133Sdim SI->setCondition(ArgValue); 88224133Sdim return true; 89224133Sdim} 90224133Sdim 91224133Sdim 92224133Sdimbool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) { 93224133Sdim if (BI->isUnconditional()) 94224133Sdim return false; 95224133Sdim 96224133Sdim // Handle non-optimized IR code like: 97224133Sdim // %expval = call i64 @llvm.expect.i64.i64(i64 %conv1, i64 1) 98224133Sdim // %tobool = icmp ne i64 %expval, 0 99224133Sdim // br i1 %tobool, label %if.then, label %if.end 100224133Sdim 101224133Sdim ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition()); 102224133Sdim if (!CmpI || CmpI->getPredicate() != CmpInst::ICMP_NE) 103224133Sdim return false; 104224133Sdim 105224133Sdim CallInst *CI = dyn_cast<CallInst>(CmpI->getOperand(0)); 106224133Sdim if (!CI) 107224133Sdim return false; 108224133Sdim 109224133Sdim Function *Fn = CI->getCalledFunction(); 110224133Sdim if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 111224133Sdim return false; 112224133Sdim 113224133Sdim Value *ArgValue = CI->getArgOperand(0); 114224133Sdim ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 115224133Sdim if (!ExpectedValue) 116224133Sdim return false; 117224133Sdim 118239462Sdim MDBuilder MDB(CI->getContext()); 119239462Sdim MDNode *Node; 120224133Sdim 121224133Sdim // If expect value is equal to 1 it means that we are more likely to take 122224133Sdim // branch 0, in other case more likely is branch 1. 123239462Sdim if (ExpectedValue->isOne()) 124239462Sdim Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); 125239462Sdim else 126239462Sdim Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); 127224133Sdim 128239462Sdim BI->setMetadata(LLVMContext::MD_prof, Node); 129224133Sdim 130224133Sdim CmpI->setOperand(0, ArgValue); 131224133Sdim return true; 132224133Sdim} 133224133Sdim 134224133Sdim 135224133Sdimbool LowerExpectIntrinsic::runOnFunction(Function &F) { 136224133Sdim for (Function::iterator I = F.begin(), E = F.end(); I != E;) { 137224133Sdim BasicBlock *BB = I++; 138224133Sdim 139224133Sdim // Create "block_weights" metadata. 140224133Sdim if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) { 141224133Sdim if (HandleIfExpect(BI)) 142224133Sdim IfHandled++; 143224133Sdim } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) { 144224133Sdim if (HandleSwitchExpect(SI)) 145224133Sdim IfHandled++; 146224133Sdim } 147224133Sdim 148224133Sdim // remove llvm.expect intrinsics. 149224133Sdim for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); 150224133Sdim BI != BE; ) { 151224133Sdim CallInst *CI = dyn_cast<CallInst>(BI++); 152224133Sdim if (!CI) 153224133Sdim continue; 154224133Sdim 155224133Sdim Function *Fn = CI->getCalledFunction(); 156224133Sdim if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { 157224133Sdim Value *Exp = CI->getArgOperand(0); 158224133Sdim CI->replaceAllUsesWith(Exp); 159224133Sdim CI->eraseFromParent(); 160224133Sdim } 161224133Sdim } 162224133Sdim } 163224133Sdim 164224133Sdim return false; 165224133Sdim} 166224133Sdim 167224133Sdim 168224133Sdimchar LowerExpectIntrinsic::ID = 0; 169224133SdimINITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' " 170224133Sdim "Intrinsics", false, false) 171224133Sdim 172224133SdimFunctionPass *llvm::createLowerExpectIntrinsicPass() { 173224133Sdim return new LowerExpectIntrinsic(); 174224133Sdim} 175