1199989Srdivacky//===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===// 2199989Srdivacky// 3199989Srdivacky// The LLVM Compiler Infrastructure 4199989Srdivacky// 5199989Srdivacky// This file is distributed under the University of Illinois Open Source 6199989Srdivacky// License. See LICENSE.TXT for details. 7199989Srdivacky// 8199989Srdivacky//===----------------------------------------------------------------------===// 9199989Srdivacky// 10199989Srdivacky// This implements routines for translating from LLVM IR into SelectionDAG IR. 11199989Srdivacky// 12199989Srdivacky//===----------------------------------------------------------------------===// 13199989Srdivacky 14199989Srdivacky#define DEBUG_TYPE "isel" 15249423Sdim#include "SelectionDAGBuilder.h" 16205218Srdivacky#include "SDNodeDbgValue.h" 17199989Srdivacky#include "llvm/ADT/BitVector.h" 18263508Sdim#include "llvm/ADT/Optional.h" 19199989Srdivacky#include "llvm/ADT/SmallSet.h" 20199989Srdivacky#include "llvm/Analysis/AliasAnalysis.h" 21249423Sdim#include "llvm/Analysis/BranchProbabilityInfo.h" 22201360Srdivacky#include "llvm/Analysis/ConstantFolding.h" 23243830Sdim#include "llvm/Analysis/ValueTracking.h" 24207618Srdivacky#include "llvm/CodeGen/Analysis.h" 25199989Srdivacky#include "llvm/CodeGen/FastISel.h" 26210299Sed#include "llvm/CodeGen/FunctionLoweringInfo.h" 27249423Sdim#include "llvm/CodeGen/GCMetadata.h" 28199989Srdivacky#include "llvm/CodeGen/GCStrategy.h" 29249423Sdim#include "llvm/CodeGen/MachineFrameInfo.h" 30199989Srdivacky#include "llvm/CodeGen/MachineFunction.h" 31199989Srdivacky#include "llvm/CodeGen/MachineInstrBuilder.h" 32199989Srdivacky#include "llvm/CodeGen/MachineJumpTableInfo.h" 33199989Srdivacky#include "llvm/CodeGen/MachineModuleInfo.h" 34199989Srdivacky#include "llvm/CodeGen/MachineRegisterInfo.h" 35199989Srdivacky#include "llvm/CodeGen/SelectionDAG.h" 36263508Sdim#include "llvm/CodeGen/StackMaps.h" 37249423Sdim#include "llvm/DebugInfo.h" 38249423Sdim#include "llvm/IR/CallingConv.h" 39249423Sdim#include "llvm/IR/Constants.h" 40249423Sdim#include "llvm/IR/DataLayout.h" 41249423Sdim#include "llvm/IR/DerivedTypes.h" 42249423Sdim#include "llvm/IR/Function.h" 43249423Sdim#include "llvm/IR/GlobalVariable.h" 44249423Sdim#include "llvm/IR/InlineAsm.h" 45249423Sdim#include "llvm/IR/Instructions.h" 46249423Sdim#include "llvm/IR/IntrinsicInst.h" 47249423Sdim#include "llvm/IR/Intrinsics.h" 48249423Sdim#include "llvm/IR/LLVMContext.h" 49249423Sdim#include "llvm/IR/Module.h" 50249423Sdim#include "llvm/Support/CommandLine.h" 51249423Sdim#include "llvm/Support/Debug.h" 52249423Sdim#include "llvm/Support/ErrorHandling.h" 53249423Sdim#include "llvm/Support/MathExtras.h" 54249423Sdim#include "llvm/Support/raw_ostream.h" 55218893Sdim#include "llvm/Target/TargetFrameLowering.h" 56199989Srdivacky#include "llvm/Target/TargetInstrInfo.h" 57199989Srdivacky#include "llvm/Target/TargetIntrinsicInfo.h" 58234353Sdim#include "llvm/Target/TargetLibraryInfo.h" 59199989Srdivacky#include "llvm/Target/TargetLowering.h" 60199989Srdivacky#include "llvm/Target/TargetOptions.h" 61263508Sdim#include "llvm/Target/TargetSelectionDAGInfo.h" 62199989Srdivacky#include <algorithm> 63199989Srdivackyusing namespace llvm; 64199989Srdivacky 65199989Srdivacky/// LimitFloatPrecision - Generate low-precision inline sequences for 66199989Srdivacky/// some float libcalls (6, 8 or 12 bits). 67199989Srdivackystatic unsigned LimitFloatPrecision; 68199989Srdivacky 69199989Srdivackystatic cl::opt<unsigned, true> 70199989SrdivackyLimitFPPrecision("limit-float-precision", 71199989Srdivacky cl::desc("Generate low-precision inline sequences " 72199989Srdivacky "for some float libcalls"), 73199989Srdivacky cl::location(LimitFloatPrecision), 74199989Srdivacky cl::init(0)); 75199989Srdivacky 76218893Sdim// Limit the width of DAG chains. This is important in general to prevent 77218893Sdim// prevent DAG-based analysis from blowing up. For example, alias analysis and 78218893Sdim// load clustering may not complete in reasonable time. It is difficult to 79218893Sdim// recognize and avoid this situation within each individual analysis, and 80218893Sdim// future analyses are likely to have the same behavior. Limiting DAG width is 81218893Sdim// the safe approach, and will be especially important with global DAGs. 82218893Sdim// 83218893Sdim// MaxParallelChains default is arbitrarily high to avoid affecting 84218893Sdim// optimization, but could be lowered to improve compile time. Any ld-ld-st-st 85218893Sdim// sequence over this should have been converted to llvm.memcpy by the 86218893Sdim// frontend. It easy to induce this behavior with .ll code such as: 87218893Sdim// %buffer = alloca [4096 x i8] 88218893Sdim// %data = load [4096 x i8]* %argPtr 89218893Sdim// store [4096 x i8] %data, [4096 x i8]* %buffer 90221345Sdimstatic const unsigned MaxParallelChains = 64; 91218893Sdim 92263508Sdimstatic SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, 93212904Sdim const SDValue *Parts, unsigned NumParts, 94249423Sdim MVT PartVT, EVT ValueVT, const Value *V); 95218893Sdim 96199989Srdivacky/// getCopyFromParts - Create a value that contains the specified legal parts 97199989Srdivacky/// combined into the value they represent. If the parts combine to a type 98199989Srdivacky/// larger then ValueVT then AssertOp can be used to specify whether the extra 99199989Srdivacky/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT 100199989Srdivacky/// (ISD::AssertSext). 101263508Sdimstatic SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL, 102199989Srdivacky const SDValue *Parts, 103249423Sdim unsigned NumParts, MVT PartVT, EVT ValueVT, 104243830Sdim const Value *V, 105199989Srdivacky ISD::NodeType AssertOp = ISD::DELETED_NODE) { 106212904Sdim if (ValueVT.isVector()) 107243830Sdim return getCopyFromPartsVector(DAG, DL, Parts, NumParts, 108243830Sdim PartVT, ValueVT, V); 109218893Sdim 110199989Srdivacky assert(NumParts > 0 && "No parts to assemble!"); 111199989Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 112199989Srdivacky SDValue Val = Parts[0]; 113199989Srdivacky 114199989Srdivacky if (NumParts > 1) { 115199989Srdivacky // Assemble the value from multiple parts. 116212904Sdim if (ValueVT.isInteger()) { 117199989Srdivacky unsigned PartBits = PartVT.getSizeInBits(); 118199989Srdivacky unsigned ValueBits = ValueVT.getSizeInBits(); 119199989Srdivacky 120199989Srdivacky // Assemble the power of 2 part. 121199989Srdivacky unsigned RoundParts = NumParts & (NumParts - 1) ? 122199989Srdivacky 1 << Log2_32(NumParts) : NumParts; 123199989Srdivacky unsigned RoundBits = PartBits * RoundParts; 124199989Srdivacky EVT RoundVT = RoundBits == ValueBits ? 125199989Srdivacky ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); 126199989Srdivacky SDValue Lo, Hi; 127199989Srdivacky 128199989Srdivacky EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); 129199989Srdivacky 130199989Srdivacky if (RoundParts > 2) { 131212904Sdim Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, 132243830Sdim PartVT, HalfVT, V); 133212904Sdim Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, 134243830Sdim RoundParts / 2, PartVT, HalfVT, V); 135199989Srdivacky } else { 136218893Sdim Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); 137218893Sdim Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); 138199989Srdivacky } 139201360Srdivacky 140199989Srdivacky if (TLI.isBigEndian()) 141199989Srdivacky std::swap(Lo, Hi); 142201360Srdivacky 143212904Sdim Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); 144199989Srdivacky 145199989Srdivacky if (RoundParts < NumParts) { 146199989Srdivacky // Assemble the trailing non-power-of-2 part. 147199989Srdivacky unsigned OddParts = NumParts - RoundParts; 148199989Srdivacky EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); 149212904Sdim Hi = getCopyFromParts(DAG, DL, 150243830Sdim Parts + RoundParts, OddParts, PartVT, OddVT, V); 151199989Srdivacky 152199989Srdivacky // Combine the round and odd parts. 153199989Srdivacky Lo = Val; 154199989Srdivacky if (TLI.isBigEndian()) 155199989Srdivacky std::swap(Lo, Hi); 156199989Srdivacky EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 157212904Sdim Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); 158212904Sdim Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi, 159199989Srdivacky DAG.getConstant(Lo.getValueType().getSizeInBits(), 160199989Srdivacky TLI.getPointerTy())); 161212904Sdim Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); 162212904Sdim Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); 163199989Srdivacky } 164199989Srdivacky } else if (PartVT.isFloatingPoint()) { 165199989Srdivacky // FP split into multiple FP parts (for ppcf128) 166249423Sdim assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && 167199989Srdivacky "Unexpected split"); 168199989Srdivacky SDValue Lo, Hi; 169218893Sdim Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); 170218893Sdim Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); 171199989Srdivacky if (TLI.isBigEndian()) 172199989Srdivacky std::swap(Lo, Hi); 173212904Sdim Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); 174199989Srdivacky } else { 175199989Srdivacky // FP split into integer parts (soft fp) 176199989Srdivacky assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && 177199989Srdivacky !PartVT.isVector() && "Unexpected split"); 178199989Srdivacky EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); 179243830Sdim Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V); 180199989Srdivacky } 181199989Srdivacky } 182199989Srdivacky 183199989Srdivacky // There is now one part, held in Val. Correct it to match ValueVT. 184249423Sdim EVT PartEVT = Val.getValueType(); 185199989Srdivacky 186249423Sdim if (PartEVT == ValueVT) 187199989Srdivacky return Val; 188199989Srdivacky 189249423Sdim if (PartEVT.isInteger() && ValueVT.isInteger()) { 190249423Sdim if (ValueVT.bitsLT(PartEVT)) { 191199989Srdivacky // For a truncate, see if we have any information to 192199989Srdivacky // indicate whether the truncated bits will always be 193199989Srdivacky // zero or sign-extension. 194199989Srdivacky if (AssertOp != ISD::DELETED_NODE) 195249423Sdim Val = DAG.getNode(AssertOp, DL, PartEVT, Val, 196199989Srdivacky DAG.getValueType(ValueVT)); 197212904Sdim return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 198199989Srdivacky } 199212904Sdim return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); 200199989Srdivacky } 201199989Srdivacky 202249423Sdim if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 203212904Sdim // FP_ROUND's are always exact here. 204212904Sdim if (ValueVT.bitsLT(Val.getValueType())) 205212904Sdim return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, 206234353Sdim DAG.getTargetConstant(1, TLI.getPointerTy())); 207201360Srdivacky 208212904Sdim return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); 209199989Srdivacky } 210199989Srdivacky 211249423Sdim if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) 212218893Sdim return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 213199989Srdivacky 214199989Srdivacky llvm_unreachable("Unknown mismatch!"); 215199989Srdivacky} 216199989Srdivacky 217243830Sdim/// getCopyFromPartsVector - Create a value that contains the specified legal 218243830Sdim/// parts combined into the value they represent. If the parts combine to a 219243830Sdim/// type larger then ValueVT then AssertOp can be used to specify whether the 220243830Sdim/// extra bits are known to be zero (ISD::AssertZext) or sign extended from 221243830Sdim/// ValueVT (ISD::AssertSext). 222263508Sdimstatic SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, 223212904Sdim const SDValue *Parts, unsigned NumParts, 224249423Sdim MVT PartVT, EVT ValueVT, const Value *V) { 225212904Sdim assert(ValueVT.isVector() && "Not a vector value"); 226212904Sdim assert(NumParts > 0 && "No parts to assemble!"); 227212904Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 228212904Sdim SDValue Val = Parts[0]; 229218893Sdim 230212904Sdim // Handle a multi-element vector. 231212904Sdim if (NumParts > 1) { 232249423Sdim EVT IntermediateVT; 233249423Sdim MVT RegisterVT; 234212904Sdim unsigned NumIntermediates; 235212904Sdim unsigned NumRegs = 236212904Sdim TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, 237212904Sdim NumIntermediates, RegisterVT); 238212904Sdim assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 239212904Sdim NumParts = NumRegs; // Silence a compiler warning. 240212904Sdim assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 241249423Sdim assert(RegisterVT == Parts[0].getSimpleValueType() && 242212904Sdim "Part type doesn't match part!"); 243218893Sdim 244212904Sdim // Assemble the parts into intermediate operands. 245212904Sdim SmallVector<SDValue, 8> Ops(NumIntermediates); 246212904Sdim if (NumIntermediates == NumParts) { 247212904Sdim // If the register was not expanded, truncate or copy the value, 248212904Sdim // as appropriate. 249212904Sdim for (unsigned i = 0; i != NumParts; ++i) 250212904Sdim Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, 251243830Sdim PartVT, IntermediateVT, V); 252212904Sdim } else if (NumParts > 0) { 253212904Sdim // If the intermediate type was expanded, build the intermediate 254212904Sdim // operands from the parts. 255212904Sdim assert(NumParts % NumIntermediates == 0 && 256212904Sdim "Must expand into a divisible number of parts!"); 257212904Sdim unsigned Factor = NumParts / NumIntermediates; 258212904Sdim for (unsigned i = 0; i != NumIntermediates; ++i) 259212904Sdim Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, 260243830Sdim PartVT, IntermediateVT, V); 261212904Sdim } 262218893Sdim 263212904Sdim // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the 264212904Sdim // intermediate operands. 265212904Sdim Val = DAG.getNode(IntermediateVT.isVector() ? 266212904Sdim ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, DL, 267212904Sdim ValueVT, &Ops[0], NumIntermediates); 268212904Sdim } 269218893Sdim 270212904Sdim // There is now one part, held in Val. Correct it to match ValueVT. 271249423Sdim EVT PartEVT = Val.getValueType(); 272218893Sdim 273249423Sdim if (PartEVT == ValueVT) 274212904Sdim return Val; 275218893Sdim 276249423Sdim if (PartEVT.isVector()) { 277212904Sdim // If the element type of the source/dest vectors are the same, but the 278212904Sdim // parts vector has more elements than the value vector, then we have a 279212904Sdim // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the 280212904Sdim // elements we want. 281249423Sdim if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) { 282249423Sdim assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && 283212904Sdim "Cannot narrow, it would be a lossy transformation"); 284212904Sdim return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, 285263508Sdim DAG.getConstant(0, TLI.getVectorIdxTy())); 286218893Sdim } 287218893Sdim 288212904Sdim // Vector/Vector bitcast. 289249423Sdim if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) 290223017Sdim return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 291223017Sdim 292249423Sdim assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && 293223017Sdim "Cannot handle this kind of promotion"); 294223017Sdim // Promoted vector extract 295249423Sdim bool Smaller = ValueVT.bitsLE(PartEVT); 296224145Sdim return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 297224145Sdim DL, ValueVT, Val); 298223017Sdim 299212904Sdim } 300218893Sdim 301223017Sdim // Trivial bitcast if the types are the same size and the destination 302223017Sdim // vector type is legal. 303249423Sdim if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && 304223017Sdim TLI.isTypeLegal(ValueVT)) 305223017Sdim return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 306223017Sdim 307224145Sdim // Handle cases such as i8 -> <1 x i1> 308243830Sdim if (ValueVT.getVectorNumElements() != 1) { 309243830Sdim LLVMContext &Ctx = *DAG.getContext(); 310243830Sdim Twine ErrMsg("non-trivial scalar-to-vector conversion"); 311243830Sdim if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) { 312243830Sdim if (const CallInst *CI = dyn_cast<CallInst>(I)) 313243830Sdim if (isa<InlineAsm>(CI->getCalledValue())) 314243830Sdim ErrMsg = ErrMsg + ", possible invalid constraint for vector type"; 315243830Sdim Ctx.emitError(I, ErrMsg); 316243830Sdim } else { 317243830Sdim Ctx.emitError(ErrMsg); 318243830Sdim } 319251662Sdim return DAG.getUNDEF(ValueVT); 320243830Sdim } 321224145Sdim 322224145Sdim if (ValueVT.getVectorNumElements() == 1 && 323249423Sdim ValueVT.getVectorElementType() != PartEVT) { 324249423Sdim bool Smaller = ValueVT.bitsLE(PartEVT); 325224145Sdim Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 326224145Sdim DL, ValueVT.getScalarType(), Val); 327224145Sdim } 328224145Sdim 329212904Sdim return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val); 330212904Sdim} 331212904Sdim 332263508Sdimstatic void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl, 333212904Sdim SDValue Val, SDValue *Parts, unsigned NumParts, 334249423Sdim MVT PartVT, const Value *V); 335218893Sdim 336199989Srdivacky/// getCopyToParts - Create a series of nodes that contain the specified value 337199989Srdivacky/// split into legal parts. If the parts contain more bits than Val, then, for 338199989Srdivacky/// integers, ExtendKind can be used to specify how to generate the extra bits. 339263508Sdimstatic void getCopyToParts(SelectionDAG &DAG, SDLoc DL, 340201360Srdivacky SDValue Val, SDValue *Parts, unsigned NumParts, 341249423Sdim MVT PartVT, const Value *V, 342199989Srdivacky ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { 343212904Sdim EVT ValueVT = Val.getValueType(); 344218893Sdim 345212904Sdim // Handle the vector case separately. 346212904Sdim if (ValueVT.isVector()) 347243830Sdim return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V); 348218893Sdim 349199989Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 350199989Srdivacky unsigned PartBits = PartVT.getSizeInBits(); 351199989Srdivacky unsigned OrigNumParts = NumParts; 352199989Srdivacky assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!"); 353199989Srdivacky 354212904Sdim if (NumParts == 0) 355199989Srdivacky return; 356199989Srdivacky 357212904Sdim assert(!ValueVT.isVector() && "Vector case handled elsewhere"); 358249423Sdim EVT PartEVT = PartVT; 359249423Sdim if (PartEVT == ValueVT) { 360212904Sdim assert(NumParts == 1 && "No-op copy with multiple parts!"); 361212904Sdim Parts[0] = Val; 362212904Sdim return; 363212904Sdim } 364199989Srdivacky 365212904Sdim if (NumParts * PartBits > ValueVT.getSizeInBits()) { 366212904Sdim // If the parts cover more bits than the value has, promote the value. 367212904Sdim if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 368212904Sdim assert(NumParts == 1 && "Do not know what to promote to!"); 369212904Sdim Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); 370212904Sdim } else { 371234353Sdim assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 372234353Sdim ValueVT.isInteger() && 373218893Sdim "Unknown mismatch!"); 374212904Sdim ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 375212904Sdim Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); 376234353Sdim if (PartVT == MVT::x86mmx) 377234353Sdim Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 378199989Srdivacky } 379212904Sdim } else if (PartBits == ValueVT.getSizeInBits()) { 380212904Sdim // Different types of the same size. 381249423Sdim assert(NumParts == 1 && PartEVT != ValueVT); 382218893Sdim Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 383212904Sdim } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { 384212904Sdim // If the parts cover less bits than value has, truncate the value. 385234353Sdim assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 386234353Sdim ValueVT.isInteger() && 387212904Sdim "Unknown mismatch!"); 388212904Sdim ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 389212904Sdim Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 390234353Sdim if (PartVT == MVT::x86mmx) 391234353Sdim Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 392212904Sdim } 393199989Srdivacky 394212904Sdim // The value may have changed - recompute ValueVT. 395212904Sdim ValueVT = Val.getValueType(); 396212904Sdim assert(NumParts * PartBits == ValueVT.getSizeInBits() && 397212904Sdim "Failed to tile the value with PartVT!"); 398199989Srdivacky 399212904Sdim if (NumParts == 1) { 400249423Sdim if (PartEVT != ValueVT) { 401243830Sdim LLVMContext &Ctx = *DAG.getContext(); 402243830Sdim Twine ErrMsg("scalar-to-vector conversion failed"); 403243830Sdim if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) { 404243830Sdim if (const CallInst *CI = dyn_cast<CallInst>(I)) 405243830Sdim if (isa<InlineAsm>(CI->getCalledValue())) 406243830Sdim ErrMsg = ErrMsg + ", possible invalid constraint for vector type"; 407243830Sdim Ctx.emitError(I, ErrMsg); 408243830Sdim } else { 409243830Sdim Ctx.emitError(ErrMsg); 410243830Sdim } 411243830Sdim } 412243830Sdim 413212904Sdim Parts[0] = Val; 414212904Sdim return; 415212904Sdim } 416199989Srdivacky 417212904Sdim // Expand the value into multiple parts. 418212904Sdim if (NumParts & (NumParts - 1)) { 419212904Sdim // The number of parts is not a power of 2. Split off and copy the tail. 420212904Sdim assert(PartVT.isInteger() && ValueVT.isInteger() && 421212904Sdim "Do not know what to expand to!"); 422212904Sdim unsigned RoundParts = 1 << Log2_32(NumParts); 423212904Sdim unsigned RoundBits = RoundParts * PartBits; 424212904Sdim unsigned OddParts = NumParts - RoundParts; 425212904Sdim SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, 426212904Sdim DAG.getIntPtrConstant(RoundBits)); 427243830Sdim getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V); 428201360Srdivacky 429212904Sdim if (TLI.isBigEndian()) 430212904Sdim // The odd parts were reversed by getCopyToParts - unreverse them. 431212904Sdim std::reverse(Parts + RoundParts, Parts + NumParts); 432201360Srdivacky 433212904Sdim NumParts = RoundParts; 434212904Sdim ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 435212904Sdim Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 436212904Sdim } 437199989Srdivacky 438212904Sdim // The number of parts is a power of 2. Repeatedly bisect the value using 439212904Sdim // EXTRACT_ELEMENT. 440218893Sdim Parts[0] = DAG.getNode(ISD::BITCAST, DL, 441212904Sdim EVT::getIntegerVT(*DAG.getContext(), 442212904Sdim ValueVT.getSizeInBits()), 443212904Sdim Val); 444201360Srdivacky 445212904Sdim for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { 446212904Sdim for (unsigned i = 0; i < NumParts; i += StepSize) { 447212904Sdim unsigned ThisBits = StepSize * PartBits / 2; 448212904Sdim EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); 449212904Sdim SDValue &Part0 = Parts[i]; 450212904Sdim SDValue &Part1 = Parts[i+StepSize/2]; 451199989Srdivacky 452212904Sdim Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 453212904Sdim ThisVT, Part0, DAG.getIntPtrConstant(1)); 454212904Sdim Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 455212904Sdim ThisVT, Part0, DAG.getIntPtrConstant(0)); 456199989Srdivacky 457212904Sdim if (ThisBits == PartBits && ThisVT != PartVT) { 458218893Sdim Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); 459218893Sdim Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); 460199989Srdivacky } 461199989Srdivacky } 462212904Sdim } 463199989Srdivacky 464212904Sdim if (TLI.isBigEndian()) 465212904Sdim std::reverse(Parts, Parts + OrigNumParts); 466212904Sdim} 467199989Srdivacky 468199989Srdivacky 469212904Sdim/// getCopyToPartsVector - Create a series of nodes that contain the specified 470212904Sdim/// value split into legal parts. 471263508Sdimstatic void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL, 472212904Sdim SDValue Val, SDValue *Parts, unsigned NumParts, 473249423Sdim MVT PartVT, const Value *V) { 474212904Sdim EVT ValueVT = Val.getValueType(); 475212904Sdim assert(ValueVT.isVector() && "Not a vector"); 476212904Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 477218893Sdim 478199989Srdivacky if (NumParts == 1) { 479249423Sdim EVT PartEVT = PartVT; 480249423Sdim if (PartEVT == ValueVT) { 481212904Sdim // Nothing to do. 482212904Sdim } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { 483212904Sdim // Bitconvert vector->vector case. 484218893Sdim Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 485212904Sdim } else if (PartVT.isVector() && 486249423Sdim PartEVT.getVectorElementType() == ValueVT.getVectorElementType() && 487249423Sdim PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) { 488212904Sdim EVT ElementVT = PartVT.getVectorElementType(); 489212904Sdim // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in 490212904Sdim // undef elements. 491212904Sdim SmallVector<SDValue, 16> Ops; 492212904Sdim for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i) 493212904Sdim Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 494263508Sdim ElementVT, Val, DAG.getConstant(i, 495263508Sdim TLI.getVectorIdxTy()))); 496218893Sdim 497212904Sdim for (unsigned i = ValueVT.getVectorNumElements(), 498212904Sdim e = PartVT.getVectorNumElements(); i != e; ++i) 499212904Sdim Ops.push_back(DAG.getUNDEF(ElementVT)); 500212904Sdim 501212904Sdim Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, &Ops[0], Ops.size()); 502212904Sdim 503212904Sdim // FIXME: Use CONCAT for 2x -> 4x. 504218893Sdim 505212904Sdim //SDValue UndefElts = DAG.getUNDEF(VectorTy); 506212904Sdim //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts); 507223017Sdim } else if (PartVT.isVector() && 508249423Sdim PartEVT.getVectorElementType().bitsGE( 509224145Sdim ValueVT.getVectorElementType()) && 510249423Sdim PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { 511223017Sdim 512223017Sdim // Promoted vector extract 513249423Sdim bool Smaller = PartEVT.bitsLE(ValueVT); 514224145Sdim Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 515224145Sdim DL, PartVT, Val); 516223017Sdim } else{ 517212904Sdim // Vector -> scalar conversion. 518224145Sdim assert(ValueVT.getVectorNumElements() == 1 && 519212904Sdim "Only trivial vector-to-scalar conversions should get here!"); 520212904Sdim Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 521263508Sdim PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy())); 522224145Sdim 523224145Sdim bool Smaller = ValueVT.bitsLE(PartVT); 524224145Sdim Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 525224145Sdim DL, PartVT, Val); 526199989Srdivacky } 527218893Sdim 528199989Srdivacky Parts[0] = Val; 529199989Srdivacky return; 530199989Srdivacky } 531218893Sdim 532199989Srdivacky // Handle a multi-element vector. 533249423Sdim EVT IntermediateVT; 534249423Sdim MVT RegisterVT; 535199989Srdivacky unsigned NumIntermediates; 536199989Srdivacky unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, 537212904Sdim IntermediateVT, 538212904Sdim NumIntermediates, RegisterVT); 539199989Srdivacky unsigned NumElements = ValueVT.getVectorNumElements(); 540218893Sdim 541199989Srdivacky assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 542199989Srdivacky NumParts = NumRegs; // Silence a compiler warning. 543199989Srdivacky assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 544218893Sdim 545199989Srdivacky // Split the vector into intermediate operands. 546199989Srdivacky SmallVector<SDValue, 8> Ops(NumIntermediates); 547201360Srdivacky for (unsigned i = 0; i != NumIntermediates; ++i) { 548199989Srdivacky if (IntermediateVT.isVector()) 549212904Sdim Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, 550199989Srdivacky IntermediateVT, Val, 551263508Sdim DAG.getConstant(i * (NumElements / NumIntermediates), 552263508Sdim TLI.getVectorIdxTy())); 553199989Srdivacky else 554212904Sdim Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 555263508Sdim IntermediateVT, Val, 556263508Sdim DAG.getConstant(i, TLI.getVectorIdxTy())); 557201360Srdivacky } 558218893Sdim 559199989Srdivacky // Split the intermediate operands into legal parts. 560199989Srdivacky if (NumParts == NumIntermediates) { 561199989Srdivacky // If the register was not expanded, promote or copy the value, 562199989Srdivacky // as appropriate. 563199989Srdivacky for (unsigned i = 0; i != NumParts; ++i) 564243830Sdim getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V); 565199989Srdivacky } else if (NumParts > 0) { 566199989Srdivacky // If the intermediate type was expanded, split each the value into 567199989Srdivacky // legal parts. 568199989Srdivacky assert(NumParts % NumIntermediates == 0 && 569199989Srdivacky "Must expand into a divisible number of parts!"); 570199989Srdivacky unsigned Factor = NumParts / NumIntermediates; 571199989Srdivacky for (unsigned i = 0; i != NumIntermediates; ++i) 572243830Sdim getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V); 573199989Srdivacky } 574199989Srdivacky} 575199989Srdivacky 576210299Sednamespace { 577210299Sed /// RegsForValue - This struct represents the registers (physical or virtual) 578210299Sed /// that a particular set of values is assigned, and the type information 579210299Sed /// about the value. The most common situation is to represent one value at a 580210299Sed /// time, but struct or array values are handled element-wise as multiple 581210299Sed /// values. The splitting of aggregates is performed recursively, so that we 582210299Sed /// never have aggregate-typed registers. The values at this point do not 583210299Sed /// necessarily have legal types, so each value may require one or more 584210299Sed /// registers of some legal type. 585210299Sed /// 586210299Sed struct RegsForValue { 587210299Sed /// ValueVTs - The value types of the values, which may not be legal, and 588210299Sed /// may need be promoted or synthesized from one or more registers. 589210299Sed /// 590210299Sed SmallVector<EVT, 4> ValueVTs; 591199989Srdivacky 592210299Sed /// RegVTs - The value types of the registers. This is the same size as 593210299Sed /// ValueVTs and it records, for each value, what the type of the assigned 594210299Sed /// register or registers are. (Individual values are never synthesized 595210299Sed /// from more than one type of register.) 596210299Sed /// 597210299Sed /// With virtual registers, the contents of RegVTs is redundant with TLI's 598210299Sed /// getRegisterType member function, however when with physical registers 599210299Sed /// it is necessary to have a separate record of the types. 600210299Sed /// 601249423Sdim SmallVector<MVT, 4> RegVTs; 602210299Sed 603210299Sed /// Regs - This list holds the registers assigned to the values. 604210299Sed /// Each legal or promoted value requires one register, and each 605210299Sed /// expanded value requires multiple registers. 606210299Sed /// 607210299Sed SmallVector<unsigned, 4> Regs; 608210299Sed 609210299Sed RegsForValue() {} 610210299Sed 611210299Sed RegsForValue(const SmallVector<unsigned, 4> ®s, 612249423Sdim MVT regvt, EVT valuevt) 613210299Sed : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} 614210299Sed 615210299Sed RegsForValue(LLVMContext &Context, const TargetLowering &tli, 616226633Sdim unsigned Reg, Type *Ty) { 617210299Sed ComputeValueVTs(tli, Ty, ValueVTs); 618210299Sed 619210299Sed for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { 620210299Sed EVT ValueVT = ValueVTs[Value]; 621210299Sed unsigned NumRegs = tli.getNumRegisters(Context, ValueVT); 622249423Sdim MVT RegisterVT = tli.getRegisterType(Context, ValueVT); 623210299Sed for (unsigned i = 0; i != NumRegs; ++i) 624210299Sed Regs.push_back(Reg + i); 625210299Sed RegVTs.push_back(RegisterVT); 626210299Sed Reg += NumRegs; 627210299Sed } 628210299Sed } 629210299Sed 630210299Sed /// areValueTypesLegal - Return true if types of all the values are legal. 631210299Sed bool areValueTypesLegal(const TargetLowering &TLI) { 632210299Sed for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { 633249423Sdim MVT RegisterVT = RegVTs[Value]; 634210299Sed if (!TLI.isTypeLegal(RegisterVT)) 635210299Sed return false; 636210299Sed } 637210299Sed return true; 638210299Sed } 639210299Sed 640210299Sed /// append - Add the specified values to this one. 641210299Sed void append(const RegsForValue &RHS) { 642210299Sed ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); 643210299Sed RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); 644210299Sed Regs.append(RHS.Regs.begin(), RHS.Regs.end()); 645210299Sed } 646210299Sed 647210299Sed /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 648210299Sed /// this value and returns the result as a ValueVTs value. This uses 649210299Sed /// Chain/Flag as the input and updates them for the output Chain/Flag. 650210299Sed /// If the Flag pointer is NULL, no flag is used. 651210299Sed SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, 652263508Sdim SDLoc dl, 653243830Sdim SDValue &Chain, SDValue *Flag, 654243830Sdim const Value *V = 0) const; 655210299Sed 656210299Sed /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 657210299Sed /// specified value into the registers specified by this object. This uses 658210299Sed /// Chain/Flag as the input and updates them for the output Chain/Flag. 659210299Sed /// If the Flag pointer is NULL, no flag is used. 660263508Sdim void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, 661243830Sdim SDValue &Chain, SDValue *Flag, const Value *V) const; 662210299Sed 663210299Sed /// AddInlineAsmOperands - Add this value to the specified inlineasm node 664210299Sed /// operand list. This adds the code marker, matching input operand index 665210299Sed /// (if applicable), and includes the number of values added into it. 666210299Sed void AddInlineAsmOperands(unsigned Kind, 667210299Sed bool HasMatching, unsigned MatchingIdx, 668210299Sed SelectionDAG &DAG, 669210299Sed std::vector<SDValue> &Ops) const; 670210299Sed }; 671210299Sed} 672210299Sed 673210299Sed/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 674210299Sed/// this value and returns the result as a ValueVT value. This uses 675210299Sed/// Chain/Flag as the input and updates them for the output Chain/Flag. 676210299Sed/// If the Flag pointer is NULL, no flag is used. 677210299SedSDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, 678210299Sed FunctionLoweringInfo &FuncInfo, 679263508Sdim SDLoc dl, 680243830Sdim SDValue &Chain, SDValue *Flag, 681243830Sdim const Value *V) const { 682212904Sdim // A Value with type {} or [0 x %t] needs no registers. 683212904Sdim if (ValueVTs.empty()) 684212904Sdim return SDValue(); 685212904Sdim 686210299Sed const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 687210299Sed 688210299Sed // Assemble the legal parts into the final values. 689210299Sed SmallVector<SDValue, 4> Values(ValueVTs.size()); 690210299Sed SmallVector<SDValue, 8> Parts; 691210299Sed for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 692210299Sed // Copy the legal parts from the registers. 693210299Sed EVT ValueVT = ValueVTs[Value]; 694210299Sed unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT); 695249423Sdim MVT RegisterVT = RegVTs[Value]; 696210299Sed 697210299Sed Parts.resize(NumRegs); 698210299Sed for (unsigned i = 0; i != NumRegs; ++i) { 699210299Sed SDValue P; 700210299Sed if (Flag == 0) { 701210299Sed P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); 702210299Sed } else { 703210299Sed P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); 704210299Sed *Flag = P.getValue(2); 705210299Sed } 706210299Sed 707210299Sed Chain = P.getValue(1); 708218893Sdim Parts[i] = P; 709210299Sed 710210299Sed // If the source register was virtual and if we know something about it, 711210299Sed // add an assert node. 712218893Sdim if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) || 713219077Sdim !RegisterVT.isInteger() || RegisterVT.isVector()) 714218893Sdim continue; 715210299Sed 716219077Sdim const FunctionLoweringInfo::LiveOutInfo *LOI = 717219077Sdim FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); 718219077Sdim if (!LOI) 719219077Sdim continue; 720219077Sdim 721218893Sdim unsigned RegSize = RegisterVT.getSizeInBits(); 722219077Sdim unsigned NumSignBits = LOI->NumSignBits; 723219077Sdim unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes(); 724210299Sed 725263508Sdim if (NumZeroBits == RegSize) { 726263508Sdim // The current value is a zero. 727263508Sdim // Explicitly express that as it would be easier for 728263508Sdim // optimizations to kick in. 729263508Sdim Parts[i] = DAG.getConstant(0, RegisterVT); 730263508Sdim continue; 731263508Sdim } 732263508Sdim 733218893Sdim // FIXME: We capture more information than the dag can represent. For 734218893Sdim // now, just use the tightest assertzext/assertsext possible. 735218893Sdim bool isSExt = true; 736218893Sdim EVT FromVT(MVT::Other); 737218893Sdim if (NumSignBits == RegSize) 738218893Sdim isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1 739218893Sdim else if (NumZeroBits >= RegSize-1) 740218893Sdim isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1 741218893Sdim else if (NumSignBits > RegSize-8) 742218893Sdim isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8 743218893Sdim else if (NumZeroBits >= RegSize-8) 744218893Sdim isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8 745218893Sdim else if (NumSignBits > RegSize-16) 746218893Sdim isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16 747218893Sdim else if (NumZeroBits >= RegSize-16) 748218893Sdim isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16 749218893Sdim else if (NumSignBits > RegSize-32) 750218893Sdim isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32 751218893Sdim else if (NumZeroBits >= RegSize-32) 752218893Sdim isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32 753218893Sdim else 754218893Sdim continue; 755210299Sed 756218893Sdim // Add an assertion node. 757218893Sdim assert(FromVT != MVT::Other); 758218893Sdim Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, 759218893Sdim RegisterVT, P, DAG.getValueType(FromVT)); 760210299Sed } 761210299Sed 762210299Sed Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), 763243830Sdim NumRegs, RegisterVT, ValueVT, V); 764210299Sed Part += NumRegs; 765210299Sed Parts.clear(); 766210299Sed } 767210299Sed 768210299Sed return DAG.getNode(ISD::MERGE_VALUES, dl, 769210299Sed DAG.getVTList(&ValueVTs[0], ValueVTs.size()), 770210299Sed &Values[0], ValueVTs.size()); 771210299Sed} 772210299Sed 773210299Sed/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 774210299Sed/// specified value into the registers specified by this object. This uses 775210299Sed/// Chain/Flag as the input and updates them for the output Chain/Flag. 776210299Sed/// If the Flag pointer is NULL, no flag is used. 777263508Sdimvoid RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, 778243830Sdim SDValue &Chain, SDValue *Flag, 779243830Sdim const Value *V) const { 780210299Sed const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 781210299Sed 782210299Sed // Get the list of the values's legal parts. 783210299Sed unsigned NumRegs = Regs.size(); 784210299Sed SmallVector<SDValue, 8> Parts(NumRegs); 785210299Sed for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 786210299Sed EVT ValueVT = ValueVTs[Value]; 787210299Sed unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT); 788249423Sdim MVT RegisterVT = RegVTs[Value]; 789249423Sdim ISD::NodeType ExtendKind = 790249423Sdim TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND; 791210299Sed 792212904Sdim getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), 793249423Sdim &Parts[Part], NumParts, RegisterVT, V, ExtendKind); 794210299Sed Part += NumParts; 795210299Sed } 796210299Sed 797210299Sed // Copy the parts into the registers. 798210299Sed SmallVector<SDValue, 8> Chains(NumRegs); 799210299Sed for (unsigned i = 0; i != NumRegs; ++i) { 800210299Sed SDValue Part; 801210299Sed if (Flag == 0) { 802210299Sed Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); 803210299Sed } else { 804210299Sed Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); 805210299Sed *Flag = Part.getValue(1); 806210299Sed } 807210299Sed 808210299Sed Chains[i] = Part.getValue(0); 809210299Sed } 810210299Sed 811210299Sed if (NumRegs == 1 || Flag) 812210299Sed // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is 813210299Sed // flagged to it. That is the CopyToReg nodes and the user are considered 814210299Sed // a single scheduling unit. If we create a TokenFactor and return it as 815210299Sed // chain, then the TokenFactor is both a predecessor (operand) of the 816210299Sed // user as well as a successor (the TF operands are flagged to the user). 817210299Sed // c1, f1 = CopyToReg 818210299Sed // c2, f2 = CopyToReg 819210299Sed // c3 = TokenFactor c1, c2 820210299Sed // ... 821210299Sed // = op c3, ..., f2 822210299Sed Chain = Chains[NumRegs-1]; 823210299Sed else 824210299Sed Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], NumRegs); 825210299Sed} 826210299Sed 827210299Sed/// AddInlineAsmOperands - Add this value to the specified inlineasm node 828210299Sed/// operand list. This adds the code marker and includes the number of 829210299Sed/// values added into it. 830210299Sedvoid RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, 831210299Sed unsigned MatchingIdx, 832210299Sed SelectionDAG &DAG, 833210299Sed std::vector<SDValue> &Ops) const { 834210299Sed const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 835210299Sed 836210299Sed unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); 837210299Sed if (HasMatching) 838210299Sed Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); 839226633Sdim else if (!Regs.empty() && 840226633Sdim TargetRegisterInfo::isVirtualRegister(Regs.front())) { 841226633Sdim // Put the register class of the virtual registers in the flag word. That 842226633Sdim // way, later passes can recompute register class constraints for inline 843226633Sdim // assembly as well as normal instructions. 844226633Sdim // Don't do this for tied operands that can use the regclass information 845226633Sdim // from the def. 846226633Sdim const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 847226633Sdim const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); 848226633Sdim Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); 849226633Sdim } 850226633Sdim 851210299Sed SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); 852210299Sed Ops.push_back(Res); 853210299Sed 854263765Sdim unsigned SP = TLI.getStackPointerRegisterToSaveRestore(); 855210299Sed for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { 856210299Sed unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]); 857249423Sdim MVT RegisterVT = RegVTs[Value]; 858210299Sed for (unsigned i = 0; i != NumRegs; ++i) { 859210299Sed assert(Reg < Regs.size() && "Mismatch in # registers expected"); 860263765Sdim unsigned TheReg = Regs[Reg++]; 861263765Sdim Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); 862263765Sdim 863263765Sdim if (TheReg == SP && Code == InlineAsm::Kind_Clobber) { 864263765Sdim // If we clobbered the stack pointer, MFI should know about it. 865263765Sdim assert(DAG.getMachineFunction().getFrameInfo()-> 866263765Sdim hasInlineAsmWithSPAdjust()); 867263765Sdim } 868210299Sed } 869210299Sed } 870210299Sed} 871210299Sed 872234353Sdimvoid SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa, 873234353Sdim const TargetLibraryInfo *li) { 874199989Srdivacky AA = &aa; 875199989Srdivacky GFI = gfi; 876234353Sdim LibInfo = li; 877243830Sdim TD = DAG.getTarget().getDataLayout(); 878243830Sdim Context = DAG.getContext(); 879226633Sdim LPadToCallSiteMap.clear(); 880199989Srdivacky} 881199989Srdivacky 882207618Srdivacky/// clear - Clear out the current SelectionDAG and the associated 883199989Srdivacky/// state and prepare this SelectionDAGBuilder object to be used 884199989Srdivacky/// for a new block. This doesn't clear out information about 885199989Srdivacky/// additional blocks that are needed to complete switch lowering 886199989Srdivacky/// or PHI node updating; that information is cleared out as it is 887199989Srdivacky/// consumed. 888199989Srdivackyvoid SelectionDAGBuilder::clear() { 889199989Srdivacky NodeMap.clear(); 890210299Sed UnusedArgNodeMap.clear(); 891199989Srdivacky PendingLoads.clear(); 892199989Srdivacky PendingExports.clear(); 893263508Sdim CurInst = NULL; 894199989Srdivacky HasTailCall = false; 895199989Srdivacky} 896199989Srdivacky 897223017Sdim/// clearDanglingDebugInfo - Clear the dangling debug information 898239462Sdim/// map. This function is separated from the clear so that debug 899223017Sdim/// information that is dangling in a basic block can be properly 900223017Sdim/// resolved in a different basic block. This allows the 901223017Sdim/// SelectionDAG to resolve dangling debug information attached 902223017Sdim/// to PHI nodes. 903223017Sdimvoid SelectionDAGBuilder::clearDanglingDebugInfo() { 904223017Sdim DanglingDebugInfoMap.clear(); 905223017Sdim} 906223017Sdim 907199989Srdivacky/// getRoot - Return the current virtual root of the Selection DAG, 908199989Srdivacky/// flushing any PendingLoad items. This must be done before emitting 909199989Srdivacky/// a store or any other node that may need to be ordered after any 910199989Srdivacky/// prior load instructions. 911199989Srdivacky/// 912199989SrdivackySDValue SelectionDAGBuilder::getRoot() { 913199989Srdivacky if (PendingLoads.empty()) 914199989Srdivacky return DAG.getRoot(); 915199989Srdivacky 916199989Srdivacky if (PendingLoads.size() == 1) { 917199989Srdivacky SDValue Root = PendingLoads[0]; 918199989Srdivacky DAG.setRoot(Root); 919199989Srdivacky PendingLoads.clear(); 920199989Srdivacky return Root; 921199989Srdivacky } 922199989Srdivacky 923199989Srdivacky // Otherwise, we have to make a token factor node. 924263508Sdim SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 925199989Srdivacky &PendingLoads[0], PendingLoads.size()); 926199989Srdivacky PendingLoads.clear(); 927199989Srdivacky DAG.setRoot(Root); 928199989Srdivacky return Root; 929199989Srdivacky} 930199989Srdivacky 931199989Srdivacky/// getControlRoot - Similar to getRoot, but instead of flushing all the 932199989Srdivacky/// PendingLoad items, flush all the PendingExports items. It is necessary 933199989Srdivacky/// to do this before emitting a terminator instruction. 934199989Srdivacky/// 935199989SrdivackySDValue SelectionDAGBuilder::getControlRoot() { 936199989Srdivacky SDValue Root = DAG.getRoot(); 937199989Srdivacky 938199989Srdivacky if (PendingExports.empty()) 939199989Srdivacky return Root; 940199989Srdivacky 941199989Srdivacky // Turn all of the CopyToReg chains into one factored node. 942199989Srdivacky if (Root.getOpcode() != ISD::EntryToken) { 943199989Srdivacky unsigned i = 0, e = PendingExports.size(); 944199989Srdivacky for (; i != e; ++i) { 945199989Srdivacky assert(PendingExports[i].getNode()->getNumOperands() > 1); 946199989Srdivacky if (PendingExports[i].getNode()->getOperand(0) == Root) 947199989Srdivacky break; // Don't add the root if we already indirectly depend on it. 948199989Srdivacky } 949199989Srdivacky 950199989Srdivacky if (i == e) 951199989Srdivacky PendingExports.push_back(Root); 952199989Srdivacky } 953199989Srdivacky 954263508Sdim Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 955199989Srdivacky &PendingExports[0], 956199989Srdivacky PendingExports.size()); 957199989Srdivacky PendingExports.clear(); 958199989Srdivacky DAG.setRoot(Root); 959199989Srdivacky return Root; 960199989Srdivacky} 961199989Srdivacky 962207618Srdivackyvoid SelectionDAGBuilder::visit(const Instruction &I) { 963207618Srdivacky // Set up outgoing PHI node register values before emitting the terminator. 964207618Srdivacky if (isa<TerminatorInst>(&I)) 965207618Srdivacky HandlePHINodesInSuccessorBlocks(I.getParent()); 966207618Srdivacky 967263508Sdim ++SDNodeOrder; 968207618Srdivacky 969263508Sdim CurInst = &I; 970263508Sdim 971199989Srdivacky visit(I.getOpcode(), I); 972207618Srdivacky 973207618Srdivacky if (!isa<TerminatorInst>(&I) && !HasTailCall) 974207618Srdivacky CopyToExportRegsIfNeeded(&I); 975207618Srdivacky 976263508Sdim CurInst = NULL; 977199989Srdivacky} 978199989Srdivacky 979207618Srdivackyvoid SelectionDAGBuilder::visitPHI(const PHINode &) { 980207618Srdivacky llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!"); 981207618Srdivacky} 982207618Srdivacky 983207618Srdivackyvoid SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { 984199989Srdivacky // Note: this doesn't use InstVisitor, because it has to work with 985199989Srdivacky // ConstantExpr's in addition to instructions. 986199989Srdivacky switch (Opcode) { 987199989Srdivacky default: llvm_unreachable("Unknown instruction type encountered!"); 988199989Srdivacky // Build the switch statement using the Instruction.def file. 989199989Srdivacky#define HANDLE_INST(NUM, OPCODE, CLASS) \ 990239462Sdim case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; 991249423Sdim#include "llvm/IR/Instruction.def" 992199989Srdivacky } 993199989Srdivacky} 994199989Srdivacky 995212904Sdim// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, 996212904Sdim// generate the debug data structures now that we've seen its definition. 997212904Sdimvoid SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, 998212904Sdim SDValue Val) { 999212904Sdim DanglingDebugInfo &DDI = DanglingDebugInfoMap[V]; 1000212904Sdim if (DDI.getDI()) { 1001212904Sdim const DbgValueInst *DI = DDI.getDI(); 1002212904Sdim DebugLoc dl = DDI.getdl(); 1003212904Sdim unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); 1004212904Sdim MDNode *Variable = DI->getVariable(); 1005212904Sdim uint64_t Offset = DI->getOffset(); 1006212904Sdim SDDbgValue *SDV; 1007212904Sdim if (Val.getNode()) { 1008212904Sdim if (!EmitFuncArgumentDbgValue(V, Variable, Offset, Val)) { 1009212904Sdim SDV = DAG.getDbgValue(Variable, Val.getNode(), 1010212904Sdim Val.getResNo(), Offset, dl, DbgSDNodeOrder); 1011212904Sdim DAG.AddDbgValue(SDV, Val.getNode(), false); 1012212904Sdim } 1013219077Sdim } else 1014263508Sdim DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); 1015212904Sdim DanglingDebugInfoMap[V] = DanglingDebugInfo(); 1016212904Sdim } 1017212904Sdim} 1018212904Sdim 1019226633Sdim/// getValue - Return an SDValue for the given Value. 1020199989SrdivackySDValue SelectionDAGBuilder::getValue(const Value *V) { 1021210299Sed // If we already have an SDValue for this value, use it. It's important 1022210299Sed // to do this first, so that we don't create a CopyFromReg if we already 1023210299Sed // have a regular SDValue. 1024199989Srdivacky SDValue &N = NodeMap[V]; 1025199989Srdivacky if (N.getNode()) return N; 1026199989Srdivacky 1027210299Sed // If there's a virtual register allocated and initialized for this 1028210299Sed // value, use it. 1029210299Sed DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V); 1030210299Sed if (It != FuncInfo.ValueMap.end()) { 1031210299Sed unsigned InReg = It->second; 1032263508Sdim RegsForValue RFV(*DAG.getContext(), *TM.getTargetLowering(), 1033263508Sdim InReg, V->getType()); 1034210299Sed SDValue Chain = DAG.getEntryNode(); 1035263508Sdim N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, NULL, V); 1036218893Sdim resolveDanglingDebugInfo(V, N); 1037218893Sdim return N; 1038210299Sed } 1039210299Sed 1040210299Sed // Otherwise create a new SDValue and remember it. 1041210299Sed SDValue Val = getValueImpl(V); 1042210299Sed NodeMap[V] = Val; 1043212904Sdim resolveDanglingDebugInfo(V, Val); 1044210299Sed return Val; 1045210299Sed} 1046210299Sed 1047210299Sed/// getNonRegisterValue - Return an SDValue for the given Value, but 1048210299Sed/// don't look in FuncInfo.ValueMap for a virtual register. 1049210299SedSDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { 1050210299Sed // If we already have an SDValue for this value, use it. 1051210299Sed SDValue &N = NodeMap[V]; 1052210299Sed if (N.getNode()) return N; 1053210299Sed 1054210299Sed // Otherwise create a new SDValue and remember it. 1055210299Sed SDValue Val = getValueImpl(V); 1056210299Sed NodeMap[V] = Val; 1057212904Sdim resolveDanglingDebugInfo(V, Val); 1058210299Sed return Val; 1059210299Sed} 1060210299Sed 1061212904Sdim/// getValueImpl - Helper function for getValue and getNonRegisterValue. 1062210299Sed/// Create an SDValue for the given value. 1063210299SedSDValue SelectionDAGBuilder::getValueImpl(const Value *V) { 1064263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1065263508Sdim 1066207618Srdivacky if (const Constant *C = dyn_cast<Constant>(V)) { 1067263508Sdim EVT VT = TLI->getValueType(V->getType(), true); 1068199989Srdivacky 1069207618Srdivacky if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) 1070210299Sed return DAG.getConstant(*CI, VT); 1071199989Srdivacky 1072207618Srdivacky if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) 1073263508Sdim return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); 1074199989Srdivacky 1075263508Sdim if (isa<ConstantPointerNull>(C)) { 1076263508Sdim unsigned AS = V->getType()->getPointerAddressSpace(); 1077263508Sdim return DAG.getConstant(0, TLI->getPointerTy(AS)); 1078263508Sdim } 1079199989Srdivacky 1080207618Srdivacky if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) 1081210299Sed return DAG.getConstantFP(*CFP, VT); 1082199989Srdivacky 1083199989Srdivacky if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) 1084210299Sed return DAG.getUNDEF(VT); 1085199989Srdivacky 1086207618Srdivacky if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 1087199989Srdivacky visit(CE->getOpcode(), *CE); 1088199989Srdivacky SDValue N1 = NodeMap[V]; 1089207618Srdivacky assert(N1.getNode() && "visit didn't populate the NodeMap!"); 1090199989Srdivacky return N1; 1091199989Srdivacky } 1092199989Srdivacky 1093199989Srdivacky if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { 1094199989Srdivacky SmallVector<SDValue, 4> Constants; 1095199989Srdivacky for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); 1096199989Srdivacky OI != OE; ++OI) { 1097199989Srdivacky SDNode *Val = getValue(*OI).getNode(); 1098199989Srdivacky // If the operand is an empty aggregate, there are no values. 1099199989Srdivacky if (!Val) continue; 1100199989Srdivacky // Add each leaf value from the operand to the Constants list 1101199989Srdivacky // to form a flattened list of all the values. 1102199989Srdivacky for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1103199989Srdivacky Constants.push_back(SDValue(Val, i)); 1104199989Srdivacky } 1105201360Srdivacky 1106203954Srdivacky return DAG.getMergeValues(&Constants[0], Constants.size(), 1107263508Sdim getCurSDLoc()); 1108199989Srdivacky } 1109263508Sdim 1110234353Sdim if (const ConstantDataSequential *CDS = 1111234353Sdim dyn_cast<ConstantDataSequential>(C)) { 1112234353Sdim SmallVector<SDValue, 4> Ops; 1113234353Sdim for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { 1114234353Sdim SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); 1115234353Sdim // Add each leaf value from the operand to the Constants list 1116234353Sdim // to form a flattened list of all the values. 1117234353Sdim for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1118234353Sdim Ops.push_back(SDValue(Val, i)); 1119234353Sdim } 1120199989Srdivacky 1121234353Sdim if (isa<ArrayType>(CDS->getType())) 1122263508Sdim return DAG.getMergeValues(&Ops[0], Ops.size(), getCurSDLoc()); 1123263508Sdim return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 1124234353Sdim VT, &Ops[0], Ops.size()); 1125234353Sdim } 1126234353Sdim 1127204642Srdivacky if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { 1128199989Srdivacky assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && 1129199989Srdivacky "Unknown struct or array constant!"); 1130199989Srdivacky 1131199989Srdivacky SmallVector<EVT, 4> ValueVTs; 1132263508Sdim ComputeValueVTs(*TLI, C->getType(), ValueVTs); 1133199989Srdivacky unsigned NumElts = ValueVTs.size(); 1134199989Srdivacky if (NumElts == 0) 1135199989Srdivacky return SDValue(); // empty struct 1136199989Srdivacky SmallVector<SDValue, 4> Constants(NumElts); 1137199989Srdivacky for (unsigned i = 0; i != NumElts; ++i) { 1138199989Srdivacky EVT EltVT = ValueVTs[i]; 1139199989Srdivacky if (isa<UndefValue>(C)) 1140199989Srdivacky Constants[i] = DAG.getUNDEF(EltVT); 1141199989Srdivacky else if (EltVT.isFloatingPoint()) 1142199989Srdivacky Constants[i] = DAG.getConstantFP(0, EltVT); 1143199989Srdivacky else 1144199989Srdivacky Constants[i] = DAG.getConstant(0, EltVT); 1145199989Srdivacky } 1146201360Srdivacky 1147203954Srdivacky return DAG.getMergeValues(&Constants[0], NumElts, 1148263508Sdim getCurSDLoc()); 1149199989Srdivacky } 1150199989Srdivacky 1151207618Srdivacky if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) 1152199989Srdivacky return DAG.getBlockAddress(BA, VT); 1153199989Srdivacky 1154226633Sdim VectorType *VecTy = cast<VectorType>(V->getType()); 1155199989Srdivacky unsigned NumElements = VecTy->getNumElements(); 1156199989Srdivacky 1157199989Srdivacky // Now that we know the number and type of the elements, get that number of 1158199989Srdivacky // elements into the Ops array based on what kind of constant it is. 1159199989Srdivacky SmallVector<SDValue, 16> Ops; 1160234353Sdim if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { 1161199989Srdivacky for (unsigned i = 0; i != NumElements; ++i) 1162234353Sdim Ops.push_back(getValue(CV->getOperand(i))); 1163199989Srdivacky } else { 1164199989Srdivacky assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!"); 1165263508Sdim EVT EltVT = TLI->getValueType(VecTy->getElementType()); 1166199989Srdivacky 1167199989Srdivacky SDValue Op; 1168199989Srdivacky if (EltVT.isFloatingPoint()) 1169199989Srdivacky Op = DAG.getConstantFP(0, EltVT); 1170199989Srdivacky else 1171199989Srdivacky Op = DAG.getConstant(0, EltVT); 1172199989Srdivacky Ops.assign(NumElements, Op); 1173199989Srdivacky } 1174199989Srdivacky 1175199989Srdivacky // Create a BUILD_VECTOR node. 1176263508Sdim return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 1177203954Srdivacky VT, &Ops[0], Ops.size()); 1178199989Srdivacky } 1179199989Srdivacky 1180199989Srdivacky // If this is a static alloca, generate it as the frameindex instead of 1181199989Srdivacky // computation. 1182199989Srdivacky if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 1183199989Srdivacky DenseMap<const AllocaInst*, int>::iterator SI = 1184199989Srdivacky FuncInfo.StaticAllocaMap.find(AI); 1185199989Srdivacky if (SI != FuncInfo.StaticAllocaMap.end()) 1186263508Sdim return DAG.getFrameIndex(SI->second, TLI->getPointerTy()); 1187199989Srdivacky } 1188199989Srdivacky 1189210299Sed // If this is an instruction which fast-isel has deferred, select it now. 1190210299Sed if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 1191210299Sed unsigned InReg = FuncInfo.InitializeRegForValue(Inst); 1192263508Sdim RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType()); 1193210299Sed SDValue Chain = DAG.getEntryNode(); 1194263508Sdim return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, NULL, V); 1195210299Sed } 1196199989Srdivacky 1197210299Sed llvm_unreachable("Can't get register for value!"); 1198199989Srdivacky} 1199199989Srdivacky 1200207618Srdivackyvoid SelectionDAGBuilder::visitRet(const ReturnInst &I) { 1201263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1202199989Srdivacky SDValue Chain = getControlRoot(); 1203199989Srdivacky SmallVector<ISD::OutputArg, 8> Outs; 1204210299Sed SmallVector<SDValue, 8> OutVals; 1205201360Srdivacky 1206210299Sed if (!FuncInfo.CanLowerReturn) { 1207210299Sed unsigned DemoteReg = FuncInfo.DemoteRegister; 1208199989Srdivacky const Function *F = I.getParent()->getParent(); 1209199989Srdivacky 1210199989Srdivacky // Emit a store of the return value through the virtual register. 1211199989Srdivacky // Leave Outs empty so that LowerReturn won't try to load return 1212199989Srdivacky // registers the usual way. 1213199989Srdivacky SmallVector<EVT, 1> PtrValueVTs; 1214263508Sdim ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()), 1215199989Srdivacky PtrValueVTs); 1216199989Srdivacky 1217199989Srdivacky SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]); 1218199989Srdivacky SDValue RetOp = getValue(I.getOperand(0)); 1219201360Srdivacky 1220199989Srdivacky SmallVector<EVT, 4> ValueVTs; 1221199989Srdivacky SmallVector<uint64_t, 4> Offsets; 1222263508Sdim ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets); 1223199989Srdivacky unsigned NumValues = ValueVTs.size(); 1224199989Srdivacky 1225199989Srdivacky SmallVector<SDValue, 4> Chains(NumValues); 1226201360Srdivacky for (unsigned i = 0; i != NumValues; ++i) { 1227263508Sdim SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), 1228212904Sdim RetPtr.getValueType(), RetPtr, 1229212904Sdim DAG.getIntPtrConstant(Offsets[i])); 1230201360Srdivacky Chains[i] = 1231263508Sdim DAG.getStore(Chain, getCurSDLoc(), 1232201360Srdivacky SDValue(RetOp.getNode(), RetOp.getResNo() + i), 1233218893Sdim // FIXME: better loc info would be nice. 1234218893Sdim Add, MachinePointerInfo(), false, false, 0); 1235201360Srdivacky } 1236201360Srdivacky 1237263508Sdim Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 1238199989Srdivacky MVT::Other, &Chains[0], NumValues); 1239204642Srdivacky } else if (I.getNumOperands() != 0) { 1240204642Srdivacky SmallVector<EVT, 4> ValueVTs; 1241263508Sdim ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs); 1242204642Srdivacky unsigned NumValues = ValueVTs.size(); 1243204642Srdivacky if (NumValues) { 1244204642Srdivacky SDValue RetOp = getValue(I.getOperand(0)); 1245199989Srdivacky for (unsigned j = 0, f = NumValues; j != f; ++j) { 1246199989Srdivacky EVT VT = ValueVTs[j]; 1247199989Srdivacky 1248199989Srdivacky ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 1249199989Srdivacky 1250199989Srdivacky const Function *F = I.getParent()->getParent(); 1251249423Sdim if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1252249423Sdim Attribute::SExt)) 1253199989Srdivacky ExtendKind = ISD::SIGN_EXTEND; 1254249423Sdim else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1255249423Sdim Attribute::ZExt)) 1256199989Srdivacky ExtendKind = ISD::ZERO_EXTEND; 1257199989Srdivacky 1258221345Sdim if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) 1259263508Sdim VT = TLI->getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind); 1260199989Srdivacky 1261263508Sdim unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT); 1262263508Sdim MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT); 1263199989Srdivacky SmallVector<SDValue, 4> Parts(NumParts); 1264263508Sdim getCopyToParts(DAG, getCurSDLoc(), 1265199989Srdivacky SDValue(RetOp.getNode(), RetOp.getResNo() + j), 1266243830Sdim &Parts[0], NumParts, PartVT, &I, ExtendKind); 1267199989Srdivacky 1268199989Srdivacky // 'inreg' on function refers to return value 1269199989Srdivacky ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 1270249423Sdim if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1271249423Sdim Attribute::InReg)) 1272199989Srdivacky Flags.setInReg(); 1273199989Srdivacky 1274199989Srdivacky // Propagate extension type if any 1275221345Sdim if (ExtendKind == ISD::SIGN_EXTEND) 1276199989Srdivacky Flags.setSExt(); 1277221345Sdim else if (ExtendKind == ISD::ZERO_EXTEND) 1278199989Srdivacky Flags.setZExt(); 1279199989Srdivacky 1280210299Sed for (unsigned i = 0; i < NumParts; ++i) { 1281210299Sed Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(), 1282263508Sdim VT, /*isfixed=*/true, 0, 0)); 1283210299Sed OutVals.push_back(Parts[i]); 1284210299Sed } 1285199989Srdivacky } 1286199989Srdivacky } 1287199989Srdivacky } 1288199989Srdivacky 1289199989Srdivacky bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 1290199989Srdivacky CallingConv::ID CallConv = 1291199989Srdivacky DAG.getMachineFunction().getFunction()->getCallingConv(); 1292263508Sdim Chain = TM.getTargetLowering()->LowerReturn(Chain, CallConv, isVarArg, 1293263508Sdim Outs, OutVals, getCurSDLoc(), 1294263508Sdim DAG); 1295199989Srdivacky 1296199989Srdivacky // Verify that the target's LowerReturn behaved as expected. 1297199989Srdivacky assert(Chain.getNode() && Chain.getValueType() == MVT::Other && 1298199989Srdivacky "LowerReturn didn't return a valid chain!"); 1299199989Srdivacky 1300199989Srdivacky // Update the DAG with the new chain value resulting from return lowering. 1301199989Srdivacky DAG.setRoot(Chain); 1302199989Srdivacky} 1303199989Srdivacky 1304199989Srdivacky/// CopyToExportRegsIfNeeded - If the given value has virtual registers 1305199989Srdivacky/// created for it, emit nodes to copy the value into the virtual 1306199989Srdivacky/// registers. 1307207618Srdivackyvoid SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { 1308223017Sdim // Skip empty types 1309223017Sdim if (V->getType()->isEmptyTy()) 1310223017Sdim return; 1311223017Sdim 1312207618Srdivacky DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); 1313207618Srdivacky if (VMI != FuncInfo.ValueMap.end()) { 1314207618Srdivacky assert(!V->use_empty() && "Unused value assigned virtual registers!"); 1315207618Srdivacky CopyValueToVirtualRegister(V, VMI->second); 1316199989Srdivacky } 1317199989Srdivacky} 1318199989Srdivacky 1319199989Srdivacky/// ExportFromCurrentBlock - If this condition isn't known to be exported from 1320199989Srdivacky/// the current basic block, add it to ValueMap now so that we'll get a 1321199989Srdivacky/// CopyTo/FromReg. 1322207618Srdivackyvoid SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { 1323199989Srdivacky // No need to export constants. 1324199989Srdivacky if (!isa<Instruction>(V) && !isa<Argument>(V)) return; 1325199989Srdivacky 1326199989Srdivacky // Already exported? 1327199989Srdivacky if (FuncInfo.isExportedInst(V)) return; 1328199989Srdivacky 1329199989Srdivacky unsigned Reg = FuncInfo.InitializeRegForValue(V); 1330199989Srdivacky CopyValueToVirtualRegister(V, Reg); 1331199989Srdivacky} 1332199989Srdivacky 1333207618Srdivackybool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, 1334199989Srdivacky const BasicBlock *FromBB) { 1335199989Srdivacky // The operands of the setcc have to be in this block. We don't know 1336199989Srdivacky // how to export them from some other block. 1337207618Srdivacky if (const Instruction *VI = dyn_cast<Instruction>(V)) { 1338199989Srdivacky // Can export from current BB. 1339199989Srdivacky if (VI->getParent() == FromBB) 1340199989Srdivacky return true; 1341199989Srdivacky 1342199989Srdivacky // Is already exported, noop. 1343199989Srdivacky return FuncInfo.isExportedInst(V); 1344199989Srdivacky } 1345199989Srdivacky 1346199989Srdivacky // If this is an argument, we can export it if the BB is the entry block or 1347199989Srdivacky // if it is already exported. 1348199989Srdivacky if (isa<Argument>(V)) { 1349199989Srdivacky if (FromBB == &FromBB->getParent()->getEntryBlock()) 1350199989Srdivacky return true; 1351199989Srdivacky 1352199989Srdivacky // Otherwise, can only export this if it is already exported. 1353199989Srdivacky return FuncInfo.isExportedInst(V); 1354199989Srdivacky } 1355199989Srdivacky 1356199989Srdivacky // Otherwise, constants can always be exported. 1357199989Srdivacky return true; 1358199989Srdivacky} 1359199989Srdivacky 1360224145Sdim/// Return branch probability calculated by BranchProbabilityInfo for IR blocks. 1361234353Sdimuint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src, 1362234353Sdim const MachineBasicBlock *Dst) const { 1363224145Sdim BranchProbabilityInfo *BPI = FuncInfo.BPI; 1364224145Sdim if (!BPI) 1365224145Sdim return 0; 1366226633Sdim const BasicBlock *SrcBB = Src->getBasicBlock(); 1367226633Sdim const BasicBlock *DstBB = Dst->getBasicBlock(); 1368224145Sdim return BPI->getEdgeWeight(SrcBB, DstBB); 1369224145Sdim} 1370224145Sdim 1371226633Sdimvoid SelectionDAGBuilder:: 1372226633SdimaddSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, 1373226633Sdim uint32_t Weight /* = 0 */) { 1374226633Sdim if (!Weight) 1375226633Sdim Weight = getEdgeWeight(Src, Dst); 1376226633Sdim Src->addSuccessor(Dst, Weight); 1377224145Sdim} 1378224145Sdim 1379224145Sdim 1380199989Srdivackystatic bool InBlock(const Value *V, const BasicBlock *BB) { 1381199989Srdivacky if (const Instruction *I = dyn_cast<Instruction>(V)) 1382199989Srdivacky return I->getParent() == BB; 1383199989Srdivacky return true; 1384199989Srdivacky} 1385199989Srdivacky 1386199989Srdivacky/// EmitBranchForMergedCondition - Helper method for FindMergedConditions. 1387199989Srdivacky/// This function emits a branch and is used at the leaves of an OR or an 1388199989Srdivacky/// AND operator tree. 1389199989Srdivacky/// 1390199989Srdivackyvoid 1391207618SrdivackySelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, 1392199989Srdivacky MachineBasicBlock *TBB, 1393199989Srdivacky MachineBasicBlock *FBB, 1394207618Srdivacky MachineBasicBlock *CurBB, 1395207618Srdivacky MachineBasicBlock *SwitchBB) { 1396199989Srdivacky const BasicBlock *BB = CurBB->getBasicBlock(); 1397199989Srdivacky 1398199989Srdivacky // If the leaf of the tree is a comparison, merge the condition into 1399199989Srdivacky // the caseblock. 1400207618Srdivacky if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { 1401199989Srdivacky // The operands of the cmp have to be in this block. We don't know 1402199989Srdivacky // how to export them from some other block. If this is the first block 1403199989Srdivacky // of the sequence, no exporting is needed. 1404207618Srdivacky if (CurBB == SwitchBB || 1405199989Srdivacky (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && 1406199989Srdivacky isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { 1407199989Srdivacky ISD::CondCode Condition; 1408207618Srdivacky if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { 1409199989Srdivacky Condition = getICmpCondCode(IC->getPredicate()); 1410207618Srdivacky } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) { 1411199989Srdivacky Condition = getFCmpCondCode(FC->getPredicate()); 1412234353Sdim if (TM.Options.NoNaNsFPMath) 1413234353Sdim Condition = getFCmpCodeWithoutNaN(Condition); 1414199989Srdivacky } else { 1415199989Srdivacky Condition = ISD::SETEQ; // silence warning. 1416199989Srdivacky llvm_unreachable("Unknown compare instruction"); 1417199989Srdivacky } 1418199989Srdivacky 1419199989Srdivacky CaseBlock CB(Condition, BOp->getOperand(0), 1420199989Srdivacky BOp->getOperand(1), NULL, TBB, FBB, CurBB); 1421199989Srdivacky SwitchCases.push_back(CB); 1422199989Srdivacky return; 1423199989Srdivacky } 1424199989Srdivacky } 1425199989Srdivacky 1426199989Srdivacky // Create a CaseBlock record representing this branch. 1427199989Srdivacky CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()), 1428199989Srdivacky NULL, TBB, FBB, CurBB); 1429199989Srdivacky SwitchCases.push_back(CB); 1430199989Srdivacky} 1431199989Srdivacky 1432199989Srdivacky/// FindMergedConditions - If Cond is an expression like 1433207618Srdivackyvoid SelectionDAGBuilder::FindMergedConditions(const Value *Cond, 1434199989Srdivacky MachineBasicBlock *TBB, 1435199989Srdivacky MachineBasicBlock *FBB, 1436199989Srdivacky MachineBasicBlock *CurBB, 1437207618Srdivacky MachineBasicBlock *SwitchBB, 1438199989Srdivacky unsigned Opc) { 1439199989Srdivacky // If this node is not part of the or/and tree, emit it as a branch. 1440207618Srdivacky const Instruction *BOp = dyn_cast<Instruction>(Cond); 1441199989Srdivacky if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) || 1442199989Srdivacky (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() || 1443199989Srdivacky BOp->getParent() != CurBB->getBasicBlock() || 1444199989Srdivacky !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) || 1445199989Srdivacky !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) { 1446207618Srdivacky EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB); 1447199989Srdivacky return; 1448199989Srdivacky } 1449199989Srdivacky 1450199989Srdivacky // Create TmpBB after CurBB. 1451199989Srdivacky MachineFunction::iterator BBI = CurBB; 1452199989Srdivacky MachineFunction &MF = DAG.getMachineFunction(); 1453199989Srdivacky MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); 1454199989Srdivacky CurBB->getParent()->insert(++BBI, TmpBB); 1455199989Srdivacky 1456199989Srdivacky if (Opc == Instruction::Or) { 1457199989Srdivacky // Codegen X | Y as: 1458199989Srdivacky // jmp_if_X TBB 1459199989Srdivacky // jmp TmpBB 1460199989Srdivacky // TmpBB: 1461199989Srdivacky // jmp_if_Y TBB 1462199989Srdivacky // jmp FBB 1463199989Srdivacky // 1464199989Srdivacky 1465199989Srdivacky // Emit the LHS condition. 1466207618Srdivacky FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc); 1467199989Srdivacky 1468199989Srdivacky // Emit the RHS condition into TmpBB. 1469207618Srdivacky FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc); 1470199989Srdivacky } else { 1471199989Srdivacky assert(Opc == Instruction::And && "Unknown merge op!"); 1472199989Srdivacky // Codegen X & Y as: 1473199989Srdivacky // jmp_if_X TmpBB 1474199989Srdivacky // jmp FBB 1475199989Srdivacky // TmpBB: 1476199989Srdivacky // jmp_if_Y TBB 1477199989Srdivacky // jmp FBB 1478199989Srdivacky // 1479199989Srdivacky // This requires creation of TmpBB after CurBB. 1480199989Srdivacky 1481199989Srdivacky // Emit the LHS condition. 1482207618Srdivacky FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc); 1483199989Srdivacky 1484199989Srdivacky // Emit the RHS condition into TmpBB. 1485207618Srdivacky FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc); 1486199989Srdivacky } 1487199989Srdivacky} 1488199989Srdivacky 1489199989Srdivacky/// If the set of cases should be emitted as a series of branches, return true. 1490199989Srdivacky/// If we should emit this as a bunch of and/or'd together conditions, return 1491199989Srdivacky/// false. 1492199989Srdivackybool 1493263508SdimSelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { 1494199989Srdivacky if (Cases.size() != 2) return true; 1495199989Srdivacky 1496199989Srdivacky // If this is two comparisons of the same values or'd or and'd together, they 1497199989Srdivacky // will get folded into a single comparison, so don't emit two blocks. 1498199989Srdivacky if ((Cases[0].CmpLHS == Cases[1].CmpLHS && 1499199989Srdivacky Cases[0].CmpRHS == Cases[1].CmpRHS) || 1500199989Srdivacky (Cases[0].CmpRHS == Cases[1].CmpLHS && 1501199989Srdivacky Cases[0].CmpLHS == Cases[1].CmpRHS)) { 1502199989Srdivacky return false; 1503199989Srdivacky } 1504199989Srdivacky 1505202375Srdivacky // Handle: (X != null) | (Y != null) --> (X|Y) != 0 1506202375Srdivacky // Handle: (X == null) & (Y == null) --> (X|Y) == 0 1507202375Srdivacky if (Cases[0].CmpRHS == Cases[1].CmpRHS && 1508202375Srdivacky Cases[0].CC == Cases[1].CC && 1509202375Srdivacky isa<Constant>(Cases[0].CmpRHS) && 1510202375Srdivacky cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { 1511202375Srdivacky if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) 1512202375Srdivacky return false; 1513202375Srdivacky if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) 1514202375Srdivacky return false; 1515202375Srdivacky } 1516218893Sdim 1517199989Srdivacky return true; 1518199989Srdivacky} 1519199989Srdivacky 1520207618Srdivackyvoid SelectionDAGBuilder::visitBr(const BranchInst &I) { 1521210299Sed MachineBasicBlock *BrMBB = FuncInfo.MBB; 1522207618Srdivacky 1523199989Srdivacky // Update machine-CFG edges. 1524199989Srdivacky MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; 1525199989Srdivacky 1526199989Srdivacky // Figure out which block is immediately after the current one. 1527199989Srdivacky MachineBasicBlock *NextBlock = 0; 1528207618Srdivacky MachineFunction::iterator BBI = BrMBB; 1529199989Srdivacky if (++BBI != FuncInfo.MF->end()) 1530199989Srdivacky NextBlock = BBI; 1531199989Srdivacky 1532199989Srdivacky if (I.isUnconditional()) { 1533199989Srdivacky // Update machine-CFG edges. 1534207618Srdivacky BrMBB->addSuccessor(Succ0MBB); 1535199989Srdivacky 1536199989Srdivacky // If this is not a fall-through branch, emit the branch. 1537203954Srdivacky if (Succ0MBB != NextBlock) 1538263508Sdim DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 1539199989Srdivacky MVT::Other, getControlRoot(), 1540203954Srdivacky DAG.getBasicBlock(Succ0MBB))); 1541201360Srdivacky 1542199989Srdivacky return; 1543199989Srdivacky } 1544199989Srdivacky 1545199989Srdivacky // If this condition is one of the special cases we handle, do special stuff 1546199989Srdivacky // now. 1547207618Srdivacky const Value *CondVal = I.getCondition(); 1548199989Srdivacky MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; 1549199989Srdivacky 1550199989Srdivacky // If this is a series of conditions that are or'd or and'd together, emit 1551199989Srdivacky // this as a sequence of branches instead of setcc's with and/or operations. 1552218893Sdim // As long as jumps are not expensive, this should improve performance. 1553199989Srdivacky // For example, instead of something like: 1554199989Srdivacky // cmp A, B 1555199989Srdivacky // C = seteq 1556199989Srdivacky // cmp D, E 1557199989Srdivacky // F = setle 1558199989Srdivacky // or C, F 1559199989Srdivacky // jnz foo 1560199989Srdivacky // Emit: 1561199989Srdivacky // cmp A, B 1562199989Srdivacky // je foo 1563199989Srdivacky // cmp D, E 1564199989Srdivacky // jle foo 1565199989Srdivacky // 1566207618Srdivacky if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) { 1567263508Sdim if (!TM.getTargetLowering()->isJumpExpensive() && 1568218893Sdim BOp->hasOneUse() && 1569199989Srdivacky (BOp->getOpcode() == Instruction::And || 1570199989Srdivacky BOp->getOpcode() == Instruction::Or)) { 1571207618Srdivacky FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, 1572207618Srdivacky BOp->getOpcode()); 1573199989Srdivacky // If the compares in later blocks need to use values not currently 1574199989Srdivacky // exported from this block, export them now. This block should always 1575199989Srdivacky // be the first entry. 1576207618Srdivacky assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); 1577199989Srdivacky 1578199989Srdivacky // Allow some cases to be rejected. 1579199989Srdivacky if (ShouldEmitAsBranches(SwitchCases)) { 1580199989Srdivacky for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) { 1581199989Srdivacky ExportFromCurrentBlock(SwitchCases[i].CmpLHS); 1582199989Srdivacky ExportFromCurrentBlock(SwitchCases[i].CmpRHS); 1583199989Srdivacky } 1584199989Srdivacky 1585199989Srdivacky // Emit the branch for this block. 1586207618Srdivacky visitSwitchCase(SwitchCases[0], BrMBB); 1587199989Srdivacky SwitchCases.erase(SwitchCases.begin()); 1588199989Srdivacky return; 1589199989Srdivacky } 1590199989Srdivacky 1591199989Srdivacky // Okay, we decided not to do this, remove any inserted MBB's and clear 1592199989Srdivacky // SwitchCases. 1593199989Srdivacky for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) 1594199989Srdivacky FuncInfo.MF->erase(SwitchCases[i].ThisBB); 1595199989Srdivacky 1596199989Srdivacky SwitchCases.clear(); 1597199989Srdivacky } 1598199989Srdivacky } 1599199989Srdivacky 1600199989Srdivacky // Create a CaseBlock record representing this branch. 1601199989Srdivacky CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), 1602207618Srdivacky NULL, Succ0MBB, Succ1MBB, BrMBB); 1603201360Srdivacky 1604199989Srdivacky // Use visitSwitchCase to actually insert the fast branch sequence for this 1605199989Srdivacky // cond branch. 1606207618Srdivacky visitSwitchCase(CB, BrMBB); 1607199989Srdivacky} 1608199989Srdivacky 1609199989Srdivacky/// visitSwitchCase - Emits the necessary code to represent a single node in 1610199989Srdivacky/// the binary search tree resulting from lowering a switch instruction. 1611207618Srdivackyvoid SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, 1612207618Srdivacky MachineBasicBlock *SwitchBB) { 1613199989Srdivacky SDValue Cond; 1614199989Srdivacky SDValue CondLHS = getValue(CB.CmpLHS); 1615263508Sdim SDLoc dl = getCurSDLoc(); 1616199989Srdivacky 1617199989Srdivacky // Build the setcc now. 1618199989Srdivacky if (CB.CmpMHS == NULL) { 1619199989Srdivacky // Fold "(X == true)" to X and "(X == false)" to !X to 1620199989Srdivacky // handle common cases produced by branch lowering. 1621199989Srdivacky if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && 1622199989Srdivacky CB.CC == ISD::SETEQ) 1623199989Srdivacky Cond = CondLHS; 1624199989Srdivacky else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && 1625199989Srdivacky CB.CC == ISD::SETEQ) { 1626199989Srdivacky SDValue True = DAG.getConstant(1, CondLHS.getValueType()); 1627199989Srdivacky Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); 1628199989Srdivacky } else 1629199989Srdivacky Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); 1630199989Srdivacky } else { 1631263508Sdim assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); 1632199989Srdivacky 1633199989Srdivacky const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); 1634199989Srdivacky const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); 1635199989Srdivacky 1636199989Srdivacky SDValue CmpOp = getValue(CB.CmpMHS); 1637199989Srdivacky EVT VT = CmpOp.getValueType(); 1638263508Sdim 1639263508Sdim if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { 1640199989Srdivacky Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT), 1641263508Sdim ISD::SETLE); 1642199989Srdivacky } else { 1643199989Srdivacky SDValue SUB = DAG.getNode(ISD::SUB, dl, 1644199989Srdivacky VT, CmpOp, DAG.getConstant(Low, VT)); 1645199989Srdivacky Cond = DAG.getSetCC(dl, MVT::i1, SUB, 1646199989Srdivacky DAG.getConstant(High-Low, VT), ISD::SETULE); 1647199989Srdivacky } 1648199989Srdivacky } 1649199989Srdivacky 1650199989Srdivacky // Update successor info 1651226633Sdim addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight); 1652243830Sdim // TrueBB and FalseBB are always different unless the incoming IR is 1653243830Sdim // degenerate. This only happens when running llc on weird IR. 1654243830Sdim if (CB.TrueBB != CB.FalseBB) 1655243830Sdim addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight); 1656199989Srdivacky 1657199989Srdivacky // Set NextBlock to be the MBB immediately after the current one, if any. 1658199989Srdivacky // This is used to avoid emitting unnecessary branches to the next block. 1659199989Srdivacky MachineBasicBlock *NextBlock = 0; 1660207618Srdivacky MachineFunction::iterator BBI = SwitchBB; 1661199989Srdivacky if (++BBI != FuncInfo.MF->end()) 1662199989Srdivacky NextBlock = BBI; 1663199989Srdivacky 1664199989Srdivacky // If the lhs block is the next block, invert the condition so that we can 1665199989Srdivacky // fall through to the lhs instead of the rhs block. 1666199989Srdivacky if (CB.TrueBB == NextBlock) { 1667199989Srdivacky std::swap(CB.TrueBB, CB.FalseBB); 1668199989Srdivacky SDValue True = DAG.getConstant(1, Cond.getValueType()); 1669199989Srdivacky Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); 1670199989Srdivacky } 1671201360Srdivacky 1672199989Srdivacky SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, 1673199989Srdivacky MVT::Other, getControlRoot(), Cond, 1674199989Srdivacky DAG.getBasicBlock(CB.TrueBB)); 1675199989Srdivacky 1676218893Sdim // Insert the false branch. Do this even if it's a fall through branch, 1677218893Sdim // this makes it easier to do DAG optimizations which require inverting 1678218893Sdim // the branch condition. 1679218893Sdim BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, 1680218893Sdim DAG.getBasicBlock(CB.FalseBB)); 1681199989Srdivacky 1682201360Srdivacky DAG.setRoot(BrCond); 1683199989Srdivacky} 1684199989Srdivacky 1685199989Srdivacky/// visitJumpTable - Emit JumpTable node in the current MBB 1686199989Srdivackyvoid SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { 1687199989Srdivacky // Emit the code for the jump table 1688199989Srdivacky assert(JT.Reg != -1U && "Should lower JT Header first!"); 1689263508Sdim EVT PTy = TM.getTargetLowering()->getPointerTy(); 1690263508Sdim SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 1691199989Srdivacky JT.Reg, PTy); 1692199989Srdivacky SDValue Table = DAG.getJumpTable(JT.JTI, PTy); 1693263508Sdim SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), 1694201360Srdivacky MVT::Other, Index.getValue(1), 1695201360Srdivacky Table, Index); 1696201360Srdivacky DAG.setRoot(BrJumpTable); 1697199989Srdivacky} 1698199989Srdivacky 1699199989Srdivacky/// visitJumpTableHeader - This function emits necessary code to produce index 1700199989Srdivacky/// in the JumpTable from switch case. 1701199989Srdivackyvoid SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, 1702207618Srdivacky JumpTableHeader &JTH, 1703207618Srdivacky MachineBasicBlock *SwitchBB) { 1704199989Srdivacky // Subtract the lowest switch case value from the value being switched on and 1705199989Srdivacky // conditional branch to default mbb if the result is greater than the 1706199989Srdivacky // difference between smallest and largest cases. 1707199989Srdivacky SDValue SwitchOp = getValue(JTH.SValue); 1708199989Srdivacky EVT VT = SwitchOp.getValueType(); 1709263508Sdim SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp, 1710199989Srdivacky DAG.getConstant(JTH.First, VT)); 1711199989Srdivacky 1712199989Srdivacky // The SDNode we just created, which holds the value being switched on minus 1713203954Srdivacky // the smallest case value, needs to be copied to a virtual register so it 1714199989Srdivacky // can be used as an index into the jump table in a subsequent basic block. 1715199989Srdivacky // This value may be smaller or larger than the target's pointer type, and 1716199989Srdivacky // therefore require extension or truncating. 1717263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1718263508Sdim SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy()); 1719199989Srdivacky 1720263508Sdim unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy()); 1721263508Sdim SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), 1722199989Srdivacky JumpTableReg, SwitchOp); 1723199989Srdivacky JT.Reg = JumpTableReg; 1724199989Srdivacky 1725199989Srdivacky // Emit the range check for the jump table, and branch to the default block 1726199989Srdivacky // for the switch statement if the value being switched on exceeds the largest 1727199989Srdivacky // case in the switch. 1728263508Sdim SDValue CMP = DAG.getSetCC(getCurSDLoc(), 1729263508Sdim TLI->getSetCCResultType(*DAG.getContext(), 1730263508Sdim Sub.getValueType()), 1731263508Sdim Sub, 1732263508Sdim DAG.getConstant(JTH.Last - JTH.First,VT), 1733199989Srdivacky ISD::SETUGT); 1734199989Srdivacky 1735199989Srdivacky // Set NextBlock to be the MBB immediately after the current one, if any. 1736199989Srdivacky // This is used to avoid emitting unnecessary branches to the next block. 1737199989Srdivacky MachineBasicBlock *NextBlock = 0; 1738207618Srdivacky MachineFunction::iterator BBI = SwitchBB; 1739201360Srdivacky 1740199989Srdivacky if (++BBI != FuncInfo.MF->end()) 1741199989Srdivacky NextBlock = BBI; 1742199989Srdivacky 1743263508Sdim SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1744199989Srdivacky MVT::Other, CopyTo, CMP, 1745199989Srdivacky DAG.getBasicBlock(JT.Default)); 1746199989Srdivacky 1747203954Srdivacky if (JT.MBB != NextBlock) 1748263508Sdim BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond, 1749201360Srdivacky DAG.getBasicBlock(JT.MBB)); 1750201360Srdivacky 1751201360Srdivacky DAG.setRoot(BrCond); 1752199989Srdivacky} 1753199989Srdivacky 1754263508Sdim/// Codegen a new tail for a stack protector check ParentMBB which has had its 1755263508Sdim/// tail spliced into a stack protector check success bb. 1756263508Sdim/// 1757263508Sdim/// For a high level explanation of how this fits into the stack protector 1758263508Sdim/// generation see the comment on the declaration of class 1759263508Sdim/// StackProtectorDescriptor. 1760263508Sdimvoid SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, 1761263508Sdim MachineBasicBlock *ParentBB) { 1762263508Sdim 1763263508Sdim // First create the loads to the guard/stack slot for the comparison. 1764263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1765263508Sdim EVT PtrTy = TLI->getPointerTy(); 1766263508Sdim 1767263508Sdim MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo(); 1768263508Sdim int FI = MFI->getStackProtectorIndex(); 1769263508Sdim 1770263508Sdim const Value *IRGuard = SPD.getGuard(); 1771263508Sdim SDValue GuardPtr = getValue(IRGuard); 1772263508Sdim SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); 1773263508Sdim 1774263508Sdim unsigned Align = 1775263508Sdim TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType()); 1776263508Sdim SDValue Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(), 1777263508Sdim GuardPtr, MachinePointerInfo(IRGuard, 0), 1778263508Sdim true, false, false, Align); 1779263508Sdim 1780263508Sdim SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(), 1781263508Sdim StackSlotPtr, 1782263508Sdim MachinePointerInfo::getFixedStack(FI), 1783263508Sdim true, false, false, Align); 1784263508Sdim 1785263508Sdim // Perform the comparison via a subtract/getsetcc. 1786263508Sdim EVT VT = Guard.getValueType(); 1787263508Sdim SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot); 1788263508Sdim 1789263508Sdim SDValue Cmp = DAG.getSetCC(getCurSDLoc(), 1790263508Sdim TLI->getSetCCResultType(*DAG.getContext(), 1791263508Sdim Sub.getValueType()), 1792263508Sdim Sub, DAG.getConstant(0, VT), 1793263508Sdim ISD::SETNE); 1794263508Sdim 1795263508Sdim // If the sub is not 0, then we know the guard/stackslot do not equal, so 1796263508Sdim // branch to failure MBB. 1797263508Sdim SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1798263508Sdim MVT::Other, StackSlot.getOperand(0), 1799263508Sdim Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); 1800263508Sdim // Otherwise branch to success MBB. 1801263508Sdim SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(), 1802263508Sdim MVT::Other, BrCond, 1803263508Sdim DAG.getBasicBlock(SPD.getSuccessMBB())); 1804263508Sdim 1805263508Sdim DAG.setRoot(Br); 1806263508Sdim} 1807263508Sdim 1808263508Sdim/// Codegen the failure basic block for a stack protector check. 1809263508Sdim/// 1810263508Sdim/// A failure stack protector machine basic block consists simply of a call to 1811263508Sdim/// __stack_chk_fail(). 1812263508Sdim/// 1813263508Sdim/// For a high level explanation of how this fits into the stack protector 1814263508Sdim/// generation see the comment on the declaration of class 1815263508Sdim/// StackProtectorDescriptor. 1816263508Sdimvoid 1817263508SdimSelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { 1818263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1819263508Sdim SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, 1820263508Sdim MVT::isVoid, 0, 0, false, getCurSDLoc(), 1821263508Sdim false, false).second; 1822263508Sdim DAG.setRoot(Chain); 1823263508Sdim} 1824263508Sdim 1825199989Srdivacky/// visitBitTestHeader - This function emits necessary code to produce value 1826199989Srdivacky/// suitable for "bit tests" 1827207618Srdivackyvoid SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, 1828207618Srdivacky MachineBasicBlock *SwitchBB) { 1829199989Srdivacky // Subtract the minimum value 1830199989Srdivacky SDValue SwitchOp = getValue(B.SValue); 1831199989Srdivacky EVT VT = SwitchOp.getValueType(); 1832263508Sdim SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp, 1833199989Srdivacky DAG.getConstant(B.First, VT)); 1834199989Srdivacky 1835199989Srdivacky // Check range 1836263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1837263508Sdim SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(), 1838263508Sdim TLI->getSetCCResultType(*DAG.getContext(), 1839263508Sdim Sub.getValueType()), 1840201360Srdivacky Sub, DAG.getConstant(B.Range, VT), 1841199989Srdivacky ISD::SETUGT); 1842199989Srdivacky 1843218893Sdim // Determine the type of the test operands. 1844218893Sdim bool UsePtrType = false; 1845263508Sdim if (!TLI->isTypeLegal(VT)) 1846218893Sdim UsePtrType = true; 1847218893Sdim else { 1848218893Sdim for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) 1849226633Sdim if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { 1850218893Sdim // Switch table case range are encoded into series of masks. 1851218893Sdim // Just use pointer type, it's guaranteed to fit. 1852218893Sdim UsePtrType = true; 1853218893Sdim break; 1854218893Sdim } 1855218893Sdim } 1856218893Sdim if (UsePtrType) { 1857263508Sdim VT = TLI->getPointerTy(); 1858263508Sdim Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT); 1859218893Sdim } 1860199989Srdivacky 1861249423Sdim B.RegVT = VT.getSimpleVT(); 1862249423Sdim B.Reg = FuncInfo.CreateReg(B.RegVT); 1863263508Sdim SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), 1864218893Sdim B.Reg, Sub); 1865199989Srdivacky 1866199989Srdivacky // Set NextBlock to be the MBB immediately after the current one, if any. 1867199989Srdivacky // This is used to avoid emitting unnecessary branches to the next block. 1868199989Srdivacky MachineBasicBlock *NextBlock = 0; 1869207618Srdivacky MachineFunction::iterator BBI = SwitchBB; 1870199989Srdivacky if (++BBI != FuncInfo.MF->end()) 1871199989Srdivacky NextBlock = BBI; 1872199989Srdivacky 1873199989Srdivacky MachineBasicBlock* MBB = B.Cases[0].ThisBB; 1874199989Srdivacky 1875224145Sdim addSuccessorWithWeight(SwitchBB, B.Default); 1876224145Sdim addSuccessorWithWeight(SwitchBB, MBB); 1877199989Srdivacky 1878263508Sdim SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1879199989Srdivacky MVT::Other, CopyTo, RangeCmp, 1880199989Srdivacky DAG.getBasicBlock(B.Default)); 1881199989Srdivacky 1882203954Srdivacky if (MBB != NextBlock) 1883263508Sdim BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo, 1884201360Srdivacky DAG.getBasicBlock(MBB)); 1885201360Srdivacky 1886201360Srdivacky DAG.setRoot(BrRange); 1887199989Srdivacky} 1888199989Srdivacky 1889199989Srdivacky/// visitBitTestCase - this function produces one "bit test" 1890218893Sdimvoid SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, 1891218893Sdim MachineBasicBlock* NextMBB, 1892243830Sdim uint32_t BranchWeightToNext, 1893199989Srdivacky unsigned Reg, 1894207618Srdivacky BitTestCase &B, 1895207618Srdivacky MachineBasicBlock *SwitchBB) { 1896249423Sdim MVT VT = BB.RegVT; 1897263508Sdim SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 1898218893Sdim Reg, VT); 1899210299Sed SDValue Cmp; 1900224145Sdim unsigned PopCount = CountPopulation_64(B.Mask); 1901263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1902224145Sdim if (PopCount == 1) { 1903210299Sed // Testing for a single bit; just compare the shift count with what it 1904210299Sed // would need to be to shift a 1 bit in that position. 1905263508Sdim Cmp = DAG.getSetCC(getCurSDLoc(), 1906263508Sdim TLI->getSetCCResultType(*DAG.getContext(), VT), 1907210299Sed ShiftOp, 1908263508Sdim DAG.getConstant(countTrailingZeros(B.Mask), VT), 1909210299Sed ISD::SETEQ); 1910224145Sdim } else if (PopCount == BB.Range) { 1911224145Sdim // There is only one zero bit in the range, test for it directly. 1912263508Sdim Cmp = DAG.getSetCC(getCurSDLoc(), 1913263508Sdim TLI->getSetCCResultType(*DAG.getContext(), VT), 1914224145Sdim ShiftOp, 1915224145Sdim DAG.getConstant(CountTrailingOnes_64(B.Mask), VT), 1916224145Sdim ISD::SETNE); 1917210299Sed } else { 1918210299Sed // Make desired shift 1919263508Sdim SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT, 1920218893Sdim DAG.getConstant(1, VT), ShiftOp); 1921199989Srdivacky 1922210299Sed // Emit bit tests and jumps 1923263508Sdim SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(), 1924218893Sdim VT, SwitchVal, DAG.getConstant(B.Mask, VT)); 1925263508Sdim Cmp = DAG.getSetCC(getCurSDLoc(), 1926263508Sdim TLI->getSetCCResultType(*DAG.getContext(), VT), 1927218893Sdim AndOp, DAG.getConstant(0, VT), 1928210299Sed ISD::SETNE); 1929210299Sed } 1930199989Srdivacky 1931243830Sdim // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight. 1932243830Sdim addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight); 1933243830Sdim // The branch weight from SwitchBB to NextMBB is BranchWeightToNext. 1934243830Sdim addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext); 1935199989Srdivacky 1936263508Sdim SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1937199989Srdivacky MVT::Other, getControlRoot(), 1938210299Sed Cmp, DAG.getBasicBlock(B.TargetBB)); 1939199989Srdivacky 1940199989Srdivacky // Set NextBlock to be the MBB immediately after the current one, if any. 1941199989Srdivacky // This is used to avoid emitting unnecessary branches to the next block. 1942199989Srdivacky MachineBasicBlock *NextBlock = 0; 1943207618Srdivacky MachineFunction::iterator BBI = SwitchBB; 1944199989Srdivacky if (++BBI != FuncInfo.MF->end()) 1945199989Srdivacky NextBlock = BBI; 1946199989Srdivacky 1947203954Srdivacky if (NextMBB != NextBlock) 1948263508Sdim BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd, 1949201360Srdivacky DAG.getBasicBlock(NextMBB)); 1950201360Srdivacky 1951201360Srdivacky DAG.setRoot(BrAnd); 1952199989Srdivacky} 1953199989Srdivacky 1954207618Srdivackyvoid SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { 1955210299Sed MachineBasicBlock *InvokeMBB = FuncInfo.MBB; 1956207618Srdivacky 1957199989Srdivacky // Retrieve successors. 1958199989Srdivacky MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; 1959199989Srdivacky MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)]; 1960199989Srdivacky 1961199989Srdivacky const Value *Callee(I.getCalledValue()); 1962239462Sdim const Function *Fn = dyn_cast<Function>(Callee); 1963199989Srdivacky if (isa<InlineAsm>(Callee)) 1964199989Srdivacky visitInlineAsm(&I); 1965239462Sdim else if (Fn && Fn->isIntrinsic()) { 1966239462Sdim assert(Fn->getIntrinsicID() == Intrinsic::donothing); 1967239462Sdim // Ignore invokes to @llvm.donothing: jump directly to the next BB. 1968239462Sdim } else 1969199989Srdivacky LowerCallTo(&I, getValue(Callee), false, LandingPad); 1970199989Srdivacky 1971199989Srdivacky // If the value of the invoke is used outside of its defining block, make it 1972199989Srdivacky // available as a virtual register. 1973199989Srdivacky CopyToExportRegsIfNeeded(&I); 1974199989Srdivacky 1975199989Srdivacky // Update successor info 1976234353Sdim addSuccessorWithWeight(InvokeMBB, Return); 1977234353Sdim addSuccessorWithWeight(InvokeMBB, LandingPad); 1978199989Srdivacky 1979199989Srdivacky // Drop into normal successor. 1980263508Sdim DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 1981203954Srdivacky MVT::Other, getControlRoot(), 1982203954Srdivacky DAG.getBasicBlock(Return))); 1983199989Srdivacky} 1984199989Srdivacky 1985226633Sdimvoid SelectionDAGBuilder::visitResume(const ResumeInst &RI) { 1986226633Sdim llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!"); 1987226633Sdim} 1988226633Sdim 1989226633Sdimvoid SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { 1990226633Sdim assert(FuncInfo.MBB->isLandingPad() && 1991226633Sdim "Call to landingpad not in landing pad!"); 1992226633Sdim 1993226633Sdim MachineBasicBlock *MBB = FuncInfo.MBB; 1994226633Sdim MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 1995226633Sdim AddLandingPadInfo(LP, MMI, MBB); 1996226633Sdim 1997234353Sdim // If there aren't registers to copy the values into (e.g., during SjLj 1998234353Sdim // exceptions), then don't bother to create these DAG nodes. 1999263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2000263508Sdim if (TLI->getExceptionPointerRegister() == 0 && 2001263508Sdim TLI->getExceptionSelectorRegister() == 0) 2002234353Sdim return; 2003234353Sdim 2004226633Sdim SmallVector<EVT, 2> ValueVTs; 2005263508Sdim ComputeValueVTs(*TLI, LP.getType(), ValueVTs); 2006252720Sdim assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported"); 2007226633Sdim 2008252720Sdim // Get the two live-in registers as SDValues. The physregs have already been 2009252720Sdim // copied into virtual registers. 2010226633Sdim SDValue Ops[2]; 2011252720Sdim Ops[0] = DAG.getZExtOrTrunc( 2012263508Sdim DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), 2013263508Sdim FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()), 2014263508Sdim getCurSDLoc(), ValueVTs[0]); 2015252720Sdim Ops[1] = DAG.getZExtOrTrunc( 2016263508Sdim DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), 2017263508Sdim FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()), 2018263508Sdim getCurSDLoc(), ValueVTs[1]); 2019226633Sdim 2020252720Sdim // Merge into one. 2021263508Sdim SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 2022226633Sdim DAG.getVTList(&ValueVTs[0], ValueVTs.size()), 2023226633Sdim &Ops[0], 2); 2024252720Sdim setValue(&LP, Res); 2025226633Sdim} 2026226633Sdim 2027199989Srdivacky/// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for 2028199989Srdivacky/// small case ranges). 2029199989Srdivackybool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, 2030199989Srdivacky CaseRecVector& WorkList, 2031207618Srdivacky const Value* SV, 2032207618Srdivacky MachineBasicBlock *Default, 2033207618Srdivacky MachineBasicBlock *SwitchBB) { 2034199989Srdivacky // Size is the number of Cases represented by this range. 2035199989Srdivacky size_t Size = CR.Range.second - CR.Range.first; 2036199989Srdivacky if (Size > 3) 2037199989Srdivacky return false; 2038199989Srdivacky 2039199989Srdivacky // Get the MachineFunction which holds the current MBB. This is used when 2040199989Srdivacky // inserting any additional MBBs necessary to represent the switch. 2041199989Srdivacky MachineFunction *CurMF = FuncInfo.MF; 2042199989Srdivacky 2043199989Srdivacky // Figure out which block is immediately after the current one. 2044199989Srdivacky MachineBasicBlock *NextBlock = 0; 2045199989Srdivacky MachineFunction::iterator BBI = CR.CaseBB; 2046199989Srdivacky 2047199989Srdivacky if (++BBI != FuncInfo.MF->end()) 2048199989Srdivacky NextBlock = BBI; 2049199989Srdivacky 2050243830Sdim BranchProbabilityInfo *BPI = FuncInfo.BPI; 2051218893Sdim // If any two of the cases has the same destination, and if one value 2052199989Srdivacky // is the same as the other, but has one bit unset that the other has set, 2053199989Srdivacky // use bit manipulation to do two compares at once. For example: 2054199989Srdivacky // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" 2055218893Sdim // TODO: This could be extended to merge any 2 cases in switches with 3 cases. 2056218893Sdim // TODO: Handle cases where CR.CaseBB != SwitchBB. 2057218893Sdim if (Size == 2 && CR.CaseBB == SwitchBB) { 2058218893Sdim Case &Small = *CR.Range.first; 2059218893Sdim Case &Big = *(CR.Range.second-1); 2060199989Srdivacky 2061218893Sdim if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) { 2062218893Sdim const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue(); 2063218893Sdim const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue(); 2064218893Sdim 2065218893Sdim // Check that there is only one bit different. 2066218893Sdim if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 && 2067218893Sdim (SmallValue | BigValue) == BigValue) { 2068218893Sdim // Isolate the common bit. 2069218893Sdim APInt CommonBit = BigValue & ~SmallValue; 2070218893Sdim assert((SmallValue | CommonBit) == BigValue && 2071218893Sdim CommonBit.countPopulation() == 1 && "Not a common bit?"); 2072218893Sdim 2073218893Sdim SDValue CondLHS = getValue(SV); 2074218893Sdim EVT VT = CondLHS.getValueType(); 2075263508Sdim SDLoc DL = getCurSDLoc(); 2076218893Sdim 2077218893Sdim SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, 2078218893Sdim DAG.getConstant(CommonBit, VT)); 2079218893Sdim SDValue Cond = DAG.getSetCC(DL, MVT::i1, 2080218893Sdim Or, DAG.getConstant(BigValue, VT), 2081218893Sdim ISD::SETEQ); 2082218893Sdim 2083218893Sdim // Update successor info. 2084243830Sdim // Both Small and Big will jump to Small.BB, so we sum up the weights. 2085243830Sdim addSuccessorWithWeight(SwitchBB, Small.BB, 2086243830Sdim Small.ExtraWeight + Big.ExtraWeight); 2087243830Sdim addSuccessorWithWeight(SwitchBB, Default, 2088243830Sdim // The default destination is the first successor in IR. 2089243830Sdim BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0); 2090218893Sdim 2091218893Sdim // Insert the true branch. 2092218893Sdim SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other, 2093218893Sdim getControlRoot(), Cond, 2094218893Sdim DAG.getBasicBlock(Small.BB)); 2095218893Sdim 2096218893Sdim // Insert the false branch. 2097218893Sdim BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, 2098218893Sdim DAG.getBasicBlock(Default)); 2099218893Sdim 2100218893Sdim DAG.setRoot(BrCond); 2101218893Sdim return true; 2102218893Sdim } 2103218893Sdim } 2104218893Sdim } 2105218893Sdim 2106239462Sdim // Order cases by weight so the most likely case will be checked first. 2107243830Sdim uint32_t UnhandledWeights = 0; 2108239462Sdim if (BPI) { 2109239462Sdim for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) { 2110243830Sdim uint32_t IWeight = I->ExtraWeight; 2111243830Sdim UnhandledWeights += IWeight; 2112239462Sdim for (CaseItr J = CR.Range.first; J < I; ++J) { 2113243830Sdim uint32_t JWeight = J->ExtraWeight; 2114239462Sdim if (IWeight > JWeight) 2115239462Sdim std::swap(*I, *J); 2116239462Sdim } 2117239462Sdim } 2118239462Sdim } 2119199989Srdivacky // Rearrange the case blocks so that the last one falls through if possible. 2120239462Sdim Case &BackCase = *(CR.Range.second-1); 2121239462Sdim if (Size > 1 && 2122239462Sdim NextBlock && Default != NextBlock && BackCase.BB != NextBlock) { 2123199989Srdivacky // The last case block won't fall through into 'NextBlock' if we emit the 2124199989Srdivacky // branches in this order. See if rearranging a case value would help. 2125239462Sdim // We start at the bottom as it's the case with the least weight. 2126263508Sdim for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I) 2127199989Srdivacky if (I->BB == NextBlock) { 2128199989Srdivacky std::swap(*I, BackCase); 2129199989Srdivacky break; 2130199989Srdivacky } 2131199989Srdivacky } 2132199989Srdivacky 2133199989Srdivacky // Create a CaseBlock record representing a conditional branch to 2134199989Srdivacky // the Case's target mbb if the value being switched on SV is equal 2135199989Srdivacky // to C. 2136199989Srdivacky MachineBasicBlock *CurBlock = CR.CaseBB; 2137199989Srdivacky for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { 2138199989Srdivacky MachineBasicBlock *FallThrough; 2139199989Srdivacky if (I != E-1) { 2140199989Srdivacky FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock()); 2141199989Srdivacky CurMF->insert(BBI, FallThrough); 2142199989Srdivacky 2143199989Srdivacky // Put SV in a virtual register to make it available from the new blocks. 2144199989Srdivacky ExportFromCurrentBlock(SV); 2145199989Srdivacky } else { 2146199989Srdivacky // If the last case doesn't match, go to the default block. 2147199989Srdivacky FallThrough = Default; 2148199989Srdivacky } 2149199989Srdivacky 2150207618Srdivacky const Value *RHS, *LHS, *MHS; 2151199989Srdivacky ISD::CondCode CC; 2152199989Srdivacky if (I->High == I->Low) { 2153199989Srdivacky // This is just small small case range :) containing exactly 1 case 2154199989Srdivacky CC = ISD::SETEQ; 2155199989Srdivacky LHS = SV; RHS = I->High; MHS = NULL; 2156199989Srdivacky } else { 2157263508Sdim CC = ISD::SETLE; 2158199989Srdivacky LHS = I->Low; MHS = SV; RHS = I->High; 2159199989Srdivacky } 2160199989Srdivacky 2161243830Sdim // The false weight should be sum of all un-handled cases. 2162243830Sdim UnhandledWeights -= I->ExtraWeight; 2163226633Sdim CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough, 2164226633Sdim /* me */ CurBlock, 2165243830Sdim /* trueweight */ I->ExtraWeight, 2166243830Sdim /* falseweight */ UnhandledWeights); 2167226633Sdim 2168199989Srdivacky // If emitting the first comparison, just call visitSwitchCase to emit the 2169199989Srdivacky // code into the current block. Otherwise, push the CaseBlock onto the 2170199989Srdivacky // vector to be later processed by SDISel, and insert the node's MBB 2171199989Srdivacky // before the next MBB. 2172207618Srdivacky if (CurBlock == SwitchBB) 2173207618Srdivacky visitSwitchCase(CB, SwitchBB); 2174199989Srdivacky else 2175199989Srdivacky SwitchCases.push_back(CB); 2176199989Srdivacky 2177199989Srdivacky CurBlock = FallThrough; 2178199989Srdivacky } 2179199989Srdivacky 2180199989Srdivacky return true; 2181199989Srdivacky} 2182199989Srdivacky 2183199989Srdivackystatic inline bool areJTsAllowed(const TargetLowering &TLI) { 2184239462Sdim return TLI.supportJumpTables() && 2185199989Srdivacky (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || 2186199989Srdivacky TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other)); 2187199989Srdivacky} 2188199989Srdivacky 2189199989Srdivackystatic APInt ComputeRange(const APInt &First, const APInt &Last) { 2190199989Srdivacky uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1; 2191263508Sdim APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth); 2192199989Srdivacky return (LastExt - FirstExt + 1ULL); 2193199989Srdivacky} 2194199989Srdivacky 2195199989Srdivacky/// handleJTSwitchCase - Emit jumptable for current switch case range 2196226633Sdimbool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, 2197226633Sdim CaseRecVector &WorkList, 2198226633Sdim const Value *SV, 2199226633Sdim MachineBasicBlock *Default, 2200207618Srdivacky MachineBasicBlock *SwitchBB) { 2201199989Srdivacky Case& FrontCase = *CR.Range.first; 2202199989Srdivacky Case& BackCase = *(CR.Range.second-1); 2203199989Srdivacky 2204199989Srdivacky const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); 2205199989Srdivacky const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); 2206199989Srdivacky 2207199989Srdivacky APInt TSize(First.getBitWidth(), 0); 2208226633Sdim for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) 2209199989Srdivacky TSize += I->size(); 2210199989Srdivacky 2211263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2212263508Sdim if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries())) 2213199989Srdivacky return false; 2214199989Srdivacky 2215199989Srdivacky APInt Range = ComputeRange(First, Last); 2216228379Sdim // The density is TSize / Range. Require at least 40%. 2217228379Sdim // It should not be possible for IntTSize to saturate for sane code, but make 2218228379Sdim // sure we handle Range saturation correctly. 2219228379Sdim uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10); 2220228379Sdim uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10); 2221228379Sdim if (IntTSize * 10 < IntRange * 4) 2222199989Srdivacky return false; 2223199989Srdivacky 2224202375Srdivacky DEBUG(dbgs() << "Lowering jump table\n" 2225199989Srdivacky << "First entry: " << First << ". Last entry: " << Last << '\n' 2226228379Sdim << "Range: " << Range << ". Size: " << TSize << ".\n\n"); 2227199989Srdivacky 2228199989Srdivacky // Get the MachineFunction which holds the current MBB. This is used when 2229199989Srdivacky // inserting any additional MBBs necessary to represent the switch. 2230199989Srdivacky MachineFunction *CurMF = FuncInfo.MF; 2231199989Srdivacky 2232199989Srdivacky // Figure out which block is immediately after the current one. 2233199989Srdivacky MachineFunction::iterator BBI = CR.CaseBB; 2234199989Srdivacky ++BBI; 2235199989Srdivacky 2236199989Srdivacky const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2237199989Srdivacky 2238199989Srdivacky // Create a new basic block to hold the code for loading the address 2239199989Srdivacky // of the jump table, and jumping to it. Update successor information; 2240199989Srdivacky // we will either branch to the default case for the switch, or the jump 2241199989Srdivacky // table. 2242199989Srdivacky MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2243199989Srdivacky CurMF->insert(BBI, JumpTableBB); 2244199989Srdivacky 2245224145Sdim addSuccessorWithWeight(CR.CaseBB, Default); 2246224145Sdim addSuccessorWithWeight(CR.CaseBB, JumpTableBB); 2247224145Sdim 2248199989Srdivacky // Build a vector of destination BBs, corresponding to each target 2249199989Srdivacky // of the jump table. If the value of the jump table slot corresponds to 2250199989Srdivacky // a case statement, push the case's BB onto the vector, otherwise, push 2251199989Srdivacky // the default BB. 2252199989Srdivacky std::vector<MachineBasicBlock*> DestBBs; 2253199989Srdivacky APInt TEI = First; 2254199989Srdivacky for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) { 2255203954Srdivacky const APInt &Low = cast<ConstantInt>(I->Low)->getValue(); 2256203954Srdivacky const APInt &High = cast<ConstantInt>(I->High)->getValue(); 2257199989Srdivacky 2258263508Sdim if (Low.sle(TEI) && TEI.sle(High)) { 2259199989Srdivacky DestBBs.push_back(I->BB); 2260199989Srdivacky if (TEI==High) 2261199989Srdivacky ++I; 2262199989Srdivacky } else { 2263199989Srdivacky DestBBs.push_back(Default); 2264199989Srdivacky } 2265199989Srdivacky } 2266199989Srdivacky 2267243830Sdim // Calculate weight for each unique destination in CR. 2268243830Sdim DenseMap<MachineBasicBlock*, uint32_t> DestWeights; 2269243830Sdim if (FuncInfo.BPI) 2270243830Sdim for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { 2271243830Sdim DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr = 2272243830Sdim DestWeights.find(I->BB); 2273263508Sdim if (Itr != DestWeights.end()) 2274243830Sdim Itr->second += I->ExtraWeight; 2275243830Sdim else 2276243830Sdim DestWeights[I->BB] = I->ExtraWeight; 2277243830Sdim } 2278243830Sdim 2279199989Srdivacky // Update successor info. Add one edge to each unique successor. 2280199989Srdivacky BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs()); 2281199989Srdivacky for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(), 2282199989Srdivacky E = DestBBs.end(); I != E; ++I) { 2283199989Srdivacky if (!SuccsHandled[(*I)->getNumber()]) { 2284199989Srdivacky SuccsHandled[(*I)->getNumber()] = true; 2285243830Sdim DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr = 2286243830Sdim DestWeights.find(*I); 2287243830Sdim addSuccessorWithWeight(JumpTableBB, *I, 2288243830Sdim Itr != DestWeights.end() ? Itr->second : 0); 2289199989Srdivacky } 2290199989Srdivacky } 2291199989Srdivacky 2292205407Srdivacky // Create a jump table index for this jump table. 2293263508Sdim unsigned JTEncoding = TLI->getJumpTableEncoding(); 2294203954Srdivacky unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding) 2295205407Srdivacky ->createJumpTableIndex(DestBBs); 2296199989Srdivacky 2297199989Srdivacky // Set the jump table information so that we can codegen it as a second 2298199989Srdivacky // MachineBasicBlock 2299199989Srdivacky JumpTable JT(-1U, JTI, JumpTableBB, Default); 2300207618Srdivacky JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB)); 2301207618Srdivacky if (CR.CaseBB == SwitchBB) 2302207618Srdivacky visitJumpTableHeader(JT, JTH, SwitchBB); 2303199989Srdivacky 2304199989Srdivacky JTCases.push_back(JumpTableBlock(JTH, JT)); 2305199989Srdivacky return true; 2306199989Srdivacky} 2307199989Srdivacky 2308199989Srdivacky/// handleBTSplitSwitchCase - emit comparison and split binary search tree into 2309199989Srdivacky/// 2 subtrees. 2310199989Srdivackybool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, 2311199989Srdivacky CaseRecVector& WorkList, 2312207618Srdivacky const Value* SV, 2313263508Sdim MachineBasicBlock* Default, 2314263508Sdim MachineBasicBlock* SwitchBB) { 2315199989Srdivacky // Get the MachineFunction which holds the current MBB. This is used when 2316199989Srdivacky // inserting any additional MBBs necessary to represent the switch. 2317199989Srdivacky MachineFunction *CurMF = FuncInfo.MF; 2318199989Srdivacky 2319199989Srdivacky // Figure out which block is immediately after the current one. 2320199989Srdivacky MachineFunction::iterator BBI = CR.CaseBB; 2321199989Srdivacky ++BBI; 2322199989Srdivacky 2323199989Srdivacky Case& FrontCase = *CR.Range.first; 2324199989Srdivacky Case& BackCase = *(CR.Range.second-1); 2325199989Srdivacky const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2326199989Srdivacky 2327199989Srdivacky // Size is the number of Cases represented by this range. 2328199989Srdivacky unsigned Size = CR.Range.second - CR.Range.first; 2329199989Srdivacky 2330199989Srdivacky const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); 2331199989Srdivacky const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); 2332199989Srdivacky double FMetric = 0; 2333199989Srdivacky CaseItr Pivot = CR.Range.first + Size/2; 2334199989Srdivacky 2335199989Srdivacky // Select optimal pivot, maximizing sum density of LHS and RHS. This will 2336199989Srdivacky // (heuristically) allow us to emit JumpTable's later. 2337199989Srdivacky APInt TSize(First.getBitWidth(), 0); 2338199989Srdivacky for (CaseItr I = CR.Range.first, E = CR.Range.second; 2339199989Srdivacky I!=E; ++I) 2340199989Srdivacky TSize += I->size(); 2341199989Srdivacky 2342199989Srdivacky APInt LSize = FrontCase.size(); 2343199989Srdivacky APInt RSize = TSize-LSize; 2344202375Srdivacky DEBUG(dbgs() << "Selecting best pivot: \n" 2345199989Srdivacky << "First: " << First << ", Last: " << Last <<'\n' 2346199989Srdivacky << "LSize: " << LSize << ", RSize: " << RSize << '\n'); 2347199989Srdivacky for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second; 2348199989Srdivacky J!=E; ++I, ++J) { 2349199989Srdivacky const APInt &LEnd = cast<ConstantInt>(I->High)->getValue(); 2350199989Srdivacky const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue(); 2351199989Srdivacky APInt Range = ComputeRange(LEnd, RBegin); 2352199989Srdivacky assert((Range - 2ULL).isNonNegative() && 2353199989Srdivacky "Invalid case distance"); 2354221345Sdim // Use volatile double here to avoid excess precision issues on some hosts, 2355221345Sdim // e.g. that use 80-bit X87 registers. 2356221345Sdim volatile double LDensity = 2357221345Sdim (double)LSize.roundToDouble() / 2358199989Srdivacky (LEnd - First + 1ULL).roundToDouble(); 2359221345Sdim volatile double RDensity = 2360221345Sdim (double)RSize.roundToDouble() / 2361199989Srdivacky (Last - RBegin + 1ULL).roundToDouble(); 2362199989Srdivacky double Metric = Range.logBase2()*(LDensity+RDensity); 2363199989Srdivacky // Should always split in some non-trivial place 2364202375Srdivacky DEBUG(dbgs() <<"=>Step\n" 2365199989Srdivacky << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n' 2366199989Srdivacky << "LDensity: " << LDensity 2367199989Srdivacky << ", RDensity: " << RDensity << '\n' 2368199989Srdivacky << "Metric: " << Metric << '\n'); 2369199989Srdivacky if (FMetric < Metric) { 2370199989Srdivacky Pivot = J; 2371199989Srdivacky FMetric = Metric; 2372202375Srdivacky DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n'); 2373199989Srdivacky } 2374199989Srdivacky 2375199989Srdivacky LSize += J->size(); 2376199989Srdivacky RSize -= J->size(); 2377199989Srdivacky } 2378263508Sdim 2379263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2380263508Sdim if (areJTsAllowed(*TLI)) { 2381199989Srdivacky // If our case is dense we *really* should handle it earlier! 2382199989Srdivacky assert((FMetric > 0) && "Should handle dense range earlier!"); 2383199989Srdivacky } else { 2384199989Srdivacky Pivot = CR.Range.first + Size/2; 2385199989Srdivacky } 2386199989Srdivacky 2387199989Srdivacky CaseRange LHSR(CR.Range.first, Pivot); 2388199989Srdivacky CaseRange RHSR(Pivot, CR.Range.second); 2389234353Sdim const Constant *C = Pivot->Low; 2390199989Srdivacky MachineBasicBlock *FalseBB = 0, *TrueBB = 0; 2391199989Srdivacky 2392199989Srdivacky // We know that we branch to the LHS if the Value being switched on is 2393199989Srdivacky // less than the Pivot value, C. We use this to optimize our binary 2394199989Srdivacky // tree a bit, by recognizing that if SV is greater than or equal to the 2395199989Srdivacky // LHS's Case Value, and that Case Value is exactly one less than the 2396199989Srdivacky // Pivot's Value, then we can branch directly to the LHS's Target, 2397199989Srdivacky // rather than creating a leaf node for it. 2398199989Srdivacky if ((LHSR.second - LHSR.first) == 1 && 2399199989Srdivacky LHSR.first->High == CR.GE && 2400199989Srdivacky cast<ConstantInt>(C)->getValue() == 2401199989Srdivacky (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) { 2402199989Srdivacky TrueBB = LHSR.first->BB; 2403199989Srdivacky } else { 2404199989Srdivacky TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2405199989Srdivacky CurMF->insert(BBI, TrueBB); 2406199989Srdivacky WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR)); 2407199989Srdivacky 2408199989Srdivacky // Put SV in a virtual register to make it available from the new blocks. 2409199989Srdivacky ExportFromCurrentBlock(SV); 2410199989Srdivacky } 2411199989Srdivacky 2412199989Srdivacky // Similar to the optimization above, if the Value being switched on is 2413199989Srdivacky // known to be less than the Constant CR.LT, and the current Case Value 2414199989Srdivacky // is CR.LT - 1, then we can branch directly to the target block for 2415199989Srdivacky // the current Case Value, rather than emitting a RHS leaf node for it. 2416199989Srdivacky if ((RHSR.second - RHSR.first) == 1 && CR.LT && 2417199989Srdivacky cast<ConstantInt>(RHSR.first->Low)->getValue() == 2418199989Srdivacky (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) { 2419199989Srdivacky FalseBB = RHSR.first->BB; 2420199989Srdivacky } else { 2421199989Srdivacky FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2422199989Srdivacky CurMF->insert(BBI, FalseBB); 2423199989Srdivacky WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR)); 2424199989Srdivacky 2425199989Srdivacky // Put SV in a virtual register to make it available from the new blocks. 2426199989Srdivacky ExportFromCurrentBlock(SV); 2427199989Srdivacky } 2428199989Srdivacky 2429199989Srdivacky // Create a CaseBlock record representing a conditional branch to 2430199989Srdivacky // the LHS node if the value being switched on SV is less than C. 2431199989Srdivacky // Otherwise, branch to LHS. 2432263508Sdim CaseBlock CB(ISD::SETLT, SV, C, NULL, TrueBB, FalseBB, CR.CaseBB); 2433199989Srdivacky 2434207618Srdivacky if (CR.CaseBB == SwitchBB) 2435207618Srdivacky visitSwitchCase(CB, SwitchBB); 2436199989Srdivacky else 2437199989Srdivacky SwitchCases.push_back(CB); 2438199989Srdivacky 2439199989Srdivacky return true; 2440199989Srdivacky} 2441199989Srdivacky 2442199989Srdivacky/// handleBitTestsSwitchCase - if current case range has few destination and 2443199989Srdivacky/// range span less, than machine word bitwidth, encode case range into series 2444199989Srdivacky/// of masks and emit bit tests with these masks. 2445199989Srdivackybool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, 2446199989Srdivacky CaseRecVector& WorkList, 2447207618Srdivacky const Value* SV, 2448207618Srdivacky MachineBasicBlock* Default, 2449263508Sdim MachineBasicBlock* SwitchBB) { 2450263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2451263508Sdim EVT PTy = TLI->getPointerTy(); 2452199989Srdivacky unsigned IntPtrBits = PTy.getSizeInBits(); 2453199989Srdivacky 2454199989Srdivacky Case& FrontCase = *CR.Range.first; 2455199989Srdivacky Case& BackCase = *(CR.Range.second-1); 2456199989Srdivacky 2457199989Srdivacky // Get the MachineFunction which holds the current MBB. This is used when 2458199989Srdivacky // inserting any additional MBBs necessary to represent the switch. 2459199989Srdivacky MachineFunction *CurMF = FuncInfo.MF; 2460199989Srdivacky 2461199989Srdivacky // If target does not have legal shift left, do not emit bit tests at all. 2462263508Sdim if (!TLI->isOperationLegal(ISD::SHL, PTy)) 2463199989Srdivacky return false; 2464199989Srdivacky 2465199989Srdivacky size_t numCmps = 0; 2466199989Srdivacky for (CaseItr I = CR.Range.first, E = CR.Range.second; 2467199989Srdivacky I!=E; ++I) { 2468199989Srdivacky // Single case counts one, case range - two. 2469199989Srdivacky numCmps += (I->Low == I->High ? 1 : 2); 2470199989Srdivacky } 2471199989Srdivacky 2472199989Srdivacky // Count unique destinations 2473199989Srdivacky SmallSet<MachineBasicBlock*, 4> Dests; 2474199989Srdivacky for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) { 2475199989Srdivacky Dests.insert(I->BB); 2476199989Srdivacky if (Dests.size() > 3) 2477199989Srdivacky // Don't bother the code below, if there are too much unique destinations 2478199989Srdivacky return false; 2479199989Srdivacky } 2480202375Srdivacky DEBUG(dbgs() << "Total number of unique destinations: " 2481201360Srdivacky << Dests.size() << '\n' 2482201360Srdivacky << "Total number of comparisons: " << numCmps << '\n'); 2483199989Srdivacky 2484199989Srdivacky // Compute span of values. 2485199989Srdivacky const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue(); 2486199989Srdivacky const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue(); 2487199989Srdivacky APInt cmpRange = maxValue - minValue; 2488199989Srdivacky 2489202375Srdivacky DEBUG(dbgs() << "Compare range: " << cmpRange << '\n' 2490199989Srdivacky << "Low bound: " << minValue << '\n' 2491199989Srdivacky << "High bound: " << maxValue << '\n'); 2492199989Srdivacky 2493207618Srdivacky if (cmpRange.uge(IntPtrBits) || 2494199989Srdivacky (!(Dests.size() == 1 && numCmps >= 3) && 2495199989Srdivacky !(Dests.size() == 2 && numCmps >= 5) && 2496199989Srdivacky !(Dests.size() >= 3 && numCmps >= 6))) 2497199989Srdivacky return false; 2498199989Srdivacky 2499202375Srdivacky DEBUG(dbgs() << "Emitting bit tests\n"); 2500199989Srdivacky APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth()); 2501199989Srdivacky 2502199989Srdivacky // Optimize the case where all the case values fit in a 2503199989Srdivacky // word without having to subtract minValue. In this case, 2504199989Srdivacky // we can optimize away the subtraction. 2505263508Sdim if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) { 2506199989Srdivacky cmpRange = maxValue; 2507199989Srdivacky } else { 2508199989Srdivacky lowBound = minValue; 2509199989Srdivacky } 2510199989Srdivacky 2511199989Srdivacky CaseBitsVector CasesBits; 2512199989Srdivacky unsigned i, count = 0; 2513199989Srdivacky 2514199989Srdivacky for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) { 2515199989Srdivacky MachineBasicBlock* Dest = I->BB; 2516199989Srdivacky for (i = 0; i < count; ++i) 2517199989Srdivacky if (Dest == CasesBits[i].BB) 2518199989Srdivacky break; 2519199989Srdivacky 2520199989Srdivacky if (i == count) { 2521199989Srdivacky assert((count < 3) && "Too much destinations to test!"); 2522243830Sdim CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/)); 2523199989Srdivacky count++; 2524199989Srdivacky } 2525199989Srdivacky 2526199989Srdivacky const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue(); 2527199989Srdivacky const APInt& highValue = cast<ConstantInt>(I->High)->getValue(); 2528199989Srdivacky 2529199989Srdivacky uint64_t lo = (lowValue - lowBound).getZExtValue(); 2530199989Srdivacky uint64_t hi = (highValue - lowBound).getZExtValue(); 2531243830Sdim CasesBits[i].ExtraWeight += I->ExtraWeight; 2532199989Srdivacky 2533199989Srdivacky for (uint64_t j = lo; j <= hi; j++) { 2534199989Srdivacky CasesBits[i].Mask |= 1ULL << j; 2535199989Srdivacky CasesBits[i].Bits++; 2536199989Srdivacky } 2537199989Srdivacky 2538199989Srdivacky } 2539199989Srdivacky std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp()); 2540199989Srdivacky 2541199989Srdivacky BitTestInfo BTC; 2542199989Srdivacky 2543199989Srdivacky // Figure out which block is immediately after the current one. 2544199989Srdivacky MachineFunction::iterator BBI = CR.CaseBB; 2545199989Srdivacky ++BBI; 2546199989Srdivacky 2547199989Srdivacky const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2548199989Srdivacky 2549202375Srdivacky DEBUG(dbgs() << "Cases:\n"); 2550199989Srdivacky for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) { 2551202375Srdivacky DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask 2552199989Srdivacky << ", Bits: " << CasesBits[i].Bits 2553199989Srdivacky << ", BB: " << CasesBits[i].BB << '\n'); 2554199989Srdivacky 2555199989Srdivacky MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2556199989Srdivacky CurMF->insert(BBI, CaseBB); 2557199989Srdivacky BTC.push_back(BitTestCase(CasesBits[i].Mask, 2558199989Srdivacky CaseBB, 2559243830Sdim CasesBits[i].BB, CasesBits[i].ExtraWeight)); 2560199989Srdivacky 2561199989Srdivacky // Put SV in a virtual register to make it available from the new blocks. 2562199989Srdivacky ExportFromCurrentBlock(SV); 2563199989Srdivacky } 2564199989Srdivacky 2565199989Srdivacky BitTestBlock BTB(lowBound, cmpRange, SV, 2566218893Sdim -1U, MVT::Other, (CR.CaseBB == SwitchBB), 2567199989Srdivacky CR.CaseBB, Default, BTC); 2568199989Srdivacky 2569207618Srdivacky if (CR.CaseBB == SwitchBB) 2570207618Srdivacky visitBitTestHeader(BTB, SwitchBB); 2571199989Srdivacky 2572199989Srdivacky BitTestCases.push_back(BTB); 2573199989Srdivacky 2574199989Srdivacky return true; 2575199989Srdivacky} 2576199989Srdivacky 2577199989Srdivacky/// Clusterify - Transform simple list of Cases into list of CaseRange's 2578199989Srdivackysize_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, 2579199989Srdivacky const SwitchInst& SI) { 2580263508Sdim size_t numCmps = 0; 2581199989Srdivacky 2582243830Sdim BranchProbabilityInfo *BPI = FuncInfo.BPI; 2583199989Srdivacky // Start with "simple" cases 2584234353Sdim for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end(); 2585234353Sdim i != e; ++i) { 2586234353Sdim const BasicBlock *SuccBB = i.getCaseSuccessor(); 2587226633Sdim MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB]; 2588226633Sdim 2589263508Sdim uint32_t ExtraWeight = 2590263508Sdim BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0; 2591263508Sdim 2592263508Sdim Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(), 2593263508Sdim SMBB, ExtraWeight)); 2594199989Srdivacky } 2595263508Sdim std::sort(Cases.begin(), Cases.end(), CaseCmp()); 2596199989Srdivacky 2597263508Sdim // Merge case into clusters 2598263508Sdim if (Cases.size() >= 2) 2599263508Sdim // Must recompute end() each iteration because it may be 2600263508Sdim // invalidated by erase if we hold on to it 2601263508Sdim for (CaseItr I = Cases.begin(), J = llvm::next(Cases.begin()); 2602263508Sdim J != Cases.end(); ) { 2603263508Sdim const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue(); 2604263508Sdim const APInt& currentValue = cast<ConstantInt>(I->High)->getValue(); 2605263508Sdim MachineBasicBlock* nextBB = J->BB; 2606263508Sdim MachineBasicBlock* currentBB = I->BB; 2607263508Sdim 2608263508Sdim // If the two neighboring cases go to the same destination, merge them 2609263508Sdim // into a single case. 2610263508Sdim if ((nextValue - currentValue == 1) && (currentBB == nextBB)) { 2611263508Sdim I->High = J->High; 2612263508Sdim I->ExtraWeight += J->ExtraWeight; 2613263508Sdim J = Cases.erase(J); 2614263508Sdim } else { 2615263508Sdim I = J++; 2616263508Sdim } 2617263508Sdim } 2618263508Sdim 2619263508Sdim for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) { 2620263508Sdim if (I->Low != I->High) 2621263508Sdim // A range counts double, since it requires two compares. 2622263508Sdim ++numCmps; 2623199989Srdivacky } 2624199989Srdivacky 2625199989Srdivacky return numCmps; 2626199989Srdivacky} 2627199989Srdivacky 2628218893Sdimvoid SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, 2629218893Sdim MachineBasicBlock *Last) { 2630218893Sdim // Update JTCases. 2631218893Sdim for (unsigned i = 0, e = JTCases.size(); i != e; ++i) 2632218893Sdim if (JTCases[i].first.HeaderBB == First) 2633218893Sdim JTCases[i].first.HeaderBB = Last; 2634218893Sdim 2635218893Sdim // Update BitTestCases. 2636218893Sdim for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) 2637218893Sdim if (BitTestCases[i].Parent == First) 2638218893Sdim BitTestCases[i].Parent = Last; 2639218893Sdim} 2640218893Sdim 2641207618Srdivackyvoid SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { 2642210299Sed MachineBasicBlock *SwitchMBB = FuncInfo.MBB; 2643207618Srdivacky 2644199989Srdivacky // Figure out which block is immediately after the current one. 2645199989Srdivacky MachineBasicBlock *NextBlock = 0; 2646199989Srdivacky MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()]; 2647199989Srdivacky 2648199989Srdivacky // If there is only the default destination, branch to it if it is not the 2649199989Srdivacky // next basic block. Otherwise, just fall through. 2650234353Sdim if (!SI.getNumCases()) { 2651199989Srdivacky // Update machine-CFG edges. 2652199989Srdivacky 2653199989Srdivacky // If this is not a fall-through branch, emit the branch. 2654207618Srdivacky SwitchMBB->addSuccessor(Default); 2655203954Srdivacky if (Default != NextBlock) 2656263508Sdim DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 2657203954Srdivacky MVT::Other, getControlRoot(), 2658203954Srdivacky DAG.getBasicBlock(Default))); 2659201360Srdivacky 2660199989Srdivacky return; 2661199989Srdivacky } 2662199989Srdivacky 2663199989Srdivacky // If there are any non-default case statements, create a vector of Cases 2664199989Srdivacky // representing each one, and sort the vector so that we can efficiently 2665199989Srdivacky // create a binary search tree from them. 2666199989Srdivacky CaseVector Cases; 2667199989Srdivacky size_t numCmps = Clusterify(Cases, SI); 2668202375Srdivacky DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size() 2669199989Srdivacky << ". Total compares: " << numCmps << '\n'); 2670226633Sdim (void)numCmps; 2671199989Srdivacky 2672199989Srdivacky // Get the Value to be switched on and default basic blocks, which will be 2673199989Srdivacky // inserted into CaseBlock records, representing basic blocks in the binary 2674199989Srdivacky // search tree. 2675226633Sdim const Value *SV = SI.getCondition(); 2676199989Srdivacky 2677199989Srdivacky // Push the initial CaseRec onto the worklist 2678199989Srdivacky CaseRecVector WorkList; 2679207618Srdivacky WorkList.push_back(CaseRec(SwitchMBB,0,0, 2680207618Srdivacky CaseRange(Cases.begin(),Cases.end()))); 2681199989Srdivacky 2682199989Srdivacky while (!WorkList.empty()) { 2683199989Srdivacky // Grab a record representing a case range to process off the worklist 2684199989Srdivacky CaseRec CR = WorkList.back(); 2685199989Srdivacky WorkList.pop_back(); 2686199989Srdivacky 2687207618Srdivacky if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) 2688199989Srdivacky continue; 2689199989Srdivacky 2690199989Srdivacky // If the range has few cases (two or less) emit a series of specific 2691199989Srdivacky // tests. 2692207618Srdivacky if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB)) 2693199989Srdivacky continue; 2694199989Srdivacky 2695243830Sdim // If the switch has more than N blocks, and is at least 40% dense, and the 2696199989Srdivacky // target supports indirect branches, then emit a jump table rather than 2697199989Srdivacky // lowering the switch to a binary tree of conditional branches. 2698243830Sdim // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries(). 2699207618Srdivacky if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) 2700199989Srdivacky continue; 2701199989Srdivacky 2702199989Srdivacky // Emit binary tree. We need to pick a pivot, and push left and right ranges 2703199989Srdivacky // onto the worklist. Leafs are handled via handleSmallSwitchRange() call. 2704207618Srdivacky handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB); 2705199989Srdivacky } 2706199989Srdivacky} 2707199989Srdivacky 2708207618Srdivackyvoid SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { 2709210299Sed MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; 2710207618Srdivacky 2711203954Srdivacky // Update machine-CFG edges with unique successors. 2712243830Sdim SmallSet<BasicBlock*, 32> Done; 2713243830Sdim for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { 2714243830Sdim BasicBlock *BB = I.getSuccessor(i); 2715243830Sdim bool Inserted = Done.insert(BB); 2716243830Sdim if (!Inserted) 2717243830Sdim continue; 2718243830Sdim 2719243830Sdim MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; 2720224145Sdim addSuccessorWithWeight(IndirectBrMBB, Succ); 2721224145Sdim } 2722199989Srdivacky 2723263508Sdim DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), 2724203954Srdivacky MVT::Other, getControlRoot(), 2725203954Srdivacky getValue(I.getAddress()))); 2726199989Srdivacky} 2727199989Srdivacky 2728207618Srdivackyvoid SelectionDAGBuilder::visitFSub(const User &I) { 2729199989Srdivacky // -0.0 - X --> fneg 2730226633Sdim Type *Ty = I.getType(); 2731218893Sdim if (isa<Constant>(I.getOperand(0)) && 2732218893Sdim I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) { 2733218893Sdim SDValue Op2 = getValue(I.getOperand(1)); 2734263508Sdim setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(), 2735218893Sdim Op2.getValueType(), Op2)); 2736218893Sdim return; 2737199989Srdivacky } 2738201360Srdivacky 2739199989Srdivacky visitBinary(I, ISD::FSUB); 2740199989Srdivacky} 2741199989Srdivacky 2742207618Srdivackyvoid SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { 2743199989Srdivacky SDValue Op1 = getValue(I.getOperand(0)); 2744199989Srdivacky SDValue Op2 = getValue(I.getOperand(1)); 2745263508Sdim setValue(&I, DAG.getNode(OpCode, getCurSDLoc(), 2746203954Srdivacky Op1.getValueType(), Op1, Op2)); 2747199989Srdivacky} 2748199989Srdivacky 2749207618Srdivackyvoid SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { 2750199989Srdivacky SDValue Op1 = getValue(I.getOperand(0)); 2751199989Srdivacky SDValue Op2 = getValue(I.getOperand(1)); 2752219077Sdim 2753263508Sdim EVT ShiftTy = TM.getTargetLowering()->getShiftAmountTy(Op2.getValueType()); 2754219077Sdim 2755218893Sdim // Coerce the shift amount to the right type if we can. 2756218893Sdim if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { 2757218893Sdim unsigned ShiftSize = ShiftTy.getSizeInBits(); 2758218893Sdim unsigned Op2Size = Op2.getValueType().getSizeInBits(); 2759263508Sdim SDLoc DL = getCurSDLoc(); 2760219077Sdim 2761199989Srdivacky // If the operand is smaller than the shift count type, promote it. 2762218893Sdim if (ShiftSize > Op2Size) 2763218893Sdim Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2); 2764219077Sdim 2765199989Srdivacky // If the operand is larger than the shift count type but the shift 2766199989Srdivacky // count type has enough bits to represent any shift value, truncate 2767199989Srdivacky // it now. This is a common case and it exposes the truncate to 2768199989Srdivacky // optimization early. 2769218893Sdim else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits())) 2770218893Sdim Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2); 2771218893Sdim // Otherwise we'll need to temporarily settle for some other convenient 2772218893Sdim // type. Type legalization will make adjustments once the shiftee is split. 2773218893Sdim else 2774218893Sdim Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32); 2775199989Srdivacky } 2776199989Srdivacky 2777263508Sdim setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), 2778203954Srdivacky Op1.getValueType(), Op1, Op2)); 2779199989Srdivacky} 2780199989Srdivacky 2781224145Sdimvoid SelectionDAGBuilder::visitSDiv(const User &I) { 2782224145Sdim SDValue Op1 = getValue(I.getOperand(0)); 2783224145Sdim SDValue Op2 = getValue(I.getOperand(1)); 2784224145Sdim 2785224145Sdim // Turn exact SDivs into multiplications. 2786224145Sdim // FIXME: This should be in DAGCombiner, but it doesn't have access to the 2787224145Sdim // exact bit. 2788224145Sdim if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() && 2789224145Sdim !isa<ConstantSDNode>(Op1) && 2790224145Sdim isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue()) 2791263508Sdim setValue(&I, TM.getTargetLowering()->BuildExactSDIV(Op1, Op2, 2792263508Sdim getCurSDLoc(), DAG)); 2793224145Sdim else 2794263508Sdim setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), 2795224145Sdim Op1, Op2)); 2796224145Sdim} 2797224145Sdim 2798207618Srdivackyvoid SelectionDAGBuilder::visitICmp(const User &I) { 2799199989Srdivacky ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; 2800207618Srdivacky if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) 2801199989Srdivacky predicate = IC->getPredicate(); 2802207618Srdivacky else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) 2803199989Srdivacky predicate = ICmpInst::Predicate(IC->getPredicate()); 2804199989Srdivacky SDValue Op1 = getValue(I.getOperand(0)); 2805199989Srdivacky SDValue Op2 = getValue(I.getOperand(1)); 2806199989Srdivacky ISD::CondCode Opcode = getICmpCondCode(predicate); 2807201360Srdivacky 2808263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2809263508Sdim setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); 2810199989Srdivacky} 2811199989Srdivacky 2812207618Srdivackyvoid SelectionDAGBuilder::visitFCmp(const User &I) { 2813199989Srdivacky FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; 2814207618Srdivacky if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) 2815199989Srdivacky predicate = FC->getPredicate(); 2816207618Srdivacky else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) 2817199989Srdivacky predicate = FCmpInst::Predicate(FC->getPredicate()); 2818199989Srdivacky SDValue Op1 = getValue(I.getOperand(0)); 2819199989Srdivacky SDValue Op2 = getValue(I.getOperand(1)); 2820199989Srdivacky ISD::CondCode Condition = getFCmpCondCode(predicate); 2821234353Sdim if (TM.Options.NoNaNsFPMath) 2822234353Sdim Condition = getFCmpCodeWithoutNaN(Condition); 2823263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2824263508Sdim setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); 2825199989Srdivacky} 2826199989Srdivacky 2827207618Srdivackyvoid SelectionDAGBuilder::visitSelect(const User &I) { 2828199989Srdivacky SmallVector<EVT, 4> ValueVTs; 2829263508Sdim ComputeValueVTs(*TM.getTargetLowering(), I.getType(), ValueVTs); 2830199989Srdivacky unsigned NumValues = ValueVTs.size(); 2831201360Srdivacky if (NumValues == 0) return; 2832199989Srdivacky 2833201360Srdivacky SmallVector<SDValue, 4> Values(NumValues); 2834201360Srdivacky SDValue Cond = getValue(I.getOperand(0)); 2835201360Srdivacky SDValue TrueVal = getValue(I.getOperand(1)); 2836201360Srdivacky SDValue FalseVal = getValue(I.getOperand(2)); 2837226633Sdim ISD::NodeType OpCode = Cond.getValueType().isVector() ? 2838226633Sdim ISD::VSELECT : ISD::SELECT; 2839199989Srdivacky 2840203954Srdivacky for (unsigned i = 0; i != NumValues; ++i) 2841263508Sdim Values[i] = DAG.getNode(OpCode, getCurSDLoc(), 2842226633Sdim TrueVal.getNode()->getValueType(TrueVal.getResNo()+i), 2843205218Srdivacky Cond, 2844201360Srdivacky SDValue(TrueVal.getNode(), 2845201360Srdivacky TrueVal.getResNo() + i), 2846201360Srdivacky SDValue(FalseVal.getNode(), 2847201360Srdivacky FalseVal.getResNo() + i)); 2848201360Srdivacky 2849263508Sdim setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 2850203954Srdivacky DAG.getVTList(&ValueVTs[0], NumValues), 2851203954Srdivacky &Values[0], NumValues)); 2852199989Srdivacky} 2853199989Srdivacky 2854207618Srdivackyvoid SelectionDAGBuilder::visitTrunc(const User &I) { 2855199989Srdivacky // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). 2856199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2857263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2858263508Sdim setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); 2859199989Srdivacky} 2860199989Srdivacky 2861207618Srdivackyvoid SelectionDAGBuilder::visitZExt(const User &I) { 2862199989Srdivacky // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 2863199989Srdivacky // ZExt also can't be a cast to bool for same reason. So, nothing much to do 2864199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2865263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2866263508Sdim setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); 2867199989Srdivacky} 2868199989Srdivacky 2869207618Srdivackyvoid SelectionDAGBuilder::visitSExt(const User &I) { 2870199989Srdivacky // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 2871199989Srdivacky // SExt also can't be a cast to bool for same reason. So, nothing much to do 2872199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2873263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2874263508Sdim setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); 2875199989Srdivacky} 2876199989Srdivacky 2877207618Srdivackyvoid SelectionDAGBuilder::visitFPTrunc(const User &I) { 2878199989Srdivacky // FPTrunc is never a no-op cast, no need to check 2879199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2880263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2881263508Sdim EVT DestVT = TLI->getValueType(I.getType()); 2882263508Sdim setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), 2883234353Sdim DestVT, N, 2884263508Sdim DAG.getTargetConstant(0, TLI->getPointerTy()))); 2885199989Srdivacky} 2886199989Srdivacky 2887263508Sdimvoid SelectionDAGBuilder::visitFPExt(const User &I) { 2888234353Sdim // FPExt is never a no-op cast, no need to check 2889199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2890263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2891263508Sdim setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); 2892199989Srdivacky} 2893199989Srdivacky 2894207618Srdivackyvoid SelectionDAGBuilder::visitFPToUI(const User &I) { 2895199989Srdivacky // FPToUI is never a no-op cast, no need to check 2896199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2897263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2898263508Sdim setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); 2899199989Srdivacky} 2900199989Srdivacky 2901207618Srdivackyvoid SelectionDAGBuilder::visitFPToSI(const User &I) { 2902199989Srdivacky // FPToSI is never a no-op cast, no need to check 2903199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2904263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2905263508Sdim setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); 2906199989Srdivacky} 2907199989Srdivacky 2908207618Srdivackyvoid SelectionDAGBuilder::visitUIToFP(const User &I) { 2909199989Srdivacky // UIToFP is never a no-op cast, no need to check 2910199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2911263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2912263508Sdim setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); 2913199989Srdivacky} 2914199989Srdivacky 2915263508Sdimvoid SelectionDAGBuilder::visitSIToFP(const User &I) { 2916199989Srdivacky // SIToFP is never a no-op cast, no need to check 2917199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2918263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2919263508Sdim setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); 2920199989Srdivacky} 2921199989Srdivacky 2922207618Srdivackyvoid SelectionDAGBuilder::visitPtrToInt(const User &I) { 2923199989Srdivacky // What to do depends on the size of the integer and the size of the pointer. 2924199989Srdivacky // We can either truncate, zero extend, or no-op, accordingly. 2925199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2926263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2927263508Sdim setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); 2928199989Srdivacky} 2929199989Srdivacky 2930207618Srdivackyvoid SelectionDAGBuilder::visitIntToPtr(const User &I) { 2931199989Srdivacky // What to do depends on the size of the integer and the size of the pointer. 2932199989Srdivacky // We can either truncate, zero extend, or no-op, accordingly. 2933199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2934263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2935263508Sdim setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); 2936199989Srdivacky} 2937199989Srdivacky 2938207618Srdivackyvoid SelectionDAGBuilder::visitBitCast(const User &I) { 2939199989Srdivacky SDValue N = getValue(I.getOperand(0)); 2940263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2941199989Srdivacky 2942201360Srdivacky // BitCast assures us that source and destination are the same size so this is 2943218893Sdim // either a BITCAST or a no-op. 2944203954Srdivacky if (DestVT != N.getValueType()) 2945263508Sdim setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(), 2946203954Srdivacky DestVT, N)); // convert types. 2947203954Srdivacky else 2948201360Srdivacky setValue(&I, N); // noop cast. 2949199989Srdivacky} 2950199989Srdivacky 2951263508Sdimvoid SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { 2952263508Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2953263508Sdim const Value *SV = I.getOperand(0); 2954263508Sdim SDValue N = getValue(SV); 2955263508Sdim EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2956263508Sdim 2957263508Sdim unsigned SrcAS = SV->getType()->getPointerAddressSpace(); 2958263508Sdim unsigned DestAS = I.getType()->getPointerAddressSpace(); 2959263508Sdim 2960263508Sdim if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS)) 2961263508Sdim N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); 2962263508Sdim 2963263508Sdim setValue(&I, N); 2964263508Sdim} 2965263508Sdim 2966207618Srdivackyvoid SelectionDAGBuilder::visitInsertElement(const User &I) { 2967263508Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2968199989Srdivacky SDValue InVec = getValue(I.getOperand(0)); 2969199989Srdivacky SDValue InVal = getValue(I.getOperand(1)); 2970263508Sdim SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), 2971263508Sdim getCurSDLoc(), TLI.getVectorIdxTy()); 2972263508Sdim setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), 2973263508Sdim TM.getTargetLowering()->getValueType(I.getType()), 2974203954Srdivacky InVec, InVal, InIdx)); 2975199989Srdivacky} 2976199989Srdivacky 2977207618Srdivackyvoid SelectionDAGBuilder::visitExtractElement(const User &I) { 2978263508Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2979199989Srdivacky SDValue InVec = getValue(I.getOperand(0)); 2980263508Sdim SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), 2981263508Sdim getCurSDLoc(), TLI.getVectorIdxTy()); 2982263508Sdim setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 2983263508Sdim TM.getTargetLowering()->getValueType(I.getType()), 2984263508Sdim InVec, InIdx)); 2985199989Srdivacky} 2986199989Srdivacky 2987234353Sdim// Utility for visitShuffleVector - Return true if every element in Mask, 2988239462Sdim// beginning from position Pos and ending in Pos+Size, falls within the 2989234353Sdim// specified sequential range [L, L+Pos). or is undef. 2990234353Sdimstatic bool isSequentialInRange(const SmallVectorImpl<int> &Mask, 2991234353Sdim unsigned Pos, unsigned Size, int Low) { 2992234353Sdim for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low) 2993234353Sdim if (Mask[i] >= 0 && Mask[i] != Low) 2994199989Srdivacky return false; 2995199989Srdivacky return true; 2996199989Srdivacky} 2997199989Srdivacky 2998207618Srdivackyvoid SelectionDAGBuilder::visitShuffleVector(const User &I) { 2999199989Srdivacky SDValue Src1 = getValue(I.getOperand(0)); 3000199989Srdivacky SDValue Src2 = getValue(I.getOperand(1)); 3001199989Srdivacky 3002234353Sdim SmallVector<int, 8> Mask; 3003234353Sdim ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask); 3004234353Sdim unsigned MaskNumElts = Mask.size(); 3005263508Sdim 3006263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3007263508Sdim EVT VT = TLI->getValueType(I.getType()); 3008199989Srdivacky EVT SrcVT = Src1.getValueType(); 3009199989Srdivacky unsigned SrcNumElts = SrcVT.getVectorNumElements(); 3010199989Srdivacky 3011199989Srdivacky if (SrcNumElts == MaskNumElts) { 3012263508Sdim setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3013203954Srdivacky &Mask[0])); 3014199989Srdivacky return; 3015199989Srdivacky } 3016199989Srdivacky 3017199989Srdivacky // Normalize the shuffle vector since mask and vector length don't match. 3018199989Srdivacky if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) { 3019199989Srdivacky // Mask is longer than the source vectors and is a multiple of the source 3020199989Srdivacky // vectors. We can use concatenate vector to make the mask and vectors 3021199989Srdivacky // lengths match. 3022234353Sdim if (SrcNumElts*2 == MaskNumElts) { 3023234353Sdim // First check for Src1 in low and Src2 in high 3024234353Sdim if (isSequentialInRange(Mask, 0, SrcNumElts, 0) && 3025234353Sdim isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) { 3026234353Sdim // The shuffle is concatenating two vectors together. 3027263508Sdim setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(), 3028234353Sdim VT, Src1, Src2)); 3029234353Sdim return; 3030234353Sdim } 3031234353Sdim // Then check for Src2 in low and Src1 in high 3032234353Sdim if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) && 3033234353Sdim isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) { 3034234353Sdim // The shuffle is concatenating two vectors together. 3035263508Sdim setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(), 3036234353Sdim VT, Src2, Src1)); 3037234353Sdim return; 3038234353Sdim } 3039199989Srdivacky } 3040199989Srdivacky 3041199989Srdivacky // Pad both vectors with undefs to make them the same length as the mask. 3042199989Srdivacky unsigned NumConcat = MaskNumElts / SrcNumElts; 3043199989Srdivacky bool Src1U = Src1.getOpcode() == ISD::UNDEF; 3044199989Srdivacky bool Src2U = Src2.getOpcode() == ISD::UNDEF; 3045199989Srdivacky SDValue UndefVal = DAG.getUNDEF(SrcVT); 3046199989Srdivacky 3047199989Srdivacky SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); 3048199989Srdivacky SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); 3049199989Srdivacky MOps1[0] = Src1; 3050199989Srdivacky MOps2[0] = Src2; 3051201360Srdivacky 3052201360Srdivacky Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 3053263508Sdim getCurSDLoc(), VT, 3054199989Srdivacky &MOps1[0], NumConcat); 3055199989Srdivacky Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 3056263508Sdim getCurSDLoc(), VT, 3057199989Srdivacky &MOps2[0], NumConcat); 3058199989Srdivacky 3059199989Srdivacky // Readjust mask for new input vector length. 3060199989Srdivacky SmallVector<int, 8> MappedOps; 3061199989Srdivacky for (unsigned i = 0; i != MaskNumElts; ++i) { 3062199989Srdivacky int Idx = Mask[i]; 3063234353Sdim if (Idx >= (int)SrcNumElts) 3064234353Sdim Idx -= SrcNumElts - MaskNumElts; 3065234353Sdim MappedOps.push_back(Idx); 3066199989Srdivacky } 3067201360Srdivacky 3068263508Sdim setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3069203954Srdivacky &MappedOps[0])); 3070199989Srdivacky return; 3071199989Srdivacky } 3072199989Srdivacky 3073199989Srdivacky if (SrcNumElts > MaskNumElts) { 3074199989Srdivacky // Analyze the access pattern of the vector to see if we can extract 3075199989Srdivacky // two subvectors and do the shuffle. The analysis is done by calculating 3076199989Srdivacky // the range of elements the mask access on both vectors. 3077234353Sdim int MinRange[2] = { static_cast<int>(SrcNumElts), 3078234353Sdim static_cast<int>(SrcNumElts)}; 3079199989Srdivacky int MaxRange[2] = {-1, -1}; 3080199989Srdivacky 3081199989Srdivacky for (unsigned i = 0; i != MaskNumElts; ++i) { 3082199989Srdivacky int Idx = Mask[i]; 3083234353Sdim unsigned Input = 0; 3084199989Srdivacky if (Idx < 0) 3085199989Srdivacky continue; 3086201360Srdivacky 3087199989Srdivacky if (Idx >= (int)SrcNumElts) { 3088199989Srdivacky Input = 1; 3089199989Srdivacky Idx -= SrcNumElts; 3090199989Srdivacky } 3091199989Srdivacky if (Idx > MaxRange[Input]) 3092199989Srdivacky MaxRange[Input] = Idx; 3093199989Srdivacky if (Idx < MinRange[Input]) 3094199989Srdivacky MinRange[Input] = Idx; 3095199989Srdivacky } 3096199989Srdivacky 3097199989Srdivacky // Check if the access is smaller than the vector size and can we find 3098199989Srdivacky // a reasonable extract index. 3099234353Sdim int RangeUse[2] = { -1, -1 }; // 0 = Unused, 1 = Extract, -1 = Can not 3100234353Sdim // Extract. 3101199989Srdivacky int StartIdx[2]; // StartIdx to extract from 3102234353Sdim for (unsigned Input = 0; Input < 2; ++Input) { 3103234353Sdim if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) { 3104199989Srdivacky RangeUse[Input] = 0; // Unused 3105199989Srdivacky StartIdx[Input] = 0; 3106234353Sdim continue; 3107199989Srdivacky } 3108234353Sdim 3109234353Sdim // Find a good start index that is a multiple of the mask length. Then 3110234353Sdim // see if the rest of the elements are in range. 3111234353Sdim StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts; 3112234353Sdim if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts && 3113234353Sdim StartIdx[Input] + MaskNumElts <= SrcNumElts) 3114234353Sdim RangeUse[Input] = 1; // Extract from a multiple of the mask length. 3115199989Srdivacky } 3116199989Srdivacky 3117199989Srdivacky if (RangeUse[0] == 0 && RangeUse[1] == 0) { 3118203954Srdivacky setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. 3119199989Srdivacky return; 3120199989Srdivacky } 3121234353Sdim if (RangeUse[0] >= 0 && RangeUse[1] >= 0) { 3122199989Srdivacky // Extract appropriate subvector and generate a vector shuffle 3123234353Sdim for (unsigned Input = 0; Input < 2; ++Input) { 3124201360Srdivacky SDValue &Src = Input == 0 ? Src1 : Src2; 3125201360Srdivacky if (RangeUse[Input] == 0) 3126199989Srdivacky Src = DAG.getUNDEF(VT); 3127201360Srdivacky else 3128263508Sdim Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, 3129263508Sdim Src, DAG.getConstant(StartIdx[Input], 3130263508Sdim TLI->getVectorIdxTy())); 3131199989Srdivacky } 3132201360Srdivacky 3133199989Srdivacky // Calculate new mask. 3134199989Srdivacky SmallVector<int, 8> MappedOps; 3135199989Srdivacky for (unsigned i = 0; i != MaskNumElts; ++i) { 3136199989Srdivacky int Idx = Mask[i]; 3137234353Sdim if (Idx >= 0) { 3138234353Sdim if (Idx < (int)SrcNumElts) 3139234353Sdim Idx -= StartIdx[0]; 3140234353Sdim else 3141234353Sdim Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; 3142234353Sdim } 3143234353Sdim MappedOps.push_back(Idx); 3144199989Srdivacky } 3145201360Srdivacky 3146263508Sdim setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3147203954Srdivacky &MappedOps[0])); 3148199989Srdivacky return; 3149199989Srdivacky } 3150199989Srdivacky } 3151199989Srdivacky 3152199989Srdivacky // We can't use either concat vectors or extract subvectors so fall back to 3153199989Srdivacky // replacing the shuffle with extract and build vector. 3154199989Srdivacky // to insert and build vector. 3155199989Srdivacky EVT EltVT = VT.getVectorElementType(); 3156263508Sdim EVT IdxVT = TLI->getVectorIdxTy(); 3157199989Srdivacky SmallVector<SDValue,8> Ops; 3158199989Srdivacky for (unsigned i = 0; i != MaskNumElts; ++i) { 3159234353Sdim int Idx = Mask[i]; 3160234353Sdim SDValue Res; 3161234353Sdim 3162234353Sdim if (Idx < 0) { 3163234353Sdim Res = DAG.getUNDEF(EltVT); 3164199989Srdivacky } else { 3165234353Sdim SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; 3166234353Sdim if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; 3167201360Srdivacky 3168263508Sdim Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 3169263508Sdim EltVT, Src, DAG.getConstant(Idx, IdxVT)); 3170234353Sdim } 3171201360Srdivacky 3172234353Sdim Ops.push_back(Res); 3173199989Srdivacky } 3174201360Srdivacky 3175263508Sdim setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 3176203954Srdivacky VT, &Ops[0], Ops.size())); 3177199989Srdivacky} 3178199989Srdivacky 3179207618Srdivackyvoid SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { 3180199989Srdivacky const Value *Op0 = I.getOperand(0); 3181199989Srdivacky const Value *Op1 = I.getOperand(1); 3182226633Sdim Type *AggTy = I.getType(); 3183226633Sdim Type *ValTy = Op1->getType(); 3184199989Srdivacky bool IntoUndef = isa<UndefValue>(Op0); 3185199989Srdivacky bool FromUndef = isa<UndefValue>(Op1); 3186199989Srdivacky 3187224145Sdim unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); 3188199989Srdivacky 3189263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3190199989Srdivacky SmallVector<EVT, 4> AggValueVTs; 3191263508Sdim ComputeValueVTs(*TLI, AggTy, AggValueVTs); 3192199989Srdivacky SmallVector<EVT, 4> ValValueVTs; 3193263508Sdim ComputeValueVTs(*TLI, ValTy, ValValueVTs); 3194199989Srdivacky 3195199989Srdivacky unsigned NumAggValues = AggValueVTs.size(); 3196199989Srdivacky unsigned NumValValues = ValValueVTs.size(); 3197199989Srdivacky SmallVector<SDValue, 4> Values(NumAggValues); 3198199989Srdivacky 3199199989Srdivacky SDValue Agg = getValue(Op0); 3200199989Srdivacky unsigned i = 0; 3201199989Srdivacky // Copy the beginning value(s) from the original aggregate. 3202199989Srdivacky for (; i != LinearIndex; ++i) 3203199989Srdivacky Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3204199989Srdivacky SDValue(Agg.getNode(), Agg.getResNo() + i); 3205199989Srdivacky // Copy values from the inserted value(s). 3206223017Sdim if (NumValValues) { 3207223017Sdim SDValue Val = getValue(Op1); 3208223017Sdim for (; i != LinearIndex + NumValValues; ++i) 3209223017Sdim Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3210223017Sdim SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); 3211223017Sdim } 3212199989Srdivacky // Copy remaining value(s) from the original aggregate. 3213199989Srdivacky for (; i != NumAggValues; ++i) 3214199989Srdivacky Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3215199989Srdivacky SDValue(Agg.getNode(), Agg.getResNo() + i); 3216199989Srdivacky 3217263508Sdim setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3218203954Srdivacky DAG.getVTList(&AggValueVTs[0], NumAggValues), 3219203954Srdivacky &Values[0], NumAggValues)); 3220199989Srdivacky} 3221199989Srdivacky 3222207618Srdivackyvoid SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { 3223199989Srdivacky const Value *Op0 = I.getOperand(0); 3224226633Sdim Type *AggTy = Op0->getType(); 3225226633Sdim Type *ValTy = I.getType(); 3226199989Srdivacky bool OutOfUndef = isa<UndefValue>(Op0); 3227199989Srdivacky 3228224145Sdim unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); 3229199989Srdivacky 3230263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3231199989Srdivacky SmallVector<EVT, 4> ValValueVTs; 3232263508Sdim ComputeValueVTs(*TLI, ValTy, ValValueVTs); 3233199989Srdivacky 3234199989Srdivacky unsigned NumValValues = ValValueVTs.size(); 3235223017Sdim 3236223017Sdim // Ignore a extractvalue that produces an empty object 3237223017Sdim if (!NumValValues) { 3238223017Sdim setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); 3239223017Sdim return; 3240223017Sdim } 3241223017Sdim 3242199989Srdivacky SmallVector<SDValue, 4> Values(NumValValues); 3243199989Srdivacky 3244199989Srdivacky SDValue Agg = getValue(Op0); 3245199989Srdivacky // Copy out the selected value(s). 3246199989Srdivacky for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) 3247199989Srdivacky Values[i - LinearIndex] = 3248199989Srdivacky OutOfUndef ? 3249199989Srdivacky DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : 3250199989Srdivacky SDValue(Agg.getNode(), Agg.getResNo() + i); 3251199989Srdivacky 3252263508Sdim setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3253203954Srdivacky DAG.getVTList(&ValValueVTs[0], NumValValues), 3254203954Srdivacky &Values[0], NumValValues)); 3255199989Srdivacky} 3256199989Srdivacky 3257207618Srdivackyvoid SelectionDAGBuilder::visitGetElementPtr(const User &I) { 3258263508Sdim Value *Op0 = I.getOperand(0); 3259234353Sdim // Note that the pointer operand may be a vector of pointers. Take the scalar 3260234353Sdim // element which holds a pointer. 3261263508Sdim Type *Ty = Op0->getType()->getScalarType(); 3262263508Sdim unsigned AS = Ty->getPointerAddressSpace(); 3263263508Sdim SDValue N = getValue(Op0); 3264199989Srdivacky 3265207618Srdivacky for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end(); 3266199989Srdivacky OI != E; ++OI) { 3267207618Srdivacky const Value *Idx = *OI; 3268226633Sdim if (StructType *StTy = dyn_cast<StructType>(Ty)) { 3269249423Sdim unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); 3270199989Srdivacky if (Field) { 3271199989Srdivacky // N = N + Offset 3272199989Srdivacky uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field); 3273263508Sdim N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, 3274249423Sdim DAG.getConstant(Offset, N.getValueType())); 3275199989Srdivacky } 3276201360Srdivacky 3277199989Srdivacky Ty = StTy->getElementType(Field); 3278199989Srdivacky } else { 3279199989Srdivacky Ty = cast<SequentialType>(Ty)->getElementType(); 3280199989Srdivacky 3281199989Srdivacky // If this is a constant subscript, handle it quickly. 3282263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3283207618Srdivacky if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { 3284210299Sed if (CI->isZero()) continue; 3285199989Srdivacky uint64_t Offs = 3286199989Srdivacky TD->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue(); 3287199989Srdivacky SDValue OffsVal; 3288263508Sdim EVT PTy = TLI->getPointerTy(AS); 3289199989Srdivacky unsigned PtrBits = PTy.getSizeInBits(); 3290201360Srdivacky if (PtrBits < 64) 3291263508Sdim OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy, 3292199989Srdivacky DAG.getConstant(Offs, MVT::i64)); 3293201360Srdivacky else 3294263508Sdim OffsVal = DAG.getConstant(Offs, PTy); 3295201360Srdivacky 3296263508Sdim N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, 3297199989Srdivacky OffsVal); 3298199989Srdivacky continue; 3299199989Srdivacky } 3300199989Srdivacky 3301199989Srdivacky // N = N + Idx * ElementSize; 3302263508Sdim APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS), 3303199989Srdivacky TD->getTypeAllocSize(Ty)); 3304199989Srdivacky SDValue IdxN = getValue(Idx); 3305199989Srdivacky 3306199989Srdivacky // If the index is smaller or larger than intptr_t, truncate or extend 3307199989Srdivacky // it. 3308263508Sdim IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType()); 3309199989Srdivacky 3310199989Srdivacky // If this is a multiply by a power of two, turn it into a shl 3311199989Srdivacky // immediately. This is a very common case. 3312199989Srdivacky if (ElementSize != 1) { 3313199989Srdivacky if (ElementSize.isPowerOf2()) { 3314199989Srdivacky unsigned Amt = ElementSize.logBase2(); 3315263508Sdim IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(), 3316199989Srdivacky N.getValueType(), IdxN, 3317234353Sdim DAG.getConstant(Amt, IdxN.getValueType())); 3318199989Srdivacky } else { 3319249423Sdim SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType()); 3320263508Sdim IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(), 3321199989Srdivacky N.getValueType(), IdxN, Scale); 3322199989Srdivacky } 3323199989Srdivacky } 3324199989Srdivacky 3325263508Sdim N = DAG.getNode(ISD::ADD, getCurSDLoc(), 3326199989Srdivacky N.getValueType(), N, IdxN); 3327199989Srdivacky } 3328199989Srdivacky } 3329201360Srdivacky 3330199989Srdivacky setValue(&I, N); 3331199989Srdivacky} 3332199989Srdivacky 3333207618Srdivackyvoid SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { 3334199989Srdivacky // If this is a fixed sized alloca in the entry block of the function, 3335199989Srdivacky // allocate it statically on the stack. 3336199989Srdivacky if (FuncInfo.StaticAllocaMap.count(&I)) 3337199989Srdivacky return; // getValue will auto-populate this. 3338199989Srdivacky 3339226633Sdim Type *Ty = I.getAllocatedType(); 3340263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3341263508Sdim uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty); 3342199989Srdivacky unsigned Align = 3343263508Sdim std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), 3344199989Srdivacky I.getAlignment()); 3345199989Srdivacky 3346199989Srdivacky SDValue AllocSize = getValue(I.getArraySize()); 3347201360Srdivacky 3348263508Sdim EVT IntPtr = TLI->getPointerTy(); 3349210299Sed if (AllocSize.getValueType() != IntPtr) 3350263508Sdim AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr); 3351210299Sed 3352263508Sdim AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr, 3353199989Srdivacky AllocSize, 3354210299Sed DAG.getConstant(TySize, IntPtr)); 3355201360Srdivacky 3356199989Srdivacky // Handle alignment. If the requested alignment is less than or equal to 3357199989Srdivacky // the stack alignment, ignore it. If the size is greater than or equal to 3358199989Srdivacky // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. 3359218893Sdim unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 3360199989Srdivacky if (Align <= StackAlign) 3361199989Srdivacky Align = 0; 3362199989Srdivacky 3363199989Srdivacky // Round the size of the allocation up to the stack alignment size 3364199989Srdivacky // by add SA-1 to the size. 3365263508Sdim AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(), 3366199989Srdivacky AllocSize.getValueType(), AllocSize, 3367199989Srdivacky DAG.getIntPtrConstant(StackAlign-1)); 3368201360Srdivacky 3369199989Srdivacky // Mask out the low bits for alignment purposes. 3370263508Sdim AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(), 3371199989Srdivacky AllocSize.getValueType(), AllocSize, 3372199989Srdivacky DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); 3373199989Srdivacky 3374199989Srdivacky SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; 3375199989Srdivacky SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); 3376263508Sdim SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), 3377199989Srdivacky VTs, Ops, 3); 3378199989Srdivacky setValue(&I, DSA); 3379199989Srdivacky DAG.setRoot(DSA.getValue(1)); 3380199989Srdivacky 3381263765Sdim assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects()); 3382199989Srdivacky} 3383199989Srdivacky 3384207618Srdivackyvoid SelectionDAGBuilder::visitLoad(const LoadInst &I) { 3385226633Sdim if (I.isAtomic()) 3386226633Sdim return visitAtomicLoad(I); 3387226633Sdim 3388199989Srdivacky const Value *SV = I.getOperand(0); 3389199989Srdivacky SDValue Ptr = getValue(SV); 3390199989Srdivacky 3391226633Sdim Type *Ty = I.getType(); 3392203954Srdivacky 3393199989Srdivacky bool isVolatile = I.isVolatile(); 3394203954Srdivacky bool isNonTemporal = I.getMetadata("nontemporal") != 0; 3395234353Sdim bool isInvariant = I.getMetadata("invariant.load") != 0; 3396199989Srdivacky unsigned Alignment = I.getAlignment(); 3397218893Sdim const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa); 3398234353Sdim const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); 3399199989Srdivacky 3400199989Srdivacky SmallVector<EVT, 4> ValueVTs; 3401199989Srdivacky SmallVector<uint64_t, 4> Offsets; 3402263508Sdim ComputeValueVTs(*TM.getTargetLowering(), Ty, ValueVTs, &Offsets); 3403199989Srdivacky unsigned NumValues = ValueVTs.size(); 3404199989Srdivacky if (NumValues == 0) 3405199989Srdivacky return; 3406199989Srdivacky 3407199989Srdivacky SDValue Root; 3408199989Srdivacky bool ConstantMemory = false; 3409218893Sdim if (I.isVolatile() || NumValues > MaxParallelChains) 3410199989Srdivacky // Serialize volatile loads with other side effects. 3411199989Srdivacky Root = getRoot(); 3412218893Sdim else if (AA->pointsToConstantMemory( 3413218893Sdim AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), TBAAInfo))) { 3414199989Srdivacky // Do not serialize (non-volatile) loads of constant memory with anything. 3415199989Srdivacky Root = DAG.getEntryNode(); 3416199989Srdivacky ConstantMemory = true; 3417199989Srdivacky } else { 3418199989Srdivacky // Do not serialize non-volatile loads against each other. 3419199989Srdivacky Root = DAG.getRoot(); 3420199989Srdivacky } 3421199989Srdivacky 3422199989Srdivacky SmallVector<SDValue, 4> Values(NumValues); 3423218893Sdim SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains), 3424218893Sdim NumValues)); 3425199989Srdivacky EVT PtrVT = Ptr.getValueType(); 3426218893Sdim unsigned ChainI = 0; 3427218893Sdim for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 3428218893Sdim // Serializing loads here may result in excessive register pressure, and 3429218893Sdim // TokenFactor places arbitrary choke points on the scheduler. SD scheduling 3430218893Sdim // could recover a bit by hoisting nodes upward in the chain by recognizing 3431218893Sdim // they are side-effect free or do not alias. The optimizer should really 3432218893Sdim // avoid this case by converting large object/array copies to llvm.memcpy 3433218893Sdim // (MaxParallelChains should always remain as failsafe). 3434218893Sdim if (ChainI == MaxParallelChains) { 3435218893Sdim assert(PendingLoads.empty() && "PendingLoads must be serialized first"); 3436263508Sdim SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3437218893Sdim MVT::Other, &Chains[0], ChainI); 3438218893Sdim Root = Chain; 3439218893Sdim ChainI = 0; 3440218893Sdim } 3441263508Sdim SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(), 3442201360Srdivacky PtrVT, Ptr, 3443201360Srdivacky DAG.getConstant(Offsets[i], PtrVT)); 3444263508Sdim SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root, 3445218893Sdim A, MachinePointerInfo(SV, Offsets[i]), isVolatile, 3446234353Sdim isNonTemporal, isInvariant, Alignment, TBAAInfo, 3447234353Sdim Ranges); 3448201360Srdivacky 3449199989Srdivacky Values[i] = L; 3450218893Sdim Chains[ChainI] = L.getValue(1); 3451199989Srdivacky } 3452199989Srdivacky 3453199989Srdivacky if (!ConstantMemory) { 3454263508Sdim SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3455218893Sdim MVT::Other, &Chains[0], ChainI); 3456199989Srdivacky if (isVolatile) 3457199989Srdivacky DAG.setRoot(Chain); 3458199989Srdivacky else 3459199989Srdivacky PendingLoads.push_back(Chain); 3460199989Srdivacky } 3461199989Srdivacky 3462263508Sdim setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3463203954Srdivacky DAG.getVTList(&ValueVTs[0], NumValues), 3464203954Srdivacky &Values[0], NumValues)); 3465199989Srdivacky} 3466199989Srdivacky 3467207618Srdivackyvoid SelectionDAGBuilder::visitStore(const StoreInst &I) { 3468226633Sdim if (I.isAtomic()) 3469226633Sdim return visitAtomicStore(I); 3470226633Sdim 3471207618Srdivacky const Value *SrcV = I.getOperand(0); 3472207618Srdivacky const Value *PtrV = I.getOperand(1); 3473199989Srdivacky 3474199989Srdivacky SmallVector<EVT, 4> ValueVTs; 3475199989Srdivacky SmallVector<uint64_t, 4> Offsets; 3476263508Sdim ComputeValueVTs(*TM.getTargetLowering(), SrcV->getType(), ValueVTs, &Offsets); 3477199989Srdivacky unsigned NumValues = ValueVTs.size(); 3478199989Srdivacky if (NumValues == 0) 3479199989Srdivacky return; 3480199989Srdivacky 3481199989Srdivacky // Get the lowered operands. Note that we do this after 3482199989Srdivacky // checking if NumResults is zero, because with zero results 3483199989Srdivacky // the operands won't have values in the map. 3484199989Srdivacky SDValue Src = getValue(SrcV); 3485199989Srdivacky SDValue Ptr = getValue(PtrV); 3486199989Srdivacky 3487199989Srdivacky SDValue Root = getRoot(); 3488218893Sdim SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains), 3489218893Sdim NumValues)); 3490199989Srdivacky EVT PtrVT = Ptr.getValueType(); 3491199989Srdivacky bool isVolatile = I.isVolatile(); 3492203954Srdivacky bool isNonTemporal = I.getMetadata("nontemporal") != 0; 3493199989Srdivacky unsigned Alignment = I.getAlignment(); 3494218893Sdim const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa); 3495201360Srdivacky 3496218893Sdim unsigned ChainI = 0; 3497218893Sdim for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 3498218893Sdim // See visitLoad comments. 3499218893Sdim if (ChainI == MaxParallelChains) { 3500263508Sdim SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3501218893Sdim MVT::Other, &Chains[0], ChainI); 3502218893Sdim Root = Chain; 3503218893Sdim ChainI = 0; 3504218893Sdim } 3505263508Sdim SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr, 3506201360Srdivacky DAG.getConstant(Offsets[i], PtrVT)); 3507263508Sdim SDValue St = DAG.getStore(Root, getCurSDLoc(), 3508218893Sdim SDValue(Src.getNode(), Src.getResNo() + i), 3509218893Sdim Add, MachinePointerInfo(PtrV, Offsets[i]), 3510218893Sdim isVolatile, isNonTemporal, Alignment, TBAAInfo); 3511218893Sdim Chains[ChainI] = St; 3512201360Srdivacky } 3513201360Srdivacky 3514263508Sdim SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3515218893Sdim MVT::Other, &Chains[0], ChainI); 3516218893Sdim DAG.setRoot(StoreNode); 3517199989Srdivacky} 3518199989Srdivacky 3519226633Sdimstatic SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order, 3520226633Sdim SynchronizationScope Scope, 3521263508Sdim bool Before, SDLoc dl, 3522226633Sdim SelectionDAG &DAG, 3523226633Sdim const TargetLowering &TLI) { 3524226633Sdim // Fence, if necessary 3525226633Sdim if (Before) { 3526226633Sdim if (Order == AcquireRelease || Order == SequentiallyConsistent) 3527226633Sdim Order = Release; 3528226633Sdim else if (Order == Acquire || Order == Monotonic) 3529226633Sdim return Chain; 3530226633Sdim } else { 3531226633Sdim if (Order == AcquireRelease) 3532226633Sdim Order = Acquire; 3533226633Sdim else if (Order == Release || Order == Monotonic) 3534226633Sdim return Chain; 3535226633Sdim } 3536226633Sdim SDValue Ops[3]; 3537226633Sdim Ops[0] = Chain; 3538226633Sdim Ops[1] = DAG.getConstant(Order, TLI.getPointerTy()); 3539226633Sdim Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy()); 3540226633Sdim return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3); 3541226633Sdim} 3542226633Sdim 3543226633Sdimvoid SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { 3544263508Sdim SDLoc dl = getCurSDLoc(); 3545226633Sdim AtomicOrdering Order = I.getOrdering(); 3546226633Sdim SynchronizationScope Scope = I.getSynchScope(); 3547226633Sdim 3548226633Sdim SDValue InChain = getRoot(); 3549226633Sdim 3550263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3551263508Sdim if (TLI->getInsertFencesForAtomic()) 3552226633Sdim InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3553263508Sdim DAG, *TLI); 3554226633Sdim 3555226633Sdim SDValue L = 3556226633Sdim DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 3557263508Sdim getValue(I.getCompareOperand()).getSimpleValueType(), 3558226633Sdim InChain, 3559226633Sdim getValue(I.getPointerOperand()), 3560226633Sdim getValue(I.getCompareOperand()), 3561226633Sdim getValue(I.getNewValOperand()), 3562226633Sdim MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, 3563263508Sdim TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3564226633Sdim Scope); 3565226633Sdim 3566226633Sdim SDValue OutChain = L.getValue(1); 3567226633Sdim 3568263508Sdim if (TLI->getInsertFencesForAtomic()) 3569226633Sdim OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3570263508Sdim DAG, *TLI); 3571226633Sdim 3572226633Sdim setValue(&I, L); 3573226633Sdim DAG.setRoot(OutChain); 3574226633Sdim} 3575226633Sdim 3576226633Sdimvoid SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { 3577263508Sdim SDLoc dl = getCurSDLoc(); 3578226633Sdim ISD::NodeType NT; 3579226633Sdim switch (I.getOperation()) { 3580234353Sdim default: llvm_unreachable("Unknown atomicrmw operation"); 3581226633Sdim case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; 3582226633Sdim case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; 3583226633Sdim case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; 3584226633Sdim case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; 3585226633Sdim case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; 3586226633Sdim case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; 3587226633Sdim case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; 3588226633Sdim case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; 3589226633Sdim case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; 3590226633Sdim case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; 3591226633Sdim case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; 3592226633Sdim } 3593226633Sdim AtomicOrdering Order = I.getOrdering(); 3594226633Sdim SynchronizationScope Scope = I.getSynchScope(); 3595226633Sdim 3596226633Sdim SDValue InChain = getRoot(); 3597226633Sdim 3598263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3599263508Sdim if (TLI->getInsertFencesForAtomic()) 3600226633Sdim InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3601263508Sdim DAG, *TLI); 3602226633Sdim 3603226633Sdim SDValue L = 3604226633Sdim DAG.getAtomic(NT, dl, 3605263508Sdim getValue(I.getValOperand()).getSimpleValueType(), 3606226633Sdim InChain, 3607226633Sdim getValue(I.getPointerOperand()), 3608226633Sdim getValue(I.getValOperand()), 3609226633Sdim I.getPointerOperand(), 0 /* Alignment */, 3610263508Sdim TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3611226633Sdim Scope); 3612226633Sdim 3613226633Sdim SDValue OutChain = L.getValue(1); 3614226633Sdim 3615263508Sdim if (TLI->getInsertFencesForAtomic()) 3616226633Sdim OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3617263508Sdim DAG, *TLI); 3618226633Sdim 3619226633Sdim setValue(&I, L); 3620226633Sdim DAG.setRoot(OutChain); 3621226633Sdim} 3622226633Sdim 3623226633Sdimvoid SelectionDAGBuilder::visitFence(const FenceInst &I) { 3624263508Sdim SDLoc dl = getCurSDLoc(); 3625263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3626226633Sdim SDValue Ops[3]; 3627226633Sdim Ops[0] = getRoot(); 3628263508Sdim Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy()); 3629263508Sdim Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy()); 3630226633Sdim DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3)); 3631226633Sdim} 3632226633Sdim 3633226633Sdimvoid SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { 3634263508Sdim SDLoc dl = getCurSDLoc(); 3635226633Sdim AtomicOrdering Order = I.getOrdering(); 3636226633Sdim SynchronizationScope Scope = I.getSynchScope(); 3637226633Sdim 3638226633Sdim SDValue InChain = getRoot(); 3639226633Sdim 3640263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3641263508Sdim EVT VT = TLI->getValueType(I.getType()); 3642226633Sdim 3643249423Sdim if (I.getAlignment() < VT.getSizeInBits() / 8) 3644226633Sdim report_fatal_error("Cannot generate unaligned atomic load"); 3645226633Sdim 3646226633Sdim SDValue L = 3647226633Sdim DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, 3648226633Sdim getValue(I.getPointerOperand()), 3649226633Sdim I.getPointerOperand(), I.getAlignment(), 3650263508Sdim TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3651226633Sdim Scope); 3652226633Sdim 3653226633Sdim SDValue OutChain = L.getValue(1); 3654226633Sdim 3655263508Sdim if (TLI->getInsertFencesForAtomic()) 3656226633Sdim OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3657263508Sdim DAG, *TLI); 3658226633Sdim 3659226633Sdim setValue(&I, L); 3660226633Sdim DAG.setRoot(OutChain); 3661226633Sdim} 3662226633Sdim 3663226633Sdimvoid SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { 3664263508Sdim SDLoc dl = getCurSDLoc(); 3665226633Sdim 3666226633Sdim AtomicOrdering Order = I.getOrdering(); 3667226633Sdim SynchronizationScope Scope = I.getSynchScope(); 3668226633Sdim 3669226633Sdim SDValue InChain = getRoot(); 3670226633Sdim 3671263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3672263508Sdim EVT VT = TLI->getValueType(I.getValueOperand()->getType()); 3673226633Sdim 3674249423Sdim if (I.getAlignment() < VT.getSizeInBits() / 8) 3675226633Sdim report_fatal_error("Cannot generate unaligned atomic store"); 3676226633Sdim 3677263508Sdim if (TLI->getInsertFencesForAtomic()) 3678226633Sdim InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3679263508Sdim DAG, *TLI); 3680226633Sdim 3681226633Sdim SDValue OutChain = 3682226633Sdim DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, 3683226633Sdim InChain, 3684226633Sdim getValue(I.getPointerOperand()), 3685226633Sdim getValue(I.getValueOperand()), 3686226633Sdim I.getPointerOperand(), I.getAlignment(), 3687263508Sdim TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3688226633Sdim Scope); 3689226633Sdim 3690263508Sdim if (TLI->getInsertFencesForAtomic()) 3691226633Sdim OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3692263508Sdim DAG, *TLI); 3693226633Sdim 3694226633Sdim DAG.setRoot(OutChain); 3695226633Sdim} 3696226633Sdim 3697199989Srdivacky/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC 3698199989Srdivacky/// node. 3699207618Srdivackyvoid SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, 3700199989Srdivacky unsigned Intrinsic) { 3701199989Srdivacky bool HasChain = !I.doesNotAccessMemory(); 3702199989Srdivacky bool OnlyLoad = HasChain && I.onlyReadsMemory(); 3703199989Srdivacky 3704199989Srdivacky // Build the operand list. 3705199989Srdivacky SmallVector<SDValue, 8> Ops; 3706199989Srdivacky if (HasChain) { // If this intrinsic has side-effects, chainify it. 3707199989Srdivacky if (OnlyLoad) { 3708199989Srdivacky // We don't need to serialize loads against other loads. 3709199989Srdivacky Ops.push_back(DAG.getRoot()); 3710199989Srdivacky } else { 3711199989Srdivacky Ops.push_back(getRoot()); 3712199989Srdivacky } 3713199989Srdivacky } 3714199989Srdivacky 3715199989Srdivacky // Info is set by getTgtMemInstrinsic 3716199989Srdivacky TargetLowering::IntrinsicInfo Info; 3717263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 3718263508Sdim bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic); 3719199989Srdivacky 3720199989Srdivacky // Add the intrinsic ID as an integer operand if it's not a target intrinsic. 3721218893Sdim if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || 3722218893Sdim Info.opc == ISD::INTRINSIC_W_CHAIN) 3723263508Sdim Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy())); 3724199989Srdivacky 3725199989Srdivacky // Add all operands of the call to the operand list. 3726210299Sed for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { 3727210299Sed SDValue Op = getValue(I.getArgOperand(i)); 3728199989Srdivacky Ops.push_back(Op); 3729199989Srdivacky } 3730199989Srdivacky 3731199989Srdivacky SmallVector<EVT, 4> ValueVTs; 3732263508Sdim ComputeValueVTs(*TLI, I.getType(), ValueVTs); 3733201360Srdivacky 3734199989Srdivacky if (HasChain) 3735199989Srdivacky ValueVTs.push_back(MVT::Other); 3736199989Srdivacky 3737199989Srdivacky SDVTList VTs = DAG.getVTList(ValueVTs.data(), ValueVTs.size()); 3738199989Srdivacky 3739199989Srdivacky // Create the node. 3740199989Srdivacky SDValue Result; 3741199989Srdivacky if (IsTgtIntrinsic) { 3742199989Srdivacky // This is target intrinsic that touches memory 3743263508Sdim Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), 3744199989Srdivacky VTs, &Ops[0], Ops.size(), 3745218893Sdim Info.memVT, 3746218893Sdim MachinePointerInfo(Info.ptrVal, Info.offset), 3747199989Srdivacky Info.align, Info.vol, 3748199989Srdivacky Info.readMem, Info.writeMem); 3749201360Srdivacky } else if (!HasChain) { 3750263508Sdim Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), 3751199989Srdivacky VTs, &Ops[0], Ops.size()); 3752202375Srdivacky } else if (!I.getType()->isVoidTy()) { 3753263508Sdim Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), 3754199989Srdivacky VTs, &Ops[0], Ops.size()); 3755201360Srdivacky } else { 3756263508Sdim Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), 3757199989Srdivacky VTs, &Ops[0], Ops.size()); 3758201360Srdivacky } 3759199989Srdivacky 3760199989Srdivacky if (HasChain) { 3761199989Srdivacky SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); 3762199989Srdivacky if (OnlyLoad) 3763199989Srdivacky PendingLoads.push_back(Chain); 3764199989Srdivacky else 3765199989Srdivacky DAG.setRoot(Chain); 3766199989Srdivacky } 3767201360Srdivacky 3768202375Srdivacky if (!I.getType()->isVoidTy()) { 3769226633Sdim if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) { 3770263508Sdim EVT VT = TLI->getValueType(PTy); 3771263508Sdim Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result); 3772199989Srdivacky } 3773201360Srdivacky 3774199989Srdivacky setValue(&I, Result); 3775199989Srdivacky } 3776199989Srdivacky} 3777199989Srdivacky 3778199989Srdivacky/// GetSignificand - Get the significand and build it into a floating-point 3779199989Srdivacky/// number with exponent of 1: 3780199989Srdivacky/// 3781199989Srdivacky/// Op = (Op & 0x007fffff) | 0x3f800000; 3782199989Srdivacky/// 3783249423Sdim/// where Op is the hexadecimal representation of floating point value. 3784199989Srdivackystatic SDValue 3785263508SdimGetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) { 3786199989Srdivacky SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 3787199989Srdivacky DAG.getConstant(0x007fffff, MVT::i32)); 3788199989Srdivacky SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, 3789199989Srdivacky DAG.getConstant(0x3f800000, MVT::i32)); 3790218893Sdim return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); 3791199989Srdivacky} 3792199989Srdivacky 3793199989Srdivacky/// GetExponent - Get the exponent: 3794199989Srdivacky/// 3795199989Srdivacky/// (float)(int)(((Op & 0x7f800000) >> 23) - 127); 3796199989Srdivacky/// 3797249423Sdim/// where Op is the hexadecimal representation of floating point value. 3798199989Srdivackystatic SDValue 3799199989SrdivackyGetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, 3800263508Sdim SDLoc dl) { 3801199989Srdivacky SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 3802199989Srdivacky DAG.getConstant(0x7f800000, MVT::i32)); 3803199989Srdivacky SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0, 3804199989Srdivacky DAG.getConstant(23, TLI.getPointerTy())); 3805199989Srdivacky SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, 3806199989Srdivacky DAG.getConstant(127, MVT::i32)); 3807203954Srdivacky return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); 3808199989Srdivacky} 3809199989Srdivacky 3810199989Srdivacky/// getF32Constant - Get 32-bit floating point constant. 3811199989Srdivackystatic SDValue 3812199989SrdivackygetF32Constant(SelectionDAG &DAG, unsigned Flt) { 3813249423Sdim return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), 3814249423Sdim MVT::f32); 3815199989Srdivacky} 3816199989Srdivacky 3817249423Sdim/// expandExp - Lower an exp intrinsic. Handles the special sequences for 3818199989Srdivacky/// limited-precision mode. 3819263508Sdimstatic SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG, 3820249423Sdim const TargetLowering &TLI) { 3821249423Sdim if (Op.getValueType() == MVT::f32 && 3822199989Srdivacky LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 3823199989Srdivacky 3824199989Srdivacky // Put the exponent in the right bit position for later addition to the 3825199989Srdivacky // final result: 3826199989Srdivacky // 3827199989Srdivacky // #define LOG2OFe 1.4426950f 3828199989Srdivacky // IntegerPartOfX = ((int32_t)(X * LOG2OFe)); 3829199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, 3830199989Srdivacky getF32Constant(DAG, 0x3fb8aa3b)); 3831199989Srdivacky SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 3832199989Srdivacky 3833199989Srdivacky // FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX; 3834199989Srdivacky SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 3835199989Srdivacky SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 3836199989Srdivacky 3837199989Srdivacky // IntegerPartOfX <<= 23; 3838199989Srdivacky IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 3839199989Srdivacky DAG.getConstant(23, TLI.getPointerTy())); 3840199989Srdivacky 3841249423Sdim SDValue TwoToFracPartOfX; 3842199989Srdivacky if (LimitFloatPrecision <= 6) { 3843199989Srdivacky // For floating-point precision of 6: 3844199989Srdivacky // 3845199989Srdivacky // TwoToFractionalPartOfX = 3846199989Srdivacky // 0.997535578f + 3847199989Srdivacky // (0.735607626f + 0.252464424f * x) * x; 3848199989Srdivacky // 3849199989Srdivacky // error 0.0144103317, which is 6 bits 3850199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3851199989Srdivacky getF32Constant(DAG, 0x3e814304)); 3852199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3853199989Srdivacky getF32Constant(DAG, 0x3f3c50c8)); 3854199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3855249423Sdim TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3856249423Sdim getF32Constant(DAG, 0x3f7f5e7e)); 3857249423Sdim } else if (LimitFloatPrecision <= 12) { 3858199989Srdivacky // For floating-point precision of 12: 3859199989Srdivacky // 3860199989Srdivacky // TwoToFractionalPartOfX = 3861199989Srdivacky // 0.999892986f + 3862199989Srdivacky // (0.696457318f + 3863199989Srdivacky // (0.224338339f + 0.792043434e-1f * x) * x) * x; 3864199989Srdivacky // 3865199989Srdivacky // 0.000107046256 error, which is 13 to 14 bits 3866199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3867199989Srdivacky getF32Constant(DAG, 0x3da235e3)); 3868199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3869199989Srdivacky getF32Constant(DAG, 0x3e65b8f3)); 3870199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3871199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3872199989Srdivacky getF32Constant(DAG, 0x3f324b07)); 3873199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3874249423Sdim TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 3875249423Sdim getF32Constant(DAG, 0x3f7ff8fd)); 3876249423Sdim } else { // LimitFloatPrecision <= 18 3877199989Srdivacky // For floating-point precision of 18: 3878199989Srdivacky // 3879199989Srdivacky // TwoToFractionalPartOfX = 3880199989Srdivacky // 0.999999982f + 3881199989Srdivacky // (0.693148872f + 3882199989Srdivacky // (0.240227044f + 3883199989Srdivacky // (0.554906021e-1f + 3884199989Srdivacky // (0.961591928e-2f + 3885199989Srdivacky // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 3886199989Srdivacky // 3887199989Srdivacky // error 2.47208000*10^(-7), which is better than 18 bits 3888199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3889199989Srdivacky getF32Constant(DAG, 0x3924b03e)); 3890199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3891199989Srdivacky getF32Constant(DAG, 0x3ab24b87)); 3892199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3893199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3894199989Srdivacky getF32Constant(DAG, 0x3c1d8c17)); 3895199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3896199989Srdivacky SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 3897199989Srdivacky getF32Constant(DAG, 0x3d634a1d)); 3898199989Srdivacky SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 3899199989Srdivacky SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 3900199989Srdivacky getF32Constant(DAG, 0x3e75fe14)); 3901199989Srdivacky SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 3902199989Srdivacky SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 3903199989Srdivacky getF32Constant(DAG, 0x3f317234)); 3904199989Srdivacky SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 3905249423Sdim TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 3906249423Sdim getF32Constant(DAG, 0x3f800000)); 3907249423Sdim } 3908199989Srdivacky 3909249423Sdim // Add the exponent into the result in integer domain. 3910249423Sdim SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX); 3911249423Sdim return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 3912249423Sdim DAG.getNode(ISD::ADD, dl, MVT::i32, 3913249423Sdim t13, IntegerPartOfX)); 3914199989Srdivacky } 3915199989Srdivacky 3916249423Sdim // No special expansion. 3917249423Sdim return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op); 3918199989Srdivacky} 3919199989Srdivacky 3920249423Sdim/// expandLog - Lower a log intrinsic. Handles the special sequences for 3921199989Srdivacky/// limited-precision mode. 3922263508Sdimstatic SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG, 3923249423Sdim const TargetLowering &TLI) { 3924249423Sdim if (Op.getValueType() == MVT::f32 && 3925199989Srdivacky LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 3926218893Sdim SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 3927199989Srdivacky 3928199989Srdivacky // Scale the exponent by log(2) [0.69314718f]. 3929204642Srdivacky SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 3930199989Srdivacky SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 3931199989Srdivacky getF32Constant(DAG, 0x3f317218)); 3932199989Srdivacky 3933199989Srdivacky // Get the significand and build it into a floating-point number with 3934199989Srdivacky // exponent of 1. 3935204642Srdivacky SDValue X = GetSignificand(DAG, Op1, dl); 3936199989Srdivacky 3937249423Sdim SDValue LogOfMantissa; 3938199989Srdivacky if (LimitFloatPrecision <= 6) { 3939199989Srdivacky // For floating-point precision of 6: 3940199989Srdivacky // 3941199989Srdivacky // LogofMantissa = 3942199989Srdivacky // -1.1609546f + 3943199989Srdivacky // (1.4034025f - 0.23903021f * x) * x; 3944199989Srdivacky // 3945199989Srdivacky // error 0.0034276066, which is better than 8 bits 3946199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3947199989Srdivacky getF32Constant(DAG, 0xbe74c456)); 3948199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 3949199989Srdivacky getF32Constant(DAG, 0x3fb3a2b1)); 3950199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 3951249423Sdim LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 3952249423Sdim getF32Constant(DAG, 0x3f949a29)); 3953249423Sdim } else if (LimitFloatPrecision <= 12) { 3954199989Srdivacky // For floating-point precision of 12: 3955199989Srdivacky // 3956199989Srdivacky // LogOfMantissa = 3957199989Srdivacky // -1.7417939f + 3958199989Srdivacky // (2.8212026f + 3959199989Srdivacky // (-1.4699568f + 3960199989Srdivacky // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; 3961199989Srdivacky // 3962199989Srdivacky // error 0.000061011436, which is 14 bits 3963199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3964199989Srdivacky getF32Constant(DAG, 0xbd67b6d6)); 3965199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 3966199989Srdivacky getF32Constant(DAG, 0x3ee4f4b8)); 3967199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 3968199989Srdivacky SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 3969199989Srdivacky getF32Constant(DAG, 0x3fbc278b)); 3970199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3971199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3972199989Srdivacky getF32Constant(DAG, 0x40348e95)); 3973199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3974249423Sdim LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 3975249423Sdim getF32Constant(DAG, 0x3fdef31a)); 3976249423Sdim } else { // LimitFloatPrecision <= 18 3977199989Srdivacky // For floating-point precision of 18: 3978199989Srdivacky // 3979199989Srdivacky // LogOfMantissa = 3980199989Srdivacky // -2.1072184f + 3981199989Srdivacky // (4.2372794f + 3982199989Srdivacky // (-3.7029485f + 3983199989Srdivacky // (2.2781945f + 3984199989Srdivacky // (-0.87823314f + 3985199989Srdivacky // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; 3986199989Srdivacky // 3987199989Srdivacky // error 0.0000023660568, which is better than 18 bits 3988199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3989199989Srdivacky getF32Constant(DAG, 0xbc91e5ac)); 3990199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 3991199989Srdivacky getF32Constant(DAG, 0x3e4350aa)); 3992199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 3993199989Srdivacky SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 3994199989Srdivacky getF32Constant(DAG, 0x3f60d3e3)); 3995199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3996199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3997199989Srdivacky getF32Constant(DAG, 0x4011cdf0)); 3998199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3999199989Srdivacky SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4000199989Srdivacky getF32Constant(DAG, 0x406cfd1c)); 4001199989Srdivacky SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4002199989Srdivacky SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4003199989Srdivacky getF32Constant(DAG, 0x408797cb)); 4004199989Srdivacky SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4005249423Sdim LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 4006249423Sdim getF32Constant(DAG, 0x4006dcab)); 4007249423Sdim } 4008199989Srdivacky 4009249423Sdim return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); 4010199989Srdivacky } 4011199989Srdivacky 4012249423Sdim // No special expansion. 4013249423Sdim return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op); 4014199989Srdivacky} 4015199989Srdivacky 4016249423Sdim/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for 4017199989Srdivacky/// limited-precision mode. 4018263508Sdimstatic SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4019249423Sdim const TargetLowering &TLI) { 4020249423Sdim if (Op.getValueType() == MVT::f32 && 4021199989Srdivacky LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4022218893Sdim SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 4023199989Srdivacky 4024199989Srdivacky // Get the exponent. 4025204642Srdivacky SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); 4026199989Srdivacky 4027199989Srdivacky // Get the significand and build it into a floating-point number with 4028199989Srdivacky // exponent of 1. 4029204642Srdivacky SDValue X = GetSignificand(DAG, Op1, dl); 4030199989Srdivacky 4031199989Srdivacky // Different possible minimax approximations of significand in 4032199989Srdivacky // floating-point for various degrees of accuracy over [1,2]. 4033249423Sdim SDValue Log2ofMantissa; 4034199989Srdivacky if (LimitFloatPrecision <= 6) { 4035199989Srdivacky // For floating-point precision of 6: 4036199989Srdivacky // 4037199989Srdivacky // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; 4038199989Srdivacky // 4039199989Srdivacky // error 0.0049451742, which is more than 7 bits 4040199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4041199989Srdivacky getF32Constant(DAG, 0xbeb08fe0)); 4042199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4043199989Srdivacky getF32Constant(DAG, 0x40019463)); 4044199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4045249423Sdim Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4046249423Sdim getF32Constant(DAG, 0x3fd6633d)); 4047249423Sdim } else if (LimitFloatPrecision <= 12) { 4048199989Srdivacky // For floating-point precision of 12: 4049199989Srdivacky // 4050199989Srdivacky // Log2ofMantissa = 4051199989Srdivacky // -2.51285454f + 4052199989Srdivacky // (4.07009056f + 4053199989Srdivacky // (-2.12067489f + 4054199989Srdivacky // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; 4055199989Srdivacky // 4056199989Srdivacky // error 0.0000876136000, which is better than 13 bits 4057199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4058199989Srdivacky getF32Constant(DAG, 0xbda7262e)); 4059199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4060199989Srdivacky getF32Constant(DAG, 0x3f25280b)); 4061199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4062199989Srdivacky SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4063199989Srdivacky getF32Constant(DAG, 0x4007b923)); 4064199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4065199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4066199989Srdivacky getF32Constant(DAG, 0x40823e2f)); 4067199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4068249423Sdim Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4069249423Sdim getF32Constant(DAG, 0x4020d29c)); 4070249423Sdim } else { // LimitFloatPrecision <= 18 4071199989Srdivacky // For floating-point precision of 18: 4072199989Srdivacky // 4073199989Srdivacky // Log2ofMantissa = 4074199989Srdivacky // -3.0400495f + 4075199989Srdivacky // (6.1129976f + 4076199989Srdivacky // (-5.3420409f + 4077199989Srdivacky // (3.2865683f + 4078199989Srdivacky // (-1.2669343f + 4079199989Srdivacky // (0.27515199f - 4080199989Srdivacky // 0.25691327e-1f * x) * x) * x) * x) * x) * x; 4081199989Srdivacky // 4082199989Srdivacky // error 0.0000018516, which is better than 18 bits 4083199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4084199989Srdivacky getF32Constant(DAG, 0xbcd2769e)); 4085199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4086199989Srdivacky getF32Constant(DAG, 0x3e8ce0b9)); 4087199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4088199989Srdivacky SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4089199989Srdivacky getF32Constant(DAG, 0x3fa22ae7)); 4090199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4091199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4092199989Srdivacky getF32Constant(DAG, 0x40525723)); 4093199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4094199989Srdivacky SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4095199989Srdivacky getF32Constant(DAG, 0x40aaf200)); 4096199989Srdivacky SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4097199989Srdivacky SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4098199989Srdivacky getF32Constant(DAG, 0x40c39dad)); 4099199989Srdivacky SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4100249423Sdim Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 4101249423Sdim getF32Constant(DAG, 0x4042902c)); 4102249423Sdim } 4103199989Srdivacky 4104249423Sdim return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); 4105199989Srdivacky } 4106199989Srdivacky 4107249423Sdim // No special expansion. 4108249423Sdim return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op); 4109199989Srdivacky} 4110199989Srdivacky 4111249423Sdim/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for 4112199989Srdivacky/// limited-precision mode. 4113263508Sdimstatic SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4114249423Sdim const TargetLowering &TLI) { 4115249423Sdim if (Op.getValueType() == MVT::f32 && 4116199989Srdivacky LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4117218893Sdim SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 4118199989Srdivacky 4119199989Srdivacky // Scale the exponent by log10(2) [0.30102999f]. 4120204642Srdivacky SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 4121199989Srdivacky SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 4122199989Srdivacky getF32Constant(DAG, 0x3e9a209a)); 4123199989Srdivacky 4124199989Srdivacky // Get the significand and build it into a floating-point number with 4125199989Srdivacky // exponent of 1. 4126204642Srdivacky SDValue X = GetSignificand(DAG, Op1, dl); 4127199989Srdivacky 4128249423Sdim SDValue Log10ofMantissa; 4129199989Srdivacky if (LimitFloatPrecision <= 6) { 4130199989Srdivacky // For floating-point precision of 6: 4131199989Srdivacky // 4132199989Srdivacky // Log10ofMantissa = 4133199989Srdivacky // -0.50419619f + 4134199989Srdivacky // (0.60948995f - 0.10380950f * x) * x; 4135199989Srdivacky // 4136199989Srdivacky // error 0.0014886165, which is 6 bits 4137199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4138199989Srdivacky getF32Constant(DAG, 0xbdd49a13)); 4139199989Srdivacky SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4140199989Srdivacky getF32Constant(DAG, 0x3f1c0789)); 4141199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4142249423Sdim Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4143249423Sdim getF32Constant(DAG, 0x3f011300)); 4144249423Sdim } else if (LimitFloatPrecision <= 12) { 4145199989Srdivacky // For floating-point precision of 12: 4146199989Srdivacky // 4147199989Srdivacky // Log10ofMantissa = 4148199989Srdivacky // -0.64831180f + 4149199989Srdivacky // (0.91751397f + 4150199989Srdivacky // (-0.31664806f + 0.47637168e-1f * x) * x) * x; 4151199989Srdivacky // 4152199989Srdivacky // error 0.00019228036, which is better than 12 bits 4153199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4154199989Srdivacky getF32Constant(DAG, 0x3d431f31)); 4155199989Srdivacky SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 4156199989Srdivacky getF32Constant(DAG, 0x3ea21fb2)); 4157199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4158199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4159199989Srdivacky getF32Constant(DAG, 0x3f6ae232)); 4160199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4161249423Sdim Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 4162249423Sdim getF32Constant(DAG, 0x3f25f7c3)); 4163249423Sdim } else { // LimitFloatPrecision <= 18 4164199989Srdivacky // For floating-point precision of 18: 4165199989Srdivacky // 4166199989Srdivacky // Log10ofMantissa = 4167199989Srdivacky // -0.84299375f + 4168199989Srdivacky // (1.5327582f + 4169199989Srdivacky // (-1.0688956f + 4170199989Srdivacky // (0.49102474f + 4171199989Srdivacky // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; 4172199989Srdivacky // 4173199989Srdivacky // error 0.0000037995730, which is better than 18 bits 4174199989Srdivacky SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4175199989Srdivacky getF32Constant(DAG, 0x3c5d51ce)); 4176199989Srdivacky SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 4177199989Srdivacky getF32Constant(DAG, 0x3e00685a)); 4178199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4179199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4180199989Srdivacky getF32Constant(DAG, 0x3efb6798)); 4181199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4182199989Srdivacky SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 4183199989Srdivacky getF32Constant(DAG, 0x3f88d192)); 4184199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4185199989Srdivacky SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4186199989Srdivacky getF32Constant(DAG, 0x3fc4316c)); 4187199989Srdivacky SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4188249423Sdim Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, 4189249423Sdim getF32Constant(DAG, 0x3f57ce70)); 4190249423Sdim } 4191199989Srdivacky 4192249423Sdim return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); 4193199989Srdivacky } 4194199989Srdivacky 4195249423Sdim // No special expansion. 4196249423Sdim return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op); 4197199989Srdivacky} 4198199989Srdivacky 4199249423Sdim/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for 4200199989Srdivacky/// limited-precision mode. 4201263508Sdimstatic SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4202249423Sdim const TargetLowering &TLI) { 4203249423Sdim if (Op.getValueType() == MVT::f32 && 4204199989Srdivacky LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4205199989Srdivacky SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); 4206199989Srdivacky 4207199989Srdivacky // FractionalPartOfX = x - (float)IntegerPartOfX; 4208199989Srdivacky SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 4209199989Srdivacky SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1); 4210199989Srdivacky 4211199989Srdivacky // IntegerPartOfX <<= 23; 4212199989Srdivacky IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 4213199989Srdivacky DAG.getConstant(23, TLI.getPointerTy())); 4214199989Srdivacky 4215249423Sdim SDValue TwoToFractionalPartOfX; 4216199989Srdivacky if (LimitFloatPrecision <= 6) { 4217199989Srdivacky // For floating-point precision of 6: 4218199989Srdivacky // 4219199989Srdivacky // TwoToFractionalPartOfX = 4220199989Srdivacky // 0.997535578f + 4221199989Srdivacky // (0.735607626f + 0.252464424f * x) * x; 4222199989Srdivacky // 4223199989Srdivacky // error 0.0144103317, which is 6 bits 4224199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4225199989Srdivacky getF32Constant(DAG, 0x3e814304)); 4226199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4227199989Srdivacky getF32Constant(DAG, 0x3f3c50c8)); 4228199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4229249423Sdim TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4230249423Sdim getF32Constant(DAG, 0x3f7f5e7e)); 4231249423Sdim } else if (LimitFloatPrecision <= 12) { 4232199989Srdivacky // For floating-point precision of 12: 4233199989Srdivacky // 4234199989Srdivacky // TwoToFractionalPartOfX = 4235199989Srdivacky // 0.999892986f + 4236199989Srdivacky // (0.696457318f + 4237199989Srdivacky // (0.224338339f + 0.792043434e-1f * x) * x) * x; 4238199989Srdivacky // 4239199989Srdivacky // error 0.000107046256, which is 13 to 14 bits 4240199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4241199989Srdivacky getF32Constant(DAG, 0x3da235e3)); 4242199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4243199989Srdivacky getF32Constant(DAG, 0x3e65b8f3)); 4244199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4245199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4246199989Srdivacky getF32Constant(DAG, 0x3f324b07)); 4247199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4248249423Sdim TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4249249423Sdim getF32Constant(DAG, 0x3f7ff8fd)); 4250249423Sdim } else { // LimitFloatPrecision <= 18 4251199989Srdivacky // For floating-point precision of 18: 4252199989Srdivacky // 4253199989Srdivacky // TwoToFractionalPartOfX = 4254199989Srdivacky // 0.999999982f + 4255199989Srdivacky // (0.693148872f + 4256199989Srdivacky // (0.240227044f + 4257199989Srdivacky // (0.554906021e-1f + 4258199989Srdivacky // (0.961591928e-2f + 4259199989Srdivacky // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 4260199989Srdivacky // error 2.47208000*10^(-7), which is better than 18 bits 4261199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4262199989Srdivacky getF32Constant(DAG, 0x3924b03e)); 4263199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4264199989Srdivacky getF32Constant(DAG, 0x3ab24b87)); 4265199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4266199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4267199989Srdivacky getF32Constant(DAG, 0x3c1d8c17)); 4268199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4269199989Srdivacky SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4270199989Srdivacky getF32Constant(DAG, 0x3d634a1d)); 4271199989Srdivacky SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4272199989Srdivacky SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4273199989Srdivacky getF32Constant(DAG, 0x3e75fe14)); 4274199989Srdivacky SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4275199989Srdivacky SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 4276199989Srdivacky getF32Constant(DAG, 0x3f317234)); 4277199989Srdivacky SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 4278249423Sdim TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 4279249423Sdim getF32Constant(DAG, 0x3f800000)); 4280249423Sdim } 4281199989Srdivacky 4282249423Sdim // Add the exponent into the result in integer domain. 4283249423Sdim SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, 4284249423Sdim TwoToFractionalPartOfX); 4285249423Sdim return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4286249423Sdim DAG.getNode(ISD::ADD, dl, MVT::i32, 4287249423Sdim t13, IntegerPartOfX)); 4288199989Srdivacky } 4289199989Srdivacky 4290249423Sdim // No special expansion. 4291249423Sdim return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op); 4292199989Srdivacky} 4293199989Srdivacky 4294199989Srdivacky/// visitPow - Lower a pow intrinsic. Handles the special sequences for 4295199989Srdivacky/// limited-precision mode with x == 10.0f. 4296263508Sdimstatic SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS, 4297249423Sdim SelectionDAG &DAG, const TargetLowering &TLI) { 4298199989Srdivacky bool IsExp10 = false; 4299263508Sdim if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && 4300199989Srdivacky LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4301249423Sdim if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { 4302249423Sdim APFloat Ten(10.0f); 4303249423Sdim IsExp10 = LHSC->isExactlyValue(Ten); 4304199989Srdivacky } 4305199989Srdivacky } 4306199989Srdivacky 4307249423Sdim if (IsExp10) { 4308199989Srdivacky // Put the exponent in the right bit position for later addition to the 4309199989Srdivacky // final result: 4310199989Srdivacky // 4311199989Srdivacky // #define LOG2OF10 3.3219281f 4312199989Srdivacky // IntegerPartOfX = (int32_t)(x * LOG2OF10); 4313249423Sdim SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, 4314199989Srdivacky getF32Constant(DAG, 0x40549a78)); 4315199989Srdivacky SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 4316199989Srdivacky 4317199989Srdivacky // FractionalPartOfX = x - (float)IntegerPartOfX; 4318199989Srdivacky SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 4319199989Srdivacky SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 4320199989Srdivacky 4321199989Srdivacky // IntegerPartOfX <<= 23; 4322199989Srdivacky IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 4323199989Srdivacky DAG.getConstant(23, TLI.getPointerTy())); 4324199989Srdivacky 4325249423Sdim SDValue TwoToFractionalPartOfX; 4326199989Srdivacky if (LimitFloatPrecision <= 6) { 4327199989Srdivacky // For floating-point precision of 6: 4328199989Srdivacky // 4329199989Srdivacky // twoToFractionalPartOfX = 4330199989Srdivacky // 0.997535578f + 4331199989Srdivacky // (0.735607626f + 0.252464424f * x) * x; 4332199989Srdivacky // 4333199989Srdivacky // error 0.0144103317, which is 6 bits 4334199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4335199989Srdivacky getF32Constant(DAG, 0x3e814304)); 4336199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4337199989Srdivacky getF32Constant(DAG, 0x3f3c50c8)); 4338199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4339249423Sdim TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4340249423Sdim getF32Constant(DAG, 0x3f7f5e7e)); 4341249423Sdim } else if (LimitFloatPrecision <= 12) { 4342199989Srdivacky // For floating-point precision of 12: 4343199989Srdivacky // 4344199989Srdivacky // TwoToFractionalPartOfX = 4345199989Srdivacky // 0.999892986f + 4346199989Srdivacky // (0.696457318f + 4347199989Srdivacky // (0.224338339f + 0.792043434e-1f * x) * x) * x; 4348199989Srdivacky // 4349199989Srdivacky // error 0.000107046256, which is 13 to 14 bits 4350199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4351199989Srdivacky getF32Constant(DAG, 0x3da235e3)); 4352199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4353199989Srdivacky getF32Constant(DAG, 0x3e65b8f3)); 4354199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4355199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4356199989Srdivacky getF32Constant(DAG, 0x3f324b07)); 4357199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4358249423Sdim TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4359249423Sdim getF32Constant(DAG, 0x3f7ff8fd)); 4360249423Sdim } else { // LimitFloatPrecision <= 18 4361199989Srdivacky // For floating-point precision of 18: 4362199989Srdivacky // 4363199989Srdivacky // TwoToFractionalPartOfX = 4364199989Srdivacky // 0.999999982f + 4365199989Srdivacky // (0.693148872f + 4366199989Srdivacky // (0.240227044f + 4367199989Srdivacky // (0.554906021e-1f + 4368199989Srdivacky // (0.961591928e-2f + 4369199989Srdivacky // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 4370199989Srdivacky // error 2.47208000*10^(-7), which is better than 18 bits 4371199989Srdivacky SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4372199989Srdivacky getF32Constant(DAG, 0x3924b03e)); 4373199989Srdivacky SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4374199989Srdivacky getF32Constant(DAG, 0x3ab24b87)); 4375199989Srdivacky SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4376199989Srdivacky SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4377199989Srdivacky getF32Constant(DAG, 0x3c1d8c17)); 4378199989Srdivacky SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4379199989Srdivacky SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4380199989Srdivacky getF32Constant(DAG, 0x3d634a1d)); 4381199989Srdivacky SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4382199989Srdivacky SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4383199989Srdivacky getF32Constant(DAG, 0x3e75fe14)); 4384199989Srdivacky SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4385199989Srdivacky SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 4386199989Srdivacky getF32Constant(DAG, 0x3f317234)); 4387199989Srdivacky SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 4388249423Sdim TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 4389249423Sdim getF32Constant(DAG, 0x3f800000)); 4390249423Sdim } 4391199989Srdivacky 4392249423Sdim SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX); 4393249423Sdim return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4394249423Sdim DAG.getNode(ISD::ADD, dl, MVT::i32, 4395249423Sdim t13, IntegerPartOfX)); 4396199989Srdivacky } 4397199989Srdivacky 4398249423Sdim // No special expansion. 4399249423Sdim return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS); 4400199989Srdivacky} 4401199989Srdivacky 4402201360Srdivacky 4403201360Srdivacky/// ExpandPowI - Expand a llvm.powi intrinsic. 4404263508Sdimstatic SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS, 4405201360Srdivacky SelectionDAG &DAG) { 4406201360Srdivacky // If RHS is a constant, we can expand this out to a multiplication tree, 4407201360Srdivacky // otherwise we end up lowering to a call to __powidf2 (for example). When 4408201360Srdivacky // optimizing for size, we only want to do this if the expansion would produce 4409201360Srdivacky // a small number of multiplies, otherwise we do the full expansion. 4410201360Srdivacky if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 4411201360Srdivacky // Get the exponent as a positive value. 4412201360Srdivacky unsigned Val = RHSC->getSExtValue(); 4413201360Srdivacky if ((int)Val < 0) Val = -Val; 4414201360Srdivacky 4415201360Srdivacky // powi(x, 0) -> 1.0 4416201360Srdivacky if (Val == 0) 4417201360Srdivacky return DAG.getConstantFP(1.0, LHS.getValueType()); 4418201360Srdivacky 4419207618Srdivacky const Function *F = DAG.getMachineFunction().getFunction(); 4420249423Sdim if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, 4421249423Sdim Attribute::OptimizeForSize) || 4422201360Srdivacky // If optimizing for size, don't insert too many multiplies. This 4423201360Srdivacky // inserts up to 5 multiplies. 4424201360Srdivacky CountPopulation_32(Val)+Log2_32(Val) < 7) { 4425201360Srdivacky // We use the simple binary decomposition method to generate the multiply 4426201360Srdivacky // sequence. There are more optimal ways to do this (for example, 4427201360Srdivacky // powi(x,15) generates one more multiply than it should), but this has 4428201360Srdivacky // the benefit of being both really simple and much better than a libcall. 4429201360Srdivacky SDValue Res; // Logically starts equal to 1.0 4430201360Srdivacky SDValue CurSquare = LHS; 4431201360Srdivacky while (Val) { 4432201360Srdivacky if (Val & 1) { 4433201360Srdivacky if (Res.getNode()) 4434201360Srdivacky Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); 4435201360Srdivacky else 4436201360Srdivacky Res = CurSquare; // 1.0*CurSquare. 4437201360Srdivacky } 4438201360Srdivacky 4439201360Srdivacky CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), 4440201360Srdivacky CurSquare, CurSquare); 4441201360Srdivacky Val >>= 1; 4442201360Srdivacky } 4443201360Srdivacky 4444201360Srdivacky // If the original was negative, invert the result, producing 1/(x*x*x). 4445201360Srdivacky if (RHSC->getSExtValue() < 0) 4446201360Srdivacky Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), 4447201360Srdivacky DAG.getConstantFP(1.0, LHS.getValueType()), Res); 4448201360Srdivacky return Res; 4449201360Srdivacky } 4450201360Srdivacky } 4451201360Srdivacky 4452201360Srdivacky // Otherwise, expand to a libcall. 4453201360Srdivacky return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); 4454201360Srdivacky} 4455201360Srdivacky 4456223017Sdim// getTruncatedArgReg - Find underlying register used for an truncated 4457223017Sdim// argument. 4458223017Sdimstatic unsigned getTruncatedArgReg(const SDValue &N) { 4459223017Sdim if (N.getOpcode() != ISD::TRUNCATE) 4460223017Sdim return 0; 4461223017Sdim 4462223017Sdim const SDValue &Ext = N.getOperand(0); 4463263508Sdim if (Ext.getOpcode() == ISD::AssertZext || 4464263508Sdim Ext.getOpcode() == ISD::AssertSext) { 4465223017Sdim const SDValue &CFR = Ext.getOperand(0); 4466223017Sdim if (CFR.getOpcode() == ISD::CopyFromReg) 4467223017Sdim return cast<RegisterSDNode>(CFR.getOperand(1))->getReg(); 4468234353Sdim if (CFR.getOpcode() == ISD::TRUNCATE) 4469234353Sdim return getTruncatedArgReg(CFR); 4470223017Sdim } 4471223017Sdim return 0; 4472223017Sdim} 4473223017Sdim 4474207618Srdivacky/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function 4475207618Srdivacky/// argument, create the corresponding DBG_VALUE machine instruction for it now. 4476207618Srdivacky/// At the end of instruction selection, they will be inserted to the entry BB. 4477207618Srdivackybool 4478212904SdimSelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable, 4479218893Sdim int64_t Offset, 4480207618Srdivacky const SDValue &N) { 4481212904Sdim const Argument *Arg = dyn_cast<Argument>(V); 4482212904Sdim if (!Arg) 4483207618Srdivacky return false; 4484201360Srdivacky 4485207618Srdivacky MachineFunction &MF = DAG.getMachineFunction(); 4486218893Sdim const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo(); 4487218893Sdim 4488207618Srdivacky // Ignore inlined function arguments here. 4489207618Srdivacky DIVariable DV(Variable); 4490207618Srdivacky if (DV.isInlinedFnArgument(MF.getFunction())) 4491207618Srdivacky return false; 4492207618Srdivacky 4493263508Sdim Optional<MachineOperand> Op; 4494226633Sdim // Some arguments' frame index is recorded during argument lowering. 4495263508Sdim if (int FI = FuncInfo.getArgumentFrameIndex(Arg)) 4496263508Sdim Op = MachineOperand::CreateFI(FI); 4497212904Sdim 4498263508Sdim if (!Op && N.getNode()) { 4499263508Sdim unsigned Reg; 4500223017Sdim if (N.getOpcode() == ISD::CopyFromReg) 4501223017Sdim Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg(); 4502223017Sdim else 4503223017Sdim Reg = getTruncatedArgReg(N); 4504223017Sdim if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) { 4505207618Srdivacky MachineRegisterInfo &RegInfo = MF.getRegInfo(); 4506207618Srdivacky unsigned PR = RegInfo.getLiveInPhysReg(Reg); 4507207618Srdivacky if (PR) 4508207618Srdivacky Reg = PR; 4509207618Srdivacky } 4510263508Sdim if (Reg) 4511263508Sdim Op = MachineOperand::CreateReg(Reg, false); 4512207618Srdivacky } 4513207618Srdivacky 4514263508Sdim if (!Op) { 4515218893Sdim // Check if ValueMap has reg number. 4516207618Srdivacky DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); 4517218893Sdim if (VMI != FuncInfo.ValueMap.end()) 4518263508Sdim Op = MachineOperand::CreateReg(VMI->second, false); 4519207618Srdivacky } 4520207618Srdivacky 4521263508Sdim if (!Op && N.getNode()) 4522218893Sdim // Check if frame index is available. 4523218893Sdim if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode())) 4524218893Sdim if (FrameIndexSDNode *FINode = 4525263508Sdim dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 4526263508Sdim Op = MachineOperand::CreateFI(FINode->getIndex()); 4527218893Sdim 4528263508Sdim if (!Op) 4529218893Sdim return false; 4530218893Sdim 4531263508Sdim // FIXME: This does not handle register-indirect values at offset 0. 4532263508Sdim bool IsIndirect = Offset != 0; 4533263508Sdim if (Op->isReg()) 4534263508Sdim FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(), 4535263508Sdim TII->get(TargetOpcode::DBG_VALUE), 4536263508Sdim IsIndirect, 4537263508Sdim Op->getReg(), Offset, Variable)); 4538263508Sdim else 4539263508Sdim FuncInfo.ArgDbgValues.push_back( 4540263508Sdim BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE)) 4541263508Sdim .addOperand(*Op).addImm(Offset).addMetadata(Variable)); 4542263508Sdim 4543207618Srdivacky return true; 4544207618Srdivacky} 4545207618Srdivacky 4546208599Srdivacky// VisualStudio defines setjmp as _setjmp 4547218893Sdim#if defined(_MSC_VER) && defined(setjmp) && \ 4548218893Sdim !defined(setjmp_undefined_for_msvc) 4549218893Sdim# pragma push_macro("setjmp") 4550218893Sdim# undef setjmp 4551218893Sdim# define setjmp_undefined_for_msvc 4552208599Srdivacky#endif 4553208599Srdivacky 4554199989Srdivacky/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If 4555199989Srdivacky/// we want to emit this as a call to a named external function, return the name 4556199989Srdivacky/// otherwise lower it and return null. 4557199989Srdivackyconst char * 4558207618SrdivackySelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { 4559263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4560263508Sdim SDLoc sdl = getCurSDLoc(); 4561199989Srdivacky DebugLoc dl = getCurDebugLoc(); 4562201360Srdivacky SDValue Res; 4563201360Srdivacky 4564199989Srdivacky switch (Intrinsic) { 4565199989Srdivacky default: 4566199989Srdivacky // By default, turn this into a target intrinsic node. 4567199989Srdivacky visitTargetIntrinsic(I, Intrinsic); 4568199989Srdivacky return 0; 4569199989Srdivacky case Intrinsic::vastart: visitVAStart(I); return 0; 4570199989Srdivacky case Intrinsic::vaend: visitVAEnd(I); return 0; 4571199989Srdivacky case Intrinsic::vacopy: visitVACopy(I); return 0; 4572199989Srdivacky case Intrinsic::returnaddress: 4573263508Sdim setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(), 4574210299Sed getValue(I.getArgOperand(0)))); 4575199989Srdivacky return 0; 4576199989Srdivacky case Intrinsic::frameaddress: 4577263508Sdim setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(), 4578210299Sed getValue(I.getArgOperand(0)))); 4579199989Srdivacky return 0; 4580199989Srdivacky case Intrinsic::setjmp: 4581263508Sdim return &"_setjmp"[!TLI->usesUnderscoreSetJmp()]; 4582199989Srdivacky case Intrinsic::longjmp: 4583263508Sdim return &"_longjmp"[!TLI->usesUnderscoreLongJmp()]; 4584199989Srdivacky case Intrinsic::memcpy: { 4585206274Srdivacky // Assert for address < 256 since we support only user defined address 4586206274Srdivacky // spaces. 4587210299Sed assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4588206274Srdivacky < 256 && 4589210299Sed cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace() 4590206274Srdivacky < 256 && 4591206274Srdivacky "Unknown address space"); 4592210299Sed SDValue Op1 = getValue(I.getArgOperand(0)); 4593210299Sed SDValue Op2 = getValue(I.getArgOperand(1)); 4594210299Sed SDValue Op3 = getValue(I.getArgOperand(2)); 4595210299Sed unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4596249423Sdim if (!Align) 4597249423Sdim Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment. 4598210299Sed bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4599263508Sdim DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false, 4600218893Sdim MachinePointerInfo(I.getArgOperand(0)), 4601218893Sdim MachinePointerInfo(I.getArgOperand(1)))); 4602199989Srdivacky return 0; 4603199989Srdivacky } 4604199989Srdivacky case Intrinsic::memset: { 4605206274Srdivacky // Assert for address < 256 since we support only user defined address 4606206274Srdivacky // spaces. 4607210299Sed assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4608206274Srdivacky < 256 && 4609206274Srdivacky "Unknown address space"); 4610210299Sed SDValue Op1 = getValue(I.getArgOperand(0)); 4611210299Sed SDValue Op2 = getValue(I.getArgOperand(1)); 4612210299Sed SDValue Op3 = getValue(I.getArgOperand(2)); 4613210299Sed unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4614249423Sdim if (!Align) 4615249423Sdim Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment. 4616210299Sed bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4617263508Sdim DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, 4618218893Sdim MachinePointerInfo(I.getArgOperand(0)))); 4619199989Srdivacky return 0; 4620199989Srdivacky } 4621199989Srdivacky case Intrinsic::memmove: { 4622206274Srdivacky // Assert for address < 256 since we support only user defined address 4623206274Srdivacky // spaces. 4624210299Sed assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4625206274Srdivacky < 256 && 4626210299Sed cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace() 4627206274Srdivacky < 256 && 4628206274Srdivacky "Unknown address space"); 4629210299Sed SDValue Op1 = getValue(I.getArgOperand(0)); 4630210299Sed SDValue Op2 = getValue(I.getArgOperand(1)); 4631210299Sed SDValue Op3 = getValue(I.getArgOperand(2)); 4632210299Sed unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4633249423Sdim if (!Align) 4634249423Sdim Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment. 4635210299Sed bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4636263508Sdim DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, 4637218893Sdim MachinePointerInfo(I.getArgOperand(0)), 4638218893Sdim MachinePointerInfo(I.getArgOperand(1)))); 4639199989Srdivacky return 0; 4640199989Srdivacky } 4641199989Srdivacky case Intrinsic::dbg_declare: { 4642207618Srdivacky const DbgDeclareInst &DI = cast<DbgDeclareInst>(I); 4643199989Srdivacky MDNode *Variable = DI.getVariable(); 4644207618Srdivacky const Value *Address = DI.getAddress(); 4645263508Sdim DIVariable DIVar(Variable); 4646263508Sdim assert((!DIVar || DIVar.isVariable()) && 4647263508Sdim "Variable in DbgDeclareInst should be either null or a DIVariable."); 4648263508Sdim if (!Address || !DIVar) { 4649234353Sdim DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4650203954Srdivacky return 0; 4651234353Sdim } 4652199989Srdivacky 4653212904Sdim // Check if address has undef value. 4654212904Sdim if (isa<UndefValue>(Address) || 4655212904Sdim (Address->use_empty() && !isa<Argument>(Address))) { 4656234353Sdim DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4657212904Sdim return 0; 4658212904Sdim } 4659212904Sdim 4660207618Srdivacky SDValue &N = NodeMap[Address]; 4661212904Sdim if (!N.getNode() && isa<Argument>(Address)) 4662212904Sdim // Check unused arguments map. 4663212904Sdim N = UnusedArgNodeMap[Address]; 4664207618Srdivacky SDDbgValue *SDV; 4665207618Srdivacky if (N.getNode()) { 4666234353Sdim if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) 4667234353Sdim Address = BCI->getOperand(0); 4668212904Sdim // Parameters are handled specially. 4669218893Sdim bool isParameter = 4670234353Sdim (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable || 4671234353Sdim isa<Argument>(Address)); 4672234353Sdim 4673212904Sdim const AllocaInst *AI = dyn_cast<AllocaInst>(Address); 4674212904Sdim 4675207618Srdivacky if (isParameter && !AI) { 4676207618Srdivacky FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); 4677207618Srdivacky if (FINode) 4678207618Srdivacky // Byval parameter. We have a frame index at this point. 4679207618Srdivacky SDV = DAG.getDbgValue(Variable, FINode->getIndex(), 4680207618Srdivacky 0, dl, SDNodeOrder); 4681218893Sdim else { 4682223017Sdim // Address is an argument, so try to emit its dbg value using 4683223017Sdim // virtual register info from the FuncInfo.ValueMap. 4684223017Sdim EmitFuncArgumentDbgValue(Address, Variable, 0, N); 4685207618Srdivacky return 0; 4686218893Sdim } 4687207618Srdivacky } else if (AI) 4688207618Srdivacky SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(), 4689207618Srdivacky 0, dl, SDNodeOrder); 4690218893Sdim else { 4691207618Srdivacky // Can't do anything with other non-AI cases yet. 4692234353Sdim DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4693234353Sdim DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t"); 4694234353Sdim DEBUG(Address->dump()); 4695207618Srdivacky return 0; 4696218893Sdim } 4697207618Srdivacky DAG.AddDbgValue(SDV, N.getNode(), isParameter); 4698207618Srdivacky } else { 4699218893Sdim // If Address is an argument then try to emit its dbg value using 4700218893Sdim // virtual register info from the FuncInfo.ValueMap. 4701212904Sdim if (!EmitFuncArgumentDbgValue(Address, Variable, 0, N)) { 4702218893Sdim // If variable is pinned by a alloca in dominating bb then 4703218893Sdim // use StaticAllocaMap. 4704218893Sdim if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) { 4705218893Sdim if (AI->getParent() != DI.getParent()) { 4706218893Sdim DenseMap<const AllocaInst*, int>::iterator SI = 4707218893Sdim FuncInfo.StaticAllocaMap.find(AI); 4708218893Sdim if (SI != FuncInfo.StaticAllocaMap.end()) { 4709218893Sdim SDV = DAG.getDbgValue(Variable, SI->second, 4710218893Sdim 0, dl, SDNodeOrder); 4711218893Sdim DAG.AddDbgValue(SDV, 0, false); 4712218893Sdim return 0; 4713218893Sdim } 4714218893Sdim } 4715218893Sdim } 4716234353Sdim DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4717212904Sdim } 4718207618Srdivacky } 4719199989Srdivacky return 0; 4720199989Srdivacky } 4721203954Srdivacky case Intrinsic::dbg_value: { 4722207618Srdivacky const DbgValueInst &DI = cast<DbgValueInst>(I); 4723263508Sdim DIVariable DIVar(DI.getVariable()); 4724263508Sdim assert((!DIVar || DIVar.isVariable()) && 4725263508Sdim "Variable in DbgValueInst should be either null or a DIVariable."); 4726263508Sdim if (!DIVar) 4727203954Srdivacky return 0; 4728203954Srdivacky 4729203954Srdivacky MDNode *Variable = DI.getVariable(); 4730205218Srdivacky uint64_t Offset = DI.getOffset(); 4731207618Srdivacky const Value *V = DI.getValue(); 4732203954Srdivacky if (!V) 4733203954Srdivacky return 0; 4734205218Srdivacky 4735207618Srdivacky SDDbgValue *SDV; 4736226633Sdim if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) { 4737207618Srdivacky SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder); 4738207618Srdivacky DAG.AddDbgValue(SDV, 0, false); 4739205218Srdivacky } else { 4740212904Sdim // Do not use getValue() in here; we don't want to generate code at 4741212904Sdim // this point if it hasn't been done yet. 4742210299Sed SDValue N = NodeMap[V]; 4743210299Sed if (!N.getNode() && isa<Argument>(V)) 4744210299Sed // Check unused arguments map. 4745210299Sed N = UnusedArgNodeMap[V]; 4746207618Srdivacky if (N.getNode()) { 4747212904Sdim if (!EmitFuncArgumentDbgValue(V, Variable, Offset, N)) { 4748207618Srdivacky SDV = DAG.getDbgValue(Variable, N.getNode(), 4749207618Srdivacky N.getResNo(), Offset, dl, SDNodeOrder); 4750207618Srdivacky DAG.AddDbgValue(SDV, N.getNode(), false); 4751207618Srdivacky } 4752218893Sdim } else if (!V->use_empty() ) { 4753212904Sdim // Do not call getValue(V) yet, as we don't want to generate code. 4754212904Sdim // Remember it for later. 4755212904Sdim DanglingDebugInfo DDI(&DI, dl, SDNodeOrder); 4756212904Sdim DanglingDebugInfoMap[V] = DDI; 4757212904Sdim } else { 4758205218Srdivacky // We may expand this to cover more cases. One case where we have no 4759218893Sdim // data available is an unreferenced parameter. 4760234353Sdim DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4761207618Srdivacky } 4762205218Srdivacky } 4763205218Srdivacky 4764205218Srdivacky // Build a debug info table entry. 4765207618Srdivacky if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V)) 4766203954Srdivacky V = BCI->getOperand(0); 4767207618Srdivacky const AllocaInst *AI = dyn_cast<AllocaInst>(V); 4768203954Srdivacky // Don't handle byval struct arguments or VLAs, for example. 4769234353Sdim if (!AI) { 4770234353Sdim DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n"); 4771234353Sdim DEBUG(dbgs() << " Last seen at:\n " << *V << "\n"); 4772203954Srdivacky return 0; 4773234353Sdim } 4774203954Srdivacky DenseMap<const AllocaInst*, int>::iterator SI = 4775203954Srdivacky FuncInfo.StaticAllocaMap.find(AI); 4776203954Srdivacky if (SI == FuncInfo.StaticAllocaMap.end()) 4777203954Srdivacky return 0; // VLAs. 4778203954Srdivacky int FI = SI->second; 4779218893Sdim 4780206274Srdivacky MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 4781206274Srdivacky if (!DI.getDebugLoc().isUnknown() && MMI.hasDebugInfo()) 4782206274Srdivacky MMI.setVariableDbgInfo(Variable, FI, DI.getDebugLoc()); 4783203954Srdivacky return 0; 4784203954Srdivacky } 4785199989Srdivacky 4786199989Srdivacky case Intrinsic::eh_typeid_for: { 4787206274Srdivacky // Find the type id for the given typeinfo. 4788210299Sed GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0)); 4789206274Srdivacky unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV); 4790206274Srdivacky Res = DAG.getConstant(TypeID, MVT::i32); 4791201360Srdivacky setValue(&I, Res); 4792199989Srdivacky return 0; 4793199989Srdivacky } 4794199989Srdivacky 4795199989Srdivacky case Intrinsic::eh_return_i32: 4796199989Srdivacky case Intrinsic::eh_return_i64: 4797206274Srdivacky DAG.getMachineFunction().getMMI().setCallsEHReturn(true); 4798263508Sdim DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, 4799206274Srdivacky MVT::Other, 4800206274Srdivacky getControlRoot(), 4801210299Sed getValue(I.getArgOperand(0)), 4802210299Sed getValue(I.getArgOperand(1)))); 4803199989Srdivacky return 0; 4804199989Srdivacky case Intrinsic::eh_unwind_init: 4805206274Srdivacky DAG.getMachineFunction().getMMI().setCallsUnwindInit(true); 4806199989Srdivacky return 0; 4807199989Srdivacky case Intrinsic::eh_dwarf_cfa: { 4808263508Sdim SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl, 4809263508Sdim TLI->getPointerTy()); 4810263508Sdim SDValue Offset = DAG.getNode(ISD::ADD, sdl, 4811263508Sdim CfaArg.getValueType(), 4812263508Sdim DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl, 4813263508Sdim CfaArg.getValueType()), 4814199989Srdivacky CfaArg); 4815263508Sdim SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, 4816263508Sdim TLI->getPointerTy(), 4817263508Sdim DAG.getConstant(0, TLI->getPointerTy())); 4818263508Sdim setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(), 4819203954Srdivacky FA, Offset)); 4820199989Srdivacky return 0; 4821199989Srdivacky } 4822203954Srdivacky case Intrinsic::eh_sjlj_callsite: { 4823206274Srdivacky MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 4824210299Sed ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0)); 4825203954Srdivacky assert(CI && "Non-constant call site value in eh.sjlj.callsite!"); 4826206274Srdivacky assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"); 4827203954Srdivacky 4828206274Srdivacky MMI.setCurrentCallSite(CI->getZExtValue()); 4829203954Srdivacky return 0; 4830203954Srdivacky } 4831226633Sdim case Intrinsic::eh_sjlj_functioncontext: { 4832226633Sdim // Get and store the index of the function context. 4833226633Sdim MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 4834226633Sdim AllocaInst *FnCtx = 4835226633Sdim cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); 4836226633Sdim int FI = FuncInfo.StaticAllocaMap[FnCtx]; 4837226633Sdim MFI->setFunctionContextIndex(FI); 4838226633Sdim return 0; 4839226633Sdim } 4840208599Srdivacky case Intrinsic::eh_sjlj_setjmp: { 4841226633Sdim SDValue Ops[2]; 4842226633Sdim Ops[0] = getRoot(); 4843226633Sdim Ops[1] = getValue(I.getArgOperand(0)); 4844263508Sdim SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, 4845226633Sdim DAG.getVTList(MVT::i32, MVT::Other), 4846226633Sdim Ops, 2); 4847226633Sdim setValue(&I, Op.getValue(0)); 4848226633Sdim DAG.setRoot(Op.getValue(1)); 4849208599Srdivacky return 0; 4850208599Srdivacky } 4851208599Srdivacky case Intrinsic::eh_sjlj_longjmp: { 4852263508Sdim DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, 4853218893Sdim getRoot(), getValue(I.getArgOperand(0)))); 4854208599Srdivacky return 0; 4855208599Srdivacky } 4856203954Srdivacky 4857218893Sdim case Intrinsic::x86_mmx_pslli_w: 4858218893Sdim case Intrinsic::x86_mmx_pslli_d: 4859218893Sdim case Intrinsic::x86_mmx_pslli_q: 4860218893Sdim case Intrinsic::x86_mmx_psrli_w: 4861218893Sdim case Intrinsic::x86_mmx_psrli_d: 4862218893Sdim case Intrinsic::x86_mmx_psrli_q: 4863218893Sdim case Intrinsic::x86_mmx_psrai_w: 4864218893Sdim case Intrinsic::x86_mmx_psrai_d: { 4865218893Sdim SDValue ShAmt = getValue(I.getArgOperand(1)); 4866218893Sdim if (isa<ConstantSDNode>(ShAmt)) { 4867218893Sdim visitTargetIntrinsic(I, Intrinsic); 4868218893Sdim return 0; 4869218893Sdim } 4870218893Sdim unsigned NewIntrinsic = 0; 4871218893Sdim EVT ShAmtVT = MVT::v2i32; 4872218893Sdim switch (Intrinsic) { 4873218893Sdim case Intrinsic::x86_mmx_pslli_w: 4874218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psll_w; 4875218893Sdim break; 4876218893Sdim case Intrinsic::x86_mmx_pslli_d: 4877218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psll_d; 4878218893Sdim break; 4879218893Sdim case Intrinsic::x86_mmx_pslli_q: 4880218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psll_q; 4881218893Sdim break; 4882218893Sdim case Intrinsic::x86_mmx_psrli_w: 4883218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psrl_w; 4884218893Sdim break; 4885218893Sdim case Intrinsic::x86_mmx_psrli_d: 4886218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psrl_d; 4887218893Sdim break; 4888218893Sdim case Intrinsic::x86_mmx_psrli_q: 4889218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psrl_q; 4890218893Sdim break; 4891218893Sdim case Intrinsic::x86_mmx_psrai_w: 4892218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psra_w; 4893218893Sdim break; 4894218893Sdim case Intrinsic::x86_mmx_psrai_d: 4895218893Sdim NewIntrinsic = Intrinsic::x86_mmx_psra_d; 4896218893Sdim break; 4897218893Sdim default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 4898218893Sdim } 4899218893Sdim 4900218893Sdim // The vector shift intrinsics with scalars uses 32b shift amounts but 4901218893Sdim // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits 4902218893Sdim // to be zero. 4903218893Sdim // We must do this early because v2i32 is not a legal type. 4904218893Sdim SDValue ShOps[2]; 4905218893Sdim ShOps[0] = ShAmt; 4906218893Sdim ShOps[1] = DAG.getConstant(0, MVT::i32); 4907263508Sdim ShAmt = DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, &ShOps[0], 2); 4908263508Sdim EVT DestVT = TLI->getValueType(I.getType()); 4909263508Sdim ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt); 4910263508Sdim Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT, 4911218893Sdim DAG.getConstant(NewIntrinsic, MVT::i32), 4912218893Sdim getValue(I.getArgOperand(0)), ShAmt); 4913218893Sdim setValue(&I, Res); 4914218893Sdim return 0; 4915218893Sdim } 4916234353Sdim case Intrinsic::x86_avx_vinsertf128_pd_256: 4917234353Sdim case Intrinsic::x86_avx_vinsertf128_ps_256: 4918234353Sdim case Intrinsic::x86_avx_vinsertf128_si_256: 4919234353Sdim case Intrinsic::x86_avx2_vinserti128: { 4920263508Sdim EVT DestVT = TLI->getValueType(I.getType()); 4921263508Sdim EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType()); 4922234353Sdim uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) * 4923234353Sdim ElVT.getVectorNumElements(); 4924263508Sdim Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT, 4925234353Sdim getValue(I.getArgOperand(0)), 4926234353Sdim getValue(I.getArgOperand(1)), 4927263508Sdim DAG.getConstant(Idx, TLI->getVectorIdxTy())); 4928234353Sdim setValue(&I, Res); 4929234353Sdim return 0; 4930234353Sdim } 4931243830Sdim case Intrinsic::x86_avx_vextractf128_pd_256: 4932243830Sdim case Intrinsic::x86_avx_vextractf128_ps_256: 4933243830Sdim case Intrinsic::x86_avx_vextractf128_si_256: 4934243830Sdim case Intrinsic::x86_avx2_vextracti128: { 4935263508Sdim EVT DestVT = TLI->getValueType(I.getType()); 4936243830Sdim uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) * 4937243830Sdim DestVT.getVectorNumElements(); 4938263508Sdim Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT, 4939243830Sdim getValue(I.getArgOperand(0)), 4940263508Sdim DAG.getConstant(Idx, TLI->getVectorIdxTy())); 4941243830Sdim setValue(&I, Res); 4942243830Sdim return 0; 4943243830Sdim } 4944199989Srdivacky case Intrinsic::convertff: 4945199989Srdivacky case Intrinsic::convertfsi: 4946199989Srdivacky case Intrinsic::convertfui: 4947199989Srdivacky case Intrinsic::convertsif: 4948199989Srdivacky case Intrinsic::convertuif: 4949199989Srdivacky case Intrinsic::convertss: 4950199989Srdivacky case Intrinsic::convertsu: 4951199989Srdivacky case Intrinsic::convertus: 4952199989Srdivacky case Intrinsic::convertuu: { 4953199989Srdivacky ISD::CvtCode Code = ISD::CVT_INVALID; 4954199989Srdivacky switch (Intrinsic) { 4955234353Sdim default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 4956199989Srdivacky case Intrinsic::convertff: Code = ISD::CVT_FF; break; 4957199989Srdivacky case Intrinsic::convertfsi: Code = ISD::CVT_FS; break; 4958199989Srdivacky case Intrinsic::convertfui: Code = ISD::CVT_FU; break; 4959199989Srdivacky case Intrinsic::convertsif: Code = ISD::CVT_SF; break; 4960199989Srdivacky case Intrinsic::convertuif: Code = ISD::CVT_UF; break; 4961199989Srdivacky case Intrinsic::convertss: Code = ISD::CVT_SS; break; 4962199989Srdivacky case Intrinsic::convertsu: Code = ISD::CVT_SU; break; 4963199989Srdivacky case Intrinsic::convertus: Code = ISD::CVT_US; break; 4964199989Srdivacky case Intrinsic::convertuu: Code = ISD::CVT_UU; break; 4965199989Srdivacky } 4966263508Sdim EVT DestVT = TLI->getValueType(I.getType()); 4967210299Sed const Value *Op1 = I.getArgOperand(0); 4968263508Sdim Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1), 4969201360Srdivacky DAG.getValueType(DestVT), 4970201360Srdivacky DAG.getValueType(getValue(Op1).getValueType()), 4971210299Sed getValue(I.getArgOperand(1)), 4972210299Sed getValue(I.getArgOperand(2)), 4973201360Srdivacky Code); 4974201360Srdivacky setValue(&I, Res); 4975199989Srdivacky return 0; 4976199989Srdivacky } 4977199989Srdivacky case Intrinsic::powi: 4978263508Sdim setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), 4979210299Sed getValue(I.getArgOperand(1)), DAG)); 4980199989Srdivacky return 0; 4981199989Srdivacky case Intrinsic::log: 4982263508Sdim setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4983199989Srdivacky return 0; 4984199989Srdivacky case Intrinsic::log2: 4985263508Sdim setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4986199989Srdivacky return 0; 4987199989Srdivacky case Intrinsic::log10: 4988263508Sdim setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4989199989Srdivacky return 0; 4990199989Srdivacky case Intrinsic::exp: 4991263508Sdim setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4992199989Srdivacky return 0; 4993199989Srdivacky case Intrinsic::exp2: 4994263508Sdim setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4995199989Srdivacky return 0; 4996199989Srdivacky case Intrinsic::pow: 4997263508Sdim setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), 4998263508Sdim getValue(I.getArgOperand(1)), DAG, *TLI)); 4999199989Srdivacky return 0; 5000249423Sdim case Intrinsic::sqrt: 5001239462Sdim case Intrinsic::fabs: 5002249423Sdim case Intrinsic::sin: 5003249423Sdim case Intrinsic::cos: 5004239462Sdim case Intrinsic::floor: 5005249423Sdim case Intrinsic::ceil: 5006249423Sdim case Intrinsic::trunc: 5007249423Sdim case Intrinsic::rint: 5008263508Sdim case Intrinsic::nearbyint: 5009263508Sdim case Intrinsic::round: { 5010249423Sdim unsigned Opcode; 5011249423Sdim switch (Intrinsic) { 5012249423Sdim default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5013249423Sdim case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 5014249423Sdim case Intrinsic::fabs: Opcode = ISD::FABS; break; 5015249423Sdim case Intrinsic::sin: Opcode = ISD::FSIN; break; 5016249423Sdim case Intrinsic::cos: Opcode = ISD::FCOS; break; 5017249423Sdim case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 5018249423Sdim case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 5019249423Sdim case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 5020249423Sdim case Intrinsic::rint: Opcode = ISD::FRINT; break; 5021249423Sdim case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 5022263508Sdim case Intrinsic::round: Opcode = ISD::FROUND; break; 5023249423Sdim } 5024249423Sdim 5025263508Sdim setValue(&I, DAG.getNode(Opcode, sdl, 5026239462Sdim getValue(I.getArgOperand(0)).getValueType(), 5027239462Sdim getValue(I.getArgOperand(0)))); 5028239462Sdim return 0; 5029249423Sdim } 5030263508Sdim case Intrinsic::copysign: 5031263508Sdim setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, 5032263508Sdim getValue(I.getArgOperand(0)).getValueType(), 5033263508Sdim getValue(I.getArgOperand(0)), 5034263508Sdim getValue(I.getArgOperand(1)))); 5035263508Sdim return 0; 5036224145Sdim case Intrinsic::fma: 5037263508Sdim setValue(&I, DAG.getNode(ISD::FMA, sdl, 5038224145Sdim getValue(I.getArgOperand(0)).getValueType(), 5039224145Sdim getValue(I.getArgOperand(0)), 5040224145Sdim getValue(I.getArgOperand(1)), 5041224145Sdim getValue(I.getArgOperand(2)))); 5042224145Sdim return 0; 5043239462Sdim case Intrinsic::fmuladd: { 5044263508Sdim EVT VT = TLI->getValueType(I.getType()); 5045239462Sdim if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && 5046263508Sdim TLI->isFMAFasterThanFMulAndFAdd(VT)) { 5047263508Sdim setValue(&I, DAG.getNode(ISD::FMA, sdl, 5048239462Sdim getValue(I.getArgOperand(0)).getValueType(), 5049239462Sdim getValue(I.getArgOperand(0)), 5050239462Sdim getValue(I.getArgOperand(1)), 5051239462Sdim getValue(I.getArgOperand(2)))); 5052239462Sdim } else { 5053263508Sdim SDValue Mul = DAG.getNode(ISD::FMUL, sdl, 5054239462Sdim getValue(I.getArgOperand(0)).getValueType(), 5055239462Sdim getValue(I.getArgOperand(0)), 5056239462Sdim getValue(I.getArgOperand(1))); 5057263508Sdim SDValue Add = DAG.getNode(ISD::FADD, sdl, 5058239462Sdim getValue(I.getArgOperand(0)).getValueType(), 5059239462Sdim Mul, 5060239462Sdim getValue(I.getArgOperand(2))); 5061239462Sdim setValue(&I, Add); 5062239462Sdim } 5063239462Sdim return 0; 5064239462Sdim } 5065205218Srdivacky case Intrinsic::convert_to_fp16: 5066263508Sdim setValue(&I, DAG.getNode(ISD::FP32_TO_FP16, sdl, 5067210299Sed MVT::i16, getValue(I.getArgOperand(0)))); 5068205218Srdivacky return 0; 5069205218Srdivacky case Intrinsic::convert_from_fp16: 5070263508Sdim setValue(&I, DAG.getNode(ISD::FP16_TO_FP32, sdl, 5071210299Sed MVT::f32, getValue(I.getArgOperand(0)))); 5072205218Srdivacky return 0; 5073199989Srdivacky case Intrinsic::pcmarker: { 5074210299Sed SDValue Tmp = getValue(I.getArgOperand(0)); 5075263508Sdim DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); 5076199989Srdivacky return 0; 5077199989Srdivacky } 5078199989Srdivacky case Intrinsic::readcyclecounter: { 5079199989Srdivacky SDValue Op = getRoot(); 5080263508Sdim Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, 5081201360Srdivacky DAG.getVTList(MVT::i64, MVT::Other), 5082201360Srdivacky &Op, 1); 5083201360Srdivacky setValue(&I, Res); 5084201360Srdivacky DAG.setRoot(Res.getValue(1)); 5085199989Srdivacky return 0; 5086199989Srdivacky } 5087199989Srdivacky case Intrinsic::bswap: 5088263508Sdim setValue(&I, DAG.getNode(ISD::BSWAP, sdl, 5089210299Sed getValue(I.getArgOperand(0)).getValueType(), 5090210299Sed getValue(I.getArgOperand(0)))); 5091199989Srdivacky return 0; 5092199989Srdivacky case Intrinsic::cttz: { 5093210299Sed SDValue Arg = getValue(I.getArgOperand(0)); 5094234353Sdim ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 5095199989Srdivacky EVT Ty = Arg.getValueType(); 5096234353Sdim setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, 5097263508Sdim sdl, Ty, Arg)); 5098199989Srdivacky return 0; 5099199989Srdivacky } 5100199989Srdivacky case Intrinsic::ctlz: { 5101210299Sed SDValue Arg = getValue(I.getArgOperand(0)); 5102234353Sdim ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 5103199989Srdivacky EVT Ty = Arg.getValueType(); 5104234353Sdim setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, 5105263508Sdim sdl, Ty, Arg)); 5106199989Srdivacky return 0; 5107199989Srdivacky } 5108199989Srdivacky case Intrinsic::ctpop: { 5109210299Sed SDValue Arg = getValue(I.getArgOperand(0)); 5110199989Srdivacky EVT Ty = Arg.getValueType(); 5111263508Sdim setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); 5112199989Srdivacky return 0; 5113199989Srdivacky } 5114199989Srdivacky case Intrinsic::stacksave: { 5115199989Srdivacky SDValue Op = getRoot(); 5116263508Sdim Res = DAG.getNode(ISD::STACKSAVE, sdl, 5117263508Sdim DAG.getVTList(TLI->getPointerTy(), MVT::Other), &Op, 1); 5118201360Srdivacky setValue(&I, Res); 5119201360Srdivacky DAG.setRoot(Res.getValue(1)); 5120199989Srdivacky return 0; 5121199989Srdivacky } 5122199989Srdivacky case Intrinsic::stackrestore: { 5123210299Sed Res = getValue(I.getArgOperand(0)); 5124263508Sdim DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); 5125199989Srdivacky return 0; 5126199989Srdivacky } 5127199989Srdivacky case Intrinsic::stackprotector: { 5128199989Srdivacky // Emit code into the DAG to store the stack guard onto the stack. 5129199989Srdivacky MachineFunction &MF = DAG.getMachineFunction(); 5130199989Srdivacky MachineFrameInfo *MFI = MF.getFrameInfo(); 5131263508Sdim EVT PtrTy = TLI->getPointerTy(); 5132199989Srdivacky 5133210299Sed SDValue Src = getValue(I.getArgOperand(0)); // The guard's value. 5134210299Sed AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); 5135199989Srdivacky 5136199989Srdivacky int FI = FuncInfo.StaticAllocaMap[Slot]; 5137199989Srdivacky MFI->setStackProtectorIndex(FI); 5138199989Srdivacky 5139199989Srdivacky SDValue FIN = DAG.getFrameIndex(FI, PtrTy); 5140199989Srdivacky 5141199989Srdivacky // Store the stack protector onto the stack. 5142263508Sdim Res = DAG.getStore(getRoot(), sdl, Src, FIN, 5143218893Sdim MachinePointerInfo::getFixedStack(FI), 5144218893Sdim true, false, 0); 5145201360Srdivacky setValue(&I, Res); 5146201360Srdivacky DAG.setRoot(Res); 5147199989Srdivacky return 0; 5148199989Srdivacky } 5149199989Srdivacky case Intrinsic::objectsize: { 5150199989Srdivacky // If we don't know by now, we're never going to know. 5151210299Sed ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1)); 5152199989Srdivacky 5153199989Srdivacky assert(CI && "Non-constant type in __builtin_object_size?"); 5154199989Srdivacky 5155210299Sed SDValue Arg = getValue(I.getCalledValue()); 5156199989Srdivacky EVT Ty = Arg.getValueType(); 5157199989Srdivacky 5158210299Sed if (CI->isZero()) 5159201360Srdivacky Res = DAG.getConstant(-1ULL, Ty); 5160199989Srdivacky else 5161201360Srdivacky Res = DAG.getConstant(0, Ty); 5162201360Srdivacky 5163201360Srdivacky setValue(&I, Res); 5164199989Srdivacky return 0; 5165199989Srdivacky } 5166251662Sdim case Intrinsic::annotation: 5167251662Sdim case Intrinsic::ptr_annotation: 5168251662Sdim // Drop the intrinsic, but forward the value 5169251662Sdim setValue(&I, getValue(I.getOperand(0))); 5170251662Sdim return 0; 5171199989Srdivacky case Intrinsic::var_annotation: 5172199989Srdivacky // Discard annotate attributes 5173199989Srdivacky return 0; 5174199989Srdivacky 5175199989Srdivacky case Intrinsic::init_trampoline: { 5176210299Sed const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts()); 5177199989Srdivacky 5178199989Srdivacky SDValue Ops[6]; 5179199989Srdivacky Ops[0] = getRoot(); 5180210299Sed Ops[1] = getValue(I.getArgOperand(0)); 5181210299Sed Ops[2] = getValue(I.getArgOperand(1)); 5182210299Sed Ops[3] = getValue(I.getArgOperand(2)); 5183210299Sed Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); 5184199989Srdivacky Ops[5] = DAG.getSrcValue(F); 5185199989Srdivacky 5186263508Sdim Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops, 6); 5187199989Srdivacky 5188226633Sdim DAG.setRoot(Res); 5189199989Srdivacky return 0; 5190199989Srdivacky } 5191226633Sdim case Intrinsic::adjust_trampoline: { 5192263508Sdim setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, 5193263508Sdim TLI->getPointerTy(), 5194226633Sdim getValue(I.getArgOperand(0)))); 5195226633Sdim return 0; 5196226633Sdim } 5197199989Srdivacky case Intrinsic::gcroot: 5198199989Srdivacky if (GFI) { 5199234982Sdim const Value *Alloca = I.getArgOperand(0)->stripPointerCasts(); 5200210299Sed const Constant *TypeMap = cast<Constant>(I.getArgOperand(1)); 5201199989Srdivacky 5202199989Srdivacky FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); 5203199989Srdivacky GFI->addStackRoot(FI->getIndex(), TypeMap); 5204199989Srdivacky } 5205199989Srdivacky return 0; 5206199989Srdivacky case Intrinsic::gcread: 5207199989Srdivacky case Intrinsic::gcwrite: 5208199989Srdivacky llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); 5209201360Srdivacky case Intrinsic::flt_rounds: 5210263508Sdim setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32)); 5211199989Srdivacky return 0; 5212224145Sdim 5213224145Sdim case Intrinsic::expect: { 5214224145Sdim // Just replace __builtin_expect(exp, c) with EXP. 5215224145Sdim setValue(&I, getValue(I.getArgOperand(0))); 5216224145Sdim return 0; 5217224145Sdim } 5218224145Sdim 5219243830Sdim case Intrinsic::debugtrap: 5220221345Sdim case Intrinsic::trap: { 5221234353Sdim StringRef TrapFuncName = TM.Options.getTrapFunctionName(); 5222221345Sdim if (TrapFuncName.empty()) { 5223263508Sdim ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ? 5224243830Sdim ISD::TRAP : ISD::DEBUGTRAP; 5225263508Sdim DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot())); 5226221345Sdim return 0; 5227221345Sdim } 5228221345Sdim TargetLowering::ArgListTy Args; 5229239462Sdim TargetLowering:: 5230239462Sdim CallLoweringInfo CLI(getRoot(), I.getType(), 5231221345Sdim false, false, false, false, 0, CallingConv::C, 5232234353Sdim /*isTailCall=*/false, 5233234353Sdim /*doesNotRet=*/false, /*isReturnValueUsed=*/true, 5234263508Sdim DAG.getExternalSymbol(TrapFuncName.data(), 5235263508Sdim TLI->getPointerTy()), 5236263508Sdim Args, DAG, sdl); 5237263508Sdim std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI); 5238221345Sdim DAG.setRoot(Result.second); 5239199989Srdivacky return 0; 5240221345Sdim } 5241243830Sdim 5242199989Srdivacky case Intrinsic::uadd_with_overflow: 5243199989Srdivacky case Intrinsic::sadd_with_overflow: 5244199989Srdivacky case Intrinsic::usub_with_overflow: 5245199989Srdivacky case Intrinsic::ssub_with_overflow: 5246199989Srdivacky case Intrinsic::umul_with_overflow: 5247234353Sdim case Intrinsic::smul_with_overflow: { 5248234353Sdim ISD::NodeType Op; 5249234353Sdim switch (Intrinsic) { 5250234353Sdim default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5251234353Sdim case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; 5252234353Sdim case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; 5253234353Sdim case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; 5254234353Sdim case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; 5255234353Sdim case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; 5256234353Sdim case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; 5257234353Sdim } 5258234353Sdim SDValue Op1 = getValue(I.getArgOperand(0)); 5259234353Sdim SDValue Op2 = getValue(I.getArgOperand(1)); 5260199989Srdivacky 5261234353Sdim SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1); 5262263508Sdim setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); 5263234353Sdim return 0; 5264234353Sdim } 5265199989Srdivacky case Intrinsic::prefetch: { 5266224145Sdim SDValue Ops[5]; 5267218893Sdim unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 5268199989Srdivacky Ops[0] = getRoot(); 5269210299Sed Ops[1] = getValue(I.getArgOperand(0)); 5270210299Sed Ops[2] = getValue(I.getArgOperand(1)); 5271210299Sed Ops[3] = getValue(I.getArgOperand(2)); 5272224145Sdim Ops[4] = getValue(I.getArgOperand(3)); 5273263508Sdim DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl, 5274218893Sdim DAG.getVTList(MVT::Other), 5275224145Sdim &Ops[0], 5, 5276218893Sdim EVT::getIntegerVT(*Context, 8), 5277218893Sdim MachinePointerInfo(I.getArgOperand(0)), 5278218893Sdim 0, /* align */ 5279218893Sdim false, /* volatile */ 5280218893Sdim rw==0, /* read */ 5281218893Sdim rw==1)); /* write */ 5282199989Srdivacky return 0; 5283199989Srdivacky } 5284243830Sdim case Intrinsic::lifetime_start: 5285243830Sdim case Intrinsic::lifetime_end: { 5286243830Sdim bool IsStart = (Intrinsic == Intrinsic::lifetime_start); 5287243830Sdim // Stack coloring is not enabled in O0, discard region information. 5288243830Sdim if (TM.getOptLevel() == CodeGenOpt::None) 5289243830Sdim return 0; 5290199989Srdivacky 5291243830Sdim SmallVector<Value *, 4> Allocas; 5292243830Sdim GetUnderlyingObjects(I.getArgOperand(1), Allocas, TD); 5293243830Sdim 5294263508Sdim for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(), 5295263508Sdim E = Allocas.end(); Object != E; ++Object) { 5296243830Sdim AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object); 5297243830Sdim 5298243830Sdim // Could not find an Alloca. 5299243830Sdim if (!LifetimeObject) 5300243830Sdim continue; 5301243830Sdim 5302243830Sdim int FI = FuncInfo.StaticAllocaMap[LifetimeObject]; 5303243830Sdim 5304243830Sdim SDValue Ops[2]; 5305243830Sdim Ops[0] = getRoot(); 5306263508Sdim Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true); 5307243830Sdim unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END); 5308243830Sdim 5309263508Sdim Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops, 2); 5310243830Sdim DAG.setRoot(Res); 5311243830Sdim } 5312249423Sdim return 0; 5313243830Sdim } 5314199989Srdivacky case Intrinsic::invariant_start: 5315199989Srdivacky // Discard region information. 5316263508Sdim setValue(&I, DAG.getUNDEF(TLI->getPointerTy())); 5317199989Srdivacky return 0; 5318199989Srdivacky case Intrinsic::invariant_end: 5319199989Srdivacky // Discard region information. 5320199989Srdivacky return 0; 5321263508Sdim case Intrinsic::stackprotectorcheck: { 5322263508Sdim // Do not actually emit anything for this basic block. Instead we initialize 5323263508Sdim // the stack protector descriptor and export the guard variable so we can 5324263508Sdim // access it in FinishBasicBlock. 5325263508Sdim const BasicBlock *BB = I.getParent(); 5326263508Sdim SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I); 5327263508Sdim ExportFromCurrentBlock(SPDescriptor.getGuard()); 5328263508Sdim 5329263508Sdim // Flush our exports since we are going to process a terminator. 5330263508Sdim (void)getControlRoot(); 5331263508Sdim return 0; 5332263508Sdim } 5333239462Sdim case Intrinsic::donothing: 5334239462Sdim // ignore 5335239462Sdim return 0; 5336263508Sdim case Intrinsic::experimental_stackmap: { 5337263508Sdim visitStackmap(I); 5338263508Sdim return 0; 5339199989Srdivacky } 5340263508Sdim case Intrinsic::experimental_patchpoint_void: 5341263508Sdim case Intrinsic::experimental_patchpoint_i64: { 5342263508Sdim visitPatchpoint(I); 5343263508Sdim return 0; 5344263508Sdim } 5345263508Sdim } 5346199989Srdivacky} 5347199989Srdivacky 5348207618Srdivackyvoid SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, 5349199989Srdivacky bool isTailCall, 5350199989Srdivacky MachineBasicBlock *LandingPad) { 5351226633Sdim PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); 5352226633Sdim FunctionType *FTy = cast<FunctionType>(PT->getElementType()); 5353226633Sdim Type *RetTy = FTy->getReturnType(); 5354206274Srdivacky MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 5355205218Srdivacky MCSymbol *BeginLabel = 0; 5356199989Srdivacky 5357199989Srdivacky TargetLowering::ArgListTy Args; 5358199989Srdivacky TargetLowering::ArgListEntry Entry; 5359199989Srdivacky Args.reserve(CS.arg_size()); 5360199989Srdivacky 5361199989Srdivacky // Check whether the function can return without sret-demotion. 5362210299Sed SmallVector<ISD::OutputArg, 4> Outs; 5363263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 5364263508Sdim GetReturnInfo(RetTy, CS.getAttributes(), Outs, *TLI); 5365199989Srdivacky 5366263508Sdim bool CanLowerReturn = TLI->CanLowerReturn(CS.getCallingConv(), 5367263508Sdim DAG.getMachineFunction(), 5368263508Sdim FTy->isVarArg(), Outs, 5369263508Sdim FTy->getContext()); 5370199989Srdivacky 5371199989Srdivacky SDValue DemoteStackSlot; 5372218893Sdim int DemoteStackIdx = -100; 5373199989Srdivacky 5374199989Srdivacky if (!CanLowerReturn) { 5375263508Sdim uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize( 5376199989Srdivacky FTy->getReturnType()); 5377263508Sdim unsigned Align = TLI->getDataLayout()->getPrefTypeAlignment( 5378199989Srdivacky FTy->getReturnType()); 5379199989Srdivacky MachineFunction &MF = DAG.getMachineFunction(); 5380218893Sdim DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); 5381226633Sdim Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType()); 5382199989Srdivacky 5383263508Sdim DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI->getPointerTy()); 5384199989Srdivacky Entry.Node = DemoteStackSlot; 5385199989Srdivacky Entry.Ty = StackSlotPtrType; 5386199989Srdivacky Entry.isSExt = false; 5387199989Srdivacky Entry.isZExt = false; 5388199989Srdivacky Entry.isInReg = false; 5389199989Srdivacky Entry.isSRet = true; 5390199989Srdivacky Entry.isNest = false; 5391199989Srdivacky Entry.isByVal = false; 5392251662Sdim Entry.isReturned = false; 5393199989Srdivacky Entry.Alignment = Align; 5394199989Srdivacky Args.push_back(Entry); 5395199989Srdivacky RetTy = Type::getVoidTy(FTy->getContext()); 5396199989Srdivacky } 5397199989Srdivacky 5398207618Srdivacky for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); 5399199989Srdivacky i != e; ++i) { 5400223017Sdim const Value *V = *i; 5401199989Srdivacky 5402223017Sdim // Skip empty types 5403223017Sdim if (V->getType()->isEmptyTy()) 5404223017Sdim continue; 5405223017Sdim 5406223017Sdim SDValue ArgNode = getValue(V); 5407223017Sdim Entry.Node = ArgNode; Entry.Ty = V->getType(); 5408223017Sdim 5409263508Sdim // Skip the first return-type Attribute to get to params. 5410263508Sdim Entry.setAttributes(&CS, i - CS.arg_begin() + 1); 5411199989Srdivacky Args.push_back(Entry); 5412199989Srdivacky } 5413199989Srdivacky 5414206274Srdivacky if (LandingPad) { 5415199989Srdivacky // Insert a label before the invoke call to mark the try range. This can be 5416199989Srdivacky // used to detect deletion of the invoke via the MachineModuleInfo. 5417206274Srdivacky BeginLabel = MMI.getContext().CreateTempSymbol(); 5418199989Srdivacky 5419203954Srdivacky // For SjLj, keep track of which landing pads go with which invokes 5420203954Srdivacky // so as to maintain the ordering of pads in the LSDA. 5421206274Srdivacky unsigned CallSiteIndex = MMI.getCurrentCallSite(); 5422203954Srdivacky if (CallSiteIndex) { 5423206274Srdivacky MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); 5424226633Sdim LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex); 5425226633Sdim 5426203954Srdivacky // Now that the call site is handled, stop tracking it. 5427206274Srdivacky MMI.setCurrentCallSite(0); 5428203954Srdivacky } 5429203954Srdivacky 5430199989Srdivacky // Both PendingLoads and PendingExports must be flushed here; 5431199989Srdivacky // this call might not return. 5432199989Srdivacky (void)getRoot(); 5433263508Sdim DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel)); 5434199989Srdivacky } 5435199989Srdivacky 5436199989Srdivacky // Check if target-independent constraints permit a tail call here. 5437263508Sdim // Target-dependent constraints are checked within TLI->LowerCallTo. 5438263508Sdim if (isTailCall && !isInTailCallPosition(CS, *TLI)) 5439199989Srdivacky isTailCall = false; 5440199989Srdivacky 5441239462Sdim TargetLowering:: 5442239462Sdim CallLoweringInfo CLI(getRoot(), RetTy, FTy, isTailCall, Callee, Args, DAG, 5443263508Sdim getCurSDLoc(), CS); 5444263508Sdim std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI); 5445199989Srdivacky assert((isTailCall || Result.second.getNode()) && 5446199989Srdivacky "Non-null chain expected with non-tail call!"); 5447199989Srdivacky assert((Result.second.getNode() || !Result.first.getNode()) && 5448199989Srdivacky "Null value expected with tail call!"); 5449201360Srdivacky if (Result.first.getNode()) { 5450199989Srdivacky setValue(CS.getInstruction(), Result.first); 5451201360Srdivacky } else if (!CanLowerReturn && Result.second.getNode()) { 5452199989Srdivacky // The instruction result is the result of loading from the 5453199989Srdivacky // hidden sret parameter. 5454199989Srdivacky SmallVector<EVT, 1> PVTs; 5455226633Sdim Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType()); 5456199989Srdivacky 5457263508Sdim ComputeValueVTs(*TLI, PtrRetTy, PVTs); 5458199989Srdivacky assert(PVTs.size() == 1 && "Pointers should fit in one register"); 5459199989Srdivacky EVT PtrVT = PVTs[0]; 5460239462Sdim 5461239462Sdim SmallVector<EVT, 4> RetTys; 5462239462Sdim SmallVector<uint64_t, 4> Offsets; 5463239462Sdim RetTy = FTy->getReturnType(); 5464263508Sdim ComputeValueVTs(*TLI, RetTy, RetTys, &Offsets); 5465239462Sdim 5466239462Sdim unsigned NumValues = RetTys.size(); 5467199989Srdivacky SmallVector<SDValue, 4> Values(NumValues); 5468199989Srdivacky SmallVector<SDValue, 4> Chains(NumValues); 5469199989Srdivacky 5470199989Srdivacky for (unsigned i = 0; i < NumValues; ++i) { 5471263508Sdim SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, 5472201360Srdivacky DemoteStackSlot, 5473201360Srdivacky DAG.getConstant(Offsets[i], PtrVT)); 5474263508Sdim SDValue L = DAG.getLoad(RetTys[i], getCurSDLoc(), Result.second, Add, 5475218893Sdim MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]), 5476234353Sdim false, false, false, 1); 5477199989Srdivacky Values[i] = L; 5478199989Srdivacky Chains[i] = L.getValue(1); 5479199989Srdivacky } 5480201360Srdivacky 5481263508Sdim SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 5482199989Srdivacky MVT::Other, &Chains[0], NumValues); 5483199989Srdivacky PendingLoads.push_back(Chain); 5484218893Sdim 5485203954Srdivacky setValue(CS.getInstruction(), 5486263508Sdim DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 5487203954Srdivacky DAG.getVTList(&RetTys[0], RetTys.size()), 5488239462Sdim &Values[0], Values.size())); 5489199989Srdivacky } 5490201360Srdivacky 5491221345Sdim if (!Result.second.getNode()) { 5492263508Sdim // As a special case, a null chain means that a tail call has been emitted 5493263508Sdim // and the DAG root is already updated. 5494221345Sdim HasTailCall = true; 5495263508Sdim 5496263508Sdim // Since there's no actual continuation from this block, nothing can be 5497263508Sdim // relying on us setting vregs for them. 5498263508Sdim PendingExports.clear(); 5499221345Sdim } else { 5500199989Srdivacky DAG.setRoot(Result.second); 5501221345Sdim } 5502199989Srdivacky 5503206274Srdivacky if (LandingPad) { 5504199989Srdivacky // Insert a label at the end of the invoke call to mark the try range. This 5505199989Srdivacky // can be used to detect deletion of the invoke via the MachineModuleInfo. 5506206274Srdivacky MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol(); 5507263508Sdim DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel)); 5508199989Srdivacky 5509199989Srdivacky // Inform MachineModuleInfo of range. 5510206274Srdivacky MMI.addInvoke(LandingPad, BeginLabel, EndLabel); 5511199989Srdivacky } 5512199989Srdivacky} 5513199989Srdivacky 5514201360Srdivacky/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the 5515201360Srdivacky/// value is equal or not-equal to zero. 5516207618Srdivackystatic bool IsOnlyUsedInZeroEqualityComparison(const Value *V) { 5517207618Srdivacky for (Value::const_use_iterator UI = V->use_begin(), E = V->use_end(); 5518201360Srdivacky UI != E; ++UI) { 5519207618Srdivacky if (const ICmpInst *IC = dyn_cast<ICmpInst>(*UI)) 5520201360Srdivacky if (IC->isEquality()) 5521207618Srdivacky if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1))) 5522201360Srdivacky if (C->isNullValue()) 5523201360Srdivacky continue; 5524201360Srdivacky // Unknown instruction. 5525201360Srdivacky return false; 5526201360Srdivacky } 5527201360Srdivacky return true; 5528201360Srdivacky} 5529199989Srdivacky 5530207618Srdivackystatic SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, 5531226633Sdim Type *LoadTy, 5532201360Srdivacky SelectionDAGBuilder &Builder) { 5533201360Srdivacky 5534201360Srdivacky // Check to see if this load can be trivially constant folded, e.g. if the 5535201360Srdivacky // input is from a string literal. 5536207618Srdivacky if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { 5537201360Srdivacky // Cast pointer to the type we really want to load. 5538207618Srdivacky LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), 5539201360Srdivacky PointerType::getUnqual(LoadTy)); 5540201360Srdivacky 5541207618Srdivacky if (const Constant *LoadCst = 5542207618Srdivacky ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput), 5543207618Srdivacky Builder.TD)) 5544201360Srdivacky return Builder.getValue(LoadCst); 5545201360Srdivacky } 5546201360Srdivacky 5547201360Srdivacky // Otherwise, we have to emit the load. If the pointer is to unfoldable but 5548201360Srdivacky // still constant memory, the input chain can be the entry node. 5549201360Srdivacky SDValue Root; 5550201360Srdivacky bool ConstantMemory = false; 5551201360Srdivacky 5552201360Srdivacky // Do not serialize (non-volatile) loads of constant memory with anything. 5553201360Srdivacky if (Builder.AA->pointsToConstantMemory(PtrVal)) { 5554201360Srdivacky Root = Builder.DAG.getEntryNode(); 5555201360Srdivacky ConstantMemory = true; 5556201360Srdivacky } else { 5557201360Srdivacky // Do not serialize non-volatile loads against each other. 5558201360Srdivacky Root = Builder.DAG.getRoot(); 5559201360Srdivacky } 5560201360Srdivacky 5561201360Srdivacky SDValue Ptr = Builder.getValue(PtrVal); 5562263508Sdim SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, 5563218893Sdim Ptr, MachinePointerInfo(PtrVal), 5564203954Srdivacky false /*volatile*/, 5565263508Sdim false /*nontemporal*/, 5566234353Sdim false /*isinvariant*/, 1 /* align=1 */); 5567201360Srdivacky 5568201360Srdivacky if (!ConstantMemory) 5569201360Srdivacky Builder.PendingLoads.push_back(LoadVal.getValue(1)); 5570201360Srdivacky return LoadVal; 5571201360Srdivacky} 5572201360Srdivacky 5573263508Sdim/// processIntegerCallValue - Record the value for an instruction that 5574263508Sdim/// produces an integer result, converting the type where necessary. 5575263508Sdimvoid SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, 5576263508Sdim SDValue Value, 5577263508Sdim bool IsSigned) { 5578263508Sdim EVT VT = TM.getTargetLowering()->getValueType(I.getType(), true); 5579263508Sdim if (IsSigned) 5580263508Sdim Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT); 5581263508Sdim else 5582263508Sdim Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT); 5583263508Sdim setValue(&I, Value); 5584263508Sdim} 5585201360Srdivacky 5586201360Srdivacky/// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form. 5587201360Srdivacky/// If so, return true and lower it, otherwise return false and it will be 5588201360Srdivacky/// lowered like a normal call. 5589207618Srdivackybool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) { 5590201360Srdivacky // Verify that the prototype makes sense. int memcmp(void*,void*,size_t) 5591210299Sed if (I.getNumArgOperands() != 3) 5592201360Srdivacky return false; 5593201360Srdivacky 5594210299Sed const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); 5595204642Srdivacky if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() || 5596210299Sed !I.getArgOperand(2)->getType()->isIntegerTy() || 5597204642Srdivacky !I.getType()->isIntegerTy()) 5598201360Srdivacky return false; 5599201360Srdivacky 5600263508Sdim const Value *Size = I.getArgOperand(2); 5601263508Sdim const ConstantInt *CSize = dyn_cast<ConstantInt>(Size); 5602263508Sdim if (CSize && CSize->getZExtValue() == 0) { 5603263508Sdim EVT CallVT = TM.getTargetLowering()->getValueType(I.getType(), true); 5604263508Sdim setValue(&I, DAG.getConstant(0, CallVT)); 5605263508Sdim return true; 5606263508Sdim } 5607201360Srdivacky 5608263508Sdim const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5609263508Sdim std::pair<SDValue, SDValue> Res = 5610263508Sdim TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(), 5611263508Sdim getValue(LHS), getValue(RHS), getValue(Size), 5612263508Sdim MachinePointerInfo(LHS), 5613263508Sdim MachinePointerInfo(RHS)); 5614263508Sdim if (Res.first.getNode()) { 5615263508Sdim processIntegerCallValue(I, Res.first, true); 5616263508Sdim PendingLoads.push_back(Res.second); 5617263508Sdim return true; 5618263508Sdim } 5619263508Sdim 5620201360Srdivacky // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 5621201360Srdivacky // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 5622263508Sdim if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) { 5623201360Srdivacky bool ActuallyDoIt = true; 5624201360Srdivacky MVT LoadVT; 5625226633Sdim Type *LoadTy; 5626263508Sdim switch (CSize->getZExtValue()) { 5627201360Srdivacky default: 5628201360Srdivacky LoadVT = MVT::Other; 5629201360Srdivacky LoadTy = 0; 5630201360Srdivacky ActuallyDoIt = false; 5631201360Srdivacky break; 5632201360Srdivacky case 2: 5633201360Srdivacky LoadVT = MVT::i16; 5634263508Sdim LoadTy = Type::getInt16Ty(CSize->getContext()); 5635201360Srdivacky break; 5636201360Srdivacky case 4: 5637201360Srdivacky LoadVT = MVT::i32; 5638263508Sdim LoadTy = Type::getInt32Ty(CSize->getContext()); 5639201360Srdivacky break; 5640201360Srdivacky case 8: 5641201360Srdivacky LoadVT = MVT::i64; 5642263508Sdim LoadTy = Type::getInt64Ty(CSize->getContext()); 5643201360Srdivacky break; 5644201360Srdivacky /* 5645201360Srdivacky case 16: 5646201360Srdivacky LoadVT = MVT::v4i32; 5647263508Sdim LoadTy = Type::getInt32Ty(CSize->getContext()); 5648201360Srdivacky LoadTy = VectorType::get(LoadTy, 4); 5649201360Srdivacky break; 5650201360Srdivacky */ 5651201360Srdivacky } 5652201360Srdivacky 5653201360Srdivacky // This turns into unaligned loads. We only do this if the target natively 5654201360Srdivacky // supports the MVT we'll be loading or if it is small enough (<= 4) that 5655201360Srdivacky // we'll only produce a small number of byte loads. 5656201360Srdivacky 5657201360Srdivacky // Require that we can find a legal MVT, and only do this if the target 5658201360Srdivacky // supports unaligned loads of that type. Expanding into byte loads would 5659201360Srdivacky // bloat the code. 5660263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 5661263508Sdim if (ActuallyDoIt && CSize->getZExtValue() > 4) { 5662201360Srdivacky // TODO: Handle 5 byte compare as 4-byte + 1 byte. 5663201360Srdivacky // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. 5664263508Sdim if (!TLI->isTypeLegal(LoadVT) ||!TLI->allowsUnalignedMemoryAccesses(LoadVT)) 5665201360Srdivacky ActuallyDoIt = false; 5666201360Srdivacky } 5667201360Srdivacky 5668201360Srdivacky if (ActuallyDoIt) { 5669201360Srdivacky SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this); 5670201360Srdivacky SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this); 5671201360Srdivacky 5672263508Sdim SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal, 5673201360Srdivacky ISD::SETNE); 5674263508Sdim processIntegerCallValue(I, Res, false); 5675201360Srdivacky return true; 5676201360Srdivacky } 5677201360Srdivacky } 5678201360Srdivacky 5679201360Srdivacky 5680201360Srdivacky return false; 5681201360Srdivacky} 5682201360Srdivacky 5683263508Sdim/// visitMemChrCall -- See if we can lower a memchr call into an optimized 5684263508Sdim/// form. If so, return true and lower it, otherwise return false and it 5685263508Sdim/// will be lowered like a normal call. 5686263508Sdimbool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { 5687263508Sdim // Verify that the prototype makes sense. void *memchr(void *, int, size_t) 5688263508Sdim if (I.getNumArgOperands() != 3) 5689263508Sdim return false; 5690263508Sdim 5691263508Sdim const Value *Src = I.getArgOperand(0); 5692263508Sdim const Value *Char = I.getArgOperand(1); 5693263508Sdim const Value *Length = I.getArgOperand(2); 5694263508Sdim if (!Src->getType()->isPointerTy() || 5695263508Sdim !Char->getType()->isIntegerTy() || 5696263508Sdim !Length->getType()->isIntegerTy() || 5697263508Sdim !I.getType()->isPointerTy()) 5698263508Sdim return false; 5699263508Sdim 5700263508Sdim const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5701263508Sdim std::pair<SDValue, SDValue> Res = 5702263508Sdim TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), 5703263508Sdim getValue(Src), getValue(Char), getValue(Length), 5704263508Sdim MachinePointerInfo(Src)); 5705263508Sdim if (Res.first.getNode()) { 5706263508Sdim setValue(&I, Res.first); 5707263508Sdim PendingLoads.push_back(Res.second); 5708263508Sdim return true; 5709263508Sdim } 5710263508Sdim 5711263508Sdim return false; 5712263508Sdim} 5713263508Sdim 5714263508Sdim/// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an 5715263508Sdim/// optimized form. If so, return true and lower it, otherwise return false 5716263508Sdim/// and it will be lowered like a normal call. 5717263508Sdimbool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { 5718263508Sdim // Verify that the prototype makes sense. char *strcpy(char *, char *) 5719263508Sdim if (I.getNumArgOperands() != 2) 5720263508Sdim return false; 5721263508Sdim 5722263508Sdim const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5723263508Sdim if (!Arg0->getType()->isPointerTy() || 5724263508Sdim !Arg1->getType()->isPointerTy() || 5725263508Sdim !I.getType()->isPointerTy()) 5726263508Sdim return false; 5727263508Sdim 5728263508Sdim const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5729263508Sdim std::pair<SDValue, SDValue> Res = 5730263508Sdim TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), 5731263508Sdim getValue(Arg0), getValue(Arg1), 5732263508Sdim MachinePointerInfo(Arg0), 5733263508Sdim MachinePointerInfo(Arg1), isStpcpy); 5734263508Sdim if (Res.first.getNode()) { 5735263508Sdim setValue(&I, Res.first); 5736263508Sdim DAG.setRoot(Res.second); 5737263508Sdim return true; 5738263508Sdim } 5739263508Sdim 5740263508Sdim return false; 5741263508Sdim} 5742263508Sdim 5743263508Sdim/// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form. 5744263508Sdim/// If so, return true and lower it, otherwise return false and it will be 5745263508Sdim/// lowered like a normal call. 5746263508Sdimbool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { 5747263508Sdim // Verify that the prototype makes sense. int strcmp(void*,void*) 5748263508Sdim if (I.getNumArgOperands() != 2) 5749263508Sdim return false; 5750263508Sdim 5751263508Sdim const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5752263508Sdim if (!Arg0->getType()->isPointerTy() || 5753263508Sdim !Arg1->getType()->isPointerTy() || 5754263508Sdim !I.getType()->isIntegerTy()) 5755263508Sdim return false; 5756263508Sdim 5757263508Sdim const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5758263508Sdim std::pair<SDValue, SDValue> Res = 5759263508Sdim TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), 5760263508Sdim getValue(Arg0), getValue(Arg1), 5761263508Sdim MachinePointerInfo(Arg0), 5762263508Sdim MachinePointerInfo(Arg1)); 5763263508Sdim if (Res.first.getNode()) { 5764263508Sdim processIntegerCallValue(I, Res.first, true); 5765263508Sdim PendingLoads.push_back(Res.second); 5766263508Sdim return true; 5767263508Sdim } 5768263508Sdim 5769263508Sdim return false; 5770263508Sdim} 5771263508Sdim 5772263508Sdim/// visitStrLenCall -- See if we can lower a strlen call into an optimized 5773263508Sdim/// form. If so, return true and lower it, otherwise return false and it 5774263508Sdim/// will be lowered like a normal call. 5775263508Sdimbool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { 5776263508Sdim // Verify that the prototype makes sense. size_t strlen(char *) 5777263508Sdim if (I.getNumArgOperands() != 1) 5778263508Sdim return false; 5779263508Sdim 5780263508Sdim const Value *Arg0 = I.getArgOperand(0); 5781263508Sdim if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy()) 5782263508Sdim return false; 5783263508Sdim 5784263508Sdim const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5785263508Sdim std::pair<SDValue, SDValue> Res = 5786263508Sdim TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), 5787263508Sdim getValue(Arg0), MachinePointerInfo(Arg0)); 5788263508Sdim if (Res.first.getNode()) { 5789263508Sdim processIntegerCallValue(I, Res.first, false); 5790263508Sdim PendingLoads.push_back(Res.second); 5791263508Sdim return true; 5792263508Sdim } 5793263508Sdim 5794263508Sdim return false; 5795263508Sdim} 5796263508Sdim 5797263508Sdim/// visitStrNLenCall -- See if we can lower a strnlen call into an optimized 5798263508Sdim/// form. If so, return true and lower it, otherwise return false and it 5799263508Sdim/// will be lowered like a normal call. 5800263508Sdimbool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { 5801263508Sdim // Verify that the prototype makes sense. size_t strnlen(char *, size_t) 5802263508Sdim if (I.getNumArgOperands() != 2) 5803263508Sdim return false; 5804263508Sdim 5805263508Sdim const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5806263508Sdim if (!Arg0->getType()->isPointerTy() || 5807263508Sdim !Arg1->getType()->isIntegerTy() || 5808263508Sdim !I.getType()->isIntegerTy()) 5809263508Sdim return false; 5810263508Sdim 5811263508Sdim const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5812263508Sdim std::pair<SDValue, SDValue> Res = 5813263508Sdim TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), 5814263508Sdim getValue(Arg0), getValue(Arg1), 5815263508Sdim MachinePointerInfo(Arg0)); 5816263508Sdim if (Res.first.getNode()) { 5817263508Sdim processIntegerCallValue(I, Res.first, false); 5818263508Sdim PendingLoads.push_back(Res.second); 5819263508Sdim return true; 5820263508Sdim } 5821263508Sdim 5822263508Sdim return false; 5823263508Sdim} 5824263508Sdim 5825239462Sdim/// visitUnaryFloatCall - If a call instruction is a unary floating-point 5826239462Sdim/// operation (as expected), translate it to an SDNode with the specified opcode 5827239462Sdim/// and return true. 5828239462Sdimbool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, 5829239462Sdim unsigned Opcode) { 5830239462Sdim // Sanity check that it really is a unary floating-point call. 5831239462Sdim if (I.getNumArgOperands() != 1 || 5832239462Sdim !I.getArgOperand(0)->getType()->isFloatingPointTy() || 5833239462Sdim I.getType() != I.getArgOperand(0)->getType() || 5834239462Sdim !I.onlyReadsMemory()) 5835239462Sdim return false; 5836201360Srdivacky 5837239462Sdim SDValue Tmp = getValue(I.getArgOperand(0)); 5838263508Sdim setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp)); 5839239462Sdim return true; 5840239462Sdim} 5841239462Sdim 5842207618Srdivackyvoid SelectionDAGBuilder::visitCall(const CallInst &I) { 5843210299Sed // Handle inline assembly differently. 5844210299Sed if (isa<InlineAsm>(I.getCalledValue())) { 5845210299Sed visitInlineAsm(&I); 5846210299Sed return; 5847210299Sed } 5848218893Sdim 5849218893Sdim MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 5850234353Sdim ComputeUsesVAFloatArgument(I, &MMI); 5851218893Sdim 5852199989Srdivacky const char *RenameFn = 0; 5853199989Srdivacky if (Function *F = I.getCalledFunction()) { 5854199989Srdivacky if (F->isDeclaration()) { 5855210299Sed if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) { 5856199989Srdivacky if (unsigned IID = II->getIntrinsicID(F)) { 5857199989Srdivacky RenameFn = visitIntrinsicCall(I, IID); 5858199989Srdivacky if (!RenameFn) 5859199989Srdivacky return; 5860199989Srdivacky } 5861199989Srdivacky } 5862199989Srdivacky if (unsigned IID = F->getIntrinsicID()) { 5863199989Srdivacky RenameFn = visitIntrinsicCall(I, IID); 5864199989Srdivacky if (!RenameFn) 5865199989Srdivacky return; 5866199989Srdivacky } 5867199989Srdivacky } 5868199989Srdivacky 5869199989Srdivacky // Check for well-known libc/libm calls. If the function is internal, it 5870199989Srdivacky // can't be a library call. 5871239462Sdim LibFunc::Func Func; 5872239462Sdim if (!F->hasLocalLinkage() && F->hasName() && 5873239462Sdim LibInfo->getLibFunc(F->getName(), Func) && 5874239462Sdim LibInfo->hasOptimizedCodeGen(Func)) { 5875239462Sdim switch (Func) { 5876239462Sdim default: break; 5877239462Sdim case LibFunc::copysign: 5878239462Sdim case LibFunc::copysignf: 5879239462Sdim case LibFunc::copysignl: 5880210299Sed if (I.getNumArgOperands() == 2 && // Basic sanity checks. 5881210299Sed I.getArgOperand(0)->getType()->isFloatingPointTy() && 5882210299Sed I.getType() == I.getArgOperand(0)->getType() && 5883239462Sdim I.getType() == I.getArgOperand(1)->getType() && 5884239462Sdim I.onlyReadsMemory()) { 5885210299Sed SDValue LHS = getValue(I.getArgOperand(0)); 5886210299Sed SDValue RHS = getValue(I.getArgOperand(1)); 5887263508Sdim setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), 5888199989Srdivacky LHS.getValueType(), LHS, RHS)); 5889199989Srdivacky return; 5890199989Srdivacky } 5891239462Sdim break; 5892239462Sdim case LibFunc::fabs: 5893239462Sdim case LibFunc::fabsf: 5894239462Sdim case LibFunc::fabsl: 5895239462Sdim if (visitUnaryFloatCall(I, ISD::FABS)) 5896199989Srdivacky return; 5897239462Sdim break; 5898239462Sdim case LibFunc::sin: 5899239462Sdim case LibFunc::sinf: 5900239462Sdim case LibFunc::sinl: 5901239462Sdim if (visitUnaryFloatCall(I, ISD::FSIN)) 5902199989Srdivacky return; 5903239462Sdim break; 5904239462Sdim case LibFunc::cos: 5905239462Sdim case LibFunc::cosf: 5906239462Sdim case LibFunc::cosl: 5907239462Sdim if (visitUnaryFloatCall(I, ISD::FCOS)) 5908199989Srdivacky return; 5909239462Sdim break; 5910239462Sdim case LibFunc::sqrt: 5911239462Sdim case LibFunc::sqrtf: 5912239462Sdim case LibFunc::sqrtl: 5913263508Sdim case LibFunc::sqrt_finite: 5914263508Sdim case LibFunc::sqrtf_finite: 5915263508Sdim case LibFunc::sqrtl_finite: 5916239462Sdim if (visitUnaryFloatCall(I, ISD::FSQRT)) 5917199989Srdivacky return; 5918239462Sdim break; 5919239462Sdim case LibFunc::floor: 5920239462Sdim case LibFunc::floorf: 5921239462Sdim case LibFunc::floorl: 5922239462Sdim if (visitUnaryFloatCall(I, ISD::FFLOOR)) 5923234353Sdim return; 5924239462Sdim break; 5925239462Sdim case LibFunc::nearbyint: 5926239462Sdim case LibFunc::nearbyintf: 5927239462Sdim case LibFunc::nearbyintl: 5928239462Sdim if (visitUnaryFloatCall(I, ISD::FNEARBYINT)) 5929234353Sdim return; 5930239462Sdim break; 5931239462Sdim case LibFunc::ceil: 5932239462Sdim case LibFunc::ceilf: 5933239462Sdim case LibFunc::ceill: 5934239462Sdim if (visitUnaryFloatCall(I, ISD::FCEIL)) 5935234353Sdim return; 5936239462Sdim break; 5937239462Sdim case LibFunc::rint: 5938239462Sdim case LibFunc::rintf: 5939239462Sdim case LibFunc::rintl: 5940239462Sdim if (visitUnaryFloatCall(I, ISD::FRINT)) 5941234353Sdim return; 5942239462Sdim break; 5943263508Sdim case LibFunc::round: 5944263508Sdim case LibFunc::roundf: 5945263508Sdim case LibFunc::roundl: 5946263508Sdim if (visitUnaryFloatCall(I, ISD::FROUND)) 5947263508Sdim return; 5948263508Sdim break; 5949239462Sdim case LibFunc::trunc: 5950239462Sdim case LibFunc::truncf: 5951239462Sdim case LibFunc::truncl: 5952239462Sdim if (visitUnaryFloatCall(I, ISD::FTRUNC)) 5953234353Sdim return; 5954239462Sdim break; 5955239462Sdim case LibFunc::log2: 5956239462Sdim case LibFunc::log2f: 5957239462Sdim case LibFunc::log2l: 5958239462Sdim if (visitUnaryFloatCall(I, ISD::FLOG2)) 5959234353Sdim return; 5960239462Sdim break; 5961239462Sdim case LibFunc::exp2: 5962239462Sdim case LibFunc::exp2f: 5963239462Sdim case LibFunc::exp2l: 5964239462Sdim if (visitUnaryFloatCall(I, ISD::FEXP2)) 5965234353Sdim return; 5966239462Sdim break; 5967239462Sdim case LibFunc::memcmp: 5968201360Srdivacky if (visitMemCmpCall(I)) 5969201360Srdivacky return; 5970239462Sdim break; 5971263508Sdim case LibFunc::memchr: 5972263508Sdim if (visitMemChrCall(I)) 5973263508Sdim return; 5974263508Sdim break; 5975263508Sdim case LibFunc::strcpy: 5976263508Sdim if (visitStrCpyCall(I, false)) 5977263508Sdim return; 5978263508Sdim break; 5979263508Sdim case LibFunc::stpcpy: 5980263508Sdim if (visitStrCpyCall(I, true)) 5981263508Sdim return; 5982263508Sdim break; 5983263508Sdim case LibFunc::strcmp: 5984263508Sdim if (visitStrCmpCall(I)) 5985263508Sdim return; 5986263508Sdim break; 5987263508Sdim case LibFunc::strlen: 5988263508Sdim if (visitStrLenCall(I)) 5989263508Sdim return; 5990263508Sdim break; 5991263508Sdim case LibFunc::strnlen: 5992263508Sdim if (visitStrNLenCall(I)) 5993263508Sdim return; 5994263508Sdim break; 5995199989Srdivacky } 5996199989Srdivacky } 5997199989Srdivacky } 5998218893Sdim 5999199989Srdivacky SDValue Callee; 6000199989Srdivacky if (!RenameFn) 6001210299Sed Callee = getValue(I.getCalledValue()); 6002199989Srdivacky else 6003263508Sdim Callee = DAG.getExternalSymbol(RenameFn, 6004263508Sdim TM.getTargetLowering()->getPointerTy()); 6005199989Srdivacky 6006201360Srdivacky // Check if we can potentially perform a tail call. More detailed checking is 6007201360Srdivacky // be done within LowerCallTo, after more information about the call is known. 6008203954Srdivacky LowerCallTo(&I, Callee, I.isTailCall()); 6009199989Srdivacky} 6010199989Srdivacky 6011221345Sdimnamespace { 6012199989Srdivacky 6013199989Srdivacky/// AsmOperandInfo - This contains information for each constraint that we are 6014199989Srdivacky/// lowering. 6015221345Sdimclass SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { 6016199989Srdivackypublic: 6017199989Srdivacky /// CallOperand - If this is the result output operand or a clobber 6018199989Srdivacky /// this is null, otherwise it is the incoming operand to the CallInst. 6019199989Srdivacky /// This gets modified as the asm is processed. 6020199989Srdivacky SDValue CallOperand; 6021199989Srdivacky 6022199989Srdivacky /// AssignedRegs - If this is a register or register class operand, this 6023199989Srdivacky /// contains the set of register corresponding to the operand. 6024199989Srdivacky RegsForValue AssignedRegs; 6025199989Srdivacky 6026218893Sdim explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) 6027199989Srdivacky : TargetLowering::AsmOperandInfo(info), CallOperand(0,0) { 6028199989Srdivacky } 6029199989Srdivacky 6030199989Srdivacky /// getCallOperandValEVT - Return the EVT of the Value* that this operand 6031199989Srdivacky /// corresponds to. If there is no Value* for this operand, it returns 6032199989Srdivacky /// MVT::Other. 6033201360Srdivacky EVT getCallOperandValEVT(LLVMContext &Context, 6034199989Srdivacky const TargetLowering &TLI, 6035243830Sdim const DataLayout *TD) const { 6036199989Srdivacky if (CallOperandVal == 0) return MVT::Other; 6037199989Srdivacky 6038199989Srdivacky if (isa<BasicBlock>(CallOperandVal)) 6039199989Srdivacky return TLI.getPointerTy(); 6040199989Srdivacky 6041226633Sdim llvm::Type *OpTy = CallOperandVal->getType(); 6042199989Srdivacky 6043223017Sdim // FIXME: code duplicated from TargetLowering::ParseConstraints(). 6044199989Srdivacky // If this is an indirect operand, the operand is a pointer to the 6045199989Srdivacky // accessed type. 6046201360Srdivacky if (isIndirect) { 6047226633Sdim llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); 6048201360Srdivacky if (!PtrTy) 6049207618Srdivacky report_fatal_error("Indirect operand for inline asm not a pointer!"); 6050201360Srdivacky OpTy = PtrTy->getElementType(); 6051201360Srdivacky } 6052199989Srdivacky 6053223017Sdim // Look for vector wrapped in a struct. e.g. { <16 x i8> }. 6054226633Sdim if (StructType *STy = dyn_cast<StructType>(OpTy)) 6055223017Sdim if (STy->getNumElements() == 1) 6056223017Sdim OpTy = STy->getElementType(0); 6057223017Sdim 6058199989Srdivacky // If OpTy is not a single value, it may be a struct/union that we 6059199989Srdivacky // can tile with integers. 6060199989Srdivacky if (!OpTy->isSingleValueType() && OpTy->isSized()) { 6061199989Srdivacky unsigned BitSize = TD->getTypeSizeInBits(OpTy); 6062199989Srdivacky switch (BitSize) { 6063199989Srdivacky default: break; 6064199989Srdivacky case 1: 6065199989Srdivacky case 8: 6066199989Srdivacky case 16: 6067199989Srdivacky case 32: 6068199989Srdivacky case 64: 6069199989Srdivacky case 128: 6070199989Srdivacky OpTy = IntegerType::get(Context, BitSize); 6071199989Srdivacky break; 6072199989Srdivacky } 6073199989Srdivacky } 6074199989Srdivacky 6075199989Srdivacky return TLI.getValueType(OpTy, true); 6076199989Srdivacky } 6077199989Srdivacky}; 6078210299Sed 6079218893Sdimtypedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector; 6080218893Sdim 6081221345Sdim} // end anonymous namespace 6082199989Srdivacky 6083199989Srdivacky/// GetRegistersForValue - Assign registers (virtual or physical) for the 6084199989Srdivacky/// specified operand. We prefer to assign virtual registers, to allow the 6085201360Srdivacky/// register allocator to handle the assignment process. However, if the asm 6086201360Srdivacky/// uses features that we can't model on machineinstrs, we have SDISel do the 6087199989Srdivacky/// allocation. This produces generally horrible, but correct, code. 6088199989Srdivacky/// 6089199989Srdivacky/// OpInfo describes the operand. 6090199989Srdivacky/// 6091221345Sdimstatic void GetRegistersForValue(SelectionDAG &DAG, 6092221345Sdim const TargetLowering &TLI, 6093263508Sdim SDLoc DL, 6094234353Sdim SDISelAsmOperandInfo &OpInfo) { 6095221345Sdim LLVMContext &Context = *DAG.getContext(); 6096199989Srdivacky 6097199989Srdivacky MachineFunction &MF = DAG.getMachineFunction(); 6098199989Srdivacky SmallVector<unsigned, 4> Regs; 6099199989Srdivacky 6100199989Srdivacky // If this is a constraint for a single physreg, or a constraint for a 6101199989Srdivacky // register class, find it. 6102199989Srdivacky std::pair<unsigned, const TargetRegisterClass*> PhysReg = 6103199989Srdivacky TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, 6104199989Srdivacky OpInfo.ConstraintVT); 6105199989Srdivacky 6106199989Srdivacky unsigned NumRegs = 1; 6107199989Srdivacky if (OpInfo.ConstraintVT != MVT::Other) { 6108199989Srdivacky // If this is a FP input in an integer register (or visa versa) insert a bit 6109199989Srdivacky // cast of the input value. More generally, handle any case where the input 6110199989Srdivacky // value disagrees with the register class we plan to stick this in. 6111199989Srdivacky if (OpInfo.Type == InlineAsm::isInput && 6112199989Srdivacky PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) { 6113199989Srdivacky // Try to convert to the first EVT that the reg class contains. If the 6114199989Srdivacky // types are identical size, use a bitcast to convert (e.g. two differing 6115199989Srdivacky // vector types). 6116249423Sdim MVT RegVT = *PhysReg.second->vt_begin(); 6117199989Srdivacky if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) { 6118221345Sdim OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, 6119199989Srdivacky RegVT, OpInfo.CallOperand); 6120199989Srdivacky OpInfo.ConstraintVT = RegVT; 6121199989Srdivacky } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) { 6122199989Srdivacky // If the input is a FP value and we want it in FP registers, do a 6123199989Srdivacky // bitcast to the corresponding integer type. This turns an f64 value 6124199989Srdivacky // into i64, which can be passed with two i32 values on a 32-bit 6125199989Srdivacky // machine. 6126249423Sdim RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); 6127221345Sdim OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, 6128199989Srdivacky RegVT, OpInfo.CallOperand); 6129199989Srdivacky OpInfo.ConstraintVT = RegVT; 6130199989Srdivacky } 6131199989Srdivacky } 6132199989Srdivacky 6133199989Srdivacky NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT); 6134199989Srdivacky } 6135199989Srdivacky 6136249423Sdim MVT RegVT; 6137199989Srdivacky EVT ValueVT = OpInfo.ConstraintVT; 6138199989Srdivacky 6139199989Srdivacky // If this is a constraint for a specific physical register, like {r17}, 6140199989Srdivacky // assign it now. 6141199989Srdivacky if (unsigned AssignedReg = PhysReg.first) { 6142199989Srdivacky const TargetRegisterClass *RC = PhysReg.second; 6143199989Srdivacky if (OpInfo.ConstraintVT == MVT::Other) 6144199989Srdivacky ValueVT = *RC->vt_begin(); 6145199989Srdivacky 6146199989Srdivacky // Get the actual register value type. This is important, because the user 6147199989Srdivacky // may have asked for (e.g.) the AX register in i32 type. We need to 6148199989Srdivacky // remember that AX is actually i16 to get the right extension. 6149199989Srdivacky RegVT = *RC->vt_begin(); 6150199989Srdivacky 6151199989Srdivacky // This is a explicit reference to a physical register. 6152199989Srdivacky Regs.push_back(AssignedReg); 6153199989Srdivacky 6154199989Srdivacky // If this is an expanded reference, add the rest of the regs to Regs. 6155199989Srdivacky if (NumRegs != 1) { 6156199989Srdivacky TargetRegisterClass::iterator I = RC->begin(); 6157199989Srdivacky for (; *I != AssignedReg; ++I) 6158199989Srdivacky assert(I != RC->end() && "Didn't find reg!"); 6159199989Srdivacky 6160199989Srdivacky // Already added the first reg. 6161199989Srdivacky --NumRegs; ++I; 6162199989Srdivacky for (; NumRegs; --NumRegs, ++I) { 6163199989Srdivacky assert(I != RC->end() && "Ran out of registers to allocate!"); 6164199989Srdivacky Regs.push_back(*I); 6165199989Srdivacky } 6166199989Srdivacky } 6167201360Srdivacky 6168210299Sed OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 6169199989Srdivacky return; 6170199989Srdivacky } 6171199989Srdivacky 6172199989Srdivacky // Otherwise, if this was a reference to an LLVM register class, create vregs 6173199989Srdivacky // for this reference. 6174199989Srdivacky if (const TargetRegisterClass *RC = PhysReg.second) { 6175199989Srdivacky RegVT = *RC->vt_begin(); 6176199989Srdivacky if (OpInfo.ConstraintVT == MVT::Other) 6177199989Srdivacky ValueVT = RegVT; 6178199989Srdivacky 6179199989Srdivacky // Create the appropriate number of virtual registers. 6180199989Srdivacky MachineRegisterInfo &RegInfo = MF.getRegInfo(); 6181199989Srdivacky for (; NumRegs; --NumRegs) 6182199989Srdivacky Regs.push_back(RegInfo.createVirtualRegister(RC)); 6183199989Srdivacky 6184210299Sed OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 6185199989Srdivacky return; 6186199989Srdivacky } 6187201360Srdivacky 6188199989Srdivacky // Otherwise, we couldn't allocate enough registers for this. 6189199989Srdivacky} 6190199989Srdivacky 6191199989Srdivacky/// visitInlineAsm - Handle a call to an InlineAsm object. 6192199989Srdivacky/// 6193207618Srdivackyvoid SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { 6194207618Srdivacky const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue()); 6195199989Srdivacky 6196199989Srdivacky /// ConstraintOperands - Information about all of the constraints. 6197218893Sdim SDISelAsmOperandInfoVector ConstraintOperands; 6198199989Srdivacky 6199263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6200223017Sdim TargetLowering::AsmOperandInfoVector 6201263508Sdim TargetConstraints = TLI->ParseConstraints(CS); 6202223017Sdim 6203218893Sdim bool hasMemory = false; 6204199989Srdivacky 6205199989Srdivacky unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. 6206199989Srdivacky unsigned ResNo = 0; // ResNo - The result number of the next output. 6207218893Sdim for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 6208218893Sdim ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i])); 6209199989Srdivacky SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); 6210199989Srdivacky 6211249423Sdim MVT OpVT = MVT::Other; 6212199989Srdivacky 6213199989Srdivacky // Compute the value type for each operand. 6214199989Srdivacky switch (OpInfo.Type) { 6215199989Srdivacky case InlineAsm::isOutput: 6216199989Srdivacky // Indirect outputs just consume an argument. 6217199989Srdivacky if (OpInfo.isIndirect) { 6218207618Srdivacky OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); 6219199989Srdivacky break; 6220199989Srdivacky } 6221199989Srdivacky 6222199989Srdivacky // The return value of the call is this value. As such, there is no 6223199989Srdivacky // corresponding argument. 6224226633Sdim assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); 6225226633Sdim if (StructType *STy = dyn_cast<StructType>(CS.getType())) { 6226263508Sdim OpVT = TLI->getSimpleValueType(STy->getElementType(ResNo)); 6227199989Srdivacky } else { 6228199989Srdivacky assert(ResNo == 0 && "Asm only has one result!"); 6229263508Sdim OpVT = TLI->getSimpleValueType(CS.getType()); 6230199989Srdivacky } 6231199989Srdivacky ++ResNo; 6232199989Srdivacky break; 6233199989Srdivacky case InlineAsm::isInput: 6234207618Srdivacky OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); 6235199989Srdivacky break; 6236199989Srdivacky case InlineAsm::isClobber: 6237199989Srdivacky // Nothing to do. 6238199989Srdivacky break; 6239199989Srdivacky } 6240199989Srdivacky 6241199989Srdivacky // If this is an input or an indirect output, process the call argument. 6242199989Srdivacky // BasicBlocks are labels, currently appearing only in asm's. 6243199989Srdivacky if (OpInfo.CallOperandVal) { 6244207618Srdivacky if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) { 6245199989Srdivacky OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); 6246199989Srdivacky } else { 6247199989Srdivacky OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); 6248199989Srdivacky } 6249199989Srdivacky 6250263508Sdim OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), *TLI, TD). 6251249423Sdim getSimpleVT(); 6252199989Srdivacky } 6253199989Srdivacky 6254199989Srdivacky OpInfo.ConstraintVT = OpVT; 6255218893Sdim 6256218893Sdim // Indirect operand accesses access memory. 6257218893Sdim if (OpInfo.isIndirect) 6258218893Sdim hasMemory = true; 6259218893Sdim else { 6260218893Sdim for (unsigned j = 0, ee = OpInfo.Codes.size(); j != ee; ++j) { 6261223017Sdim TargetLowering::ConstraintType 6262263508Sdim CType = TLI->getConstraintType(OpInfo.Codes[j]); 6263218893Sdim if (CType == TargetLowering::C_Memory) { 6264218893Sdim hasMemory = true; 6265218893Sdim break; 6266218893Sdim } 6267218893Sdim } 6268218893Sdim } 6269199989Srdivacky } 6270199989Srdivacky 6271218893Sdim SDValue Chain, Flag; 6272218893Sdim 6273218893Sdim // We won't need to flush pending loads if this asm doesn't touch 6274218893Sdim // memory and is nonvolatile. 6275218893Sdim if (hasMemory || IA->hasSideEffects()) 6276218893Sdim Chain = getRoot(); 6277218893Sdim else 6278218893Sdim Chain = DAG.getRoot(); 6279218893Sdim 6280199989Srdivacky // Second pass over the constraints: compute which constraint option to use 6281199989Srdivacky // and assign registers to constraints that want a specific physreg. 6282218893Sdim for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6283199989Srdivacky SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6284199989Srdivacky 6285199989Srdivacky // If this is an output operand with a matching input operand, look up the 6286199989Srdivacky // matching input. If their types mismatch, e.g. one is an integer, the 6287199989Srdivacky // other is floating point, or their sizes are different, flag it as an 6288199989Srdivacky // error. 6289199989Srdivacky if (OpInfo.hasMatchingInput()) { 6290199989Srdivacky SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; 6291218893Sdim 6292199989Srdivacky if (OpInfo.ConstraintVT != Input.ConstraintVT) { 6293239462Sdim std::pair<unsigned, const TargetRegisterClass*> MatchRC = 6294263508Sdim TLI->getRegForInlineAsmConstraint(OpInfo.ConstraintCode, 6295263508Sdim OpInfo.ConstraintVT); 6296239462Sdim std::pair<unsigned, const TargetRegisterClass*> InputRC = 6297263508Sdim TLI->getRegForInlineAsmConstraint(Input.ConstraintCode, 6298263508Sdim Input.ConstraintVT); 6299199989Srdivacky if ((OpInfo.ConstraintVT.isInteger() != 6300199989Srdivacky Input.ConstraintVT.isInteger()) || 6301224145Sdim (MatchRC.second != InputRC.second)) { 6302207618Srdivacky report_fatal_error("Unsupported asm: input constraint" 6303207618Srdivacky " with a matching output constraint of" 6304207618Srdivacky " incompatible type!"); 6305199989Srdivacky } 6306199989Srdivacky Input.ConstraintVT = OpInfo.ConstraintVT; 6307199989Srdivacky } 6308199989Srdivacky } 6309199989Srdivacky 6310199989Srdivacky // Compute the constraint code and ConstraintType to use. 6311263508Sdim TLI->ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); 6312199989Srdivacky 6313249423Sdim if (OpInfo.ConstraintType == TargetLowering::C_Memory && 6314249423Sdim OpInfo.Type == InlineAsm::isClobber) 6315249423Sdim continue; 6316249423Sdim 6317199989Srdivacky // If this is a memory input, and if the operand is not indirect, do what we 6318199989Srdivacky // need to to provide an address for the memory input. 6319199989Srdivacky if (OpInfo.ConstraintType == TargetLowering::C_Memory && 6320199989Srdivacky !OpInfo.isIndirect) { 6321223017Sdim assert((OpInfo.isMultipleAlternative || 6322223017Sdim (OpInfo.Type == InlineAsm::isInput)) && 6323199989Srdivacky "Can only indirectify direct input operands!"); 6324199989Srdivacky 6325199989Srdivacky // Memory operands really want the address of the value. If we don't have 6326199989Srdivacky // an indirect input, put it in the constpool if we can, otherwise spill 6327199989Srdivacky // it to a stack slot. 6328223017Sdim // TODO: This isn't quite right. We need to handle these according to 6329223017Sdim // the addressing mode that the constraint wants. Also, this may take 6330223017Sdim // an additional register for the computation and we don't want that 6331223017Sdim // either. 6332199989Srdivacky 6333199989Srdivacky // If the operand is a float, integer, or vector constant, spill to a 6334199989Srdivacky // constant pool entry to get its address. 6335207618Srdivacky const Value *OpVal = OpInfo.CallOperandVal; 6336199989Srdivacky if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || 6337234353Sdim isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) { 6338199989Srdivacky OpInfo.CallOperand = DAG.getConstantPool(cast<Constant>(OpVal), 6339263508Sdim TLI->getPointerTy()); 6340199989Srdivacky } else { 6341199989Srdivacky // Otherwise, create a stack slot and emit a store to it before the 6342199989Srdivacky // asm. 6343226633Sdim Type *Ty = OpVal->getType(); 6344263508Sdim uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty); 6345263508Sdim unsigned Align = TLI->getDataLayout()->getPrefTypeAlignment(Ty); 6346199989Srdivacky MachineFunction &MF = DAG.getMachineFunction(); 6347199989Srdivacky int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); 6348263508Sdim SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI->getPointerTy()); 6349263508Sdim Chain = DAG.getStore(Chain, getCurSDLoc(), 6350218893Sdim OpInfo.CallOperand, StackSlot, 6351218893Sdim MachinePointerInfo::getFixedStack(SSFI), 6352203954Srdivacky false, false, 0); 6353199989Srdivacky OpInfo.CallOperand = StackSlot; 6354199989Srdivacky } 6355199989Srdivacky 6356199989Srdivacky // There is no longer a Value* corresponding to this operand. 6357199989Srdivacky OpInfo.CallOperandVal = 0; 6358201360Srdivacky 6359199989Srdivacky // It is now an indirect operand. 6360199989Srdivacky OpInfo.isIndirect = true; 6361199989Srdivacky } 6362199989Srdivacky 6363199989Srdivacky // If this constraint is for a specific register, allocate it before 6364199989Srdivacky // anything else. 6365199989Srdivacky if (OpInfo.ConstraintType == TargetLowering::C_Register) 6366263508Sdim GetRegistersForValue(DAG, *TLI, getCurSDLoc(), OpInfo); 6367199989Srdivacky } 6368201360Srdivacky 6369199989Srdivacky // Second pass - Loop over all of the operands, assigning virtual or physregs 6370199989Srdivacky // to register class operands. 6371199989Srdivacky for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6372199989Srdivacky SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6373199989Srdivacky 6374199989Srdivacky // C_Register operands have already been allocated, Other/Memory don't need 6375199989Srdivacky // to be. 6376199989Srdivacky if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass) 6377263508Sdim GetRegistersForValue(DAG, *TLI, getCurSDLoc(), OpInfo); 6378199989Srdivacky } 6379199989Srdivacky 6380199989Srdivacky // AsmNodeOperands - The operands for the ISD::INLINEASM node. 6381199989Srdivacky std::vector<SDValue> AsmNodeOperands; 6382199989Srdivacky AsmNodeOperands.push_back(SDValue()); // reserve space for input chain 6383199989Srdivacky AsmNodeOperands.push_back( 6384202375Srdivacky DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), 6385263508Sdim TLI->getPointerTy())); 6386199989Srdivacky 6387207618Srdivacky // If we have a !srcloc metadata node associated with it, we want to attach 6388207618Srdivacky // this to the ultimately generated inline asm machineinstr. To do this, we 6389207618Srdivacky // pass in the third operand as this (potentially null) inline asm MDNode. 6390207618Srdivacky const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc"); 6391207618Srdivacky AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); 6392199989Srdivacky 6393243830Sdim // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore 6394243830Sdim // bits as operand 3. 6395218893Sdim unsigned ExtraInfo = 0; 6396218893Sdim if (IA->hasSideEffects()) 6397218893Sdim ExtraInfo |= InlineAsm::Extra_HasSideEffects; 6398218893Sdim if (IA->isAlignStack()) 6399218893Sdim ExtraInfo |= InlineAsm::Extra_IsAlignStack; 6400243830Sdim // Set the asm dialect. 6401243830Sdim ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect; 6402243830Sdim 6403243830Sdim // Determine if this InlineAsm MayLoad or MayStore based on the constraints. 6404243830Sdim for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 6405243830Sdim TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 6406243830Sdim 6407243830Sdim // Compute the constraint code and ConstraintType to use. 6408263508Sdim TLI->ComputeConstraintToUse(OpInfo, SDValue()); 6409243830Sdim 6410243830Sdim // Ideally, we would only check against memory constraints. However, the 6411243830Sdim // meaning of an other constraint can be target-specific and we can't easily 6412243830Sdim // reason about it. Therefore, be conservative and set MayLoad/MayStore 6413243830Sdim // for other constriants as well. 6414243830Sdim if (OpInfo.ConstraintType == TargetLowering::C_Memory || 6415243830Sdim OpInfo.ConstraintType == TargetLowering::C_Other) { 6416243830Sdim if (OpInfo.Type == InlineAsm::isInput) 6417243830Sdim ExtraInfo |= InlineAsm::Extra_MayLoad; 6418243830Sdim else if (OpInfo.Type == InlineAsm::isOutput) 6419243830Sdim ExtraInfo |= InlineAsm::Extra_MayStore; 6420249423Sdim else if (OpInfo.Type == InlineAsm::isClobber) 6421249423Sdim ExtraInfo |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); 6422243830Sdim } 6423243830Sdim } 6424243830Sdim 6425218893Sdim AsmNodeOperands.push_back(DAG.getTargetConstant(ExtraInfo, 6426263508Sdim TLI->getPointerTy())); 6427210299Sed 6428199989Srdivacky // Loop over all of the inputs, copying the operand values into the 6429199989Srdivacky // appropriate registers and processing the output regs. 6430199989Srdivacky RegsForValue RetValRegs; 6431199989Srdivacky 6432199989Srdivacky // IndirectStoresToEmit - The set of stores to emit after the inline asm node. 6433199989Srdivacky std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit; 6434199989Srdivacky 6435199989Srdivacky for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6436199989Srdivacky SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6437199989Srdivacky 6438199989Srdivacky switch (OpInfo.Type) { 6439199989Srdivacky case InlineAsm::isOutput: { 6440199989Srdivacky if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass && 6441199989Srdivacky OpInfo.ConstraintType != TargetLowering::C_Register) { 6442199989Srdivacky // Memory output, or 'other' output (e.g. 'X' constraint). 6443199989Srdivacky assert(OpInfo.isIndirect && "Memory output must be indirect operand"); 6444199989Srdivacky 6445199989Srdivacky // Add information to the INLINEASM node to know about this output. 6446207618Srdivacky unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 6447207618Srdivacky AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, 6448263508Sdim TLI->getPointerTy())); 6449199989Srdivacky AsmNodeOperands.push_back(OpInfo.CallOperand); 6450199989Srdivacky break; 6451199989Srdivacky } 6452199989Srdivacky 6453199989Srdivacky // Otherwise, this is a register or register class output. 6454199989Srdivacky 6455199989Srdivacky // Copy the output from the appropriate register. Find a register that 6456199989Srdivacky // we can use. 6457234353Sdim if (OpInfo.AssignedRegs.Regs.empty()) { 6458234353Sdim LLVMContext &Ctx = *DAG.getContext(); 6459263508Sdim Ctx.emitError(CS.getInstruction(), 6460234353Sdim "couldn't allocate output register for constraint '" + 6461263508Sdim Twine(OpInfo.ConstraintCode) + "'"); 6462263508Sdim return; 6463234353Sdim } 6464199989Srdivacky 6465199989Srdivacky // If this is an indirect operand, store through the pointer after the 6466199989Srdivacky // asm. 6467199989Srdivacky if (OpInfo.isIndirect) { 6468199989Srdivacky IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, 6469199989Srdivacky OpInfo.CallOperandVal)); 6470199989Srdivacky } else { 6471199989Srdivacky // This is the result value of the call. 6472202375Srdivacky assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); 6473199989Srdivacky // Concatenate this output onto the outputs list. 6474199989Srdivacky RetValRegs.append(OpInfo.AssignedRegs); 6475199989Srdivacky } 6476199989Srdivacky 6477199989Srdivacky // Add information to the INLINEASM node to know that this register is 6478199989Srdivacky // set. 6479263508Sdim OpInfo.AssignedRegs 6480263508Sdim .AddInlineAsmOperands(OpInfo.isEarlyClobber 6481263508Sdim ? InlineAsm::Kind_RegDefEarlyClobber 6482263508Sdim : InlineAsm::Kind_RegDef, 6483263508Sdim false, 0, DAG, AsmNodeOperands); 6484199989Srdivacky break; 6485199989Srdivacky } 6486199989Srdivacky case InlineAsm::isInput: { 6487199989Srdivacky SDValue InOperandVal = OpInfo.CallOperand; 6488199989Srdivacky 6489199989Srdivacky if (OpInfo.isMatchingInputConstraint()) { // Matching constraint? 6490199989Srdivacky // If this is required to match an output register we have already set, 6491199989Srdivacky // just use its register. 6492199989Srdivacky unsigned OperandNo = OpInfo.getMatchedOperand(); 6493199989Srdivacky 6494199989Srdivacky // Scan until we find the definition we already emitted of this operand. 6495199989Srdivacky // When we find it, create a RegsForValue operand. 6496207618Srdivacky unsigned CurOp = InlineAsm::Op_FirstOperand; 6497199989Srdivacky for (; OperandNo; --OperandNo) { 6498199989Srdivacky // Advance to the next operand. 6499199989Srdivacky unsigned OpFlag = 6500199989Srdivacky cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 6501207618Srdivacky assert((InlineAsm::isRegDefKind(OpFlag) || 6502207618Srdivacky InlineAsm::isRegDefEarlyClobberKind(OpFlag) || 6503207618Srdivacky InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"); 6504199989Srdivacky CurOp += InlineAsm::getNumOperandRegisters(OpFlag)+1; 6505199989Srdivacky } 6506199989Srdivacky 6507199989Srdivacky unsigned OpFlag = 6508199989Srdivacky cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 6509207618Srdivacky if (InlineAsm::isRegDefKind(OpFlag) || 6510207618Srdivacky InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { 6511199989Srdivacky // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. 6512199989Srdivacky if (OpInfo.isIndirect) { 6513207618Srdivacky // This happens on gcc/testsuite/gcc.dg/pr8788-1.c 6514207618Srdivacky LLVMContext &Ctx = *DAG.getContext(); 6515263508Sdim Ctx.emitError(CS.getInstruction(), "inline asm not supported yet:" 6516263508Sdim " don't know how to handle tied " 6517263508Sdim "indirect register inputs"); 6518263508Sdim return; 6519199989Srdivacky } 6520218893Sdim 6521199989Srdivacky RegsForValue MatchedRegs; 6522199989Srdivacky MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); 6523249423Sdim MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType(); 6524199989Srdivacky MatchedRegs.RegVTs.push_back(RegVT); 6525199989Srdivacky MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo(); 6526199989Srdivacky for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag); 6527251662Sdim i != e; ++i) { 6528263508Sdim if (const TargetRegisterClass *RC = TLI->getRegClassFor(RegVT)) 6529251662Sdim MatchedRegs.Regs.push_back(RegInfo.createVirtualRegister(RC)); 6530251662Sdim else { 6531251662Sdim LLVMContext &Ctx = *DAG.getContext(); 6532263508Sdim Ctx.emitError(CS.getInstruction(), 6533263508Sdim "inline asm error: This value" 6534251662Sdim " type register class is not natively supported!"); 6535263508Sdim return; 6536251662Sdim } 6537251662Sdim } 6538199989Srdivacky // Use the produced MatchedRegs object to 6539263508Sdim MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(), 6540243830Sdim Chain, &Flag, CS.getInstruction()); 6541207618Srdivacky MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, 6542199989Srdivacky true, OpInfo.getMatchedOperand(), 6543204642Srdivacky DAG, AsmNodeOperands); 6544199989Srdivacky break; 6545199989Srdivacky } 6546218893Sdim 6547207618Srdivacky assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"); 6548207618Srdivacky assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 && 6549207618Srdivacky "Unexpected number of operands"); 6550207618Srdivacky // Add information to the INLINEASM node to know about this input. 6551207618Srdivacky // See InlineAsm.h isUseOperandTiedToDef. 6552207618Srdivacky OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, 6553207618Srdivacky OpInfo.getMatchedOperand()); 6554207618Srdivacky AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag, 6555263508Sdim TLI->getPointerTy())); 6556207618Srdivacky AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); 6557207618Srdivacky break; 6558199989Srdivacky } 6559199989Srdivacky 6560210299Sed // Treat indirect 'X' constraint as memory. 6561218893Sdim if (OpInfo.ConstraintType == TargetLowering::C_Other && 6562218893Sdim OpInfo.isIndirect) 6563210299Sed OpInfo.ConstraintType = TargetLowering::C_Memory; 6564210299Sed 6565199989Srdivacky if (OpInfo.ConstraintType == TargetLowering::C_Other) { 6566199989Srdivacky std::vector<SDValue> Ops; 6567263508Sdim TLI->LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, 6568263508Sdim Ops, DAG); 6569234353Sdim if (Ops.empty()) { 6570234353Sdim LLVMContext &Ctx = *DAG.getContext(); 6571234353Sdim Ctx.emitError(CS.getInstruction(), 6572234353Sdim "invalid operand for inline asm constraint '" + 6573263508Sdim Twine(OpInfo.ConstraintCode) + "'"); 6574263508Sdim return; 6575234353Sdim } 6576199989Srdivacky 6577199989Srdivacky // Add information to the INLINEASM node to know about this input. 6578207618Srdivacky unsigned ResOpType = 6579207618Srdivacky InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); 6580199989Srdivacky AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 6581263508Sdim TLI->getPointerTy())); 6582199989Srdivacky AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end()); 6583199989Srdivacky break; 6584207618Srdivacky } 6585218893Sdim 6586207618Srdivacky if (OpInfo.ConstraintType == TargetLowering::C_Memory) { 6587199989Srdivacky assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!"); 6588263508Sdim assert(InOperandVal.getValueType() == TLI->getPointerTy() && 6589199989Srdivacky "Memory operands expect pointer values"); 6590199989Srdivacky 6591199989Srdivacky // Add information to the INLINEASM node to know about this input. 6592207618Srdivacky unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 6593199989Srdivacky AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 6594263508Sdim TLI->getPointerTy())); 6595199989Srdivacky AsmNodeOperands.push_back(InOperandVal); 6596199989Srdivacky break; 6597199989Srdivacky } 6598199989Srdivacky 6599199989Srdivacky assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass || 6600199989Srdivacky OpInfo.ConstraintType == TargetLowering::C_Register) && 6601199989Srdivacky "Unknown constraint type!"); 6602199989Srdivacky 6603239462Sdim // TODO: Support this. 6604239462Sdim if (OpInfo.isIndirect) { 6605239462Sdim LLVMContext &Ctx = *DAG.getContext(); 6606239462Sdim Ctx.emitError(CS.getInstruction(), 6607239462Sdim "Don't know how to handle indirect register inputs yet " 6608263508Sdim "for constraint '" + 6609263508Sdim Twine(OpInfo.ConstraintCode) + "'"); 6610263508Sdim return; 6611239462Sdim } 6612239462Sdim 6613199989Srdivacky // Copy the input into the appropriate registers. 6614234353Sdim if (OpInfo.AssignedRegs.Regs.empty()) { 6615234353Sdim LLVMContext &Ctx = *DAG.getContext(); 6616263508Sdim Ctx.emitError(CS.getInstruction(), 6617234353Sdim "couldn't allocate input reg for constraint '" + 6618263508Sdim Twine(OpInfo.ConstraintCode) + "'"); 6619263508Sdim return; 6620234353Sdim } 6621199989Srdivacky 6622263508Sdim OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(), 6623243830Sdim Chain, &Flag, CS.getInstruction()); 6624199989Srdivacky 6625207618Srdivacky OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, 6626204642Srdivacky DAG, AsmNodeOperands); 6627199989Srdivacky break; 6628199989Srdivacky } 6629199989Srdivacky case InlineAsm::isClobber: { 6630199989Srdivacky // Add the clobbered value to the operand list, so that the register 6631199989Srdivacky // allocator is aware that the physreg got clobbered. 6632199989Srdivacky if (!OpInfo.AssignedRegs.Regs.empty()) 6633224145Sdim OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, 6634204642Srdivacky false, 0, DAG, 6635201360Srdivacky AsmNodeOperands); 6636199989Srdivacky break; 6637199989Srdivacky } 6638199989Srdivacky } 6639199989Srdivacky } 6640199989Srdivacky 6641207618Srdivacky // Finish up input operands. Set the input chain and add the flag last. 6642210299Sed AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 6643199989Srdivacky if (Flag.getNode()) AsmNodeOperands.push_back(Flag); 6644199989Srdivacky 6645263508Sdim Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(), 6646218893Sdim DAG.getVTList(MVT::Other, MVT::Glue), 6647199989Srdivacky &AsmNodeOperands[0], AsmNodeOperands.size()); 6648199989Srdivacky Flag = Chain.getValue(1); 6649199989Srdivacky 6650199989Srdivacky // If this asm returns a register value, copy the result from that register 6651199989Srdivacky // and set it as the value of the call. 6652199989Srdivacky if (!RetValRegs.Regs.empty()) { 6653263508Sdim SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 6654243830Sdim Chain, &Flag, CS.getInstruction()); 6655199989Srdivacky 6656199989Srdivacky // FIXME: Why don't we do this for inline asms with MRVs? 6657199989Srdivacky if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) { 6658263508Sdim EVT ResultType = TLI->getValueType(CS.getType()); 6659199989Srdivacky 6660199989Srdivacky // If any of the results of the inline asm is a vector, it may have the 6661199989Srdivacky // wrong width/num elts. This can happen for register classes that can 6662199989Srdivacky // contain multiple different value types. The preg or vreg allocated may 6663199989Srdivacky // not have the same VT as was expected. Convert it to the right type 6664199989Srdivacky // with bit_convert. 6665199989Srdivacky if (ResultType != Val.getValueType() && Val.getValueType().isVector()) { 6666263508Sdim Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), 6667199989Srdivacky ResultType, Val); 6668199989Srdivacky 6669199989Srdivacky } else if (ResultType != Val.getValueType() && 6670199989Srdivacky ResultType.isInteger() && Val.getValueType().isInteger()) { 6671199989Srdivacky // If a result value was tied to an input value, the computed result may 6672199989Srdivacky // have a wider width than the expected result. Extract the relevant 6673199989Srdivacky // portion. 6674263508Sdim Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val); 6675199989Srdivacky } 6676199989Srdivacky 6677199989Srdivacky assert(ResultType == Val.getValueType() && "Asm result value mismatch!"); 6678199989Srdivacky } 6679199989Srdivacky 6680199989Srdivacky setValue(CS.getInstruction(), Val); 6681199989Srdivacky // Don't need to use this as a chain in this case. 6682199989Srdivacky if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty()) 6683199989Srdivacky return; 6684199989Srdivacky } 6685199989Srdivacky 6686207618Srdivacky std::vector<std::pair<SDValue, const Value *> > StoresToEmit; 6687199989Srdivacky 6688199989Srdivacky // Process indirect outputs, first output all of the flagged copies out of 6689199989Srdivacky // physregs. 6690199989Srdivacky for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { 6691199989Srdivacky RegsForValue &OutRegs = IndirectStoresToEmit[i].first; 6692207618Srdivacky const Value *Ptr = IndirectStoresToEmit[i].second; 6693263508Sdim SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 6694243830Sdim Chain, &Flag, IA); 6695199989Srdivacky StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); 6696199989Srdivacky } 6697199989Srdivacky 6698199989Srdivacky // Emit the non-flagged stores from the physregs. 6699199989Srdivacky SmallVector<SDValue, 8> OutChains; 6700201360Srdivacky for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) { 6701263508Sdim SDValue Val = DAG.getStore(Chain, getCurSDLoc(), 6702201360Srdivacky StoresToEmit[i].first, 6703201360Srdivacky getValue(StoresToEmit[i].second), 6704218893Sdim MachinePointerInfo(StoresToEmit[i].second), 6705203954Srdivacky false, false, 0); 6706201360Srdivacky OutChains.push_back(Val); 6707201360Srdivacky } 6708201360Srdivacky 6709199989Srdivacky if (!OutChains.empty()) 6710263508Sdim Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 6711199989Srdivacky &OutChains[0], OutChains.size()); 6712201360Srdivacky 6713199989Srdivacky DAG.setRoot(Chain); 6714199989Srdivacky} 6715199989Srdivacky 6716207618Srdivackyvoid SelectionDAGBuilder::visitVAStart(const CallInst &I) { 6717263508Sdim DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), 6718199989Srdivacky MVT::Other, getRoot(), 6719210299Sed getValue(I.getArgOperand(0)), 6720210299Sed DAG.getSrcValue(I.getArgOperand(0)))); 6721199989Srdivacky} 6722199989Srdivacky 6723207618Srdivackyvoid SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { 6724263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6725263508Sdim const DataLayout &TD = *TLI->getDataLayout(); 6726263508Sdim SDValue V = DAG.getVAArg(TLI->getValueType(I.getType()), getCurSDLoc(), 6727199989Srdivacky getRoot(), getValue(I.getOperand(0)), 6728210299Sed DAG.getSrcValue(I.getOperand(0)), 6729210299Sed TD.getABITypeAlignment(I.getType())); 6730199989Srdivacky setValue(&I, V); 6731199989Srdivacky DAG.setRoot(V.getValue(1)); 6732199989Srdivacky} 6733199989Srdivacky 6734207618Srdivackyvoid SelectionDAGBuilder::visitVAEnd(const CallInst &I) { 6735263508Sdim DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), 6736199989Srdivacky MVT::Other, getRoot(), 6737210299Sed getValue(I.getArgOperand(0)), 6738210299Sed DAG.getSrcValue(I.getArgOperand(0)))); 6739199989Srdivacky} 6740199989Srdivacky 6741207618Srdivackyvoid SelectionDAGBuilder::visitVACopy(const CallInst &I) { 6742263508Sdim DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), 6743199989Srdivacky MVT::Other, getRoot(), 6744210299Sed getValue(I.getArgOperand(0)), 6745210299Sed getValue(I.getArgOperand(1)), 6746210299Sed DAG.getSrcValue(I.getArgOperand(0)), 6747210299Sed DAG.getSrcValue(I.getArgOperand(1)))); 6748199989Srdivacky} 6749199989Srdivacky 6750263508Sdim/// \brief Lower an argument list according to the target calling convention. 6751263508Sdim/// 6752263508Sdim/// \return A tuple of <return-value, token-chain> 6753263508Sdim/// 6754263508Sdim/// This is a helper for lowering intrinsics that follow a target calling 6755263508Sdim/// convention or require stack pointer adjustment. Only a subset of the 6756263508Sdim/// intrinsic's operands need to participate in the calling convention. 6757263508Sdimstd::pair<SDValue, SDValue> 6758263508SdimSelectionDAGBuilder::LowerCallOperands(const CallInst &CI, unsigned ArgIdx, 6759263508Sdim unsigned NumArgs, SDValue Callee, 6760263508Sdim bool useVoidTy) { 6761263508Sdim TargetLowering::ArgListTy Args; 6762263508Sdim Args.reserve(NumArgs); 6763263508Sdim 6764263508Sdim // Populate the argument list. 6765263508Sdim // Attributes for args start at offset 1, after the return attribute. 6766263508Sdim ImmutableCallSite CS(&CI); 6767263508Sdim for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1; 6768263508Sdim ArgI != ArgE; ++ArgI) { 6769263508Sdim const Value *V = CI.getOperand(ArgI); 6770263508Sdim 6771263508Sdim assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); 6772263508Sdim 6773263508Sdim TargetLowering::ArgListEntry Entry; 6774263508Sdim Entry.Node = getValue(V); 6775263508Sdim Entry.Ty = V->getType(); 6776263508Sdim Entry.setAttributes(&CS, AttrI); 6777263508Sdim Args.push_back(Entry); 6778263508Sdim } 6779263508Sdim 6780263508Sdim Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CI.getType(); 6781263508Sdim TargetLowering::CallLoweringInfo CLI(getRoot(), retTy, /*retSExt*/ false, 6782263508Sdim /*retZExt*/ false, /*isVarArg*/ false, /*isInReg*/ false, NumArgs, 6783263508Sdim CI.getCallingConv(), /*isTailCall*/ false, /*doesNotReturn*/ false, 6784263508Sdim /*isReturnValueUsed*/ CI.use_empty(), Callee, Args, DAG, getCurSDLoc()); 6785263508Sdim 6786263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6787263508Sdim return TLI->LowerCallTo(CLI); 6788263508Sdim} 6789263508Sdim 6790263508Sdim/// \brief Lower llvm.experimental.stackmap directly to its target opcode. 6791263508Sdimvoid SelectionDAGBuilder::visitStackmap(const CallInst &CI) { 6792263508Sdim // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>, 6793263508Sdim // [live variables...]) 6794263508Sdim 6795263508Sdim assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value."); 6796263508Sdim 6797263508Sdim SDValue Callee = getValue(CI.getCalledValue()); 6798263508Sdim 6799263508Sdim // Lower into a call sequence with no args and no return value. 6800263508Sdim std::pair<SDValue, SDValue> Result = LowerCallOperands(CI, 0, 0, Callee); 6801263508Sdim // Set the root to the target-lowered call chain. 6802263508Sdim SDValue Chain = Result.second; 6803263508Sdim DAG.setRoot(Chain); 6804263508Sdim 6805263508Sdim /// Get a call instruction from the call sequence chain. 6806263508Sdim /// Tail calls are not allowed. 6807263508Sdim SDNode *CallEnd = Chain.getNode(); 6808263508Sdim assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && 6809263508Sdim "Expected a callseq node."); 6810263508Sdim SDNode *Call = CallEnd->getOperand(0).getNode(); 6811263508Sdim bool hasGlue = Call->getGluedNode(); 6812263508Sdim 6813263508Sdim // Replace the target specific call node with the stackmap intrinsic. 6814263508Sdim SmallVector<SDValue, 8> Ops; 6815263508Sdim 6816263508Sdim // Add the <id> and <numShadowBytes> constants. 6817263508Sdim for (unsigned i = 0; i < 2; ++i) { 6818263508Sdim SDValue tmp = getValue(CI.getOperand(i)); 6819263508Sdim Ops.push_back(DAG.getTargetConstant( 6820263508Sdim cast<ConstantSDNode>(tmp)->getZExtValue(), MVT::i32)); 6821263508Sdim } 6822263508Sdim // Push live variables for the stack map. 6823263508Sdim for (unsigned i = 2, e = CI.getNumArgOperands(); i != e; ++i) 6824263508Sdim Ops.push_back(getValue(CI.getArgOperand(i))); 6825263508Sdim 6826263508Sdim // Push the chain (this is originally the first operand of the call, but 6827263508Sdim // becomes now the last or second to last operand). 6828263508Sdim Ops.push_back(*(Call->op_begin())); 6829263508Sdim 6830263508Sdim // Push the glue flag (last operand). 6831263508Sdim if (hasGlue) 6832263508Sdim Ops.push_back(*(Call->op_end()-1)); 6833263508Sdim 6834263508Sdim SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6835263508Sdim 6836263508Sdim // Replace the target specific call node with a STACKMAP node. 6837263508Sdim MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::STACKMAP, getCurSDLoc(), 6838263508Sdim NodeTys, Ops); 6839263508Sdim 6840263508Sdim // StackMap generates no value, so nothing goes in the NodeMap. 6841263508Sdim 6842263508Sdim // Fixup the consumers of the intrinsic. The chain and glue may be used in the 6843263508Sdim // call sequence. 6844263508Sdim DAG.ReplaceAllUsesWith(Call, MN); 6845263508Sdim 6846263508Sdim DAG.DeleteNode(Call); 6847263508Sdim} 6848263508Sdim 6849263508Sdim/// \brief Lower llvm.experimental.patchpoint directly to its target opcode. 6850263508Sdimvoid SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) { 6851263508Sdim // void|i64 @llvm.experimental.patchpoint.void|i64(i32 <id>, 6852263508Sdim // i32 <numBytes>, 6853263508Sdim // i8* <target>, 6854263508Sdim // i32 <numArgs>, 6855263508Sdim // [Args...], 6856263508Sdim // [live variables...]) 6857263508Sdim 6858263508Sdim CallingConv::ID CC = CI.getCallingConv(); 6859263508Sdim bool isAnyRegCC = CC == CallingConv::AnyReg; 6860263508Sdim bool hasDef = !CI.getType()->isVoidTy(); 6861263508Sdim SDValue Callee = getValue(CI.getOperand(2)); // <target> 6862263508Sdim 6863263508Sdim // Get the real number of arguments participating in the call <numArgs> 6864263508Sdim unsigned NumArgs = 6865263508Sdim cast<ConstantSDNode>(getValue(CI.getArgOperand(3)))->getZExtValue(); 6866263508Sdim 6867263508Sdim // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> 6868263508Sdim assert(CI.getNumArgOperands() >= NumArgs + 4 && 6869263508Sdim "Not enough arguments provided to the patchpoint intrinsic"); 6870263508Sdim 6871263508Sdim // For AnyRegCC the arguments are lowered later on manually. 6872263508Sdim unsigned NumCallArgs = isAnyRegCC ? 0 : NumArgs; 6873263508Sdim std::pair<SDValue, SDValue> Result = 6874263508Sdim LowerCallOperands(CI, 4, NumCallArgs, Callee, isAnyRegCC); 6875263508Sdim 6876263508Sdim // Set the root to the target-lowered call chain. 6877263508Sdim SDValue Chain = Result.second; 6878263508Sdim DAG.setRoot(Chain); 6879263508Sdim 6880263508Sdim SDNode *CallEnd = Chain.getNode(); 6881263508Sdim if (hasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) 6882263508Sdim CallEnd = CallEnd->getOperand(0).getNode(); 6883263508Sdim 6884263508Sdim /// Get a call instruction from the call sequence chain. 6885263508Sdim /// Tail calls are not allowed. 6886263508Sdim assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && 6887263508Sdim "Expected a callseq node."); 6888263508Sdim SDNode *Call = CallEnd->getOperand(0).getNode(); 6889263508Sdim bool hasGlue = Call->getGluedNode(); 6890263508Sdim 6891263508Sdim // Replace the target specific call node with the patchable intrinsic. 6892263508Sdim SmallVector<SDValue, 8> Ops; 6893263508Sdim 6894263508Sdim // Add the <id> and <numNopBytes> constants. 6895263508Sdim for (unsigned i = 0; i < 2; ++i) { 6896263508Sdim SDValue tmp = getValue(CI.getOperand(i)); 6897263508Sdim Ops.push_back(DAG.getTargetConstant( 6898263508Sdim cast<ConstantSDNode>(tmp)->getZExtValue(), MVT::i32)); 6899263508Sdim } 6900263508Sdim // Assume that the Callee is a constant address. 6901263508Sdim Ops.push_back( 6902263508Sdim DAG.getIntPtrConstant(cast<ConstantSDNode>(Callee)->getZExtValue(), 6903263508Sdim /*isTarget=*/true)); 6904263508Sdim 6905263508Sdim // Adjust <numArgs> to account for any arguments that have been passed on the 6906263508Sdim // stack instead. 6907263508Sdim // Call Node: Chain, Target, {Args}, RegMask, [Glue] 6908263508Sdim unsigned NumCallRegArgs = Call->getNumOperands() - (hasGlue ? 4 : 3); 6909263508Sdim NumCallRegArgs = isAnyRegCC ? NumArgs : NumCallRegArgs; 6910263508Sdim Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32)); 6911263508Sdim 6912263508Sdim // Add the calling convention 6913263508Sdim Ops.push_back(DAG.getTargetConstant((unsigned)CC, MVT::i32)); 6914263508Sdim 6915263508Sdim // Add the arguments we omitted previously. The register allocator should 6916263508Sdim // place these in any free register. 6917263508Sdim if (isAnyRegCC) 6918263508Sdim for (unsigned i = 4, e = NumArgs + 4; i != e; ++i) 6919263508Sdim Ops.push_back(getValue(CI.getArgOperand(i))); 6920263508Sdim 6921263508Sdim // Push the arguments from the call instruction. 6922263508Sdim SDNode::op_iterator e = hasGlue ? Call->op_end()-2 : Call->op_end()-1; 6923263508Sdim for (SDNode::op_iterator i = Call->op_begin()+2; i != e; ++i) 6924263508Sdim Ops.push_back(*i); 6925263508Sdim 6926263508Sdim // Push live variables for the stack map. 6927263508Sdim for (unsigned i = NumArgs + 4, e = CI.getNumArgOperands(); i != e; ++i) { 6928263508Sdim SDValue OpVal = getValue(CI.getArgOperand(i)); 6929263508Sdim if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) { 6930263508Sdim Ops.push_back( 6931263508Sdim DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); 6932263508Sdim Ops.push_back( 6933263508Sdim DAG.getTargetConstant(C->getSExtValue(), MVT::i64)); 6934263508Sdim } else 6935263508Sdim Ops.push_back(OpVal); 6936263508Sdim } 6937263508Sdim 6938263508Sdim // Push the register mask info. 6939263508Sdim if (hasGlue) 6940263508Sdim Ops.push_back(*(Call->op_end()-2)); 6941263508Sdim else 6942263508Sdim Ops.push_back(*(Call->op_end()-1)); 6943263508Sdim 6944263508Sdim // Push the chain (this is originally the first operand of the call, but 6945263508Sdim // becomes now the last or second to last operand). 6946263508Sdim Ops.push_back(*(Call->op_begin())); 6947263508Sdim 6948263508Sdim // Push the glue flag (last operand). 6949263508Sdim if (hasGlue) 6950263508Sdim Ops.push_back(*(Call->op_end()-1)); 6951263508Sdim 6952263508Sdim SDVTList NodeTys; 6953263508Sdim if (isAnyRegCC && hasDef) { 6954263508Sdim // Create the return types based on the intrinsic definition 6955263508Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6956263508Sdim SmallVector<EVT, 3> ValueVTs; 6957263508Sdim ComputeValueVTs(TLI, CI.getType(), ValueVTs); 6958263508Sdim assert(ValueVTs.size() == 1 && "Expected only one return value type."); 6959263508Sdim 6960263508Sdim // There is always a chain and a glue type at the end 6961263508Sdim ValueVTs.push_back(MVT::Other); 6962263508Sdim ValueVTs.push_back(MVT::Glue); 6963263508Sdim NodeTys = DAG.getVTList(ValueVTs.data(), ValueVTs.size()); 6964263508Sdim } else 6965263508Sdim NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6966263508Sdim 6967263508Sdim // Replace the target specific call node with a PATCHPOINT node. 6968263508Sdim MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT, 6969263508Sdim getCurSDLoc(), NodeTys, Ops); 6970263508Sdim 6971263508Sdim // Update the NodeMap. 6972263508Sdim if (hasDef) { 6973263508Sdim if (isAnyRegCC) 6974263508Sdim setValue(&CI, SDValue(MN, 0)); 6975263508Sdim else 6976263508Sdim setValue(&CI, Result.first); 6977263508Sdim } 6978263508Sdim 6979263508Sdim // Fixup the consumers of the intrinsic. The chain and glue may be used in the 6980263508Sdim // call sequence. Furthermore the location of the chain and glue can change 6981263508Sdim // when the AnyReg calling convention is used and the intrinsic returns a 6982263508Sdim // value. 6983263508Sdim if (isAnyRegCC && hasDef) { 6984263508Sdim SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; 6985263508Sdim SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)}; 6986263508Sdim DAG.ReplaceAllUsesOfValuesWith(From, To, 2); 6987263508Sdim } else 6988263508Sdim DAG.ReplaceAllUsesWith(Call, MN); 6989263508Sdim DAG.DeleteNode(Call); 6990263508Sdim} 6991263508Sdim 6992199989Srdivacky/// TargetLowering::LowerCallTo - This is the default LowerCallTo 6993199989Srdivacky/// implementation, which just calls LowerCall. 6994199989Srdivacky/// FIXME: When all targets are 6995199989Srdivacky/// migrated to using LowerCall, this hook should be integrated into SDISel. 6996199989Srdivackystd::pair<SDValue, SDValue> 6997239462SdimTargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { 6998251662Sdim // Handle the incoming return values from the call. 6999251662Sdim CLI.Ins.clear(); 7000251662Sdim SmallVector<EVT, 4> RetTys; 7001251662Sdim ComputeValueVTs(*this, CLI.RetTy, RetTys); 7002251662Sdim for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 7003251662Sdim EVT VT = RetTys[I]; 7004251662Sdim MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); 7005251662Sdim unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); 7006251662Sdim for (unsigned i = 0; i != NumRegs; ++i) { 7007251662Sdim ISD::InputArg MyFlags; 7008251662Sdim MyFlags.VT = RegisterVT; 7009263508Sdim MyFlags.ArgVT = VT; 7010251662Sdim MyFlags.Used = CLI.IsReturnValueUsed; 7011251662Sdim if (CLI.RetSExt) 7012251662Sdim MyFlags.Flags.setSExt(); 7013251662Sdim if (CLI.RetZExt) 7014251662Sdim MyFlags.Flags.setZExt(); 7015251662Sdim if (CLI.IsInReg) 7016251662Sdim MyFlags.Flags.setInReg(); 7017251662Sdim CLI.Ins.push_back(MyFlags); 7018251662Sdim } 7019251662Sdim } 7020251662Sdim 7021199989Srdivacky // Handle all of the outgoing arguments. 7022239462Sdim CLI.Outs.clear(); 7023239462Sdim CLI.OutVals.clear(); 7024239462Sdim ArgListTy &Args = CLI.Args; 7025199989Srdivacky for (unsigned i = 0, e = Args.size(); i != e; ++i) { 7026199989Srdivacky SmallVector<EVT, 4> ValueVTs; 7027199989Srdivacky ComputeValueVTs(*this, Args[i].Ty, ValueVTs); 7028199989Srdivacky for (unsigned Value = 0, NumValues = ValueVTs.size(); 7029199989Srdivacky Value != NumValues; ++Value) { 7030199989Srdivacky EVT VT = ValueVTs[Value]; 7031239462Sdim Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext()); 7032199989Srdivacky SDValue Op = SDValue(Args[i].Node.getNode(), 7033199989Srdivacky Args[i].Node.getResNo() + Value); 7034199989Srdivacky ISD::ArgFlagsTy Flags; 7035199989Srdivacky unsigned OriginalAlignment = 7036243830Sdim getDataLayout()->getABITypeAlignment(ArgTy); 7037199989Srdivacky 7038199989Srdivacky if (Args[i].isZExt) 7039199989Srdivacky Flags.setZExt(); 7040199989Srdivacky if (Args[i].isSExt) 7041199989Srdivacky Flags.setSExt(); 7042199989Srdivacky if (Args[i].isInReg) 7043199989Srdivacky Flags.setInReg(); 7044199989Srdivacky if (Args[i].isSRet) 7045199989Srdivacky Flags.setSRet(); 7046199989Srdivacky if (Args[i].isByVal) { 7047199989Srdivacky Flags.setByVal(); 7048226633Sdim PointerType *Ty = cast<PointerType>(Args[i].Ty); 7049226633Sdim Type *ElementTy = Ty->getElementType(); 7050243830Sdim Flags.setByValSize(getDataLayout()->getTypeAllocSize(ElementTy)); 7051199989Srdivacky // For ByVal, alignment should come from FE. BE will guess if this 7052199989Srdivacky // info is not there but there are cases it cannot get right. 7053223017Sdim unsigned FrameAlign; 7054199989Srdivacky if (Args[i].Alignment) 7055199989Srdivacky FrameAlign = Args[i].Alignment; 7056223017Sdim else 7057223017Sdim FrameAlign = getByValTypeAlignment(ElementTy); 7058199989Srdivacky Flags.setByValAlign(FrameAlign); 7059199989Srdivacky } 7060199989Srdivacky if (Args[i].isNest) 7061199989Srdivacky Flags.setNest(); 7062199989Srdivacky Flags.setOrigAlign(OriginalAlignment); 7063199989Srdivacky 7064249423Sdim MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT); 7065239462Sdim unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT); 7066199989Srdivacky SmallVector<SDValue, 4> Parts(NumParts); 7067199989Srdivacky ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 7068199989Srdivacky 7069199989Srdivacky if (Args[i].isSExt) 7070199989Srdivacky ExtendKind = ISD::SIGN_EXTEND; 7071199989Srdivacky else if (Args[i].isZExt) 7072199989Srdivacky ExtendKind = ISD::ZERO_EXTEND; 7073199989Srdivacky 7074251662Sdim // Conservatively only handle 'returned' on non-vectors for now 7075251662Sdim if (Args[i].isReturned && !Op.getValueType().isVector()) { 7076251662Sdim assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues && 7077251662Sdim "unexpected use of 'returned'"); 7078251662Sdim // Before passing 'returned' to the target lowering code, ensure that 7079251662Sdim // either the register MVT and the actual EVT are the same size or that 7080251662Sdim // the return value and argument are extended in the same way; in these 7081251662Sdim // cases it's safe to pass the argument register value unchanged as the 7082251662Sdim // return register value (although it's at the target's option whether 7083251662Sdim // to do so) 7084251662Sdim // TODO: allow code generation to take advantage of partially preserved 7085251662Sdim // registers rather than clobbering the entire register when the 7086251662Sdim // parameter extension method is not compatible with the return 7087251662Sdim // extension method 7088251662Sdim if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) || 7089251662Sdim (ExtendKind != ISD::ANY_EXTEND && 7090251662Sdim CLI.RetSExt == Args[i].isSExt && CLI.RetZExt == Args[i].isZExt)) 7091251662Sdim Flags.setReturned(); 7092251662Sdim } 7093251662Sdim 7094239462Sdim getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, 7095243830Sdim PartVT, CLI.CS ? CLI.CS->getInstruction() : 0, ExtendKind); 7096199989Srdivacky 7097199989Srdivacky for (unsigned j = 0; j != NumParts; ++j) { 7098199989Srdivacky // if it isn't first piece, alignment must be 1 7099263508Sdim ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT, 7100243830Sdim i < CLI.NumFixedArgs, 7101243830Sdim i, j*Parts[j].getValueType().getStoreSize()); 7102199989Srdivacky if (NumParts > 1 && j == 0) 7103199989Srdivacky MyFlags.Flags.setSplit(); 7104199989Srdivacky else if (j != 0) 7105199989Srdivacky MyFlags.Flags.setOrigAlign(1); 7106199989Srdivacky 7107239462Sdim CLI.Outs.push_back(MyFlags); 7108239462Sdim CLI.OutVals.push_back(Parts[j]); 7109199989Srdivacky } 7110199989Srdivacky } 7111199989Srdivacky } 7112199989Srdivacky 7113199989Srdivacky SmallVector<SDValue, 4> InVals; 7114239462Sdim CLI.Chain = LowerCall(CLI, InVals); 7115199989Srdivacky 7116199989Srdivacky // Verify that the target's LowerCall behaved as expected. 7117239462Sdim assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && 7118199989Srdivacky "LowerCall didn't return a valid chain!"); 7119239462Sdim assert((!CLI.IsTailCall || InVals.empty()) && 7120199989Srdivacky "LowerCall emitted a return value for a tail call!"); 7121239462Sdim assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && 7122199989Srdivacky "LowerCall didn't emit the correct number of values!"); 7123199989Srdivacky 7124199989Srdivacky // For a tail call, the return value is merely live-out and there aren't 7125199989Srdivacky // any nodes in the DAG representing it. Return a special value to 7126199989Srdivacky // indicate that a tail call has been emitted and no more Instructions 7127199989Srdivacky // should be processed in the current block. 7128239462Sdim if (CLI.IsTailCall) { 7129239462Sdim CLI.DAG.setRoot(CLI.Chain); 7130199989Srdivacky return std::make_pair(SDValue(), SDValue()); 7131199989Srdivacky } 7132199989Srdivacky 7133239462Sdim DEBUG(for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) { 7134205218Srdivacky assert(InVals[i].getNode() && 7135205218Srdivacky "LowerCall emitted a null value!"); 7136239462Sdim assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && 7137205218Srdivacky "LowerCall emitted a value with the wrong type!"); 7138205218Srdivacky }); 7139205218Srdivacky 7140199989Srdivacky // Collect the legal value parts into potentially illegal values 7141199989Srdivacky // that correspond to the original function's return values. 7142199989Srdivacky ISD::NodeType AssertOp = ISD::DELETED_NODE; 7143239462Sdim if (CLI.RetSExt) 7144199989Srdivacky AssertOp = ISD::AssertSext; 7145239462Sdim else if (CLI.RetZExt) 7146199989Srdivacky AssertOp = ISD::AssertZext; 7147199989Srdivacky SmallVector<SDValue, 4> ReturnValues; 7148199989Srdivacky unsigned CurReg = 0; 7149199989Srdivacky for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 7150199989Srdivacky EVT VT = RetTys[I]; 7151249423Sdim MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); 7152239462Sdim unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); 7153199989Srdivacky 7154239462Sdim ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], 7155243830Sdim NumRegs, RegisterVT, VT, NULL, 7156203954Srdivacky AssertOp)); 7157199989Srdivacky CurReg += NumRegs; 7158199989Srdivacky } 7159199989Srdivacky 7160199989Srdivacky // For a function returning void, there is no return value. We can't create 7161199989Srdivacky // such a node, so we just return a null return value in that case. In 7162221345Sdim // that case, nothing will actually look at the value. 7163199989Srdivacky if (ReturnValues.empty()) 7164239462Sdim return std::make_pair(SDValue(), CLI.Chain); 7165199989Srdivacky 7166239462Sdim SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, 7167239462Sdim CLI.DAG.getVTList(&RetTys[0], RetTys.size()), 7168199989Srdivacky &ReturnValues[0], ReturnValues.size()); 7169239462Sdim return std::make_pair(Res, CLI.Chain); 7170199989Srdivacky} 7171199989Srdivacky 7172199989Srdivackyvoid TargetLowering::LowerOperationWrapper(SDNode *N, 7173199989Srdivacky SmallVectorImpl<SDValue> &Results, 7174207618Srdivacky SelectionDAG &DAG) const { 7175199989Srdivacky SDValue Res = LowerOperation(SDValue(N, 0), DAG); 7176199989Srdivacky if (Res.getNode()) 7177199989Srdivacky Results.push_back(Res); 7178199989Srdivacky} 7179199989Srdivacky 7180207618SrdivackySDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 7181199989Srdivacky llvm_unreachable("LowerOperation not implemented for this target!"); 7182199989Srdivacky} 7183199989Srdivacky 7184207618Srdivackyvoid 7185207618SrdivackySelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) { 7186210299Sed SDValue Op = getNonRegisterValue(V); 7187199989Srdivacky assert((Op.getOpcode() != ISD::CopyFromReg || 7188199989Srdivacky cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && 7189199989Srdivacky "Copy from a reg to the same reg!"); 7190199989Srdivacky assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg"); 7191199989Srdivacky 7192263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 7193263508Sdim RegsForValue RFV(V->getContext(), *TLI, Reg, V->getType()); 7194199989Srdivacky SDValue Chain = DAG.getEntryNode(); 7195263508Sdim RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, 0, V); 7196199989Srdivacky PendingExports.push_back(Chain); 7197199989Srdivacky} 7198199989Srdivacky 7199199989Srdivacky#include "llvm/CodeGen/SelectionDAGISel.h" 7200199989Srdivacky 7201223017Sdim/// isOnlyUsedInEntryBlock - If the specified argument is only used in the 7202223017Sdim/// entry block, return true. This includes arguments used by switches, since 7203223017Sdim/// the switch may expand into multiple basic blocks. 7204234353Sdimstatic bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { 7205223017Sdim // With FastISel active, we may be splitting blocks, so force creation 7206223017Sdim // of virtual registers for all non-dead arguments. 7207234353Sdim if (FastISel) 7208223017Sdim return A->use_empty(); 7209223017Sdim 7210223017Sdim const BasicBlock *Entry = A->getParent()->begin(); 7211223017Sdim for (Value::const_use_iterator UI = A->use_begin(), E = A->use_end(); 7212223017Sdim UI != E; ++UI) { 7213223017Sdim const User *U = *UI; 7214223017Sdim if (cast<Instruction>(U)->getParent() != Entry || isa<SwitchInst>(U)) 7215223017Sdim return false; // Use not in entry block. 7216223017Sdim } 7217223017Sdim return true; 7218223017Sdim} 7219223017Sdim 7220249423Sdimvoid SelectionDAGISel::LowerArguments(const Function &F) { 7221199989Srdivacky SelectionDAG &DAG = SDB->DAG; 7222263508Sdim SDLoc dl = SDB->getCurSDLoc(); 7223263508Sdim const TargetLowering *TLI = getTargetLowering(); 7224263508Sdim const DataLayout *TD = TLI->getDataLayout(); 7225199989Srdivacky SmallVector<ISD::InputArg, 16> Ins; 7226199989Srdivacky 7227210299Sed if (!FuncInfo->CanLowerReturn) { 7228199989Srdivacky // Put in an sret pointer parameter before all the other parameters. 7229199989Srdivacky SmallVector<EVT, 1> ValueVTs; 7230263508Sdim ComputeValueVTs(*getTargetLowering(), 7231263508Sdim PointerType::getUnqual(F.getReturnType()), ValueVTs); 7232199989Srdivacky 7233199989Srdivacky // NOTE: Assuming that a pointer will never break down to more than one VT 7234199989Srdivacky // or one register. 7235199989Srdivacky ISD::ArgFlagsTy Flags; 7236199989Srdivacky Flags.setSRet(); 7237263508Sdim MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); 7238263508Sdim ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true, 0, 0); 7239199989Srdivacky Ins.push_back(RetArg); 7240199989Srdivacky } 7241199989Srdivacky 7242199989Srdivacky // Set up the incoming argument description vector. 7243199989Srdivacky unsigned Idx = 1; 7244207618Srdivacky for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); 7245199989Srdivacky I != E; ++I, ++Idx) { 7246199989Srdivacky SmallVector<EVT, 4> ValueVTs; 7247263508Sdim ComputeValueVTs(*TLI, I->getType(), ValueVTs); 7248199989Srdivacky bool isArgValueUsed = !I->use_empty(); 7249263508Sdim unsigned PartBase = 0; 7250199989Srdivacky for (unsigned Value = 0, NumValues = ValueVTs.size(); 7251199989Srdivacky Value != NumValues; ++Value) { 7252199989Srdivacky EVT VT = ValueVTs[Value]; 7253226633Sdim Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); 7254199989Srdivacky ISD::ArgFlagsTy Flags; 7255199989Srdivacky unsigned OriginalAlignment = 7256199989Srdivacky TD->getABITypeAlignment(ArgTy); 7257199989Srdivacky 7258249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) 7259199989Srdivacky Flags.setZExt(); 7260249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) 7261199989Srdivacky Flags.setSExt(); 7262249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::InReg)) 7263199989Srdivacky Flags.setInReg(); 7264249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::StructRet)) 7265199989Srdivacky Flags.setSRet(); 7266249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal)) { 7267199989Srdivacky Flags.setByVal(); 7268226633Sdim PointerType *Ty = cast<PointerType>(I->getType()); 7269226633Sdim Type *ElementTy = Ty->getElementType(); 7270223017Sdim Flags.setByValSize(TD->getTypeAllocSize(ElementTy)); 7271199989Srdivacky // For ByVal, alignment should be passed from FE. BE will guess if 7272199989Srdivacky // this info is not there but there are cases it cannot get right. 7273223017Sdim unsigned FrameAlign; 7274199989Srdivacky if (F.getParamAlignment(Idx)) 7275199989Srdivacky FrameAlign = F.getParamAlignment(Idx); 7276223017Sdim else 7277263508Sdim FrameAlign = TLI->getByValTypeAlignment(ElementTy); 7278199989Srdivacky Flags.setByValAlign(FrameAlign); 7279199989Srdivacky } 7280249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::Nest)) 7281199989Srdivacky Flags.setNest(); 7282199989Srdivacky Flags.setOrigAlign(OriginalAlignment); 7283199989Srdivacky 7284263508Sdim MVT RegisterVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7285263508Sdim unsigned NumRegs = TLI->getNumRegisters(*CurDAG->getContext(), VT); 7286199989Srdivacky for (unsigned i = 0; i != NumRegs; ++i) { 7287263508Sdim ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed, 7288263508Sdim Idx-1, PartBase+i*RegisterVT.getStoreSize()); 7289199989Srdivacky if (NumRegs > 1 && i == 0) 7290199989Srdivacky MyFlags.Flags.setSplit(); 7291199989Srdivacky // if it isn't first piece, alignment must be 1 7292199989Srdivacky else if (i > 0) 7293199989Srdivacky MyFlags.Flags.setOrigAlign(1); 7294199989Srdivacky Ins.push_back(MyFlags); 7295199989Srdivacky } 7296263508Sdim PartBase += VT.getStoreSize(); 7297199989Srdivacky } 7298199989Srdivacky } 7299199989Srdivacky 7300199989Srdivacky // Call the target to set up the argument values. 7301199989Srdivacky SmallVector<SDValue, 8> InVals; 7302263508Sdim SDValue NewRoot = TLI->LowerFormalArguments(DAG.getRoot(), F.getCallingConv(), 7303263508Sdim F.isVarArg(), Ins, 7304263508Sdim dl, DAG, InVals); 7305199989Srdivacky 7306199989Srdivacky // Verify that the target's LowerFormalArguments behaved as expected. 7307199989Srdivacky assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && 7308199989Srdivacky "LowerFormalArguments didn't return a valid chain!"); 7309199989Srdivacky assert(InVals.size() == Ins.size() && 7310199989Srdivacky "LowerFormalArguments didn't emit the correct number of values!"); 7311201360Srdivacky DEBUG({ 7312201360Srdivacky for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 7313201360Srdivacky assert(InVals[i].getNode() && 7314201360Srdivacky "LowerFormalArguments emitted a null value!"); 7315218893Sdim assert(EVT(Ins[i].VT) == InVals[i].getValueType() && 7316201360Srdivacky "LowerFormalArguments emitted a value with the wrong type!"); 7317201360Srdivacky } 7318201360Srdivacky }); 7319199989Srdivacky 7320199989Srdivacky // Update the DAG with the new chain value resulting from argument lowering. 7321199989Srdivacky DAG.setRoot(NewRoot); 7322199989Srdivacky 7323199989Srdivacky // Set up the argument values. 7324199989Srdivacky unsigned i = 0; 7325199989Srdivacky Idx = 1; 7326210299Sed if (!FuncInfo->CanLowerReturn) { 7327199989Srdivacky // Create a virtual register for the sret pointer, and put in a copy 7328199989Srdivacky // from the sret argument into it. 7329199989Srdivacky SmallVector<EVT, 1> ValueVTs; 7330263508Sdim ComputeValueVTs(*TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs); 7331249423Sdim MVT VT = ValueVTs[0].getSimpleVT(); 7332263508Sdim MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7333199989Srdivacky ISD::NodeType AssertOp = ISD::DELETED_NODE; 7334204642Srdivacky SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, 7335243830Sdim RegVT, VT, NULL, AssertOp); 7336199989Srdivacky 7337199989Srdivacky MachineFunction& MF = SDB->DAG.getMachineFunction(); 7338199989Srdivacky MachineRegisterInfo& RegInfo = MF.getRegInfo(); 7339263508Sdim unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); 7340210299Sed FuncInfo->DemoteRegister = SRetReg; 7341263508Sdim NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), 7342201360Srdivacky SRetReg, ArgValue); 7343199989Srdivacky DAG.setRoot(NewRoot); 7344201360Srdivacky 7345199989Srdivacky // i indexes lowered arguments. Bump it past the hidden sret argument. 7346199989Srdivacky // Idx indexes LLVM arguments. Don't touch it. 7347199989Srdivacky ++i; 7348199989Srdivacky } 7349201360Srdivacky 7350207618Srdivacky for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; 7351199989Srdivacky ++I, ++Idx) { 7352199989Srdivacky SmallVector<SDValue, 4> ArgValues; 7353199989Srdivacky SmallVector<EVT, 4> ValueVTs; 7354263508Sdim ComputeValueVTs(*TLI, I->getType(), ValueVTs); 7355199989Srdivacky unsigned NumValues = ValueVTs.size(); 7356210299Sed 7357210299Sed // If this argument is unused then remember its value. It is used to generate 7358210299Sed // debugging information. 7359263508Sdim if (I->use_empty() && NumValues) { 7360210299Sed SDB->setUnusedArgValue(I, InVals[i]); 7361210299Sed 7362263508Sdim // Also remember any frame index for use in FastISel. 7363263508Sdim if (FrameIndexSDNode *FI = 7364263508Sdim dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) 7365263508Sdim FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7366263508Sdim } 7367263508Sdim 7368223017Sdim for (unsigned Val = 0; Val != NumValues; ++Val) { 7369223017Sdim EVT VT = ValueVTs[Val]; 7370263508Sdim MVT PartVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7371263508Sdim unsigned NumParts = TLI->getNumRegisters(*CurDAG->getContext(), VT); 7372199989Srdivacky 7373199989Srdivacky if (!I->use_empty()) { 7374199989Srdivacky ISD::NodeType AssertOp = ISD::DELETED_NODE; 7375249423Sdim if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) 7376199989Srdivacky AssertOp = ISD::AssertSext; 7377249423Sdim else if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) 7378199989Srdivacky AssertOp = ISD::AssertZext; 7379199989Srdivacky 7380204642Srdivacky ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], 7381201360Srdivacky NumParts, PartVT, VT, 7382243830Sdim NULL, AssertOp)); 7383199989Srdivacky } 7384201360Srdivacky 7385199989Srdivacky i += NumParts; 7386199989Srdivacky } 7387201360Srdivacky 7388223017Sdim // We don't need to do anything else for unused arguments. 7389223017Sdim if (ArgValues.empty()) 7390223017Sdim continue; 7391223017Sdim 7392226633Sdim // Note down frame index. 7393226633Sdim if (FrameIndexSDNode *FI = 7394239462Sdim dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) 7395226633Sdim FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7396212904Sdim 7397223017Sdim SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues, 7398263508Sdim SDB->getCurSDLoc()); 7399226633Sdim 7400223017Sdim SDB->setValue(I, Res); 7401234353Sdim if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { 7402263508Sdim if (LoadSDNode *LNode = 7403226633Sdim dyn_cast<LoadSDNode>(Res.getOperand(0).getNode())) 7404226633Sdim if (FrameIndexSDNode *FI = 7405226633Sdim dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 7406226633Sdim FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7407226633Sdim } 7408201360Srdivacky 7409223017Sdim // If this argument is live outside of the entry block, insert a copy from 7410223017Sdim // wherever we got it to the vreg that other BB's will reference it as. 7411234353Sdim if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) { 7412223017Sdim // If we can, though, try to skip creating an unnecessary vreg. 7413223017Sdim // FIXME: This isn't very clean... it would be nice to make this more 7414223017Sdim // general. It's also subtly incompatible with the hacks FastISel 7415223017Sdim // uses with vregs. 7416223017Sdim unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); 7417223017Sdim if (TargetRegisterInfo::isVirtualRegister(Reg)) { 7418223017Sdim FuncInfo->ValueMap[I] = Reg; 7419223017Sdim continue; 7420223017Sdim } 7421223017Sdim } 7422234353Sdim if (!isOnlyUsedInEntryBlock(I, TM.Options.EnableFastISel)) { 7423223017Sdim FuncInfo->InitializeRegForValue(I); 7424199989Srdivacky SDB->CopyToExportRegsIfNeeded(I); 7425199989Srdivacky } 7426199989Srdivacky } 7427201360Srdivacky 7428199989Srdivacky assert(i == InVals.size() && "Argument register count mismatch!"); 7429199989Srdivacky 7430199989Srdivacky // Finally, if the target has anything special to do, allow it to do so. 7431199989Srdivacky // FIXME: this should insert code into the DAG! 7432207618Srdivacky EmitFunctionEntryCode(); 7433199989Srdivacky} 7434199989Srdivacky 7435199989Srdivacky/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to 7436199989Srdivacky/// ensure constants are generated when needed. Remember the virtual registers 7437199989Srdivacky/// that need to be added to the Machine PHI nodes as input. We cannot just 7438199989Srdivacky/// directly add them, because expansion might result in multiple MBB's for one 7439199989Srdivacky/// BB. As such, the start of the BB might correspond to a different MBB than 7440199989Srdivacky/// the end. 7441199989Srdivacky/// 7442199989Srdivackyvoid 7443207618SrdivackySelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { 7444207618Srdivacky const TerminatorInst *TI = LLVMBB->getTerminator(); 7445199989Srdivacky 7446199989Srdivacky SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; 7447199989Srdivacky 7448199989Srdivacky // Check successor nodes' PHI nodes that expect a constant to be available 7449199989Srdivacky // from this block. 7450199989Srdivacky for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { 7451207618Srdivacky const BasicBlock *SuccBB = TI->getSuccessor(succ); 7452199989Srdivacky if (!isa<PHINode>(SuccBB->begin())) continue; 7453207618Srdivacky MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; 7454199989Srdivacky 7455199989Srdivacky // If this terminator has multiple identical successors (common for 7456199989Srdivacky // switches), only handle each succ once. 7457199989Srdivacky if (!SuccsHandled.insert(SuccMBB)) continue; 7458199989Srdivacky 7459199989Srdivacky MachineBasicBlock::iterator MBBI = SuccMBB->begin(); 7460199989Srdivacky 7461199989Srdivacky // At this point we know that there is a 1-1 correspondence between LLVM PHI 7462199989Srdivacky // nodes and Machine PHI nodes, but the incoming operands have not been 7463199989Srdivacky // emitted yet. 7464207618Srdivacky for (BasicBlock::const_iterator I = SuccBB->begin(); 7465207618Srdivacky const PHINode *PN = dyn_cast<PHINode>(I); ++I) { 7466199989Srdivacky // Ignore dead phi's. 7467199989Srdivacky if (PN->use_empty()) continue; 7468199989Srdivacky 7469223017Sdim // Skip empty types 7470223017Sdim if (PN->getType()->isEmptyTy()) 7471223017Sdim continue; 7472223017Sdim 7473199989Srdivacky unsigned Reg; 7474207618Srdivacky const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); 7475199989Srdivacky 7476207618Srdivacky if (const Constant *C = dyn_cast<Constant>(PHIOp)) { 7477207618Srdivacky unsigned &RegOut = ConstantsOut[C]; 7478199989Srdivacky if (RegOut == 0) { 7479210299Sed RegOut = FuncInfo.CreateRegs(C->getType()); 7480207618Srdivacky CopyValueToVirtualRegister(C, RegOut); 7481199989Srdivacky } 7482199989Srdivacky Reg = RegOut; 7483199989Srdivacky } else { 7484210299Sed DenseMap<const Value *, unsigned>::iterator I = 7485210299Sed FuncInfo.ValueMap.find(PHIOp); 7486210299Sed if (I != FuncInfo.ValueMap.end()) 7487210299Sed Reg = I->second; 7488210299Sed else { 7489199989Srdivacky assert(isa<AllocaInst>(PHIOp) && 7490207618Srdivacky FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && 7491199989Srdivacky "Didn't codegen value into a register!??"); 7492210299Sed Reg = FuncInfo.CreateRegs(PHIOp->getType()); 7493207618Srdivacky CopyValueToVirtualRegister(PHIOp, Reg); 7494199989Srdivacky } 7495199989Srdivacky } 7496199989Srdivacky 7497199989Srdivacky // Remember that this register needs to added to the machine PHI node as 7498199989Srdivacky // the input for this MBB. 7499199989Srdivacky SmallVector<EVT, 4> ValueVTs; 7500263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 7501263508Sdim ComputeValueVTs(*TLI, PN->getType(), ValueVTs); 7502199989Srdivacky for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { 7503199989Srdivacky EVT VT = ValueVTs[vti]; 7504263508Sdim unsigned NumRegisters = TLI->getNumRegisters(*DAG.getContext(), VT); 7505199989Srdivacky for (unsigned i = 0, e = NumRegisters; i != e; ++i) 7506207618Srdivacky FuncInfo.PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); 7507199989Srdivacky Reg += NumRegisters; 7508199989Srdivacky } 7509199989Srdivacky } 7510199989Srdivacky } 7511263508Sdim 7512207618Srdivacky ConstantsOut.clear(); 7513199989Srdivacky} 7514263508Sdim 7515263508Sdim/// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB 7516263508Sdim/// is 0. 7517263508SdimMachineBasicBlock * 7518263508SdimSelectionDAGBuilder::StackProtectorDescriptor:: 7519263508SdimAddSuccessorMBB(const BasicBlock *BB, 7520263508Sdim MachineBasicBlock *ParentMBB, 7521263508Sdim MachineBasicBlock *SuccMBB) { 7522263508Sdim // If SuccBB has not been created yet, create it. 7523263508Sdim if (!SuccMBB) { 7524263508Sdim MachineFunction *MF = ParentMBB->getParent(); 7525263508Sdim MachineFunction::iterator BBI = ParentMBB; 7526263508Sdim SuccMBB = MF->CreateMachineBasicBlock(BB); 7527263508Sdim MF->insert(++BBI, SuccMBB); 7528263508Sdim } 7529263508Sdim // Add it as a successor of ParentMBB. 7530263508Sdim ParentMBB->addSuccessor(SuccMBB); 7531263508Sdim return SuccMBB; 7532263508Sdim} 7533