1//===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements routines for translating from LLVM IR into SelectionDAG IR. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "isel" 15#include "SelectionDAGBuilder.h" 16#include "SDNodeDbgValue.h" 17#include "llvm/ADT/BitVector.h" 18#include "llvm/ADT/Optional.h" 19#include "llvm/ADT/SmallSet.h" 20#include "llvm/Analysis/AliasAnalysis.h" 21#include "llvm/Analysis/BranchProbabilityInfo.h" 22#include "llvm/Analysis/ConstantFolding.h" 23#include "llvm/Analysis/ValueTracking.h" 24#include "llvm/CodeGen/Analysis.h" 25#include "llvm/CodeGen/FastISel.h" 26#include "llvm/CodeGen/FunctionLoweringInfo.h" 27#include "llvm/CodeGen/GCMetadata.h" 28#include "llvm/CodeGen/GCStrategy.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineJumpTableInfo.h" 33#include "llvm/CodeGen/MachineModuleInfo.h" 34#include "llvm/CodeGen/MachineRegisterInfo.h" 35#include "llvm/CodeGen/SelectionDAG.h" 36#include "llvm/CodeGen/StackMaps.h" 37#include "llvm/DebugInfo.h" 38#include "llvm/IR/CallingConv.h" 39#include "llvm/IR/Constants.h" 40#include "llvm/IR/DataLayout.h" 41#include "llvm/IR/DerivedTypes.h" 42#include "llvm/IR/Function.h" 43#include "llvm/IR/GlobalVariable.h" 44#include "llvm/IR/InlineAsm.h" 45#include "llvm/IR/Instructions.h" 46#include "llvm/IR/IntrinsicInst.h" 47#include "llvm/IR/Intrinsics.h" 48#include "llvm/IR/LLVMContext.h" 49#include "llvm/IR/Module.h" 50#include "llvm/Support/CommandLine.h" 51#include "llvm/Support/Debug.h" 52#include "llvm/Support/ErrorHandling.h" 53#include "llvm/Support/MathExtras.h" 54#include "llvm/Support/raw_ostream.h" 55#include "llvm/Target/TargetFrameLowering.h" 56#include "llvm/Target/TargetInstrInfo.h" 57#include "llvm/Target/TargetIntrinsicInfo.h" 58#include "llvm/Target/TargetLibraryInfo.h" 59#include "llvm/Target/TargetLowering.h" 60#include "llvm/Target/TargetOptions.h" 61#include "llvm/Target/TargetSelectionDAGInfo.h" 62#include <algorithm> 63using namespace llvm; 64 65/// LimitFloatPrecision - Generate low-precision inline sequences for 66/// some float libcalls (6, 8 or 12 bits). 67static unsigned LimitFloatPrecision; 68 69static cl::opt<unsigned, true> 70LimitFPPrecision("limit-float-precision", 71 cl::desc("Generate low-precision inline sequences " 72 "for some float libcalls"), 73 cl::location(LimitFloatPrecision), 74 cl::init(0)); 75 76// Limit the width of DAG chains. This is important in general to prevent 77// prevent DAG-based analysis from blowing up. For example, alias analysis and 78// load clustering may not complete in reasonable time. It is difficult to 79// recognize and avoid this situation within each individual analysis, and 80// future analyses are likely to have the same behavior. Limiting DAG width is 81// the safe approach, and will be especially important with global DAGs. 82// 83// MaxParallelChains default is arbitrarily high to avoid affecting 84// optimization, but could be lowered to improve compile time. Any ld-ld-st-st 85// sequence over this should have been converted to llvm.memcpy by the 86// frontend. It easy to induce this behavior with .ll code such as: 87// %buffer = alloca [4096 x i8] 88// %data = load [4096 x i8]* %argPtr 89// store [4096 x i8] %data, [4096 x i8]* %buffer 90static const unsigned MaxParallelChains = 64; 91 92static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, 93 const SDValue *Parts, unsigned NumParts, 94 MVT PartVT, EVT ValueVT, const Value *V); 95 96/// getCopyFromParts - Create a value that contains the specified legal parts 97/// combined into the value they represent. If the parts combine to a type 98/// larger then ValueVT then AssertOp can be used to specify whether the extra 99/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT 100/// (ISD::AssertSext). 101static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL, 102 const SDValue *Parts, 103 unsigned NumParts, MVT PartVT, EVT ValueVT, 104 const Value *V, 105 ISD::NodeType AssertOp = ISD::DELETED_NODE) { 106 if (ValueVT.isVector()) 107 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, 108 PartVT, ValueVT, V); 109 110 assert(NumParts > 0 && "No parts to assemble!"); 111 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 112 SDValue Val = Parts[0]; 113 114 if (NumParts > 1) { 115 // Assemble the value from multiple parts. 116 if (ValueVT.isInteger()) { 117 unsigned PartBits = PartVT.getSizeInBits(); 118 unsigned ValueBits = ValueVT.getSizeInBits(); 119 120 // Assemble the power of 2 part. 121 unsigned RoundParts = NumParts & (NumParts - 1) ? 122 1 << Log2_32(NumParts) : NumParts; 123 unsigned RoundBits = PartBits * RoundParts; 124 EVT RoundVT = RoundBits == ValueBits ? 125 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); 126 SDValue Lo, Hi; 127 128 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); 129 130 if (RoundParts > 2) { 131 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, 132 PartVT, HalfVT, V); 133 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, 134 RoundParts / 2, PartVT, HalfVT, V); 135 } else { 136 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); 137 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); 138 } 139 140 if (TLI.isBigEndian()) 141 std::swap(Lo, Hi); 142 143 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); 144 145 if (RoundParts < NumParts) { 146 // Assemble the trailing non-power-of-2 part. 147 unsigned OddParts = NumParts - RoundParts; 148 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); 149 Hi = getCopyFromParts(DAG, DL, 150 Parts + RoundParts, OddParts, PartVT, OddVT, V); 151 152 // Combine the round and odd parts. 153 Lo = Val; 154 if (TLI.isBigEndian()) 155 std::swap(Lo, Hi); 156 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 157 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); 158 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi, 159 DAG.getConstant(Lo.getValueType().getSizeInBits(), 160 TLI.getPointerTy())); 161 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); 162 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); 163 } 164 } else if (PartVT.isFloatingPoint()) { 165 // FP split into multiple FP parts (for ppcf128) 166 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && 167 "Unexpected split"); 168 SDValue Lo, Hi; 169 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); 170 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); 171 if (TLI.isBigEndian()) 172 std::swap(Lo, Hi); 173 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); 174 } else { 175 // FP split into integer parts (soft fp) 176 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && 177 !PartVT.isVector() && "Unexpected split"); 178 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); 179 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V); 180 } 181 } 182 183 // There is now one part, held in Val. Correct it to match ValueVT. 184 EVT PartEVT = Val.getValueType(); 185 186 if (PartEVT == ValueVT) 187 return Val; 188 189 if (PartEVT.isInteger() && ValueVT.isInteger()) { 190 if (ValueVT.bitsLT(PartEVT)) { 191 // For a truncate, see if we have any information to 192 // indicate whether the truncated bits will always be 193 // zero or sign-extension. 194 if (AssertOp != ISD::DELETED_NODE) 195 Val = DAG.getNode(AssertOp, DL, PartEVT, Val, 196 DAG.getValueType(ValueVT)); 197 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 198 } 199 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); 200 } 201 202 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 203 // FP_ROUND's are always exact here. 204 if (ValueVT.bitsLT(Val.getValueType())) 205 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, 206 DAG.getTargetConstant(1, TLI.getPointerTy())); 207 208 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); 209 } 210 211 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) 212 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 213 214 llvm_unreachable("Unknown mismatch!"); 215} 216 217/// getCopyFromPartsVector - Create a value that contains the specified legal 218/// parts combined into the value they represent. If the parts combine to a 219/// type larger then ValueVT then AssertOp can be used to specify whether the 220/// extra bits are known to be zero (ISD::AssertZext) or sign extended from 221/// ValueVT (ISD::AssertSext). 222static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL, 223 const SDValue *Parts, unsigned NumParts, 224 MVT PartVT, EVT ValueVT, const Value *V) { 225 assert(ValueVT.isVector() && "Not a vector value"); 226 assert(NumParts > 0 && "No parts to assemble!"); 227 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 228 SDValue Val = Parts[0]; 229 230 // Handle a multi-element vector. 231 if (NumParts > 1) { 232 EVT IntermediateVT; 233 MVT RegisterVT; 234 unsigned NumIntermediates; 235 unsigned NumRegs = 236 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, 237 NumIntermediates, RegisterVT); 238 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 239 NumParts = NumRegs; // Silence a compiler warning. 240 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 241 assert(RegisterVT == Parts[0].getSimpleValueType() && 242 "Part type doesn't match part!"); 243 244 // Assemble the parts into intermediate operands. 245 SmallVector<SDValue, 8> Ops(NumIntermediates); 246 if (NumIntermediates == NumParts) { 247 // If the register was not expanded, truncate or copy the value, 248 // as appropriate. 249 for (unsigned i = 0; i != NumParts; ++i) 250 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, 251 PartVT, IntermediateVT, V); 252 } else if (NumParts > 0) { 253 // If the intermediate type was expanded, build the intermediate 254 // operands from the parts. 255 assert(NumParts % NumIntermediates == 0 && 256 "Must expand into a divisible number of parts!"); 257 unsigned Factor = NumParts / NumIntermediates; 258 for (unsigned i = 0; i != NumIntermediates; ++i) 259 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, 260 PartVT, IntermediateVT, V); 261 } 262 263 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the 264 // intermediate operands. 265 Val = DAG.getNode(IntermediateVT.isVector() ? 266 ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, DL, 267 ValueVT, &Ops[0], NumIntermediates); 268 } 269 270 // There is now one part, held in Val. Correct it to match ValueVT. 271 EVT PartEVT = Val.getValueType(); 272 273 if (PartEVT == ValueVT) 274 return Val; 275 276 if (PartEVT.isVector()) { 277 // If the element type of the source/dest vectors are the same, but the 278 // parts vector has more elements than the value vector, then we have a 279 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the 280 // elements we want. 281 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) { 282 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && 283 "Cannot narrow, it would be a lossy transformation"); 284 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, 285 DAG.getConstant(0, TLI.getVectorIdxTy())); 286 } 287 288 // Vector/Vector bitcast. 289 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) 290 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 291 292 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && 293 "Cannot handle this kind of promotion"); 294 // Promoted vector extract 295 bool Smaller = ValueVT.bitsLE(PartEVT); 296 return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 297 DL, ValueVT, Val); 298 299 } 300 301 // Trivial bitcast if the types are the same size and the destination 302 // vector type is legal. 303 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && 304 TLI.isTypeLegal(ValueVT)) 305 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 306 307 // Handle cases such as i8 -> <1 x i1> 308 if (ValueVT.getVectorNumElements() != 1) { 309 LLVMContext &Ctx = *DAG.getContext(); 310 Twine ErrMsg("non-trivial scalar-to-vector conversion"); 311 if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) { 312 if (const CallInst *CI = dyn_cast<CallInst>(I)) 313 if (isa<InlineAsm>(CI->getCalledValue())) 314 ErrMsg = ErrMsg + ", possible invalid constraint for vector type"; 315 Ctx.emitError(I, ErrMsg); 316 } else { 317 Ctx.emitError(ErrMsg); 318 } 319 return DAG.getUNDEF(ValueVT); 320 } 321 322 if (ValueVT.getVectorNumElements() == 1 && 323 ValueVT.getVectorElementType() != PartEVT) { 324 bool Smaller = ValueVT.bitsLE(PartEVT); 325 Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 326 DL, ValueVT.getScalarType(), Val); 327 } 328 329 return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val); 330} 331 332static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl, 333 SDValue Val, SDValue *Parts, unsigned NumParts, 334 MVT PartVT, const Value *V); 335 336/// getCopyToParts - Create a series of nodes that contain the specified value 337/// split into legal parts. If the parts contain more bits than Val, then, for 338/// integers, ExtendKind can be used to specify how to generate the extra bits. 339static void getCopyToParts(SelectionDAG &DAG, SDLoc DL, 340 SDValue Val, SDValue *Parts, unsigned NumParts, 341 MVT PartVT, const Value *V, 342 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { 343 EVT ValueVT = Val.getValueType(); 344 345 // Handle the vector case separately. 346 if (ValueVT.isVector()) 347 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V); 348 349 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 350 unsigned PartBits = PartVT.getSizeInBits(); 351 unsigned OrigNumParts = NumParts; 352 assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!"); 353 354 if (NumParts == 0) 355 return; 356 357 assert(!ValueVT.isVector() && "Vector case handled elsewhere"); 358 EVT PartEVT = PartVT; 359 if (PartEVT == ValueVT) { 360 assert(NumParts == 1 && "No-op copy with multiple parts!"); 361 Parts[0] = Val; 362 return; 363 } 364 365 if (NumParts * PartBits > ValueVT.getSizeInBits()) { 366 // If the parts cover more bits than the value has, promote the value. 367 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 368 assert(NumParts == 1 && "Do not know what to promote to!"); 369 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); 370 } else { 371 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 372 ValueVT.isInteger() && 373 "Unknown mismatch!"); 374 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 375 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); 376 if (PartVT == MVT::x86mmx) 377 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 378 } 379 } else if (PartBits == ValueVT.getSizeInBits()) { 380 // Different types of the same size. 381 assert(NumParts == 1 && PartEVT != ValueVT); 382 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 383 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { 384 // If the parts cover less bits than value has, truncate the value. 385 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 386 ValueVT.isInteger() && 387 "Unknown mismatch!"); 388 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 389 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 390 if (PartVT == MVT::x86mmx) 391 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 392 } 393 394 // The value may have changed - recompute ValueVT. 395 ValueVT = Val.getValueType(); 396 assert(NumParts * PartBits == ValueVT.getSizeInBits() && 397 "Failed to tile the value with PartVT!"); 398 399 if (NumParts == 1) { 400 if (PartEVT != ValueVT) { 401 LLVMContext &Ctx = *DAG.getContext(); 402 Twine ErrMsg("scalar-to-vector conversion failed"); 403 if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) { 404 if (const CallInst *CI = dyn_cast<CallInst>(I)) 405 if (isa<InlineAsm>(CI->getCalledValue())) 406 ErrMsg = ErrMsg + ", possible invalid constraint for vector type"; 407 Ctx.emitError(I, ErrMsg); 408 } else { 409 Ctx.emitError(ErrMsg); 410 } 411 } 412 413 Parts[0] = Val; 414 return; 415 } 416 417 // Expand the value into multiple parts. 418 if (NumParts & (NumParts - 1)) { 419 // The number of parts is not a power of 2. Split off and copy the tail. 420 assert(PartVT.isInteger() && ValueVT.isInteger() && 421 "Do not know what to expand to!"); 422 unsigned RoundParts = 1 << Log2_32(NumParts); 423 unsigned RoundBits = RoundParts * PartBits; 424 unsigned OddParts = NumParts - RoundParts; 425 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, 426 DAG.getIntPtrConstant(RoundBits)); 427 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V); 428 429 if (TLI.isBigEndian()) 430 // The odd parts were reversed by getCopyToParts - unreverse them. 431 std::reverse(Parts + RoundParts, Parts + NumParts); 432 433 NumParts = RoundParts; 434 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 435 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 436 } 437 438 // The number of parts is a power of 2. Repeatedly bisect the value using 439 // EXTRACT_ELEMENT. 440 Parts[0] = DAG.getNode(ISD::BITCAST, DL, 441 EVT::getIntegerVT(*DAG.getContext(), 442 ValueVT.getSizeInBits()), 443 Val); 444 445 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { 446 for (unsigned i = 0; i < NumParts; i += StepSize) { 447 unsigned ThisBits = StepSize * PartBits / 2; 448 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); 449 SDValue &Part0 = Parts[i]; 450 SDValue &Part1 = Parts[i+StepSize/2]; 451 452 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 453 ThisVT, Part0, DAG.getIntPtrConstant(1)); 454 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 455 ThisVT, Part0, DAG.getIntPtrConstant(0)); 456 457 if (ThisBits == PartBits && ThisVT != PartVT) { 458 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); 459 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); 460 } 461 } 462 } 463 464 if (TLI.isBigEndian()) 465 std::reverse(Parts, Parts + OrigNumParts); 466} 467 468 469/// getCopyToPartsVector - Create a series of nodes that contain the specified 470/// value split into legal parts. 471static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL, 472 SDValue Val, SDValue *Parts, unsigned NumParts, 473 MVT PartVT, const Value *V) { 474 EVT ValueVT = Val.getValueType(); 475 assert(ValueVT.isVector() && "Not a vector"); 476 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 477 478 if (NumParts == 1) { 479 EVT PartEVT = PartVT; 480 if (PartEVT == ValueVT) { 481 // Nothing to do. 482 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { 483 // Bitconvert vector->vector case. 484 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 485 } else if (PartVT.isVector() && 486 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() && 487 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) { 488 EVT ElementVT = PartVT.getVectorElementType(); 489 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in 490 // undef elements. 491 SmallVector<SDValue, 16> Ops; 492 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i) 493 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 494 ElementVT, Val, DAG.getConstant(i, 495 TLI.getVectorIdxTy()))); 496 497 for (unsigned i = ValueVT.getVectorNumElements(), 498 e = PartVT.getVectorNumElements(); i != e; ++i) 499 Ops.push_back(DAG.getUNDEF(ElementVT)); 500 501 Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, &Ops[0], Ops.size()); 502 503 // FIXME: Use CONCAT for 2x -> 4x. 504 505 //SDValue UndefElts = DAG.getUNDEF(VectorTy); 506 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts); 507 } else if (PartVT.isVector() && 508 PartEVT.getVectorElementType().bitsGE( 509 ValueVT.getVectorElementType()) && 510 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { 511 512 // Promoted vector extract 513 bool Smaller = PartEVT.bitsLE(ValueVT); 514 Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 515 DL, PartVT, Val); 516 } else{ 517 // Vector -> scalar conversion. 518 assert(ValueVT.getVectorNumElements() == 1 && 519 "Only trivial vector-to-scalar conversions should get here!"); 520 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 521 PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy())); 522 523 bool Smaller = ValueVT.bitsLE(PartVT); 524 Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), 525 DL, PartVT, Val); 526 } 527 528 Parts[0] = Val; 529 return; 530 } 531 532 // Handle a multi-element vector. 533 EVT IntermediateVT; 534 MVT RegisterVT; 535 unsigned NumIntermediates; 536 unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, 537 IntermediateVT, 538 NumIntermediates, RegisterVT); 539 unsigned NumElements = ValueVT.getVectorNumElements(); 540 541 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 542 NumParts = NumRegs; // Silence a compiler warning. 543 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 544 545 // Split the vector into intermediate operands. 546 SmallVector<SDValue, 8> Ops(NumIntermediates); 547 for (unsigned i = 0; i != NumIntermediates; ++i) { 548 if (IntermediateVT.isVector()) 549 Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, 550 IntermediateVT, Val, 551 DAG.getConstant(i * (NumElements / NumIntermediates), 552 TLI.getVectorIdxTy())); 553 else 554 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, 555 IntermediateVT, Val, 556 DAG.getConstant(i, TLI.getVectorIdxTy())); 557 } 558 559 // Split the intermediate operands into legal parts. 560 if (NumParts == NumIntermediates) { 561 // If the register was not expanded, promote or copy the value, 562 // as appropriate. 563 for (unsigned i = 0; i != NumParts; ++i) 564 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V); 565 } else if (NumParts > 0) { 566 // If the intermediate type was expanded, split each the value into 567 // legal parts. 568 assert(NumParts % NumIntermediates == 0 && 569 "Must expand into a divisible number of parts!"); 570 unsigned Factor = NumParts / NumIntermediates; 571 for (unsigned i = 0; i != NumIntermediates; ++i) 572 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V); 573 } 574} 575 576namespace { 577 /// RegsForValue - This struct represents the registers (physical or virtual) 578 /// that a particular set of values is assigned, and the type information 579 /// about the value. The most common situation is to represent one value at a 580 /// time, but struct or array values are handled element-wise as multiple 581 /// values. The splitting of aggregates is performed recursively, so that we 582 /// never have aggregate-typed registers. The values at this point do not 583 /// necessarily have legal types, so each value may require one or more 584 /// registers of some legal type. 585 /// 586 struct RegsForValue { 587 /// ValueVTs - The value types of the values, which may not be legal, and 588 /// may need be promoted or synthesized from one or more registers. 589 /// 590 SmallVector<EVT, 4> ValueVTs; 591 592 /// RegVTs - The value types of the registers. This is the same size as 593 /// ValueVTs and it records, for each value, what the type of the assigned 594 /// register or registers are. (Individual values are never synthesized 595 /// from more than one type of register.) 596 /// 597 /// With virtual registers, the contents of RegVTs is redundant with TLI's 598 /// getRegisterType member function, however when with physical registers 599 /// it is necessary to have a separate record of the types. 600 /// 601 SmallVector<MVT, 4> RegVTs; 602 603 /// Regs - This list holds the registers assigned to the values. 604 /// Each legal or promoted value requires one register, and each 605 /// expanded value requires multiple registers. 606 /// 607 SmallVector<unsigned, 4> Regs; 608 609 RegsForValue() {} 610 611 RegsForValue(const SmallVector<unsigned, 4> ®s, 612 MVT regvt, EVT valuevt) 613 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} 614 615 RegsForValue(LLVMContext &Context, const TargetLowering &tli, 616 unsigned Reg, Type *Ty) { 617 ComputeValueVTs(tli, Ty, ValueVTs); 618 619 for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { 620 EVT ValueVT = ValueVTs[Value]; 621 unsigned NumRegs = tli.getNumRegisters(Context, ValueVT); 622 MVT RegisterVT = tli.getRegisterType(Context, ValueVT); 623 for (unsigned i = 0; i != NumRegs; ++i) 624 Regs.push_back(Reg + i); 625 RegVTs.push_back(RegisterVT); 626 Reg += NumRegs; 627 } 628 } 629 630 /// areValueTypesLegal - Return true if types of all the values are legal. 631 bool areValueTypesLegal(const TargetLowering &TLI) { 632 for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { 633 MVT RegisterVT = RegVTs[Value]; 634 if (!TLI.isTypeLegal(RegisterVT)) 635 return false; 636 } 637 return true; 638 } 639 640 /// append - Add the specified values to this one. 641 void append(const RegsForValue &RHS) { 642 ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); 643 RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); 644 Regs.append(RHS.Regs.begin(), RHS.Regs.end()); 645 } 646 647 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 648 /// this value and returns the result as a ValueVTs value. This uses 649 /// Chain/Flag as the input and updates them for the output Chain/Flag. 650 /// If the Flag pointer is NULL, no flag is used. 651 SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, 652 SDLoc dl, 653 SDValue &Chain, SDValue *Flag, 654 const Value *V = 0) const; 655 656 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 657 /// specified value into the registers specified by this object. This uses 658 /// Chain/Flag as the input and updates them for the output Chain/Flag. 659 /// If the Flag pointer is NULL, no flag is used. 660 void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, 661 SDValue &Chain, SDValue *Flag, const Value *V) const; 662 663 /// AddInlineAsmOperands - Add this value to the specified inlineasm node 664 /// operand list. This adds the code marker, matching input operand index 665 /// (if applicable), and includes the number of values added into it. 666 void AddInlineAsmOperands(unsigned Kind, 667 bool HasMatching, unsigned MatchingIdx, 668 SelectionDAG &DAG, 669 std::vector<SDValue> &Ops) const; 670 }; 671} 672 673/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 674/// this value and returns the result as a ValueVT value. This uses 675/// Chain/Flag as the input and updates them for the output Chain/Flag. 676/// If the Flag pointer is NULL, no flag is used. 677SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, 678 FunctionLoweringInfo &FuncInfo, 679 SDLoc dl, 680 SDValue &Chain, SDValue *Flag, 681 const Value *V) const { 682 // A Value with type {} or [0 x %t] needs no registers. 683 if (ValueVTs.empty()) 684 return SDValue(); 685 686 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 687 688 // Assemble the legal parts into the final values. 689 SmallVector<SDValue, 4> Values(ValueVTs.size()); 690 SmallVector<SDValue, 8> Parts; 691 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 692 // Copy the legal parts from the registers. 693 EVT ValueVT = ValueVTs[Value]; 694 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT); 695 MVT RegisterVT = RegVTs[Value]; 696 697 Parts.resize(NumRegs); 698 for (unsigned i = 0; i != NumRegs; ++i) { 699 SDValue P; 700 if (Flag == 0) { 701 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); 702 } else { 703 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); 704 *Flag = P.getValue(2); 705 } 706 707 Chain = P.getValue(1); 708 Parts[i] = P; 709 710 // If the source register was virtual and if we know something about it, 711 // add an assert node. 712 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) || 713 !RegisterVT.isInteger() || RegisterVT.isVector()) 714 continue; 715 716 const FunctionLoweringInfo::LiveOutInfo *LOI = 717 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); 718 if (!LOI) 719 continue; 720 721 unsigned RegSize = RegisterVT.getSizeInBits(); 722 unsigned NumSignBits = LOI->NumSignBits; 723 unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes(); 724 725 if (NumZeroBits == RegSize) { 726 // The current value is a zero. 727 // Explicitly express that as it would be easier for 728 // optimizations to kick in. 729 Parts[i] = DAG.getConstant(0, RegisterVT); 730 continue; 731 } 732 733 // FIXME: We capture more information than the dag can represent. For 734 // now, just use the tightest assertzext/assertsext possible. 735 bool isSExt = true; 736 EVT FromVT(MVT::Other); 737 if (NumSignBits == RegSize) 738 isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1 739 else if (NumZeroBits >= RegSize-1) 740 isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1 741 else if (NumSignBits > RegSize-8) 742 isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8 743 else if (NumZeroBits >= RegSize-8) 744 isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8 745 else if (NumSignBits > RegSize-16) 746 isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16 747 else if (NumZeroBits >= RegSize-16) 748 isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16 749 else if (NumSignBits > RegSize-32) 750 isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32 751 else if (NumZeroBits >= RegSize-32) 752 isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32 753 else 754 continue; 755 756 // Add an assertion node. 757 assert(FromVT != MVT::Other); 758 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, 759 RegisterVT, P, DAG.getValueType(FromVT)); 760 } 761 762 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), 763 NumRegs, RegisterVT, ValueVT, V); 764 Part += NumRegs; 765 Parts.clear(); 766 } 767 768 return DAG.getNode(ISD::MERGE_VALUES, dl, 769 DAG.getVTList(&ValueVTs[0], ValueVTs.size()), 770 &Values[0], ValueVTs.size()); 771} 772 773/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 774/// specified value into the registers specified by this object. This uses 775/// Chain/Flag as the input and updates them for the output Chain/Flag. 776/// If the Flag pointer is NULL, no flag is used. 777void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, 778 SDValue &Chain, SDValue *Flag, 779 const Value *V) const { 780 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 781 782 // Get the list of the values's legal parts. 783 unsigned NumRegs = Regs.size(); 784 SmallVector<SDValue, 8> Parts(NumRegs); 785 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 786 EVT ValueVT = ValueVTs[Value]; 787 unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT); 788 MVT RegisterVT = RegVTs[Value]; 789 ISD::NodeType ExtendKind = 790 TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND; 791 792 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), 793 &Parts[Part], NumParts, RegisterVT, V, ExtendKind); 794 Part += NumParts; 795 } 796 797 // Copy the parts into the registers. 798 SmallVector<SDValue, 8> Chains(NumRegs); 799 for (unsigned i = 0; i != NumRegs; ++i) { 800 SDValue Part; 801 if (Flag == 0) { 802 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); 803 } else { 804 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); 805 *Flag = Part.getValue(1); 806 } 807 808 Chains[i] = Part.getValue(0); 809 } 810 811 if (NumRegs == 1 || Flag) 812 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is 813 // flagged to it. That is the CopyToReg nodes and the user are considered 814 // a single scheduling unit. If we create a TokenFactor and return it as 815 // chain, then the TokenFactor is both a predecessor (operand) of the 816 // user as well as a successor (the TF operands are flagged to the user). 817 // c1, f1 = CopyToReg 818 // c2, f2 = CopyToReg 819 // c3 = TokenFactor c1, c2 820 // ... 821 // = op c3, ..., f2 822 Chain = Chains[NumRegs-1]; 823 else 824 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], NumRegs); 825} 826 827/// AddInlineAsmOperands - Add this value to the specified inlineasm node 828/// operand list. This adds the code marker and includes the number of 829/// values added into it. 830void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, 831 unsigned MatchingIdx, 832 SelectionDAG &DAG, 833 std::vector<SDValue> &Ops) const { 834 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 835 836 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); 837 if (HasMatching) 838 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); 839 else if (!Regs.empty() && 840 TargetRegisterInfo::isVirtualRegister(Regs.front())) { 841 // Put the register class of the virtual registers in the flag word. That 842 // way, later passes can recompute register class constraints for inline 843 // assembly as well as normal instructions. 844 // Don't do this for tied operands that can use the regclass information 845 // from the def. 846 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 847 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); 848 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); 849 } 850 851 SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); 852 Ops.push_back(Res); 853 854 unsigned SP = TLI.getStackPointerRegisterToSaveRestore(); 855 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { 856 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]); 857 MVT RegisterVT = RegVTs[Value]; 858 for (unsigned i = 0; i != NumRegs; ++i) { 859 assert(Reg < Regs.size() && "Mismatch in # registers expected"); 860 unsigned TheReg = Regs[Reg++]; 861 Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); 862 863 if (TheReg == SP && Code == InlineAsm::Kind_Clobber) { 864 // If we clobbered the stack pointer, MFI should know about it. 865 assert(DAG.getMachineFunction().getFrameInfo()-> 866 hasInlineAsmWithSPAdjust()); 867 } 868 } 869 } 870} 871 872void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa, 873 const TargetLibraryInfo *li) { 874 AA = &aa; 875 GFI = gfi; 876 LibInfo = li; 877 TD = DAG.getTarget().getDataLayout(); 878 Context = DAG.getContext(); 879 LPadToCallSiteMap.clear(); 880} 881 882/// clear - Clear out the current SelectionDAG and the associated 883/// state and prepare this SelectionDAGBuilder object to be used 884/// for a new block. This doesn't clear out information about 885/// additional blocks that are needed to complete switch lowering 886/// or PHI node updating; that information is cleared out as it is 887/// consumed. 888void SelectionDAGBuilder::clear() { 889 NodeMap.clear(); 890 UnusedArgNodeMap.clear(); 891 PendingLoads.clear(); 892 PendingExports.clear(); 893 CurInst = NULL; 894 HasTailCall = false; 895} 896 897/// clearDanglingDebugInfo - Clear the dangling debug information 898/// map. This function is separated from the clear so that debug 899/// information that is dangling in a basic block can be properly 900/// resolved in a different basic block. This allows the 901/// SelectionDAG to resolve dangling debug information attached 902/// to PHI nodes. 903void SelectionDAGBuilder::clearDanglingDebugInfo() { 904 DanglingDebugInfoMap.clear(); 905} 906 907/// getRoot - Return the current virtual root of the Selection DAG, 908/// flushing any PendingLoad items. This must be done before emitting 909/// a store or any other node that may need to be ordered after any 910/// prior load instructions. 911/// 912SDValue SelectionDAGBuilder::getRoot() { 913 if (PendingLoads.empty()) 914 return DAG.getRoot(); 915 916 if (PendingLoads.size() == 1) { 917 SDValue Root = PendingLoads[0]; 918 DAG.setRoot(Root); 919 PendingLoads.clear(); 920 return Root; 921 } 922 923 // Otherwise, we have to make a token factor node. 924 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 925 &PendingLoads[0], PendingLoads.size()); 926 PendingLoads.clear(); 927 DAG.setRoot(Root); 928 return Root; 929} 930 931/// getControlRoot - Similar to getRoot, but instead of flushing all the 932/// PendingLoad items, flush all the PendingExports items. It is necessary 933/// to do this before emitting a terminator instruction. 934/// 935SDValue SelectionDAGBuilder::getControlRoot() { 936 SDValue Root = DAG.getRoot(); 937 938 if (PendingExports.empty()) 939 return Root; 940 941 // Turn all of the CopyToReg chains into one factored node. 942 if (Root.getOpcode() != ISD::EntryToken) { 943 unsigned i = 0, e = PendingExports.size(); 944 for (; i != e; ++i) { 945 assert(PendingExports[i].getNode()->getNumOperands() > 1); 946 if (PendingExports[i].getNode()->getOperand(0) == Root) 947 break; // Don't add the root if we already indirectly depend on it. 948 } 949 950 if (i == e) 951 PendingExports.push_back(Root); 952 } 953 954 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 955 &PendingExports[0], 956 PendingExports.size()); 957 PendingExports.clear(); 958 DAG.setRoot(Root); 959 return Root; 960} 961 962void SelectionDAGBuilder::visit(const Instruction &I) { 963 // Set up outgoing PHI node register values before emitting the terminator. 964 if (isa<TerminatorInst>(&I)) 965 HandlePHINodesInSuccessorBlocks(I.getParent()); 966 967 ++SDNodeOrder; 968 969 CurInst = &I; 970 971 visit(I.getOpcode(), I); 972 973 if (!isa<TerminatorInst>(&I) && !HasTailCall) 974 CopyToExportRegsIfNeeded(&I); 975 976 CurInst = NULL; 977} 978 979void SelectionDAGBuilder::visitPHI(const PHINode &) { 980 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!"); 981} 982 983void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { 984 // Note: this doesn't use InstVisitor, because it has to work with 985 // ConstantExpr's in addition to instructions. 986 switch (Opcode) { 987 default: llvm_unreachable("Unknown instruction type encountered!"); 988 // Build the switch statement using the Instruction.def file. 989#define HANDLE_INST(NUM, OPCODE, CLASS) \ 990 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; 991#include "llvm/IR/Instruction.def" 992 } 993} 994 995// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, 996// generate the debug data structures now that we've seen its definition. 997void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, 998 SDValue Val) { 999 DanglingDebugInfo &DDI = DanglingDebugInfoMap[V]; 1000 if (DDI.getDI()) { 1001 const DbgValueInst *DI = DDI.getDI(); 1002 DebugLoc dl = DDI.getdl(); 1003 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); 1004 MDNode *Variable = DI->getVariable(); 1005 uint64_t Offset = DI->getOffset(); 1006 SDDbgValue *SDV; 1007 if (Val.getNode()) { 1008 if (!EmitFuncArgumentDbgValue(V, Variable, Offset, Val)) { 1009 SDV = DAG.getDbgValue(Variable, Val.getNode(), 1010 Val.getResNo(), Offset, dl, DbgSDNodeOrder); 1011 DAG.AddDbgValue(SDV, Val.getNode(), false); 1012 } 1013 } else 1014 DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); 1015 DanglingDebugInfoMap[V] = DanglingDebugInfo(); 1016 } 1017} 1018 1019/// getValue - Return an SDValue for the given Value. 1020SDValue SelectionDAGBuilder::getValue(const Value *V) { 1021 // If we already have an SDValue for this value, use it. It's important 1022 // to do this first, so that we don't create a CopyFromReg if we already 1023 // have a regular SDValue. 1024 SDValue &N = NodeMap[V]; 1025 if (N.getNode()) return N; 1026 1027 // If there's a virtual register allocated and initialized for this 1028 // value, use it. 1029 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V); 1030 if (It != FuncInfo.ValueMap.end()) { 1031 unsigned InReg = It->second; 1032 RegsForValue RFV(*DAG.getContext(), *TM.getTargetLowering(), 1033 InReg, V->getType()); 1034 SDValue Chain = DAG.getEntryNode(); 1035 N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, NULL, V); 1036 resolveDanglingDebugInfo(V, N); 1037 return N; 1038 } 1039 1040 // Otherwise create a new SDValue and remember it. 1041 SDValue Val = getValueImpl(V); 1042 NodeMap[V] = Val; 1043 resolveDanglingDebugInfo(V, Val); 1044 return Val; 1045} 1046 1047/// getNonRegisterValue - Return an SDValue for the given Value, but 1048/// don't look in FuncInfo.ValueMap for a virtual register. 1049SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { 1050 // If we already have an SDValue for this value, use it. 1051 SDValue &N = NodeMap[V]; 1052 if (N.getNode()) return N; 1053 1054 // Otherwise create a new SDValue and remember it. 1055 SDValue Val = getValueImpl(V); 1056 NodeMap[V] = Val; 1057 resolveDanglingDebugInfo(V, Val); 1058 return Val; 1059} 1060 1061/// getValueImpl - Helper function for getValue and getNonRegisterValue. 1062/// Create an SDValue for the given value. 1063SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { 1064 const TargetLowering *TLI = TM.getTargetLowering(); 1065 1066 if (const Constant *C = dyn_cast<Constant>(V)) { 1067 EVT VT = TLI->getValueType(V->getType(), true); 1068 1069 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) 1070 return DAG.getConstant(*CI, VT); 1071 1072 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) 1073 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); 1074 1075 if (isa<ConstantPointerNull>(C)) { 1076 unsigned AS = V->getType()->getPointerAddressSpace(); 1077 return DAG.getConstant(0, TLI->getPointerTy(AS)); 1078 } 1079 1080 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) 1081 return DAG.getConstantFP(*CFP, VT); 1082 1083 if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) 1084 return DAG.getUNDEF(VT); 1085 1086 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 1087 visit(CE->getOpcode(), *CE); 1088 SDValue N1 = NodeMap[V]; 1089 assert(N1.getNode() && "visit didn't populate the NodeMap!"); 1090 return N1; 1091 } 1092 1093 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { 1094 SmallVector<SDValue, 4> Constants; 1095 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); 1096 OI != OE; ++OI) { 1097 SDNode *Val = getValue(*OI).getNode(); 1098 // If the operand is an empty aggregate, there are no values. 1099 if (!Val) continue; 1100 // Add each leaf value from the operand to the Constants list 1101 // to form a flattened list of all the values. 1102 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1103 Constants.push_back(SDValue(Val, i)); 1104 } 1105 1106 return DAG.getMergeValues(&Constants[0], Constants.size(), 1107 getCurSDLoc()); 1108 } 1109 1110 if (const ConstantDataSequential *CDS = 1111 dyn_cast<ConstantDataSequential>(C)) { 1112 SmallVector<SDValue, 4> Ops; 1113 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { 1114 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); 1115 // Add each leaf value from the operand to the Constants list 1116 // to form a flattened list of all the values. 1117 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1118 Ops.push_back(SDValue(Val, i)); 1119 } 1120 1121 if (isa<ArrayType>(CDS->getType())) 1122 return DAG.getMergeValues(&Ops[0], Ops.size(), getCurSDLoc()); 1123 return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 1124 VT, &Ops[0], Ops.size()); 1125 } 1126 1127 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { 1128 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && 1129 "Unknown struct or array constant!"); 1130 1131 SmallVector<EVT, 4> ValueVTs; 1132 ComputeValueVTs(*TLI, C->getType(), ValueVTs); 1133 unsigned NumElts = ValueVTs.size(); 1134 if (NumElts == 0) 1135 return SDValue(); // empty struct 1136 SmallVector<SDValue, 4> Constants(NumElts); 1137 for (unsigned i = 0; i != NumElts; ++i) { 1138 EVT EltVT = ValueVTs[i]; 1139 if (isa<UndefValue>(C)) 1140 Constants[i] = DAG.getUNDEF(EltVT); 1141 else if (EltVT.isFloatingPoint()) 1142 Constants[i] = DAG.getConstantFP(0, EltVT); 1143 else 1144 Constants[i] = DAG.getConstant(0, EltVT); 1145 } 1146 1147 return DAG.getMergeValues(&Constants[0], NumElts, 1148 getCurSDLoc()); 1149 } 1150 1151 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) 1152 return DAG.getBlockAddress(BA, VT); 1153 1154 VectorType *VecTy = cast<VectorType>(V->getType()); 1155 unsigned NumElements = VecTy->getNumElements(); 1156 1157 // Now that we know the number and type of the elements, get that number of 1158 // elements into the Ops array based on what kind of constant it is. 1159 SmallVector<SDValue, 16> Ops; 1160 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { 1161 for (unsigned i = 0; i != NumElements; ++i) 1162 Ops.push_back(getValue(CV->getOperand(i))); 1163 } else { 1164 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!"); 1165 EVT EltVT = TLI->getValueType(VecTy->getElementType()); 1166 1167 SDValue Op; 1168 if (EltVT.isFloatingPoint()) 1169 Op = DAG.getConstantFP(0, EltVT); 1170 else 1171 Op = DAG.getConstant(0, EltVT); 1172 Ops.assign(NumElements, Op); 1173 } 1174 1175 // Create a BUILD_VECTOR node. 1176 return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 1177 VT, &Ops[0], Ops.size()); 1178 } 1179 1180 // If this is a static alloca, generate it as the frameindex instead of 1181 // computation. 1182 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 1183 DenseMap<const AllocaInst*, int>::iterator SI = 1184 FuncInfo.StaticAllocaMap.find(AI); 1185 if (SI != FuncInfo.StaticAllocaMap.end()) 1186 return DAG.getFrameIndex(SI->second, TLI->getPointerTy()); 1187 } 1188 1189 // If this is an instruction which fast-isel has deferred, select it now. 1190 if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 1191 unsigned InReg = FuncInfo.InitializeRegForValue(Inst); 1192 RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType()); 1193 SDValue Chain = DAG.getEntryNode(); 1194 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, NULL, V); 1195 } 1196 1197 llvm_unreachable("Can't get register for value!"); 1198} 1199 1200void SelectionDAGBuilder::visitRet(const ReturnInst &I) { 1201 const TargetLowering *TLI = TM.getTargetLowering(); 1202 SDValue Chain = getControlRoot(); 1203 SmallVector<ISD::OutputArg, 8> Outs; 1204 SmallVector<SDValue, 8> OutVals; 1205 1206 if (!FuncInfo.CanLowerReturn) { 1207 unsigned DemoteReg = FuncInfo.DemoteRegister; 1208 const Function *F = I.getParent()->getParent(); 1209 1210 // Emit a store of the return value through the virtual register. 1211 // Leave Outs empty so that LowerReturn won't try to load return 1212 // registers the usual way. 1213 SmallVector<EVT, 1> PtrValueVTs; 1214 ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()), 1215 PtrValueVTs); 1216 1217 SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]); 1218 SDValue RetOp = getValue(I.getOperand(0)); 1219 1220 SmallVector<EVT, 4> ValueVTs; 1221 SmallVector<uint64_t, 4> Offsets; 1222 ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets); 1223 unsigned NumValues = ValueVTs.size(); 1224 1225 SmallVector<SDValue, 4> Chains(NumValues); 1226 for (unsigned i = 0; i != NumValues; ++i) { 1227 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), 1228 RetPtr.getValueType(), RetPtr, 1229 DAG.getIntPtrConstant(Offsets[i])); 1230 Chains[i] = 1231 DAG.getStore(Chain, getCurSDLoc(), 1232 SDValue(RetOp.getNode(), RetOp.getResNo() + i), 1233 // FIXME: better loc info would be nice. 1234 Add, MachinePointerInfo(), false, false, 0); 1235 } 1236 1237 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 1238 MVT::Other, &Chains[0], NumValues); 1239 } else if (I.getNumOperands() != 0) { 1240 SmallVector<EVT, 4> ValueVTs; 1241 ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs); 1242 unsigned NumValues = ValueVTs.size(); 1243 if (NumValues) { 1244 SDValue RetOp = getValue(I.getOperand(0)); 1245 for (unsigned j = 0, f = NumValues; j != f; ++j) { 1246 EVT VT = ValueVTs[j]; 1247 1248 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 1249 1250 const Function *F = I.getParent()->getParent(); 1251 if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1252 Attribute::SExt)) 1253 ExtendKind = ISD::SIGN_EXTEND; 1254 else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1255 Attribute::ZExt)) 1256 ExtendKind = ISD::ZERO_EXTEND; 1257 1258 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) 1259 VT = TLI->getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind); 1260 1261 unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT); 1262 MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT); 1263 SmallVector<SDValue, 4> Parts(NumParts); 1264 getCopyToParts(DAG, getCurSDLoc(), 1265 SDValue(RetOp.getNode(), RetOp.getResNo() + j), 1266 &Parts[0], NumParts, PartVT, &I, ExtendKind); 1267 1268 // 'inreg' on function refers to return value 1269 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 1270 if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, 1271 Attribute::InReg)) 1272 Flags.setInReg(); 1273 1274 // Propagate extension type if any 1275 if (ExtendKind == ISD::SIGN_EXTEND) 1276 Flags.setSExt(); 1277 else if (ExtendKind == ISD::ZERO_EXTEND) 1278 Flags.setZExt(); 1279 1280 for (unsigned i = 0; i < NumParts; ++i) { 1281 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(), 1282 VT, /*isfixed=*/true, 0, 0)); 1283 OutVals.push_back(Parts[i]); 1284 } 1285 } 1286 } 1287 } 1288 1289 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); 1290 CallingConv::ID CallConv = 1291 DAG.getMachineFunction().getFunction()->getCallingConv(); 1292 Chain = TM.getTargetLowering()->LowerReturn(Chain, CallConv, isVarArg, 1293 Outs, OutVals, getCurSDLoc(), 1294 DAG); 1295 1296 // Verify that the target's LowerReturn behaved as expected. 1297 assert(Chain.getNode() && Chain.getValueType() == MVT::Other && 1298 "LowerReturn didn't return a valid chain!"); 1299 1300 // Update the DAG with the new chain value resulting from return lowering. 1301 DAG.setRoot(Chain); 1302} 1303 1304/// CopyToExportRegsIfNeeded - If the given value has virtual registers 1305/// created for it, emit nodes to copy the value into the virtual 1306/// registers. 1307void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { 1308 // Skip empty types 1309 if (V->getType()->isEmptyTy()) 1310 return; 1311 1312 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); 1313 if (VMI != FuncInfo.ValueMap.end()) { 1314 assert(!V->use_empty() && "Unused value assigned virtual registers!"); 1315 CopyValueToVirtualRegister(V, VMI->second); 1316 } 1317} 1318 1319/// ExportFromCurrentBlock - If this condition isn't known to be exported from 1320/// the current basic block, add it to ValueMap now so that we'll get a 1321/// CopyTo/FromReg. 1322void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { 1323 // No need to export constants. 1324 if (!isa<Instruction>(V) && !isa<Argument>(V)) return; 1325 1326 // Already exported? 1327 if (FuncInfo.isExportedInst(V)) return; 1328 1329 unsigned Reg = FuncInfo.InitializeRegForValue(V); 1330 CopyValueToVirtualRegister(V, Reg); 1331} 1332 1333bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, 1334 const BasicBlock *FromBB) { 1335 // The operands of the setcc have to be in this block. We don't know 1336 // how to export them from some other block. 1337 if (const Instruction *VI = dyn_cast<Instruction>(V)) { 1338 // Can export from current BB. 1339 if (VI->getParent() == FromBB) 1340 return true; 1341 1342 // Is already exported, noop. 1343 return FuncInfo.isExportedInst(V); 1344 } 1345 1346 // If this is an argument, we can export it if the BB is the entry block or 1347 // if it is already exported. 1348 if (isa<Argument>(V)) { 1349 if (FromBB == &FromBB->getParent()->getEntryBlock()) 1350 return true; 1351 1352 // Otherwise, can only export this if it is already exported. 1353 return FuncInfo.isExportedInst(V); 1354 } 1355 1356 // Otherwise, constants can always be exported. 1357 return true; 1358} 1359 1360/// Return branch probability calculated by BranchProbabilityInfo for IR blocks. 1361uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src, 1362 const MachineBasicBlock *Dst) const { 1363 BranchProbabilityInfo *BPI = FuncInfo.BPI; 1364 if (!BPI) 1365 return 0; 1366 const BasicBlock *SrcBB = Src->getBasicBlock(); 1367 const BasicBlock *DstBB = Dst->getBasicBlock(); 1368 return BPI->getEdgeWeight(SrcBB, DstBB); 1369} 1370 1371void SelectionDAGBuilder:: 1372addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, 1373 uint32_t Weight /* = 0 */) { 1374 if (!Weight) 1375 Weight = getEdgeWeight(Src, Dst); 1376 Src->addSuccessor(Dst, Weight); 1377} 1378 1379 1380static bool InBlock(const Value *V, const BasicBlock *BB) { 1381 if (const Instruction *I = dyn_cast<Instruction>(V)) 1382 return I->getParent() == BB; 1383 return true; 1384} 1385 1386/// EmitBranchForMergedCondition - Helper method for FindMergedConditions. 1387/// This function emits a branch and is used at the leaves of an OR or an 1388/// AND operator tree. 1389/// 1390void 1391SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, 1392 MachineBasicBlock *TBB, 1393 MachineBasicBlock *FBB, 1394 MachineBasicBlock *CurBB, 1395 MachineBasicBlock *SwitchBB) { 1396 const BasicBlock *BB = CurBB->getBasicBlock(); 1397 1398 // If the leaf of the tree is a comparison, merge the condition into 1399 // the caseblock. 1400 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { 1401 // The operands of the cmp have to be in this block. We don't know 1402 // how to export them from some other block. If this is the first block 1403 // of the sequence, no exporting is needed. 1404 if (CurBB == SwitchBB || 1405 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && 1406 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { 1407 ISD::CondCode Condition; 1408 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { 1409 Condition = getICmpCondCode(IC->getPredicate()); 1410 } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) { 1411 Condition = getFCmpCondCode(FC->getPredicate()); 1412 if (TM.Options.NoNaNsFPMath) 1413 Condition = getFCmpCodeWithoutNaN(Condition); 1414 } else { 1415 Condition = ISD::SETEQ; // silence warning. 1416 llvm_unreachable("Unknown compare instruction"); 1417 } 1418 1419 CaseBlock CB(Condition, BOp->getOperand(0), 1420 BOp->getOperand(1), NULL, TBB, FBB, CurBB); 1421 SwitchCases.push_back(CB); 1422 return; 1423 } 1424 } 1425 1426 // Create a CaseBlock record representing this branch. 1427 CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()), 1428 NULL, TBB, FBB, CurBB); 1429 SwitchCases.push_back(CB); 1430} 1431 1432/// FindMergedConditions - If Cond is an expression like 1433void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, 1434 MachineBasicBlock *TBB, 1435 MachineBasicBlock *FBB, 1436 MachineBasicBlock *CurBB, 1437 MachineBasicBlock *SwitchBB, 1438 unsigned Opc) { 1439 // If this node is not part of the or/and tree, emit it as a branch. 1440 const Instruction *BOp = dyn_cast<Instruction>(Cond); 1441 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) || 1442 (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() || 1443 BOp->getParent() != CurBB->getBasicBlock() || 1444 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) || 1445 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) { 1446 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB); 1447 return; 1448 } 1449 1450 // Create TmpBB after CurBB. 1451 MachineFunction::iterator BBI = CurBB; 1452 MachineFunction &MF = DAG.getMachineFunction(); 1453 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); 1454 CurBB->getParent()->insert(++BBI, TmpBB); 1455 1456 if (Opc == Instruction::Or) { 1457 // Codegen X | Y as: 1458 // jmp_if_X TBB 1459 // jmp TmpBB 1460 // TmpBB: 1461 // jmp_if_Y TBB 1462 // jmp FBB 1463 // 1464 1465 // Emit the LHS condition. 1466 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc); 1467 1468 // Emit the RHS condition into TmpBB. 1469 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc); 1470 } else { 1471 assert(Opc == Instruction::And && "Unknown merge op!"); 1472 // Codegen X & Y as: 1473 // jmp_if_X TmpBB 1474 // jmp FBB 1475 // TmpBB: 1476 // jmp_if_Y TBB 1477 // jmp FBB 1478 // 1479 // This requires creation of TmpBB after CurBB. 1480 1481 // Emit the LHS condition. 1482 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc); 1483 1484 // Emit the RHS condition into TmpBB. 1485 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc); 1486 } 1487} 1488 1489/// If the set of cases should be emitted as a series of branches, return true. 1490/// If we should emit this as a bunch of and/or'd together conditions, return 1491/// false. 1492bool 1493SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { 1494 if (Cases.size() != 2) return true; 1495 1496 // If this is two comparisons of the same values or'd or and'd together, they 1497 // will get folded into a single comparison, so don't emit two blocks. 1498 if ((Cases[0].CmpLHS == Cases[1].CmpLHS && 1499 Cases[0].CmpRHS == Cases[1].CmpRHS) || 1500 (Cases[0].CmpRHS == Cases[1].CmpLHS && 1501 Cases[0].CmpLHS == Cases[1].CmpRHS)) { 1502 return false; 1503 } 1504 1505 // Handle: (X != null) | (Y != null) --> (X|Y) != 0 1506 // Handle: (X == null) & (Y == null) --> (X|Y) == 0 1507 if (Cases[0].CmpRHS == Cases[1].CmpRHS && 1508 Cases[0].CC == Cases[1].CC && 1509 isa<Constant>(Cases[0].CmpRHS) && 1510 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { 1511 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) 1512 return false; 1513 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) 1514 return false; 1515 } 1516 1517 return true; 1518} 1519 1520void SelectionDAGBuilder::visitBr(const BranchInst &I) { 1521 MachineBasicBlock *BrMBB = FuncInfo.MBB; 1522 1523 // Update machine-CFG edges. 1524 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; 1525 1526 // Figure out which block is immediately after the current one. 1527 MachineBasicBlock *NextBlock = 0; 1528 MachineFunction::iterator BBI = BrMBB; 1529 if (++BBI != FuncInfo.MF->end()) 1530 NextBlock = BBI; 1531 1532 if (I.isUnconditional()) { 1533 // Update machine-CFG edges. 1534 BrMBB->addSuccessor(Succ0MBB); 1535 1536 // If this is not a fall-through branch, emit the branch. 1537 if (Succ0MBB != NextBlock) 1538 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 1539 MVT::Other, getControlRoot(), 1540 DAG.getBasicBlock(Succ0MBB))); 1541 1542 return; 1543 } 1544 1545 // If this condition is one of the special cases we handle, do special stuff 1546 // now. 1547 const Value *CondVal = I.getCondition(); 1548 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; 1549 1550 // If this is a series of conditions that are or'd or and'd together, emit 1551 // this as a sequence of branches instead of setcc's with and/or operations. 1552 // As long as jumps are not expensive, this should improve performance. 1553 // For example, instead of something like: 1554 // cmp A, B 1555 // C = seteq 1556 // cmp D, E 1557 // F = setle 1558 // or C, F 1559 // jnz foo 1560 // Emit: 1561 // cmp A, B 1562 // je foo 1563 // cmp D, E 1564 // jle foo 1565 // 1566 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) { 1567 if (!TM.getTargetLowering()->isJumpExpensive() && 1568 BOp->hasOneUse() && 1569 (BOp->getOpcode() == Instruction::And || 1570 BOp->getOpcode() == Instruction::Or)) { 1571 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, 1572 BOp->getOpcode()); 1573 // If the compares in later blocks need to use values not currently 1574 // exported from this block, export them now. This block should always 1575 // be the first entry. 1576 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); 1577 1578 // Allow some cases to be rejected. 1579 if (ShouldEmitAsBranches(SwitchCases)) { 1580 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) { 1581 ExportFromCurrentBlock(SwitchCases[i].CmpLHS); 1582 ExportFromCurrentBlock(SwitchCases[i].CmpRHS); 1583 } 1584 1585 // Emit the branch for this block. 1586 visitSwitchCase(SwitchCases[0], BrMBB); 1587 SwitchCases.erase(SwitchCases.begin()); 1588 return; 1589 } 1590 1591 // Okay, we decided not to do this, remove any inserted MBB's and clear 1592 // SwitchCases. 1593 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) 1594 FuncInfo.MF->erase(SwitchCases[i].ThisBB); 1595 1596 SwitchCases.clear(); 1597 } 1598 } 1599 1600 // Create a CaseBlock record representing this branch. 1601 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), 1602 NULL, Succ0MBB, Succ1MBB, BrMBB); 1603 1604 // Use visitSwitchCase to actually insert the fast branch sequence for this 1605 // cond branch. 1606 visitSwitchCase(CB, BrMBB); 1607} 1608 1609/// visitSwitchCase - Emits the necessary code to represent a single node in 1610/// the binary search tree resulting from lowering a switch instruction. 1611void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, 1612 MachineBasicBlock *SwitchBB) { 1613 SDValue Cond; 1614 SDValue CondLHS = getValue(CB.CmpLHS); 1615 SDLoc dl = getCurSDLoc(); 1616 1617 // Build the setcc now. 1618 if (CB.CmpMHS == NULL) { 1619 // Fold "(X == true)" to X and "(X == false)" to !X to 1620 // handle common cases produced by branch lowering. 1621 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && 1622 CB.CC == ISD::SETEQ) 1623 Cond = CondLHS; 1624 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && 1625 CB.CC == ISD::SETEQ) { 1626 SDValue True = DAG.getConstant(1, CondLHS.getValueType()); 1627 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); 1628 } else 1629 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); 1630 } else { 1631 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); 1632 1633 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); 1634 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); 1635 1636 SDValue CmpOp = getValue(CB.CmpMHS); 1637 EVT VT = CmpOp.getValueType(); 1638 1639 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { 1640 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT), 1641 ISD::SETLE); 1642 } else { 1643 SDValue SUB = DAG.getNode(ISD::SUB, dl, 1644 VT, CmpOp, DAG.getConstant(Low, VT)); 1645 Cond = DAG.getSetCC(dl, MVT::i1, SUB, 1646 DAG.getConstant(High-Low, VT), ISD::SETULE); 1647 } 1648 } 1649 1650 // Update successor info 1651 addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight); 1652 // TrueBB and FalseBB are always different unless the incoming IR is 1653 // degenerate. This only happens when running llc on weird IR. 1654 if (CB.TrueBB != CB.FalseBB) 1655 addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight); 1656 1657 // Set NextBlock to be the MBB immediately after the current one, if any. 1658 // This is used to avoid emitting unnecessary branches to the next block. 1659 MachineBasicBlock *NextBlock = 0; 1660 MachineFunction::iterator BBI = SwitchBB; 1661 if (++BBI != FuncInfo.MF->end()) 1662 NextBlock = BBI; 1663 1664 // If the lhs block is the next block, invert the condition so that we can 1665 // fall through to the lhs instead of the rhs block. 1666 if (CB.TrueBB == NextBlock) { 1667 std::swap(CB.TrueBB, CB.FalseBB); 1668 SDValue True = DAG.getConstant(1, Cond.getValueType()); 1669 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); 1670 } 1671 1672 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, 1673 MVT::Other, getControlRoot(), Cond, 1674 DAG.getBasicBlock(CB.TrueBB)); 1675 1676 // Insert the false branch. Do this even if it's a fall through branch, 1677 // this makes it easier to do DAG optimizations which require inverting 1678 // the branch condition. 1679 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, 1680 DAG.getBasicBlock(CB.FalseBB)); 1681 1682 DAG.setRoot(BrCond); 1683} 1684 1685/// visitJumpTable - Emit JumpTable node in the current MBB 1686void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { 1687 // Emit the code for the jump table 1688 assert(JT.Reg != -1U && "Should lower JT Header first!"); 1689 EVT PTy = TM.getTargetLowering()->getPointerTy(); 1690 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 1691 JT.Reg, PTy); 1692 SDValue Table = DAG.getJumpTable(JT.JTI, PTy); 1693 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), 1694 MVT::Other, Index.getValue(1), 1695 Table, Index); 1696 DAG.setRoot(BrJumpTable); 1697} 1698 1699/// visitJumpTableHeader - This function emits necessary code to produce index 1700/// in the JumpTable from switch case. 1701void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, 1702 JumpTableHeader &JTH, 1703 MachineBasicBlock *SwitchBB) { 1704 // Subtract the lowest switch case value from the value being switched on and 1705 // conditional branch to default mbb if the result is greater than the 1706 // difference between smallest and largest cases. 1707 SDValue SwitchOp = getValue(JTH.SValue); 1708 EVT VT = SwitchOp.getValueType(); 1709 SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp, 1710 DAG.getConstant(JTH.First, VT)); 1711 1712 // The SDNode we just created, which holds the value being switched on minus 1713 // the smallest case value, needs to be copied to a virtual register so it 1714 // can be used as an index into the jump table in a subsequent basic block. 1715 // This value may be smaller or larger than the target's pointer type, and 1716 // therefore require extension or truncating. 1717 const TargetLowering *TLI = TM.getTargetLowering(); 1718 SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy()); 1719 1720 unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy()); 1721 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), 1722 JumpTableReg, SwitchOp); 1723 JT.Reg = JumpTableReg; 1724 1725 // Emit the range check for the jump table, and branch to the default block 1726 // for the switch statement if the value being switched on exceeds the largest 1727 // case in the switch. 1728 SDValue CMP = DAG.getSetCC(getCurSDLoc(), 1729 TLI->getSetCCResultType(*DAG.getContext(), 1730 Sub.getValueType()), 1731 Sub, 1732 DAG.getConstant(JTH.Last - JTH.First,VT), 1733 ISD::SETUGT); 1734 1735 // Set NextBlock to be the MBB immediately after the current one, if any. 1736 // This is used to avoid emitting unnecessary branches to the next block. 1737 MachineBasicBlock *NextBlock = 0; 1738 MachineFunction::iterator BBI = SwitchBB; 1739 1740 if (++BBI != FuncInfo.MF->end()) 1741 NextBlock = BBI; 1742 1743 SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1744 MVT::Other, CopyTo, CMP, 1745 DAG.getBasicBlock(JT.Default)); 1746 1747 if (JT.MBB != NextBlock) 1748 BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond, 1749 DAG.getBasicBlock(JT.MBB)); 1750 1751 DAG.setRoot(BrCond); 1752} 1753 1754/// Codegen a new tail for a stack protector check ParentMBB which has had its 1755/// tail spliced into a stack protector check success bb. 1756/// 1757/// For a high level explanation of how this fits into the stack protector 1758/// generation see the comment on the declaration of class 1759/// StackProtectorDescriptor. 1760void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, 1761 MachineBasicBlock *ParentBB) { 1762 1763 // First create the loads to the guard/stack slot for the comparison. 1764 const TargetLowering *TLI = TM.getTargetLowering(); 1765 EVT PtrTy = TLI->getPointerTy(); 1766 1767 MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo(); 1768 int FI = MFI->getStackProtectorIndex(); 1769 1770 const Value *IRGuard = SPD.getGuard(); 1771 SDValue GuardPtr = getValue(IRGuard); 1772 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); 1773 1774 unsigned Align = 1775 TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType()); 1776 SDValue Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(), 1777 GuardPtr, MachinePointerInfo(IRGuard, 0), 1778 true, false, false, Align); 1779 1780 SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(), 1781 StackSlotPtr, 1782 MachinePointerInfo::getFixedStack(FI), 1783 true, false, false, Align); 1784 1785 // Perform the comparison via a subtract/getsetcc. 1786 EVT VT = Guard.getValueType(); 1787 SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot); 1788 1789 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), 1790 TLI->getSetCCResultType(*DAG.getContext(), 1791 Sub.getValueType()), 1792 Sub, DAG.getConstant(0, VT), 1793 ISD::SETNE); 1794 1795 // If the sub is not 0, then we know the guard/stackslot do not equal, so 1796 // branch to failure MBB. 1797 SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1798 MVT::Other, StackSlot.getOperand(0), 1799 Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); 1800 // Otherwise branch to success MBB. 1801 SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(), 1802 MVT::Other, BrCond, 1803 DAG.getBasicBlock(SPD.getSuccessMBB())); 1804 1805 DAG.setRoot(Br); 1806} 1807 1808/// Codegen the failure basic block for a stack protector check. 1809/// 1810/// A failure stack protector machine basic block consists simply of a call to 1811/// __stack_chk_fail(). 1812/// 1813/// For a high level explanation of how this fits into the stack protector 1814/// generation see the comment on the declaration of class 1815/// StackProtectorDescriptor. 1816void 1817SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { 1818 const TargetLowering *TLI = TM.getTargetLowering(); 1819 SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, 1820 MVT::isVoid, 0, 0, false, getCurSDLoc(), 1821 false, false).second; 1822 DAG.setRoot(Chain); 1823} 1824 1825/// visitBitTestHeader - This function emits necessary code to produce value 1826/// suitable for "bit tests" 1827void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, 1828 MachineBasicBlock *SwitchBB) { 1829 // Subtract the minimum value 1830 SDValue SwitchOp = getValue(B.SValue); 1831 EVT VT = SwitchOp.getValueType(); 1832 SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp, 1833 DAG.getConstant(B.First, VT)); 1834 1835 // Check range 1836 const TargetLowering *TLI = TM.getTargetLowering(); 1837 SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(), 1838 TLI->getSetCCResultType(*DAG.getContext(), 1839 Sub.getValueType()), 1840 Sub, DAG.getConstant(B.Range, VT), 1841 ISD::SETUGT); 1842 1843 // Determine the type of the test operands. 1844 bool UsePtrType = false; 1845 if (!TLI->isTypeLegal(VT)) 1846 UsePtrType = true; 1847 else { 1848 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) 1849 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { 1850 // Switch table case range are encoded into series of masks. 1851 // Just use pointer type, it's guaranteed to fit. 1852 UsePtrType = true; 1853 break; 1854 } 1855 } 1856 if (UsePtrType) { 1857 VT = TLI->getPointerTy(); 1858 Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT); 1859 } 1860 1861 B.RegVT = VT.getSimpleVT(); 1862 B.Reg = FuncInfo.CreateReg(B.RegVT); 1863 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(), 1864 B.Reg, Sub); 1865 1866 // Set NextBlock to be the MBB immediately after the current one, if any. 1867 // This is used to avoid emitting unnecessary branches to the next block. 1868 MachineBasicBlock *NextBlock = 0; 1869 MachineFunction::iterator BBI = SwitchBB; 1870 if (++BBI != FuncInfo.MF->end()) 1871 NextBlock = BBI; 1872 1873 MachineBasicBlock* MBB = B.Cases[0].ThisBB; 1874 1875 addSuccessorWithWeight(SwitchBB, B.Default); 1876 addSuccessorWithWeight(SwitchBB, MBB); 1877 1878 SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1879 MVT::Other, CopyTo, RangeCmp, 1880 DAG.getBasicBlock(B.Default)); 1881 1882 if (MBB != NextBlock) 1883 BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo, 1884 DAG.getBasicBlock(MBB)); 1885 1886 DAG.setRoot(BrRange); 1887} 1888 1889/// visitBitTestCase - this function produces one "bit test" 1890void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, 1891 MachineBasicBlock* NextMBB, 1892 uint32_t BranchWeightToNext, 1893 unsigned Reg, 1894 BitTestCase &B, 1895 MachineBasicBlock *SwitchBB) { 1896 MVT VT = BB.RegVT; 1897 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 1898 Reg, VT); 1899 SDValue Cmp; 1900 unsigned PopCount = CountPopulation_64(B.Mask); 1901 const TargetLowering *TLI = TM.getTargetLowering(); 1902 if (PopCount == 1) { 1903 // Testing for a single bit; just compare the shift count with what it 1904 // would need to be to shift a 1 bit in that position. 1905 Cmp = DAG.getSetCC(getCurSDLoc(), 1906 TLI->getSetCCResultType(*DAG.getContext(), VT), 1907 ShiftOp, 1908 DAG.getConstant(countTrailingZeros(B.Mask), VT), 1909 ISD::SETEQ); 1910 } else if (PopCount == BB.Range) { 1911 // There is only one zero bit in the range, test for it directly. 1912 Cmp = DAG.getSetCC(getCurSDLoc(), 1913 TLI->getSetCCResultType(*DAG.getContext(), VT), 1914 ShiftOp, 1915 DAG.getConstant(CountTrailingOnes_64(B.Mask), VT), 1916 ISD::SETNE); 1917 } else { 1918 // Make desired shift 1919 SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT, 1920 DAG.getConstant(1, VT), ShiftOp); 1921 1922 // Emit bit tests and jumps 1923 SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(), 1924 VT, SwitchVal, DAG.getConstant(B.Mask, VT)); 1925 Cmp = DAG.getSetCC(getCurSDLoc(), 1926 TLI->getSetCCResultType(*DAG.getContext(), VT), 1927 AndOp, DAG.getConstant(0, VT), 1928 ISD::SETNE); 1929 } 1930 1931 // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight. 1932 addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight); 1933 // The branch weight from SwitchBB to NextMBB is BranchWeightToNext. 1934 addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext); 1935 1936 SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(), 1937 MVT::Other, getControlRoot(), 1938 Cmp, DAG.getBasicBlock(B.TargetBB)); 1939 1940 // Set NextBlock to be the MBB immediately after the current one, if any. 1941 // This is used to avoid emitting unnecessary branches to the next block. 1942 MachineBasicBlock *NextBlock = 0; 1943 MachineFunction::iterator BBI = SwitchBB; 1944 if (++BBI != FuncInfo.MF->end()) 1945 NextBlock = BBI; 1946 1947 if (NextMBB != NextBlock) 1948 BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd, 1949 DAG.getBasicBlock(NextMBB)); 1950 1951 DAG.setRoot(BrAnd); 1952} 1953 1954void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { 1955 MachineBasicBlock *InvokeMBB = FuncInfo.MBB; 1956 1957 // Retrieve successors. 1958 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; 1959 MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)]; 1960 1961 const Value *Callee(I.getCalledValue()); 1962 const Function *Fn = dyn_cast<Function>(Callee); 1963 if (isa<InlineAsm>(Callee)) 1964 visitInlineAsm(&I); 1965 else if (Fn && Fn->isIntrinsic()) { 1966 assert(Fn->getIntrinsicID() == Intrinsic::donothing); 1967 // Ignore invokes to @llvm.donothing: jump directly to the next BB. 1968 } else 1969 LowerCallTo(&I, getValue(Callee), false, LandingPad); 1970 1971 // If the value of the invoke is used outside of its defining block, make it 1972 // available as a virtual register. 1973 CopyToExportRegsIfNeeded(&I); 1974 1975 // Update successor info 1976 addSuccessorWithWeight(InvokeMBB, Return); 1977 addSuccessorWithWeight(InvokeMBB, LandingPad); 1978 1979 // Drop into normal successor. 1980 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 1981 MVT::Other, getControlRoot(), 1982 DAG.getBasicBlock(Return))); 1983} 1984 1985void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { 1986 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!"); 1987} 1988 1989void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { 1990 assert(FuncInfo.MBB->isLandingPad() && 1991 "Call to landingpad not in landing pad!"); 1992 1993 MachineBasicBlock *MBB = FuncInfo.MBB; 1994 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 1995 AddLandingPadInfo(LP, MMI, MBB); 1996 1997 // If there aren't registers to copy the values into (e.g., during SjLj 1998 // exceptions), then don't bother to create these DAG nodes. 1999 const TargetLowering *TLI = TM.getTargetLowering(); 2000 if (TLI->getExceptionPointerRegister() == 0 && 2001 TLI->getExceptionSelectorRegister() == 0) 2002 return; 2003 2004 SmallVector<EVT, 2> ValueVTs; 2005 ComputeValueVTs(*TLI, LP.getType(), ValueVTs); 2006 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported"); 2007 2008 // Get the two live-in registers as SDValues. The physregs have already been 2009 // copied into virtual registers. 2010 SDValue Ops[2]; 2011 Ops[0] = DAG.getZExtOrTrunc( 2012 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), 2013 FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()), 2014 getCurSDLoc(), ValueVTs[0]); 2015 Ops[1] = DAG.getZExtOrTrunc( 2016 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), 2017 FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()), 2018 getCurSDLoc(), ValueVTs[1]); 2019 2020 // Merge into one. 2021 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 2022 DAG.getVTList(&ValueVTs[0], ValueVTs.size()), 2023 &Ops[0], 2); 2024 setValue(&LP, Res); 2025} 2026 2027/// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for 2028/// small case ranges). 2029bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, 2030 CaseRecVector& WorkList, 2031 const Value* SV, 2032 MachineBasicBlock *Default, 2033 MachineBasicBlock *SwitchBB) { 2034 // Size is the number of Cases represented by this range. 2035 size_t Size = CR.Range.second - CR.Range.first; 2036 if (Size > 3) 2037 return false; 2038 2039 // Get the MachineFunction which holds the current MBB. This is used when 2040 // inserting any additional MBBs necessary to represent the switch. 2041 MachineFunction *CurMF = FuncInfo.MF; 2042 2043 // Figure out which block is immediately after the current one. 2044 MachineBasicBlock *NextBlock = 0; 2045 MachineFunction::iterator BBI = CR.CaseBB; 2046 2047 if (++BBI != FuncInfo.MF->end()) 2048 NextBlock = BBI; 2049 2050 BranchProbabilityInfo *BPI = FuncInfo.BPI; 2051 // If any two of the cases has the same destination, and if one value 2052 // is the same as the other, but has one bit unset that the other has set, 2053 // use bit manipulation to do two compares at once. For example: 2054 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" 2055 // TODO: This could be extended to merge any 2 cases in switches with 3 cases. 2056 // TODO: Handle cases where CR.CaseBB != SwitchBB. 2057 if (Size == 2 && CR.CaseBB == SwitchBB) { 2058 Case &Small = *CR.Range.first; 2059 Case &Big = *(CR.Range.second-1); 2060 2061 if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) { 2062 const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue(); 2063 const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue(); 2064 2065 // Check that there is only one bit different. 2066 if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 && 2067 (SmallValue | BigValue) == BigValue) { 2068 // Isolate the common bit. 2069 APInt CommonBit = BigValue & ~SmallValue; 2070 assert((SmallValue | CommonBit) == BigValue && 2071 CommonBit.countPopulation() == 1 && "Not a common bit?"); 2072 2073 SDValue CondLHS = getValue(SV); 2074 EVT VT = CondLHS.getValueType(); 2075 SDLoc DL = getCurSDLoc(); 2076 2077 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, 2078 DAG.getConstant(CommonBit, VT)); 2079 SDValue Cond = DAG.getSetCC(DL, MVT::i1, 2080 Or, DAG.getConstant(BigValue, VT), 2081 ISD::SETEQ); 2082 2083 // Update successor info. 2084 // Both Small and Big will jump to Small.BB, so we sum up the weights. 2085 addSuccessorWithWeight(SwitchBB, Small.BB, 2086 Small.ExtraWeight + Big.ExtraWeight); 2087 addSuccessorWithWeight(SwitchBB, Default, 2088 // The default destination is the first successor in IR. 2089 BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0); 2090 2091 // Insert the true branch. 2092 SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other, 2093 getControlRoot(), Cond, 2094 DAG.getBasicBlock(Small.BB)); 2095 2096 // Insert the false branch. 2097 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, 2098 DAG.getBasicBlock(Default)); 2099 2100 DAG.setRoot(BrCond); 2101 return true; 2102 } 2103 } 2104 } 2105 2106 // Order cases by weight so the most likely case will be checked first. 2107 uint32_t UnhandledWeights = 0; 2108 if (BPI) { 2109 for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) { 2110 uint32_t IWeight = I->ExtraWeight; 2111 UnhandledWeights += IWeight; 2112 for (CaseItr J = CR.Range.first; J < I; ++J) { 2113 uint32_t JWeight = J->ExtraWeight; 2114 if (IWeight > JWeight) 2115 std::swap(*I, *J); 2116 } 2117 } 2118 } 2119 // Rearrange the case blocks so that the last one falls through if possible. 2120 Case &BackCase = *(CR.Range.second-1); 2121 if (Size > 1 && 2122 NextBlock && Default != NextBlock && BackCase.BB != NextBlock) { 2123 // The last case block won't fall through into 'NextBlock' if we emit the 2124 // branches in this order. See if rearranging a case value would help. 2125 // We start at the bottom as it's the case with the least weight. 2126 for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I) 2127 if (I->BB == NextBlock) { 2128 std::swap(*I, BackCase); 2129 break; 2130 } 2131 } 2132 2133 // Create a CaseBlock record representing a conditional branch to 2134 // the Case's target mbb if the value being switched on SV is equal 2135 // to C. 2136 MachineBasicBlock *CurBlock = CR.CaseBB; 2137 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { 2138 MachineBasicBlock *FallThrough; 2139 if (I != E-1) { 2140 FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock()); 2141 CurMF->insert(BBI, FallThrough); 2142 2143 // Put SV in a virtual register to make it available from the new blocks. 2144 ExportFromCurrentBlock(SV); 2145 } else { 2146 // If the last case doesn't match, go to the default block. 2147 FallThrough = Default; 2148 } 2149 2150 const Value *RHS, *LHS, *MHS; 2151 ISD::CondCode CC; 2152 if (I->High == I->Low) { 2153 // This is just small small case range :) containing exactly 1 case 2154 CC = ISD::SETEQ; 2155 LHS = SV; RHS = I->High; MHS = NULL; 2156 } else { 2157 CC = ISD::SETLE; 2158 LHS = I->Low; MHS = SV; RHS = I->High; 2159 } 2160 2161 // The false weight should be sum of all un-handled cases. 2162 UnhandledWeights -= I->ExtraWeight; 2163 CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough, 2164 /* me */ CurBlock, 2165 /* trueweight */ I->ExtraWeight, 2166 /* falseweight */ UnhandledWeights); 2167 2168 // If emitting the first comparison, just call visitSwitchCase to emit the 2169 // code into the current block. Otherwise, push the CaseBlock onto the 2170 // vector to be later processed by SDISel, and insert the node's MBB 2171 // before the next MBB. 2172 if (CurBlock == SwitchBB) 2173 visitSwitchCase(CB, SwitchBB); 2174 else 2175 SwitchCases.push_back(CB); 2176 2177 CurBlock = FallThrough; 2178 } 2179 2180 return true; 2181} 2182 2183static inline bool areJTsAllowed(const TargetLowering &TLI) { 2184 return TLI.supportJumpTables() && 2185 (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || 2186 TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other)); 2187} 2188 2189static APInt ComputeRange(const APInt &First, const APInt &Last) { 2190 uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1; 2191 APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth); 2192 return (LastExt - FirstExt + 1ULL); 2193} 2194 2195/// handleJTSwitchCase - Emit jumptable for current switch case range 2196bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, 2197 CaseRecVector &WorkList, 2198 const Value *SV, 2199 MachineBasicBlock *Default, 2200 MachineBasicBlock *SwitchBB) { 2201 Case& FrontCase = *CR.Range.first; 2202 Case& BackCase = *(CR.Range.second-1); 2203 2204 const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); 2205 const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); 2206 2207 APInt TSize(First.getBitWidth(), 0); 2208 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) 2209 TSize += I->size(); 2210 2211 const TargetLowering *TLI = TM.getTargetLowering(); 2212 if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries())) 2213 return false; 2214 2215 APInt Range = ComputeRange(First, Last); 2216 // The density is TSize / Range. Require at least 40%. 2217 // It should not be possible for IntTSize to saturate for sane code, but make 2218 // sure we handle Range saturation correctly. 2219 uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10); 2220 uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10); 2221 if (IntTSize * 10 < IntRange * 4) 2222 return false; 2223 2224 DEBUG(dbgs() << "Lowering jump table\n" 2225 << "First entry: " << First << ". Last entry: " << Last << '\n' 2226 << "Range: " << Range << ". Size: " << TSize << ".\n\n"); 2227 2228 // Get the MachineFunction which holds the current MBB. This is used when 2229 // inserting any additional MBBs necessary to represent the switch. 2230 MachineFunction *CurMF = FuncInfo.MF; 2231 2232 // Figure out which block is immediately after the current one. 2233 MachineFunction::iterator BBI = CR.CaseBB; 2234 ++BBI; 2235 2236 const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2237 2238 // Create a new basic block to hold the code for loading the address 2239 // of the jump table, and jumping to it. Update successor information; 2240 // we will either branch to the default case for the switch, or the jump 2241 // table. 2242 MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2243 CurMF->insert(BBI, JumpTableBB); 2244 2245 addSuccessorWithWeight(CR.CaseBB, Default); 2246 addSuccessorWithWeight(CR.CaseBB, JumpTableBB); 2247 2248 // Build a vector of destination BBs, corresponding to each target 2249 // of the jump table. If the value of the jump table slot corresponds to 2250 // a case statement, push the case's BB onto the vector, otherwise, push 2251 // the default BB. 2252 std::vector<MachineBasicBlock*> DestBBs; 2253 APInt TEI = First; 2254 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) { 2255 const APInt &Low = cast<ConstantInt>(I->Low)->getValue(); 2256 const APInt &High = cast<ConstantInt>(I->High)->getValue(); 2257 2258 if (Low.sle(TEI) && TEI.sle(High)) { 2259 DestBBs.push_back(I->BB); 2260 if (TEI==High) 2261 ++I; 2262 } else { 2263 DestBBs.push_back(Default); 2264 } 2265 } 2266 2267 // Calculate weight for each unique destination in CR. 2268 DenseMap<MachineBasicBlock*, uint32_t> DestWeights; 2269 if (FuncInfo.BPI) 2270 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { 2271 DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr = 2272 DestWeights.find(I->BB); 2273 if (Itr != DestWeights.end()) 2274 Itr->second += I->ExtraWeight; 2275 else 2276 DestWeights[I->BB] = I->ExtraWeight; 2277 } 2278 2279 // Update successor info. Add one edge to each unique successor. 2280 BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs()); 2281 for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(), 2282 E = DestBBs.end(); I != E; ++I) { 2283 if (!SuccsHandled[(*I)->getNumber()]) { 2284 SuccsHandled[(*I)->getNumber()] = true; 2285 DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr = 2286 DestWeights.find(*I); 2287 addSuccessorWithWeight(JumpTableBB, *I, 2288 Itr != DestWeights.end() ? Itr->second : 0); 2289 } 2290 } 2291 2292 // Create a jump table index for this jump table. 2293 unsigned JTEncoding = TLI->getJumpTableEncoding(); 2294 unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding) 2295 ->createJumpTableIndex(DestBBs); 2296 2297 // Set the jump table information so that we can codegen it as a second 2298 // MachineBasicBlock 2299 JumpTable JT(-1U, JTI, JumpTableBB, Default); 2300 JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB)); 2301 if (CR.CaseBB == SwitchBB) 2302 visitJumpTableHeader(JT, JTH, SwitchBB); 2303 2304 JTCases.push_back(JumpTableBlock(JTH, JT)); 2305 return true; 2306} 2307 2308/// handleBTSplitSwitchCase - emit comparison and split binary search tree into 2309/// 2 subtrees. 2310bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, 2311 CaseRecVector& WorkList, 2312 const Value* SV, 2313 MachineBasicBlock* Default, 2314 MachineBasicBlock* SwitchBB) { 2315 // Get the MachineFunction which holds the current MBB. This is used when 2316 // inserting any additional MBBs necessary to represent the switch. 2317 MachineFunction *CurMF = FuncInfo.MF; 2318 2319 // Figure out which block is immediately after the current one. 2320 MachineFunction::iterator BBI = CR.CaseBB; 2321 ++BBI; 2322 2323 Case& FrontCase = *CR.Range.first; 2324 Case& BackCase = *(CR.Range.second-1); 2325 const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2326 2327 // Size is the number of Cases represented by this range. 2328 unsigned Size = CR.Range.second - CR.Range.first; 2329 2330 const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); 2331 const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); 2332 double FMetric = 0; 2333 CaseItr Pivot = CR.Range.first + Size/2; 2334 2335 // Select optimal pivot, maximizing sum density of LHS and RHS. This will 2336 // (heuristically) allow us to emit JumpTable's later. 2337 APInt TSize(First.getBitWidth(), 0); 2338 for (CaseItr I = CR.Range.first, E = CR.Range.second; 2339 I!=E; ++I) 2340 TSize += I->size(); 2341 2342 APInt LSize = FrontCase.size(); 2343 APInt RSize = TSize-LSize; 2344 DEBUG(dbgs() << "Selecting best pivot: \n" 2345 << "First: " << First << ", Last: " << Last <<'\n' 2346 << "LSize: " << LSize << ", RSize: " << RSize << '\n'); 2347 for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second; 2348 J!=E; ++I, ++J) { 2349 const APInt &LEnd = cast<ConstantInt>(I->High)->getValue(); 2350 const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue(); 2351 APInt Range = ComputeRange(LEnd, RBegin); 2352 assert((Range - 2ULL).isNonNegative() && 2353 "Invalid case distance"); 2354 // Use volatile double here to avoid excess precision issues on some hosts, 2355 // e.g. that use 80-bit X87 registers. 2356 volatile double LDensity = 2357 (double)LSize.roundToDouble() / 2358 (LEnd - First + 1ULL).roundToDouble(); 2359 volatile double RDensity = 2360 (double)RSize.roundToDouble() / 2361 (Last - RBegin + 1ULL).roundToDouble(); 2362 double Metric = Range.logBase2()*(LDensity+RDensity); 2363 // Should always split in some non-trivial place 2364 DEBUG(dbgs() <<"=>Step\n" 2365 << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n' 2366 << "LDensity: " << LDensity 2367 << ", RDensity: " << RDensity << '\n' 2368 << "Metric: " << Metric << '\n'); 2369 if (FMetric < Metric) { 2370 Pivot = J; 2371 FMetric = Metric; 2372 DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n'); 2373 } 2374 2375 LSize += J->size(); 2376 RSize -= J->size(); 2377 } 2378 2379 const TargetLowering *TLI = TM.getTargetLowering(); 2380 if (areJTsAllowed(*TLI)) { 2381 // If our case is dense we *really* should handle it earlier! 2382 assert((FMetric > 0) && "Should handle dense range earlier!"); 2383 } else { 2384 Pivot = CR.Range.first + Size/2; 2385 } 2386 2387 CaseRange LHSR(CR.Range.first, Pivot); 2388 CaseRange RHSR(Pivot, CR.Range.second); 2389 const Constant *C = Pivot->Low; 2390 MachineBasicBlock *FalseBB = 0, *TrueBB = 0; 2391 2392 // We know that we branch to the LHS if the Value being switched on is 2393 // less than the Pivot value, C. We use this to optimize our binary 2394 // tree a bit, by recognizing that if SV is greater than or equal to the 2395 // LHS's Case Value, and that Case Value is exactly one less than the 2396 // Pivot's Value, then we can branch directly to the LHS's Target, 2397 // rather than creating a leaf node for it. 2398 if ((LHSR.second - LHSR.first) == 1 && 2399 LHSR.first->High == CR.GE && 2400 cast<ConstantInt>(C)->getValue() == 2401 (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) { 2402 TrueBB = LHSR.first->BB; 2403 } else { 2404 TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2405 CurMF->insert(BBI, TrueBB); 2406 WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR)); 2407 2408 // Put SV in a virtual register to make it available from the new blocks. 2409 ExportFromCurrentBlock(SV); 2410 } 2411 2412 // Similar to the optimization above, if the Value being switched on is 2413 // known to be less than the Constant CR.LT, and the current Case Value 2414 // is CR.LT - 1, then we can branch directly to the target block for 2415 // the current Case Value, rather than emitting a RHS leaf node for it. 2416 if ((RHSR.second - RHSR.first) == 1 && CR.LT && 2417 cast<ConstantInt>(RHSR.first->Low)->getValue() == 2418 (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) { 2419 FalseBB = RHSR.first->BB; 2420 } else { 2421 FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2422 CurMF->insert(BBI, FalseBB); 2423 WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR)); 2424 2425 // Put SV in a virtual register to make it available from the new blocks. 2426 ExportFromCurrentBlock(SV); 2427 } 2428 2429 // Create a CaseBlock record representing a conditional branch to 2430 // the LHS node if the value being switched on SV is less than C. 2431 // Otherwise, branch to LHS. 2432 CaseBlock CB(ISD::SETLT, SV, C, NULL, TrueBB, FalseBB, CR.CaseBB); 2433 2434 if (CR.CaseBB == SwitchBB) 2435 visitSwitchCase(CB, SwitchBB); 2436 else 2437 SwitchCases.push_back(CB); 2438 2439 return true; 2440} 2441 2442/// handleBitTestsSwitchCase - if current case range has few destination and 2443/// range span less, than machine word bitwidth, encode case range into series 2444/// of masks and emit bit tests with these masks. 2445bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, 2446 CaseRecVector& WorkList, 2447 const Value* SV, 2448 MachineBasicBlock* Default, 2449 MachineBasicBlock* SwitchBB) { 2450 const TargetLowering *TLI = TM.getTargetLowering(); 2451 EVT PTy = TLI->getPointerTy(); 2452 unsigned IntPtrBits = PTy.getSizeInBits(); 2453 2454 Case& FrontCase = *CR.Range.first; 2455 Case& BackCase = *(CR.Range.second-1); 2456 2457 // Get the MachineFunction which holds the current MBB. This is used when 2458 // inserting any additional MBBs necessary to represent the switch. 2459 MachineFunction *CurMF = FuncInfo.MF; 2460 2461 // If target does not have legal shift left, do not emit bit tests at all. 2462 if (!TLI->isOperationLegal(ISD::SHL, PTy)) 2463 return false; 2464 2465 size_t numCmps = 0; 2466 for (CaseItr I = CR.Range.first, E = CR.Range.second; 2467 I!=E; ++I) { 2468 // Single case counts one, case range - two. 2469 numCmps += (I->Low == I->High ? 1 : 2); 2470 } 2471 2472 // Count unique destinations 2473 SmallSet<MachineBasicBlock*, 4> Dests; 2474 for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) { 2475 Dests.insert(I->BB); 2476 if (Dests.size() > 3) 2477 // Don't bother the code below, if there are too much unique destinations 2478 return false; 2479 } 2480 DEBUG(dbgs() << "Total number of unique destinations: " 2481 << Dests.size() << '\n' 2482 << "Total number of comparisons: " << numCmps << '\n'); 2483 2484 // Compute span of values. 2485 const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue(); 2486 const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue(); 2487 APInt cmpRange = maxValue - minValue; 2488 2489 DEBUG(dbgs() << "Compare range: " << cmpRange << '\n' 2490 << "Low bound: " << minValue << '\n' 2491 << "High bound: " << maxValue << '\n'); 2492 2493 if (cmpRange.uge(IntPtrBits) || 2494 (!(Dests.size() == 1 && numCmps >= 3) && 2495 !(Dests.size() == 2 && numCmps >= 5) && 2496 !(Dests.size() >= 3 && numCmps >= 6))) 2497 return false; 2498 2499 DEBUG(dbgs() << "Emitting bit tests\n"); 2500 APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth()); 2501 2502 // Optimize the case where all the case values fit in a 2503 // word without having to subtract minValue. In this case, 2504 // we can optimize away the subtraction. 2505 if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) { 2506 cmpRange = maxValue; 2507 } else { 2508 lowBound = minValue; 2509 } 2510 2511 CaseBitsVector CasesBits; 2512 unsigned i, count = 0; 2513 2514 for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) { 2515 MachineBasicBlock* Dest = I->BB; 2516 for (i = 0; i < count; ++i) 2517 if (Dest == CasesBits[i].BB) 2518 break; 2519 2520 if (i == count) { 2521 assert((count < 3) && "Too much destinations to test!"); 2522 CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/)); 2523 count++; 2524 } 2525 2526 const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue(); 2527 const APInt& highValue = cast<ConstantInt>(I->High)->getValue(); 2528 2529 uint64_t lo = (lowValue - lowBound).getZExtValue(); 2530 uint64_t hi = (highValue - lowBound).getZExtValue(); 2531 CasesBits[i].ExtraWeight += I->ExtraWeight; 2532 2533 for (uint64_t j = lo; j <= hi; j++) { 2534 CasesBits[i].Mask |= 1ULL << j; 2535 CasesBits[i].Bits++; 2536 } 2537 2538 } 2539 std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp()); 2540 2541 BitTestInfo BTC; 2542 2543 // Figure out which block is immediately after the current one. 2544 MachineFunction::iterator BBI = CR.CaseBB; 2545 ++BBI; 2546 2547 const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock(); 2548 2549 DEBUG(dbgs() << "Cases:\n"); 2550 for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) { 2551 DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask 2552 << ", Bits: " << CasesBits[i].Bits 2553 << ", BB: " << CasesBits[i].BB << '\n'); 2554 2555 MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB); 2556 CurMF->insert(BBI, CaseBB); 2557 BTC.push_back(BitTestCase(CasesBits[i].Mask, 2558 CaseBB, 2559 CasesBits[i].BB, CasesBits[i].ExtraWeight)); 2560 2561 // Put SV in a virtual register to make it available from the new blocks. 2562 ExportFromCurrentBlock(SV); 2563 } 2564 2565 BitTestBlock BTB(lowBound, cmpRange, SV, 2566 -1U, MVT::Other, (CR.CaseBB == SwitchBB), 2567 CR.CaseBB, Default, BTC); 2568 2569 if (CR.CaseBB == SwitchBB) 2570 visitBitTestHeader(BTB, SwitchBB); 2571 2572 BitTestCases.push_back(BTB); 2573 2574 return true; 2575} 2576 2577/// Clusterify - Transform simple list of Cases into list of CaseRange's 2578size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, 2579 const SwitchInst& SI) { 2580 size_t numCmps = 0; 2581 2582 BranchProbabilityInfo *BPI = FuncInfo.BPI; 2583 // Start with "simple" cases 2584 for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end(); 2585 i != e; ++i) { 2586 const BasicBlock *SuccBB = i.getCaseSuccessor(); 2587 MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB]; 2588 2589 uint32_t ExtraWeight = 2590 BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0; 2591 2592 Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(), 2593 SMBB, ExtraWeight)); 2594 } 2595 std::sort(Cases.begin(), Cases.end(), CaseCmp()); 2596 2597 // Merge case into clusters 2598 if (Cases.size() >= 2) 2599 // Must recompute end() each iteration because it may be 2600 // invalidated by erase if we hold on to it 2601 for (CaseItr I = Cases.begin(), J = llvm::next(Cases.begin()); 2602 J != Cases.end(); ) { 2603 const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue(); 2604 const APInt& currentValue = cast<ConstantInt>(I->High)->getValue(); 2605 MachineBasicBlock* nextBB = J->BB; 2606 MachineBasicBlock* currentBB = I->BB; 2607 2608 // If the two neighboring cases go to the same destination, merge them 2609 // into a single case. 2610 if ((nextValue - currentValue == 1) && (currentBB == nextBB)) { 2611 I->High = J->High; 2612 I->ExtraWeight += J->ExtraWeight; 2613 J = Cases.erase(J); 2614 } else { 2615 I = J++; 2616 } 2617 } 2618 2619 for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) { 2620 if (I->Low != I->High) 2621 // A range counts double, since it requires two compares. 2622 ++numCmps; 2623 } 2624 2625 return numCmps; 2626} 2627 2628void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, 2629 MachineBasicBlock *Last) { 2630 // Update JTCases. 2631 for (unsigned i = 0, e = JTCases.size(); i != e; ++i) 2632 if (JTCases[i].first.HeaderBB == First) 2633 JTCases[i].first.HeaderBB = Last; 2634 2635 // Update BitTestCases. 2636 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) 2637 if (BitTestCases[i].Parent == First) 2638 BitTestCases[i].Parent = Last; 2639} 2640 2641void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { 2642 MachineBasicBlock *SwitchMBB = FuncInfo.MBB; 2643 2644 // Figure out which block is immediately after the current one. 2645 MachineBasicBlock *NextBlock = 0; 2646 MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()]; 2647 2648 // If there is only the default destination, branch to it if it is not the 2649 // next basic block. Otherwise, just fall through. 2650 if (!SI.getNumCases()) { 2651 // Update machine-CFG edges. 2652 2653 // If this is not a fall-through branch, emit the branch. 2654 SwitchMBB->addSuccessor(Default); 2655 if (Default != NextBlock) 2656 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 2657 MVT::Other, getControlRoot(), 2658 DAG.getBasicBlock(Default))); 2659 2660 return; 2661 } 2662 2663 // If there are any non-default case statements, create a vector of Cases 2664 // representing each one, and sort the vector so that we can efficiently 2665 // create a binary search tree from them. 2666 CaseVector Cases; 2667 size_t numCmps = Clusterify(Cases, SI); 2668 DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size() 2669 << ". Total compares: " << numCmps << '\n'); 2670 (void)numCmps; 2671 2672 // Get the Value to be switched on and default basic blocks, which will be 2673 // inserted into CaseBlock records, representing basic blocks in the binary 2674 // search tree. 2675 const Value *SV = SI.getCondition(); 2676 2677 // Push the initial CaseRec onto the worklist 2678 CaseRecVector WorkList; 2679 WorkList.push_back(CaseRec(SwitchMBB,0,0, 2680 CaseRange(Cases.begin(),Cases.end()))); 2681 2682 while (!WorkList.empty()) { 2683 // Grab a record representing a case range to process off the worklist 2684 CaseRec CR = WorkList.back(); 2685 WorkList.pop_back(); 2686 2687 if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) 2688 continue; 2689 2690 // If the range has few cases (two or less) emit a series of specific 2691 // tests. 2692 if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB)) 2693 continue; 2694 2695 // If the switch has more than N blocks, and is at least 40% dense, and the 2696 // target supports indirect branches, then emit a jump table rather than 2697 // lowering the switch to a binary tree of conditional branches. 2698 // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries(). 2699 if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) 2700 continue; 2701 2702 // Emit binary tree. We need to pick a pivot, and push left and right ranges 2703 // onto the worklist. Leafs are handled via handleSmallSwitchRange() call. 2704 handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB); 2705 } 2706} 2707 2708void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { 2709 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; 2710 2711 // Update machine-CFG edges with unique successors. 2712 SmallSet<BasicBlock*, 32> Done; 2713 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { 2714 BasicBlock *BB = I.getSuccessor(i); 2715 bool Inserted = Done.insert(BB); 2716 if (!Inserted) 2717 continue; 2718 2719 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; 2720 addSuccessorWithWeight(IndirectBrMBB, Succ); 2721 } 2722 2723 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), 2724 MVT::Other, getControlRoot(), 2725 getValue(I.getAddress()))); 2726} 2727 2728void SelectionDAGBuilder::visitFSub(const User &I) { 2729 // -0.0 - X --> fneg 2730 Type *Ty = I.getType(); 2731 if (isa<Constant>(I.getOperand(0)) && 2732 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) { 2733 SDValue Op2 = getValue(I.getOperand(1)); 2734 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(), 2735 Op2.getValueType(), Op2)); 2736 return; 2737 } 2738 2739 visitBinary(I, ISD::FSUB); 2740} 2741 2742void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { 2743 SDValue Op1 = getValue(I.getOperand(0)); 2744 SDValue Op2 = getValue(I.getOperand(1)); 2745 setValue(&I, DAG.getNode(OpCode, getCurSDLoc(), 2746 Op1.getValueType(), Op1, Op2)); 2747} 2748 2749void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { 2750 SDValue Op1 = getValue(I.getOperand(0)); 2751 SDValue Op2 = getValue(I.getOperand(1)); 2752 2753 EVT ShiftTy = TM.getTargetLowering()->getShiftAmountTy(Op2.getValueType()); 2754 2755 // Coerce the shift amount to the right type if we can. 2756 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { 2757 unsigned ShiftSize = ShiftTy.getSizeInBits(); 2758 unsigned Op2Size = Op2.getValueType().getSizeInBits(); 2759 SDLoc DL = getCurSDLoc(); 2760 2761 // If the operand is smaller than the shift count type, promote it. 2762 if (ShiftSize > Op2Size) 2763 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2); 2764 2765 // If the operand is larger than the shift count type but the shift 2766 // count type has enough bits to represent any shift value, truncate 2767 // it now. This is a common case and it exposes the truncate to 2768 // optimization early. 2769 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits())) 2770 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2); 2771 // Otherwise we'll need to temporarily settle for some other convenient 2772 // type. Type legalization will make adjustments once the shiftee is split. 2773 else 2774 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32); 2775 } 2776 2777 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), 2778 Op1.getValueType(), Op1, Op2)); 2779} 2780 2781void SelectionDAGBuilder::visitSDiv(const User &I) { 2782 SDValue Op1 = getValue(I.getOperand(0)); 2783 SDValue Op2 = getValue(I.getOperand(1)); 2784 2785 // Turn exact SDivs into multiplications. 2786 // FIXME: This should be in DAGCombiner, but it doesn't have access to the 2787 // exact bit. 2788 if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() && 2789 !isa<ConstantSDNode>(Op1) && 2790 isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue()) 2791 setValue(&I, TM.getTargetLowering()->BuildExactSDIV(Op1, Op2, 2792 getCurSDLoc(), DAG)); 2793 else 2794 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), 2795 Op1, Op2)); 2796} 2797 2798void SelectionDAGBuilder::visitICmp(const User &I) { 2799 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; 2800 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) 2801 predicate = IC->getPredicate(); 2802 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) 2803 predicate = ICmpInst::Predicate(IC->getPredicate()); 2804 SDValue Op1 = getValue(I.getOperand(0)); 2805 SDValue Op2 = getValue(I.getOperand(1)); 2806 ISD::CondCode Opcode = getICmpCondCode(predicate); 2807 2808 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2809 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); 2810} 2811 2812void SelectionDAGBuilder::visitFCmp(const User &I) { 2813 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; 2814 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) 2815 predicate = FC->getPredicate(); 2816 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) 2817 predicate = FCmpInst::Predicate(FC->getPredicate()); 2818 SDValue Op1 = getValue(I.getOperand(0)); 2819 SDValue Op2 = getValue(I.getOperand(1)); 2820 ISD::CondCode Condition = getFCmpCondCode(predicate); 2821 if (TM.Options.NoNaNsFPMath) 2822 Condition = getFCmpCodeWithoutNaN(Condition); 2823 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2824 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); 2825} 2826 2827void SelectionDAGBuilder::visitSelect(const User &I) { 2828 SmallVector<EVT, 4> ValueVTs; 2829 ComputeValueVTs(*TM.getTargetLowering(), I.getType(), ValueVTs); 2830 unsigned NumValues = ValueVTs.size(); 2831 if (NumValues == 0) return; 2832 2833 SmallVector<SDValue, 4> Values(NumValues); 2834 SDValue Cond = getValue(I.getOperand(0)); 2835 SDValue TrueVal = getValue(I.getOperand(1)); 2836 SDValue FalseVal = getValue(I.getOperand(2)); 2837 ISD::NodeType OpCode = Cond.getValueType().isVector() ? 2838 ISD::VSELECT : ISD::SELECT; 2839 2840 for (unsigned i = 0; i != NumValues; ++i) 2841 Values[i] = DAG.getNode(OpCode, getCurSDLoc(), 2842 TrueVal.getNode()->getValueType(TrueVal.getResNo()+i), 2843 Cond, 2844 SDValue(TrueVal.getNode(), 2845 TrueVal.getResNo() + i), 2846 SDValue(FalseVal.getNode(), 2847 FalseVal.getResNo() + i)); 2848 2849 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 2850 DAG.getVTList(&ValueVTs[0], NumValues), 2851 &Values[0], NumValues)); 2852} 2853 2854void SelectionDAGBuilder::visitTrunc(const User &I) { 2855 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). 2856 SDValue N = getValue(I.getOperand(0)); 2857 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2858 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); 2859} 2860 2861void SelectionDAGBuilder::visitZExt(const User &I) { 2862 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 2863 // ZExt also can't be a cast to bool for same reason. So, nothing much to do 2864 SDValue N = getValue(I.getOperand(0)); 2865 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2866 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); 2867} 2868 2869void SelectionDAGBuilder::visitSExt(const User &I) { 2870 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 2871 // SExt also can't be a cast to bool for same reason. So, nothing much to do 2872 SDValue N = getValue(I.getOperand(0)); 2873 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2874 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); 2875} 2876 2877void SelectionDAGBuilder::visitFPTrunc(const User &I) { 2878 // FPTrunc is never a no-op cast, no need to check 2879 SDValue N = getValue(I.getOperand(0)); 2880 const TargetLowering *TLI = TM.getTargetLowering(); 2881 EVT DestVT = TLI->getValueType(I.getType()); 2882 setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), 2883 DestVT, N, 2884 DAG.getTargetConstant(0, TLI->getPointerTy()))); 2885} 2886 2887void SelectionDAGBuilder::visitFPExt(const User &I) { 2888 // FPExt is never a no-op cast, no need to check 2889 SDValue N = getValue(I.getOperand(0)); 2890 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2891 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); 2892} 2893 2894void SelectionDAGBuilder::visitFPToUI(const User &I) { 2895 // FPToUI is never a no-op cast, no need to check 2896 SDValue N = getValue(I.getOperand(0)); 2897 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2898 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); 2899} 2900 2901void SelectionDAGBuilder::visitFPToSI(const User &I) { 2902 // FPToSI is never a no-op cast, no need to check 2903 SDValue N = getValue(I.getOperand(0)); 2904 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2905 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); 2906} 2907 2908void SelectionDAGBuilder::visitUIToFP(const User &I) { 2909 // UIToFP is never a no-op cast, no need to check 2910 SDValue N = getValue(I.getOperand(0)); 2911 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2912 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); 2913} 2914 2915void SelectionDAGBuilder::visitSIToFP(const User &I) { 2916 // SIToFP is never a no-op cast, no need to check 2917 SDValue N = getValue(I.getOperand(0)); 2918 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2919 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); 2920} 2921 2922void SelectionDAGBuilder::visitPtrToInt(const User &I) { 2923 // What to do depends on the size of the integer and the size of the pointer. 2924 // We can either truncate, zero extend, or no-op, accordingly. 2925 SDValue N = getValue(I.getOperand(0)); 2926 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2927 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); 2928} 2929 2930void SelectionDAGBuilder::visitIntToPtr(const User &I) { 2931 // What to do depends on the size of the integer and the size of the pointer. 2932 // We can either truncate, zero extend, or no-op, accordingly. 2933 SDValue N = getValue(I.getOperand(0)); 2934 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2935 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); 2936} 2937 2938void SelectionDAGBuilder::visitBitCast(const User &I) { 2939 SDValue N = getValue(I.getOperand(0)); 2940 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2941 2942 // BitCast assures us that source and destination are the same size so this is 2943 // either a BITCAST or a no-op. 2944 if (DestVT != N.getValueType()) 2945 setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(), 2946 DestVT, N)); // convert types. 2947 else 2948 setValue(&I, N); // noop cast. 2949} 2950 2951void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { 2952 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2953 const Value *SV = I.getOperand(0); 2954 SDValue N = getValue(SV); 2955 EVT DestVT = TM.getTargetLowering()->getValueType(I.getType()); 2956 2957 unsigned SrcAS = SV->getType()->getPointerAddressSpace(); 2958 unsigned DestAS = I.getType()->getPointerAddressSpace(); 2959 2960 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS)) 2961 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); 2962 2963 setValue(&I, N); 2964} 2965 2966void SelectionDAGBuilder::visitInsertElement(const User &I) { 2967 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2968 SDValue InVec = getValue(I.getOperand(0)); 2969 SDValue InVal = getValue(I.getOperand(1)); 2970 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), 2971 getCurSDLoc(), TLI.getVectorIdxTy()); 2972 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), 2973 TM.getTargetLowering()->getValueType(I.getType()), 2974 InVec, InVal, InIdx)); 2975} 2976 2977void SelectionDAGBuilder::visitExtractElement(const User &I) { 2978 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2979 SDValue InVec = getValue(I.getOperand(0)); 2980 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), 2981 getCurSDLoc(), TLI.getVectorIdxTy()); 2982 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 2983 TM.getTargetLowering()->getValueType(I.getType()), 2984 InVec, InIdx)); 2985} 2986 2987// Utility for visitShuffleVector - Return true if every element in Mask, 2988// beginning from position Pos and ending in Pos+Size, falls within the 2989// specified sequential range [L, L+Pos). or is undef. 2990static bool isSequentialInRange(const SmallVectorImpl<int> &Mask, 2991 unsigned Pos, unsigned Size, int Low) { 2992 for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low) 2993 if (Mask[i] >= 0 && Mask[i] != Low) 2994 return false; 2995 return true; 2996} 2997 2998void SelectionDAGBuilder::visitShuffleVector(const User &I) { 2999 SDValue Src1 = getValue(I.getOperand(0)); 3000 SDValue Src2 = getValue(I.getOperand(1)); 3001 3002 SmallVector<int, 8> Mask; 3003 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask); 3004 unsigned MaskNumElts = Mask.size(); 3005 3006 const TargetLowering *TLI = TM.getTargetLowering(); 3007 EVT VT = TLI->getValueType(I.getType()); 3008 EVT SrcVT = Src1.getValueType(); 3009 unsigned SrcNumElts = SrcVT.getVectorNumElements(); 3010 3011 if (SrcNumElts == MaskNumElts) { 3012 setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3013 &Mask[0])); 3014 return; 3015 } 3016 3017 // Normalize the shuffle vector since mask and vector length don't match. 3018 if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) { 3019 // Mask is longer than the source vectors and is a multiple of the source 3020 // vectors. We can use concatenate vector to make the mask and vectors 3021 // lengths match. 3022 if (SrcNumElts*2 == MaskNumElts) { 3023 // First check for Src1 in low and Src2 in high 3024 if (isSequentialInRange(Mask, 0, SrcNumElts, 0) && 3025 isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) { 3026 // The shuffle is concatenating two vectors together. 3027 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(), 3028 VT, Src1, Src2)); 3029 return; 3030 } 3031 // Then check for Src2 in low and Src1 in high 3032 if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) && 3033 isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) { 3034 // The shuffle is concatenating two vectors together. 3035 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(), 3036 VT, Src2, Src1)); 3037 return; 3038 } 3039 } 3040 3041 // Pad both vectors with undefs to make them the same length as the mask. 3042 unsigned NumConcat = MaskNumElts / SrcNumElts; 3043 bool Src1U = Src1.getOpcode() == ISD::UNDEF; 3044 bool Src2U = Src2.getOpcode() == ISD::UNDEF; 3045 SDValue UndefVal = DAG.getUNDEF(SrcVT); 3046 3047 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); 3048 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); 3049 MOps1[0] = Src1; 3050 MOps2[0] = Src2; 3051 3052 Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 3053 getCurSDLoc(), VT, 3054 &MOps1[0], NumConcat); 3055 Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 3056 getCurSDLoc(), VT, 3057 &MOps2[0], NumConcat); 3058 3059 // Readjust mask for new input vector length. 3060 SmallVector<int, 8> MappedOps; 3061 for (unsigned i = 0; i != MaskNumElts; ++i) { 3062 int Idx = Mask[i]; 3063 if (Idx >= (int)SrcNumElts) 3064 Idx -= SrcNumElts - MaskNumElts; 3065 MappedOps.push_back(Idx); 3066 } 3067 3068 setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3069 &MappedOps[0])); 3070 return; 3071 } 3072 3073 if (SrcNumElts > MaskNumElts) { 3074 // Analyze the access pattern of the vector to see if we can extract 3075 // two subvectors and do the shuffle. The analysis is done by calculating 3076 // the range of elements the mask access on both vectors. 3077 int MinRange[2] = { static_cast<int>(SrcNumElts), 3078 static_cast<int>(SrcNumElts)}; 3079 int MaxRange[2] = {-1, -1}; 3080 3081 for (unsigned i = 0; i != MaskNumElts; ++i) { 3082 int Idx = Mask[i]; 3083 unsigned Input = 0; 3084 if (Idx < 0) 3085 continue; 3086 3087 if (Idx >= (int)SrcNumElts) { 3088 Input = 1; 3089 Idx -= SrcNumElts; 3090 } 3091 if (Idx > MaxRange[Input]) 3092 MaxRange[Input] = Idx; 3093 if (Idx < MinRange[Input]) 3094 MinRange[Input] = Idx; 3095 } 3096 3097 // Check if the access is smaller than the vector size and can we find 3098 // a reasonable extract index. 3099 int RangeUse[2] = { -1, -1 }; // 0 = Unused, 1 = Extract, -1 = Can not 3100 // Extract. 3101 int StartIdx[2]; // StartIdx to extract from 3102 for (unsigned Input = 0; Input < 2; ++Input) { 3103 if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) { 3104 RangeUse[Input] = 0; // Unused 3105 StartIdx[Input] = 0; 3106 continue; 3107 } 3108 3109 // Find a good start index that is a multiple of the mask length. Then 3110 // see if the rest of the elements are in range. 3111 StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts; 3112 if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts && 3113 StartIdx[Input] + MaskNumElts <= SrcNumElts) 3114 RangeUse[Input] = 1; // Extract from a multiple of the mask length. 3115 } 3116 3117 if (RangeUse[0] == 0 && RangeUse[1] == 0) { 3118 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. 3119 return; 3120 } 3121 if (RangeUse[0] >= 0 && RangeUse[1] >= 0) { 3122 // Extract appropriate subvector and generate a vector shuffle 3123 for (unsigned Input = 0; Input < 2; ++Input) { 3124 SDValue &Src = Input == 0 ? Src1 : Src2; 3125 if (RangeUse[Input] == 0) 3126 Src = DAG.getUNDEF(VT); 3127 else 3128 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, 3129 Src, DAG.getConstant(StartIdx[Input], 3130 TLI->getVectorIdxTy())); 3131 } 3132 3133 // Calculate new mask. 3134 SmallVector<int, 8> MappedOps; 3135 for (unsigned i = 0; i != MaskNumElts; ++i) { 3136 int Idx = Mask[i]; 3137 if (Idx >= 0) { 3138 if (Idx < (int)SrcNumElts) 3139 Idx -= StartIdx[0]; 3140 else 3141 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; 3142 } 3143 MappedOps.push_back(Idx); 3144 } 3145 3146 setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2, 3147 &MappedOps[0])); 3148 return; 3149 } 3150 } 3151 3152 // We can't use either concat vectors or extract subvectors so fall back to 3153 // replacing the shuffle with extract and build vector. 3154 // to insert and build vector. 3155 EVT EltVT = VT.getVectorElementType(); 3156 EVT IdxVT = TLI->getVectorIdxTy(); 3157 SmallVector<SDValue,8> Ops; 3158 for (unsigned i = 0; i != MaskNumElts; ++i) { 3159 int Idx = Mask[i]; 3160 SDValue Res; 3161 3162 if (Idx < 0) { 3163 Res = DAG.getUNDEF(EltVT); 3164 } else { 3165 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; 3166 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; 3167 3168 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 3169 EltVT, Src, DAG.getConstant(Idx, IdxVT)); 3170 } 3171 3172 Ops.push_back(Res); 3173 } 3174 3175 setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), 3176 VT, &Ops[0], Ops.size())); 3177} 3178 3179void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { 3180 const Value *Op0 = I.getOperand(0); 3181 const Value *Op1 = I.getOperand(1); 3182 Type *AggTy = I.getType(); 3183 Type *ValTy = Op1->getType(); 3184 bool IntoUndef = isa<UndefValue>(Op0); 3185 bool FromUndef = isa<UndefValue>(Op1); 3186 3187 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); 3188 3189 const TargetLowering *TLI = TM.getTargetLowering(); 3190 SmallVector<EVT, 4> AggValueVTs; 3191 ComputeValueVTs(*TLI, AggTy, AggValueVTs); 3192 SmallVector<EVT, 4> ValValueVTs; 3193 ComputeValueVTs(*TLI, ValTy, ValValueVTs); 3194 3195 unsigned NumAggValues = AggValueVTs.size(); 3196 unsigned NumValValues = ValValueVTs.size(); 3197 SmallVector<SDValue, 4> Values(NumAggValues); 3198 3199 SDValue Agg = getValue(Op0); 3200 unsigned i = 0; 3201 // Copy the beginning value(s) from the original aggregate. 3202 for (; i != LinearIndex; ++i) 3203 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3204 SDValue(Agg.getNode(), Agg.getResNo() + i); 3205 // Copy values from the inserted value(s). 3206 if (NumValValues) { 3207 SDValue Val = getValue(Op1); 3208 for (; i != LinearIndex + NumValValues; ++i) 3209 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3210 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); 3211 } 3212 // Copy remaining value(s) from the original aggregate. 3213 for (; i != NumAggValues; ++i) 3214 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3215 SDValue(Agg.getNode(), Agg.getResNo() + i); 3216 3217 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3218 DAG.getVTList(&AggValueVTs[0], NumAggValues), 3219 &Values[0], NumAggValues)); 3220} 3221 3222void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { 3223 const Value *Op0 = I.getOperand(0); 3224 Type *AggTy = Op0->getType(); 3225 Type *ValTy = I.getType(); 3226 bool OutOfUndef = isa<UndefValue>(Op0); 3227 3228 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); 3229 3230 const TargetLowering *TLI = TM.getTargetLowering(); 3231 SmallVector<EVT, 4> ValValueVTs; 3232 ComputeValueVTs(*TLI, ValTy, ValValueVTs); 3233 3234 unsigned NumValValues = ValValueVTs.size(); 3235 3236 // Ignore a extractvalue that produces an empty object 3237 if (!NumValValues) { 3238 setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); 3239 return; 3240 } 3241 3242 SmallVector<SDValue, 4> Values(NumValValues); 3243 3244 SDValue Agg = getValue(Op0); 3245 // Copy out the selected value(s). 3246 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) 3247 Values[i - LinearIndex] = 3248 OutOfUndef ? 3249 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : 3250 SDValue(Agg.getNode(), Agg.getResNo() + i); 3251 3252 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3253 DAG.getVTList(&ValValueVTs[0], NumValValues), 3254 &Values[0], NumValValues)); 3255} 3256 3257void SelectionDAGBuilder::visitGetElementPtr(const User &I) { 3258 Value *Op0 = I.getOperand(0); 3259 // Note that the pointer operand may be a vector of pointers. Take the scalar 3260 // element which holds a pointer. 3261 Type *Ty = Op0->getType()->getScalarType(); 3262 unsigned AS = Ty->getPointerAddressSpace(); 3263 SDValue N = getValue(Op0); 3264 3265 for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end(); 3266 OI != E; ++OI) { 3267 const Value *Idx = *OI; 3268 if (StructType *StTy = dyn_cast<StructType>(Ty)) { 3269 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); 3270 if (Field) { 3271 // N = N + Offset 3272 uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field); 3273 N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, 3274 DAG.getConstant(Offset, N.getValueType())); 3275 } 3276 3277 Ty = StTy->getElementType(Field); 3278 } else { 3279 Ty = cast<SequentialType>(Ty)->getElementType(); 3280 3281 // If this is a constant subscript, handle it quickly. 3282 const TargetLowering *TLI = TM.getTargetLowering(); 3283 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { 3284 if (CI->isZero()) continue; 3285 uint64_t Offs = 3286 TD->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue(); 3287 SDValue OffsVal; 3288 EVT PTy = TLI->getPointerTy(AS); 3289 unsigned PtrBits = PTy.getSizeInBits(); 3290 if (PtrBits < 64) 3291 OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy, 3292 DAG.getConstant(Offs, MVT::i64)); 3293 else 3294 OffsVal = DAG.getConstant(Offs, PTy); 3295 3296 N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, 3297 OffsVal); 3298 continue; 3299 } 3300 3301 // N = N + Idx * ElementSize; 3302 APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS), 3303 TD->getTypeAllocSize(Ty)); 3304 SDValue IdxN = getValue(Idx); 3305 3306 // If the index is smaller or larger than intptr_t, truncate or extend 3307 // it. 3308 IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType()); 3309 3310 // If this is a multiply by a power of two, turn it into a shl 3311 // immediately. This is a very common case. 3312 if (ElementSize != 1) { 3313 if (ElementSize.isPowerOf2()) { 3314 unsigned Amt = ElementSize.logBase2(); 3315 IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(), 3316 N.getValueType(), IdxN, 3317 DAG.getConstant(Amt, IdxN.getValueType())); 3318 } else { 3319 SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType()); 3320 IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(), 3321 N.getValueType(), IdxN, Scale); 3322 } 3323 } 3324 3325 N = DAG.getNode(ISD::ADD, getCurSDLoc(), 3326 N.getValueType(), N, IdxN); 3327 } 3328 } 3329 3330 setValue(&I, N); 3331} 3332 3333void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { 3334 // If this is a fixed sized alloca in the entry block of the function, 3335 // allocate it statically on the stack. 3336 if (FuncInfo.StaticAllocaMap.count(&I)) 3337 return; // getValue will auto-populate this. 3338 3339 Type *Ty = I.getAllocatedType(); 3340 const TargetLowering *TLI = TM.getTargetLowering(); 3341 uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty); 3342 unsigned Align = 3343 std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), 3344 I.getAlignment()); 3345 3346 SDValue AllocSize = getValue(I.getArraySize()); 3347 3348 EVT IntPtr = TLI->getPointerTy(); 3349 if (AllocSize.getValueType() != IntPtr) 3350 AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr); 3351 3352 AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr, 3353 AllocSize, 3354 DAG.getConstant(TySize, IntPtr)); 3355 3356 // Handle alignment. If the requested alignment is less than or equal to 3357 // the stack alignment, ignore it. If the size is greater than or equal to 3358 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. 3359 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 3360 if (Align <= StackAlign) 3361 Align = 0; 3362 3363 // Round the size of the allocation up to the stack alignment size 3364 // by add SA-1 to the size. 3365 AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(), 3366 AllocSize.getValueType(), AllocSize, 3367 DAG.getIntPtrConstant(StackAlign-1)); 3368 3369 // Mask out the low bits for alignment purposes. 3370 AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(), 3371 AllocSize.getValueType(), AllocSize, 3372 DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); 3373 3374 SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; 3375 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); 3376 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), 3377 VTs, Ops, 3); 3378 setValue(&I, DSA); 3379 DAG.setRoot(DSA.getValue(1)); 3380 3381 assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects()); 3382} 3383 3384void SelectionDAGBuilder::visitLoad(const LoadInst &I) { 3385 if (I.isAtomic()) 3386 return visitAtomicLoad(I); 3387 3388 const Value *SV = I.getOperand(0); 3389 SDValue Ptr = getValue(SV); 3390 3391 Type *Ty = I.getType(); 3392 3393 bool isVolatile = I.isVolatile(); 3394 bool isNonTemporal = I.getMetadata("nontemporal") != 0; 3395 bool isInvariant = I.getMetadata("invariant.load") != 0; 3396 unsigned Alignment = I.getAlignment(); 3397 const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa); 3398 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); 3399 3400 SmallVector<EVT, 4> ValueVTs; 3401 SmallVector<uint64_t, 4> Offsets; 3402 ComputeValueVTs(*TM.getTargetLowering(), Ty, ValueVTs, &Offsets); 3403 unsigned NumValues = ValueVTs.size(); 3404 if (NumValues == 0) 3405 return; 3406 3407 SDValue Root; 3408 bool ConstantMemory = false; 3409 if (I.isVolatile() || NumValues > MaxParallelChains) 3410 // Serialize volatile loads with other side effects. 3411 Root = getRoot(); 3412 else if (AA->pointsToConstantMemory( 3413 AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), TBAAInfo))) { 3414 // Do not serialize (non-volatile) loads of constant memory with anything. 3415 Root = DAG.getEntryNode(); 3416 ConstantMemory = true; 3417 } else { 3418 // Do not serialize non-volatile loads against each other. 3419 Root = DAG.getRoot(); 3420 } 3421 3422 SmallVector<SDValue, 4> Values(NumValues); 3423 SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains), 3424 NumValues)); 3425 EVT PtrVT = Ptr.getValueType(); 3426 unsigned ChainI = 0; 3427 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 3428 // Serializing loads here may result in excessive register pressure, and 3429 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling 3430 // could recover a bit by hoisting nodes upward in the chain by recognizing 3431 // they are side-effect free or do not alias. The optimizer should really 3432 // avoid this case by converting large object/array copies to llvm.memcpy 3433 // (MaxParallelChains should always remain as failsafe). 3434 if (ChainI == MaxParallelChains) { 3435 assert(PendingLoads.empty() && "PendingLoads must be serialized first"); 3436 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3437 MVT::Other, &Chains[0], ChainI); 3438 Root = Chain; 3439 ChainI = 0; 3440 } 3441 SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(), 3442 PtrVT, Ptr, 3443 DAG.getConstant(Offsets[i], PtrVT)); 3444 SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root, 3445 A, MachinePointerInfo(SV, Offsets[i]), isVolatile, 3446 isNonTemporal, isInvariant, Alignment, TBAAInfo, 3447 Ranges); 3448 3449 Values[i] = L; 3450 Chains[ChainI] = L.getValue(1); 3451 } 3452 3453 if (!ConstantMemory) { 3454 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3455 MVT::Other, &Chains[0], ChainI); 3456 if (isVolatile) 3457 DAG.setRoot(Chain); 3458 else 3459 PendingLoads.push_back(Chain); 3460 } 3461 3462 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3463 DAG.getVTList(&ValueVTs[0], NumValues), 3464 &Values[0], NumValues)); 3465} 3466 3467void SelectionDAGBuilder::visitStore(const StoreInst &I) { 3468 if (I.isAtomic()) 3469 return visitAtomicStore(I); 3470 3471 const Value *SrcV = I.getOperand(0); 3472 const Value *PtrV = I.getOperand(1); 3473 3474 SmallVector<EVT, 4> ValueVTs; 3475 SmallVector<uint64_t, 4> Offsets; 3476 ComputeValueVTs(*TM.getTargetLowering(), SrcV->getType(), ValueVTs, &Offsets); 3477 unsigned NumValues = ValueVTs.size(); 3478 if (NumValues == 0) 3479 return; 3480 3481 // Get the lowered operands. Note that we do this after 3482 // checking if NumResults is zero, because with zero results 3483 // the operands won't have values in the map. 3484 SDValue Src = getValue(SrcV); 3485 SDValue Ptr = getValue(PtrV); 3486 3487 SDValue Root = getRoot(); 3488 SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains), 3489 NumValues)); 3490 EVT PtrVT = Ptr.getValueType(); 3491 bool isVolatile = I.isVolatile(); 3492 bool isNonTemporal = I.getMetadata("nontemporal") != 0; 3493 unsigned Alignment = I.getAlignment(); 3494 const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa); 3495 3496 unsigned ChainI = 0; 3497 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 3498 // See visitLoad comments. 3499 if (ChainI == MaxParallelChains) { 3500 SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3501 MVT::Other, &Chains[0], ChainI); 3502 Root = Chain; 3503 ChainI = 0; 3504 } 3505 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr, 3506 DAG.getConstant(Offsets[i], PtrVT)); 3507 SDValue St = DAG.getStore(Root, getCurSDLoc(), 3508 SDValue(Src.getNode(), Src.getResNo() + i), 3509 Add, MachinePointerInfo(PtrV, Offsets[i]), 3510 isVolatile, isNonTemporal, Alignment, TBAAInfo); 3511 Chains[ChainI] = St; 3512 } 3513 3514 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 3515 MVT::Other, &Chains[0], ChainI); 3516 DAG.setRoot(StoreNode); 3517} 3518 3519static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order, 3520 SynchronizationScope Scope, 3521 bool Before, SDLoc dl, 3522 SelectionDAG &DAG, 3523 const TargetLowering &TLI) { 3524 // Fence, if necessary 3525 if (Before) { 3526 if (Order == AcquireRelease || Order == SequentiallyConsistent) 3527 Order = Release; 3528 else if (Order == Acquire || Order == Monotonic) 3529 return Chain; 3530 } else { 3531 if (Order == AcquireRelease) 3532 Order = Acquire; 3533 else if (Order == Release || Order == Monotonic) 3534 return Chain; 3535 } 3536 SDValue Ops[3]; 3537 Ops[0] = Chain; 3538 Ops[1] = DAG.getConstant(Order, TLI.getPointerTy()); 3539 Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy()); 3540 return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3); 3541} 3542 3543void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { 3544 SDLoc dl = getCurSDLoc(); 3545 AtomicOrdering Order = I.getOrdering(); 3546 SynchronizationScope Scope = I.getSynchScope(); 3547 3548 SDValue InChain = getRoot(); 3549 3550 const TargetLowering *TLI = TM.getTargetLowering(); 3551 if (TLI->getInsertFencesForAtomic()) 3552 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3553 DAG, *TLI); 3554 3555 SDValue L = 3556 DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 3557 getValue(I.getCompareOperand()).getSimpleValueType(), 3558 InChain, 3559 getValue(I.getPointerOperand()), 3560 getValue(I.getCompareOperand()), 3561 getValue(I.getNewValOperand()), 3562 MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, 3563 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3564 Scope); 3565 3566 SDValue OutChain = L.getValue(1); 3567 3568 if (TLI->getInsertFencesForAtomic()) 3569 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3570 DAG, *TLI); 3571 3572 setValue(&I, L); 3573 DAG.setRoot(OutChain); 3574} 3575 3576void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { 3577 SDLoc dl = getCurSDLoc(); 3578 ISD::NodeType NT; 3579 switch (I.getOperation()) { 3580 default: llvm_unreachable("Unknown atomicrmw operation"); 3581 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; 3582 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; 3583 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; 3584 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; 3585 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; 3586 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; 3587 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; 3588 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; 3589 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; 3590 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; 3591 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; 3592 } 3593 AtomicOrdering Order = I.getOrdering(); 3594 SynchronizationScope Scope = I.getSynchScope(); 3595 3596 SDValue InChain = getRoot(); 3597 3598 const TargetLowering *TLI = TM.getTargetLowering(); 3599 if (TLI->getInsertFencesForAtomic()) 3600 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3601 DAG, *TLI); 3602 3603 SDValue L = 3604 DAG.getAtomic(NT, dl, 3605 getValue(I.getValOperand()).getSimpleValueType(), 3606 InChain, 3607 getValue(I.getPointerOperand()), 3608 getValue(I.getValOperand()), 3609 I.getPointerOperand(), 0 /* Alignment */, 3610 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3611 Scope); 3612 3613 SDValue OutChain = L.getValue(1); 3614 3615 if (TLI->getInsertFencesForAtomic()) 3616 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3617 DAG, *TLI); 3618 3619 setValue(&I, L); 3620 DAG.setRoot(OutChain); 3621} 3622 3623void SelectionDAGBuilder::visitFence(const FenceInst &I) { 3624 SDLoc dl = getCurSDLoc(); 3625 const TargetLowering *TLI = TM.getTargetLowering(); 3626 SDValue Ops[3]; 3627 Ops[0] = getRoot(); 3628 Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy()); 3629 Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy()); 3630 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3)); 3631} 3632 3633void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { 3634 SDLoc dl = getCurSDLoc(); 3635 AtomicOrdering Order = I.getOrdering(); 3636 SynchronizationScope Scope = I.getSynchScope(); 3637 3638 SDValue InChain = getRoot(); 3639 3640 const TargetLowering *TLI = TM.getTargetLowering(); 3641 EVT VT = TLI->getValueType(I.getType()); 3642 3643 if (I.getAlignment() < VT.getSizeInBits() / 8) 3644 report_fatal_error("Cannot generate unaligned atomic load"); 3645 3646 SDValue L = 3647 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, 3648 getValue(I.getPointerOperand()), 3649 I.getPointerOperand(), I.getAlignment(), 3650 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3651 Scope); 3652 3653 SDValue OutChain = L.getValue(1); 3654 3655 if (TLI->getInsertFencesForAtomic()) 3656 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3657 DAG, *TLI); 3658 3659 setValue(&I, L); 3660 DAG.setRoot(OutChain); 3661} 3662 3663void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { 3664 SDLoc dl = getCurSDLoc(); 3665 3666 AtomicOrdering Order = I.getOrdering(); 3667 SynchronizationScope Scope = I.getSynchScope(); 3668 3669 SDValue InChain = getRoot(); 3670 3671 const TargetLowering *TLI = TM.getTargetLowering(); 3672 EVT VT = TLI->getValueType(I.getValueOperand()->getType()); 3673 3674 if (I.getAlignment() < VT.getSizeInBits() / 8) 3675 report_fatal_error("Cannot generate unaligned atomic store"); 3676 3677 if (TLI->getInsertFencesForAtomic()) 3678 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, 3679 DAG, *TLI); 3680 3681 SDValue OutChain = 3682 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, 3683 InChain, 3684 getValue(I.getPointerOperand()), 3685 getValue(I.getValueOperand()), 3686 I.getPointerOperand(), I.getAlignment(), 3687 TLI->getInsertFencesForAtomic() ? Monotonic : Order, 3688 Scope); 3689 3690 if (TLI->getInsertFencesForAtomic()) 3691 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, 3692 DAG, *TLI); 3693 3694 DAG.setRoot(OutChain); 3695} 3696 3697/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC 3698/// node. 3699void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, 3700 unsigned Intrinsic) { 3701 bool HasChain = !I.doesNotAccessMemory(); 3702 bool OnlyLoad = HasChain && I.onlyReadsMemory(); 3703 3704 // Build the operand list. 3705 SmallVector<SDValue, 8> Ops; 3706 if (HasChain) { // If this intrinsic has side-effects, chainify it. 3707 if (OnlyLoad) { 3708 // We don't need to serialize loads against other loads. 3709 Ops.push_back(DAG.getRoot()); 3710 } else { 3711 Ops.push_back(getRoot()); 3712 } 3713 } 3714 3715 // Info is set by getTgtMemInstrinsic 3716 TargetLowering::IntrinsicInfo Info; 3717 const TargetLowering *TLI = TM.getTargetLowering(); 3718 bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic); 3719 3720 // Add the intrinsic ID as an integer operand if it's not a target intrinsic. 3721 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || 3722 Info.opc == ISD::INTRINSIC_W_CHAIN) 3723 Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy())); 3724 3725 // Add all operands of the call to the operand list. 3726 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { 3727 SDValue Op = getValue(I.getArgOperand(i)); 3728 Ops.push_back(Op); 3729 } 3730 3731 SmallVector<EVT, 4> ValueVTs; 3732 ComputeValueVTs(*TLI, I.getType(), ValueVTs); 3733 3734 if (HasChain) 3735 ValueVTs.push_back(MVT::Other); 3736 3737 SDVTList VTs = DAG.getVTList(ValueVTs.data(), ValueVTs.size()); 3738 3739 // Create the node. 3740 SDValue Result; 3741 if (IsTgtIntrinsic) { 3742 // This is target intrinsic that touches memory 3743 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), 3744 VTs, &Ops[0], Ops.size(), 3745 Info.memVT, 3746 MachinePointerInfo(Info.ptrVal, Info.offset), 3747 Info.align, Info.vol, 3748 Info.readMem, Info.writeMem); 3749 } else if (!HasChain) { 3750 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), 3751 VTs, &Ops[0], Ops.size()); 3752 } else if (!I.getType()->isVoidTy()) { 3753 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), 3754 VTs, &Ops[0], Ops.size()); 3755 } else { 3756 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), 3757 VTs, &Ops[0], Ops.size()); 3758 } 3759 3760 if (HasChain) { 3761 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); 3762 if (OnlyLoad) 3763 PendingLoads.push_back(Chain); 3764 else 3765 DAG.setRoot(Chain); 3766 } 3767 3768 if (!I.getType()->isVoidTy()) { 3769 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) { 3770 EVT VT = TLI->getValueType(PTy); 3771 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result); 3772 } 3773 3774 setValue(&I, Result); 3775 } 3776} 3777 3778/// GetSignificand - Get the significand and build it into a floating-point 3779/// number with exponent of 1: 3780/// 3781/// Op = (Op & 0x007fffff) | 0x3f800000; 3782/// 3783/// where Op is the hexadecimal representation of floating point value. 3784static SDValue 3785GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) { 3786 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 3787 DAG.getConstant(0x007fffff, MVT::i32)); 3788 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, 3789 DAG.getConstant(0x3f800000, MVT::i32)); 3790 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); 3791} 3792 3793/// GetExponent - Get the exponent: 3794/// 3795/// (float)(int)(((Op & 0x7f800000) >> 23) - 127); 3796/// 3797/// where Op is the hexadecimal representation of floating point value. 3798static SDValue 3799GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, 3800 SDLoc dl) { 3801 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 3802 DAG.getConstant(0x7f800000, MVT::i32)); 3803 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0, 3804 DAG.getConstant(23, TLI.getPointerTy())); 3805 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, 3806 DAG.getConstant(127, MVT::i32)); 3807 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); 3808} 3809 3810/// getF32Constant - Get 32-bit floating point constant. 3811static SDValue 3812getF32Constant(SelectionDAG &DAG, unsigned Flt) { 3813 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), 3814 MVT::f32); 3815} 3816 3817/// expandExp - Lower an exp intrinsic. Handles the special sequences for 3818/// limited-precision mode. 3819static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG, 3820 const TargetLowering &TLI) { 3821 if (Op.getValueType() == MVT::f32 && 3822 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 3823 3824 // Put the exponent in the right bit position for later addition to the 3825 // final result: 3826 // 3827 // #define LOG2OFe 1.4426950f 3828 // IntegerPartOfX = ((int32_t)(X * LOG2OFe)); 3829 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, 3830 getF32Constant(DAG, 0x3fb8aa3b)); 3831 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 3832 3833 // FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX; 3834 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 3835 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 3836 3837 // IntegerPartOfX <<= 23; 3838 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 3839 DAG.getConstant(23, TLI.getPointerTy())); 3840 3841 SDValue TwoToFracPartOfX; 3842 if (LimitFloatPrecision <= 6) { 3843 // For floating-point precision of 6: 3844 // 3845 // TwoToFractionalPartOfX = 3846 // 0.997535578f + 3847 // (0.735607626f + 0.252464424f * x) * x; 3848 // 3849 // error 0.0144103317, which is 6 bits 3850 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3851 getF32Constant(DAG, 0x3e814304)); 3852 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3853 getF32Constant(DAG, 0x3f3c50c8)); 3854 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3855 TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3856 getF32Constant(DAG, 0x3f7f5e7e)); 3857 } else if (LimitFloatPrecision <= 12) { 3858 // For floating-point precision of 12: 3859 // 3860 // TwoToFractionalPartOfX = 3861 // 0.999892986f + 3862 // (0.696457318f + 3863 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 3864 // 3865 // 0.000107046256 error, which is 13 to 14 bits 3866 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3867 getF32Constant(DAG, 0x3da235e3)); 3868 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3869 getF32Constant(DAG, 0x3e65b8f3)); 3870 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3871 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3872 getF32Constant(DAG, 0x3f324b07)); 3873 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3874 TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 3875 getF32Constant(DAG, 0x3f7ff8fd)); 3876 } else { // LimitFloatPrecision <= 18 3877 // For floating-point precision of 18: 3878 // 3879 // TwoToFractionalPartOfX = 3880 // 0.999999982f + 3881 // (0.693148872f + 3882 // (0.240227044f + 3883 // (0.554906021e-1f + 3884 // (0.961591928e-2f + 3885 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 3886 // 3887 // error 2.47208000*10^(-7), which is better than 18 bits 3888 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3889 getF32Constant(DAG, 0x3924b03e)); 3890 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 3891 getF32Constant(DAG, 0x3ab24b87)); 3892 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3893 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3894 getF32Constant(DAG, 0x3c1d8c17)); 3895 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3896 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 3897 getF32Constant(DAG, 0x3d634a1d)); 3898 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 3899 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 3900 getF32Constant(DAG, 0x3e75fe14)); 3901 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 3902 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 3903 getF32Constant(DAG, 0x3f317234)); 3904 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 3905 TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 3906 getF32Constant(DAG, 0x3f800000)); 3907 } 3908 3909 // Add the exponent into the result in integer domain. 3910 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX); 3911 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 3912 DAG.getNode(ISD::ADD, dl, MVT::i32, 3913 t13, IntegerPartOfX)); 3914 } 3915 3916 // No special expansion. 3917 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op); 3918} 3919 3920/// expandLog - Lower a log intrinsic. Handles the special sequences for 3921/// limited-precision mode. 3922static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG, 3923 const TargetLowering &TLI) { 3924 if (Op.getValueType() == MVT::f32 && 3925 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 3926 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 3927 3928 // Scale the exponent by log(2) [0.69314718f]. 3929 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 3930 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 3931 getF32Constant(DAG, 0x3f317218)); 3932 3933 // Get the significand and build it into a floating-point number with 3934 // exponent of 1. 3935 SDValue X = GetSignificand(DAG, Op1, dl); 3936 3937 SDValue LogOfMantissa; 3938 if (LimitFloatPrecision <= 6) { 3939 // For floating-point precision of 6: 3940 // 3941 // LogofMantissa = 3942 // -1.1609546f + 3943 // (1.4034025f - 0.23903021f * x) * x; 3944 // 3945 // error 0.0034276066, which is better than 8 bits 3946 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3947 getF32Constant(DAG, 0xbe74c456)); 3948 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 3949 getF32Constant(DAG, 0x3fb3a2b1)); 3950 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 3951 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 3952 getF32Constant(DAG, 0x3f949a29)); 3953 } else if (LimitFloatPrecision <= 12) { 3954 // For floating-point precision of 12: 3955 // 3956 // LogOfMantissa = 3957 // -1.7417939f + 3958 // (2.8212026f + 3959 // (-1.4699568f + 3960 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; 3961 // 3962 // error 0.000061011436, which is 14 bits 3963 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3964 getF32Constant(DAG, 0xbd67b6d6)); 3965 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 3966 getF32Constant(DAG, 0x3ee4f4b8)); 3967 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 3968 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 3969 getF32Constant(DAG, 0x3fbc278b)); 3970 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3971 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3972 getF32Constant(DAG, 0x40348e95)); 3973 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3974 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 3975 getF32Constant(DAG, 0x3fdef31a)); 3976 } else { // LimitFloatPrecision <= 18 3977 // For floating-point precision of 18: 3978 // 3979 // LogOfMantissa = 3980 // -2.1072184f + 3981 // (4.2372794f + 3982 // (-3.7029485f + 3983 // (2.2781945f + 3984 // (-0.87823314f + 3985 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; 3986 // 3987 // error 0.0000023660568, which is better than 18 bits 3988 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 3989 getF32Constant(DAG, 0xbc91e5ac)); 3990 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 3991 getF32Constant(DAG, 0x3e4350aa)); 3992 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 3993 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 3994 getF32Constant(DAG, 0x3f60d3e3)); 3995 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 3996 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 3997 getF32Constant(DAG, 0x4011cdf0)); 3998 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 3999 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4000 getF32Constant(DAG, 0x406cfd1c)); 4001 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4002 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4003 getF32Constant(DAG, 0x408797cb)); 4004 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4005 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 4006 getF32Constant(DAG, 0x4006dcab)); 4007 } 4008 4009 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); 4010 } 4011 4012 // No special expansion. 4013 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op); 4014} 4015 4016/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for 4017/// limited-precision mode. 4018static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4019 const TargetLowering &TLI) { 4020 if (Op.getValueType() == MVT::f32 && 4021 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4022 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 4023 4024 // Get the exponent. 4025 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); 4026 4027 // Get the significand and build it into a floating-point number with 4028 // exponent of 1. 4029 SDValue X = GetSignificand(DAG, Op1, dl); 4030 4031 // Different possible minimax approximations of significand in 4032 // floating-point for various degrees of accuracy over [1,2]. 4033 SDValue Log2ofMantissa; 4034 if (LimitFloatPrecision <= 6) { 4035 // For floating-point precision of 6: 4036 // 4037 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; 4038 // 4039 // error 0.0049451742, which is more than 7 bits 4040 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4041 getF32Constant(DAG, 0xbeb08fe0)); 4042 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4043 getF32Constant(DAG, 0x40019463)); 4044 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4045 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4046 getF32Constant(DAG, 0x3fd6633d)); 4047 } else if (LimitFloatPrecision <= 12) { 4048 // For floating-point precision of 12: 4049 // 4050 // Log2ofMantissa = 4051 // -2.51285454f + 4052 // (4.07009056f + 4053 // (-2.12067489f + 4054 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; 4055 // 4056 // error 0.0000876136000, which is better than 13 bits 4057 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4058 getF32Constant(DAG, 0xbda7262e)); 4059 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4060 getF32Constant(DAG, 0x3f25280b)); 4061 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4062 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4063 getF32Constant(DAG, 0x4007b923)); 4064 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4065 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4066 getF32Constant(DAG, 0x40823e2f)); 4067 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4068 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4069 getF32Constant(DAG, 0x4020d29c)); 4070 } else { // LimitFloatPrecision <= 18 4071 // For floating-point precision of 18: 4072 // 4073 // Log2ofMantissa = 4074 // -3.0400495f + 4075 // (6.1129976f + 4076 // (-5.3420409f + 4077 // (3.2865683f + 4078 // (-1.2669343f + 4079 // (0.27515199f - 4080 // 0.25691327e-1f * x) * x) * x) * x) * x) * x; 4081 // 4082 // error 0.0000018516, which is better than 18 bits 4083 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4084 getF32Constant(DAG, 0xbcd2769e)); 4085 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4086 getF32Constant(DAG, 0x3e8ce0b9)); 4087 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4088 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4089 getF32Constant(DAG, 0x3fa22ae7)); 4090 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4091 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4092 getF32Constant(DAG, 0x40525723)); 4093 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4094 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 4095 getF32Constant(DAG, 0x40aaf200)); 4096 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4097 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4098 getF32Constant(DAG, 0x40c39dad)); 4099 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4100 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 4101 getF32Constant(DAG, 0x4042902c)); 4102 } 4103 4104 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); 4105 } 4106 4107 // No special expansion. 4108 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op); 4109} 4110 4111/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for 4112/// limited-precision mode. 4113static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4114 const TargetLowering &TLI) { 4115 if (Op.getValueType() == MVT::f32 && 4116 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4117 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 4118 4119 // Scale the exponent by log10(2) [0.30102999f]. 4120 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 4121 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 4122 getF32Constant(DAG, 0x3e9a209a)); 4123 4124 // Get the significand and build it into a floating-point number with 4125 // exponent of 1. 4126 SDValue X = GetSignificand(DAG, Op1, dl); 4127 4128 SDValue Log10ofMantissa; 4129 if (LimitFloatPrecision <= 6) { 4130 // For floating-point precision of 6: 4131 // 4132 // Log10ofMantissa = 4133 // -0.50419619f + 4134 // (0.60948995f - 0.10380950f * x) * x; 4135 // 4136 // error 0.0014886165, which is 6 bits 4137 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4138 getF32Constant(DAG, 0xbdd49a13)); 4139 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 4140 getF32Constant(DAG, 0x3f1c0789)); 4141 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4142 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 4143 getF32Constant(DAG, 0x3f011300)); 4144 } else if (LimitFloatPrecision <= 12) { 4145 // For floating-point precision of 12: 4146 // 4147 // Log10ofMantissa = 4148 // -0.64831180f + 4149 // (0.91751397f + 4150 // (-0.31664806f + 0.47637168e-1f * x) * x) * x; 4151 // 4152 // error 0.00019228036, which is better than 12 bits 4153 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4154 getF32Constant(DAG, 0x3d431f31)); 4155 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 4156 getF32Constant(DAG, 0x3ea21fb2)); 4157 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4158 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4159 getF32Constant(DAG, 0x3f6ae232)); 4160 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4161 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 4162 getF32Constant(DAG, 0x3f25f7c3)); 4163 } else { // LimitFloatPrecision <= 18 4164 // For floating-point precision of 18: 4165 // 4166 // Log10ofMantissa = 4167 // -0.84299375f + 4168 // (1.5327582f + 4169 // (-1.0688956f + 4170 // (0.49102474f + 4171 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; 4172 // 4173 // error 0.0000037995730, which is better than 18 bits 4174 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4175 getF32Constant(DAG, 0x3c5d51ce)); 4176 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 4177 getF32Constant(DAG, 0x3e00685a)); 4178 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 4179 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4180 getF32Constant(DAG, 0x3efb6798)); 4181 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4182 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 4183 getF32Constant(DAG, 0x3f88d192)); 4184 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4185 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4186 getF32Constant(DAG, 0x3fc4316c)); 4187 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4188 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, 4189 getF32Constant(DAG, 0x3f57ce70)); 4190 } 4191 4192 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); 4193 } 4194 4195 // No special expansion. 4196 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op); 4197} 4198 4199/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for 4200/// limited-precision mode. 4201static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG, 4202 const TargetLowering &TLI) { 4203 if (Op.getValueType() == MVT::f32 && 4204 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4205 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); 4206 4207 // FractionalPartOfX = x - (float)IntegerPartOfX; 4208 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 4209 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1); 4210 4211 // IntegerPartOfX <<= 23; 4212 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 4213 DAG.getConstant(23, TLI.getPointerTy())); 4214 4215 SDValue TwoToFractionalPartOfX; 4216 if (LimitFloatPrecision <= 6) { 4217 // For floating-point precision of 6: 4218 // 4219 // TwoToFractionalPartOfX = 4220 // 0.997535578f + 4221 // (0.735607626f + 0.252464424f * x) * x; 4222 // 4223 // error 0.0144103317, which is 6 bits 4224 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4225 getF32Constant(DAG, 0x3e814304)); 4226 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4227 getF32Constant(DAG, 0x3f3c50c8)); 4228 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4229 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4230 getF32Constant(DAG, 0x3f7f5e7e)); 4231 } else if (LimitFloatPrecision <= 12) { 4232 // For floating-point precision of 12: 4233 // 4234 // TwoToFractionalPartOfX = 4235 // 0.999892986f + 4236 // (0.696457318f + 4237 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 4238 // 4239 // error 0.000107046256, which is 13 to 14 bits 4240 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4241 getF32Constant(DAG, 0x3da235e3)); 4242 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4243 getF32Constant(DAG, 0x3e65b8f3)); 4244 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4245 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4246 getF32Constant(DAG, 0x3f324b07)); 4247 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4248 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4249 getF32Constant(DAG, 0x3f7ff8fd)); 4250 } else { // LimitFloatPrecision <= 18 4251 // For floating-point precision of 18: 4252 // 4253 // TwoToFractionalPartOfX = 4254 // 0.999999982f + 4255 // (0.693148872f + 4256 // (0.240227044f + 4257 // (0.554906021e-1f + 4258 // (0.961591928e-2f + 4259 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 4260 // error 2.47208000*10^(-7), which is better than 18 bits 4261 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4262 getF32Constant(DAG, 0x3924b03e)); 4263 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4264 getF32Constant(DAG, 0x3ab24b87)); 4265 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4266 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4267 getF32Constant(DAG, 0x3c1d8c17)); 4268 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4269 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4270 getF32Constant(DAG, 0x3d634a1d)); 4271 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4272 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4273 getF32Constant(DAG, 0x3e75fe14)); 4274 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4275 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 4276 getF32Constant(DAG, 0x3f317234)); 4277 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 4278 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 4279 getF32Constant(DAG, 0x3f800000)); 4280 } 4281 4282 // Add the exponent into the result in integer domain. 4283 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, 4284 TwoToFractionalPartOfX); 4285 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4286 DAG.getNode(ISD::ADD, dl, MVT::i32, 4287 t13, IntegerPartOfX)); 4288 } 4289 4290 // No special expansion. 4291 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op); 4292} 4293 4294/// visitPow - Lower a pow intrinsic. Handles the special sequences for 4295/// limited-precision mode with x == 10.0f. 4296static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS, 4297 SelectionDAG &DAG, const TargetLowering &TLI) { 4298 bool IsExp10 = false; 4299 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && 4300 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 4301 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { 4302 APFloat Ten(10.0f); 4303 IsExp10 = LHSC->isExactlyValue(Ten); 4304 } 4305 } 4306 4307 if (IsExp10) { 4308 // Put the exponent in the right bit position for later addition to the 4309 // final result: 4310 // 4311 // #define LOG2OF10 3.3219281f 4312 // IntegerPartOfX = (int32_t)(x * LOG2OF10); 4313 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, 4314 getF32Constant(DAG, 0x40549a78)); 4315 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 4316 4317 // FractionalPartOfX = x - (float)IntegerPartOfX; 4318 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 4319 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 4320 4321 // IntegerPartOfX <<= 23; 4322 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 4323 DAG.getConstant(23, TLI.getPointerTy())); 4324 4325 SDValue TwoToFractionalPartOfX; 4326 if (LimitFloatPrecision <= 6) { 4327 // For floating-point precision of 6: 4328 // 4329 // twoToFractionalPartOfX = 4330 // 0.997535578f + 4331 // (0.735607626f + 0.252464424f * x) * x; 4332 // 4333 // error 0.0144103317, which is 6 bits 4334 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4335 getF32Constant(DAG, 0x3e814304)); 4336 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4337 getF32Constant(DAG, 0x3f3c50c8)); 4338 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4339 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4340 getF32Constant(DAG, 0x3f7f5e7e)); 4341 } else if (LimitFloatPrecision <= 12) { 4342 // For floating-point precision of 12: 4343 // 4344 // TwoToFractionalPartOfX = 4345 // 0.999892986f + 4346 // (0.696457318f + 4347 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 4348 // 4349 // error 0.000107046256, which is 13 to 14 bits 4350 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4351 getF32Constant(DAG, 0x3da235e3)); 4352 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4353 getF32Constant(DAG, 0x3e65b8f3)); 4354 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4355 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4356 getF32Constant(DAG, 0x3f324b07)); 4357 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4358 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4359 getF32Constant(DAG, 0x3f7ff8fd)); 4360 } else { // LimitFloatPrecision <= 18 4361 // For floating-point precision of 18: 4362 // 4363 // TwoToFractionalPartOfX = 4364 // 0.999999982f + 4365 // (0.693148872f + 4366 // (0.240227044f + 4367 // (0.554906021e-1f + 4368 // (0.961591928e-2f + 4369 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 4370 // error 2.47208000*10^(-7), which is better than 18 bits 4371 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 4372 getF32Constant(DAG, 0x3924b03e)); 4373 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 4374 getF32Constant(DAG, 0x3ab24b87)); 4375 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 4376 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 4377 getF32Constant(DAG, 0x3c1d8c17)); 4378 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 4379 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 4380 getF32Constant(DAG, 0x3d634a1d)); 4381 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 4382 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 4383 getF32Constant(DAG, 0x3e75fe14)); 4384 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 4385 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 4386 getF32Constant(DAG, 0x3f317234)); 4387 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 4388 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 4389 getF32Constant(DAG, 0x3f800000)); 4390 } 4391 4392 SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX); 4393 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 4394 DAG.getNode(ISD::ADD, dl, MVT::i32, 4395 t13, IntegerPartOfX)); 4396 } 4397 4398 // No special expansion. 4399 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS); 4400} 4401 4402 4403/// ExpandPowI - Expand a llvm.powi intrinsic. 4404static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS, 4405 SelectionDAG &DAG) { 4406 // If RHS is a constant, we can expand this out to a multiplication tree, 4407 // otherwise we end up lowering to a call to __powidf2 (for example). When 4408 // optimizing for size, we only want to do this if the expansion would produce 4409 // a small number of multiplies, otherwise we do the full expansion. 4410 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 4411 // Get the exponent as a positive value. 4412 unsigned Val = RHSC->getSExtValue(); 4413 if ((int)Val < 0) Val = -Val; 4414 4415 // powi(x, 0) -> 1.0 4416 if (Val == 0) 4417 return DAG.getConstantFP(1.0, LHS.getValueType()); 4418 4419 const Function *F = DAG.getMachineFunction().getFunction(); 4420 if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, 4421 Attribute::OptimizeForSize) || 4422 // If optimizing for size, don't insert too many multiplies. This 4423 // inserts up to 5 multiplies. 4424 CountPopulation_32(Val)+Log2_32(Val) < 7) { 4425 // We use the simple binary decomposition method to generate the multiply 4426 // sequence. There are more optimal ways to do this (for example, 4427 // powi(x,15) generates one more multiply than it should), but this has 4428 // the benefit of being both really simple and much better than a libcall. 4429 SDValue Res; // Logically starts equal to 1.0 4430 SDValue CurSquare = LHS; 4431 while (Val) { 4432 if (Val & 1) { 4433 if (Res.getNode()) 4434 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); 4435 else 4436 Res = CurSquare; // 1.0*CurSquare. 4437 } 4438 4439 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), 4440 CurSquare, CurSquare); 4441 Val >>= 1; 4442 } 4443 4444 // If the original was negative, invert the result, producing 1/(x*x*x). 4445 if (RHSC->getSExtValue() < 0) 4446 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), 4447 DAG.getConstantFP(1.0, LHS.getValueType()), Res); 4448 return Res; 4449 } 4450 } 4451 4452 // Otherwise, expand to a libcall. 4453 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); 4454} 4455 4456// getTruncatedArgReg - Find underlying register used for an truncated 4457// argument. 4458static unsigned getTruncatedArgReg(const SDValue &N) { 4459 if (N.getOpcode() != ISD::TRUNCATE) 4460 return 0; 4461 4462 const SDValue &Ext = N.getOperand(0); 4463 if (Ext.getOpcode() == ISD::AssertZext || 4464 Ext.getOpcode() == ISD::AssertSext) { 4465 const SDValue &CFR = Ext.getOperand(0); 4466 if (CFR.getOpcode() == ISD::CopyFromReg) 4467 return cast<RegisterSDNode>(CFR.getOperand(1))->getReg(); 4468 if (CFR.getOpcode() == ISD::TRUNCATE) 4469 return getTruncatedArgReg(CFR); 4470 } 4471 return 0; 4472} 4473 4474/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function 4475/// argument, create the corresponding DBG_VALUE machine instruction for it now. 4476/// At the end of instruction selection, they will be inserted to the entry BB. 4477bool 4478SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable, 4479 int64_t Offset, 4480 const SDValue &N) { 4481 const Argument *Arg = dyn_cast<Argument>(V); 4482 if (!Arg) 4483 return false; 4484 4485 MachineFunction &MF = DAG.getMachineFunction(); 4486 const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo(); 4487 4488 // Ignore inlined function arguments here. 4489 DIVariable DV(Variable); 4490 if (DV.isInlinedFnArgument(MF.getFunction())) 4491 return false; 4492 4493 Optional<MachineOperand> Op; 4494 // Some arguments' frame index is recorded during argument lowering. 4495 if (int FI = FuncInfo.getArgumentFrameIndex(Arg)) 4496 Op = MachineOperand::CreateFI(FI); 4497 4498 if (!Op && N.getNode()) { 4499 unsigned Reg; 4500 if (N.getOpcode() == ISD::CopyFromReg) 4501 Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg(); 4502 else 4503 Reg = getTruncatedArgReg(N); 4504 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) { 4505 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 4506 unsigned PR = RegInfo.getLiveInPhysReg(Reg); 4507 if (PR) 4508 Reg = PR; 4509 } 4510 if (Reg) 4511 Op = MachineOperand::CreateReg(Reg, false); 4512 } 4513 4514 if (!Op) { 4515 // Check if ValueMap has reg number. 4516 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); 4517 if (VMI != FuncInfo.ValueMap.end()) 4518 Op = MachineOperand::CreateReg(VMI->second, false); 4519 } 4520 4521 if (!Op && N.getNode()) 4522 // Check if frame index is available. 4523 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode())) 4524 if (FrameIndexSDNode *FINode = 4525 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 4526 Op = MachineOperand::CreateFI(FINode->getIndex()); 4527 4528 if (!Op) 4529 return false; 4530 4531 // FIXME: This does not handle register-indirect values at offset 0. 4532 bool IsIndirect = Offset != 0; 4533 if (Op->isReg()) 4534 FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(), 4535 TII->get(TargetOpcode::DBG_VALUE), 4536 IsIndirect, 4537 Op->getReg(), Offset, Variable)); 4538 else 4539 FuncInfo.ArgDbgValues.push_back( 4540 BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE)) 4541 .addOperand(*Op).addImm(Offset).addMetadata(Variable)); 4542 4543 return true; 4544} 4545 4546// VisualStudio defines setjmp as _setjmp 4547#if defined(_MSC_VER) && defined(setjmp) && \ 4548 !defined(setjmp_undefined_for_msvc) 4549# pragma push_macro("setjmp") 4550# undef setjmp 4551# define setjmp_undefined_for_msvc 4552#endif 4553 4554/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If 4555/// we want to emit this as a call to a named external function, return the name 4556/// otherwise lower it and return null. 4557const char * 4558SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { 4559 const TargetLowering *TLI = TM.getTargetLowering(); 4560 SDLoc sdl = getCurSDLoc(); 4561 DebugLoc dl = getCurDebugLoc(); 4562 SDValue Res; 4563 4564 switch (Intrinsic) { 4565 default: 4566 // By default, turn this into a target intrinsic node. 4567 visitTargetIntrinsic(I, Intrinsic); 4568 return 0; 4569 case Intrinsic::vastart: visitVAStart(I); return 0; 4570 case Intrinsic::vaend: visitVAEnd(I); return 0; 4571 case Intrinsic::vacopy: visitVACopy(I); return 0; 4572 case Intrinsic::returnaddress: 4573 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(), 4574 getValue(I.getArgOperand(0)))); 4575 return 0; 4576 case Intrinsic::frameaddress: 4577 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(), 4578 getValue(I.getArgOperand(0)))); 4579 return 0; 4580 case Intrinsic::setjmp: 4581 return &"_setjmp"[!TLI->usesUnderscoreSetJmp()]; 4582 case Intrinsic::longjmp: 4583 return &"_longjmp"[!TLI->usesUnderscoreLongJmp()]; 4584 case Intrinsic::memcpy: { 4585 // Assert for address < 256 since we support only user defined address 4586 // spaces. 4587 assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4588 < 256 && 4589 cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace() 4590 < 256 && 4591 "Unknown address space"); 4592 SDValue Op1 = getValue(I.getArgOperand(0)); 4593 SDValue Op2 = getValue(I.getArgOperand(1)); 4594 SDValue Op3 = getValue(I.getArgOperand(2)); 4595 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4596 if (!Align) 4597 Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment. 4598 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4599 DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false, 4600 MachinePointerInfo(I.getArgOperand(0)), 4601 MachinePointerInfo(I.getArgOperand(1)))); 4602 return 0; 4603 } 4604 case Intrinsic::memset: { 4605 // Assert for address < 256 since we support only user defined address 4606 // spaces. 4607 assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4608 < 256 && 4609 "Unknown address space"); 4610 SDValue Op1 = getValue(I.getArgOperand(0)); 4611 SDValue Op2 = getValue(I.getArgOperand(1)); 4612 SDValue Op3 = getValue(I.getArgOperand(2)); 4613 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4614 if (!Align) 4615 Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment. 4616 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4617 DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, 4618 MachinePointerInfo(I.getArgOperand(0)))); 4619 return 0; 4620 } 4621 case Intrinsic::memmove: { 4622 // Assert for address < 256 since we support only user defined address 4623 // spaces. 4624 assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace() 4625 < 256 && 4626 cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace() 4627 < 256 && 4628 "Unknown address space"); 4629 SDValue Op1 = getValue(I.getArgOperand(0)); 4630 SDValue Op2 = getValue(I.getArgOperand(1)); 4631 SDValue Op3 = getValue(I.getArgOperand(2)); 4632 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); 4633 if (!Align) 4634 Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment. 4635 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); 4636 DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, 4637 MachinePointerInfo(I.getArgOperand(0)), 4638 MachinePointerInfo(I.getArgOperand(1)))); 4639 return 0; 4640 } 4641 case Intrinsic::dbg_declare: { 4642 const DbgDeclareInst &DI = cast<DbgDeclareInst>(I); 4643 MDNode *Variable = DI.getVariable(); 4644 const Value *Address = DI.getAddress(); 4645 DIVariable DIVar(Variable); 4646 assert((!DIVar || DIVar.isVariable()) && 4647 "Variable in DbgDeclareInst should be either null or a DIVariable."); 4648 if (!Address || !DIVar) { 4649 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4650 return 0; 4651 } 4652 4653 // Check if address has undef value. 4654 if (isa<UndefValue>(Address) || 4655 (Address->use_empty() && !isa<Argument>(Address))) { 4656 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4657 return 0; 4658 } 4659 4660 SDValue &N = NodeMap[Address]; 4661 if (!N.getNode() && isa<Argument>(Address)) 4662 // Check unused arguments map. 4663 N = UnusedArgNodeMap[Address]; 4664 SDDbgValue *SDV; 4665 if (N.getNode()) { 4666 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) 4667 Address = BCI->getOperand(0); 4668 // Parameters are handled specially. 4669 bool isParameter = 4670 (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable || 4671 isa<Argument>(Address)); 4672 4673 const AllocaInst *AI = dyn_cast<AllocaInst>(Address); 4674 4675 if (isParameter && !AI) { 4676 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); 4677 if (FINode) 4678 // Byval parameter. We have a frame index at this point. 4679 SDV = DAG.getDbgValue(Variable, FINode->getIndex(), 4680 0, dl, SDNodeOrder); 4681 else { 4682 // Address is an argument, so try to emit its dbg value using 4683 // virtual register info from the FuncInfo.ValueMap. 4684 EmitFuncArgumentDbgValue(Address, Variable, 0, N); 4685 return 0; 4686 } 4687 } else if (AI) 4688 SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(), 4689 0, dl, SDNodeOrder); 4690 else { 4691 // Can't do anything with other non-AI cases yet. 4692 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4693 DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t"); 4694 DEBUG(Address->dump()); 4695 return 0; 4696 } 4697 DAG.AddDbgValue(SDV, N.getNode(), isParameter); 4698 } else { 4699 // If Address is an argument then try to emit its dbg value using 4700 // virtual register info from the FuncInfo.ValueMap. 4701 if (!EmitFuncArgumentDbgValue(Address, Variable, 0, N)) { 4702 // If variable is pinned by a alloca in dominating bb then 4703 // use StaticAllocaMap. 4704 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) { 4705 if (AI->getParent() != DI.getParent()) { 4706 DenseMap<const AllocaInst*, int>::iterator SI = 4707 FuncInfo.StaticAllocaMap.find(AI); 4708 if (SI != FuncInfo.StaticAllocaMap.end()) { 4709 SDV = DAG.getDbgValue(Variable, SI->second, 4710 0, dl, SDNodeOrder); 4711 DAG.AddDbgValue(SDV, 0, false); 4712 return 0; 4713 } 4714 } 4715 } 4716 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4717 } 4718 } 4719 return 0; 4720 } 4721 case Intrinsic::dbg_value: { 4722 const DbgValueInst &DI = cast<DbgValueInst>(I); 4723 DIVariable DIVar(DI.getVariable()); 4724 assert((!DIVar || DIVar.isVariable()) && 4725 "Variable in DbgValueInst should be either null or a DIVariable."); 4726 if (!DIVar) 4727 return 0; 4728 4729 MDNode *Variable = DI.getVariable(); 4730 uint64_t Offset = DI.getOffset(); 4731 const Value *V = DI.getValue(); 4732 if (!V) 4733 return 0; 4734 4735 SDDbgValue *SDV; 4736 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) { 4737 SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder); 4738 DAG.AddDbgValue(SDV, 0, false); 4739 } else { 4740 // Do not use getValue() in here; we don't want to generate code at 4741 // this point if it hasn't been done yet. 4742 SDValue N = NodeMap[V]; 4743 if (!N.getNode() && isa<Argument>(V)) 4744 // Check unused arguments map. 4745 N = UnusedArgNodeMap[V]; 4746 if (N.getNode()) { 4747 if (!EmitFuncArgumentDbgValue(V, Variable, Offset, N)) { 4748 SDV = DAG.getDbgValue(Variable, N.getNode(), 4749 N.getResNo(), Offset, dl, SDNodeOrder); 4750 DAG.AddDbgValue(SDV, N.getNode(), false); 4751 } 4752 } else if (!V->use_empty() ) { 4753 // Do not call getValue(V) yet, as we don't want to generate code. 4754 // Remember it for later. 4755 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder); 4756 DanglingDebugInfoMap[V] = DDI; 4757 } else { 4758 // We may expand this to cover more cases. One case where we have no 4759 // data available is an unreferenced parameter. 4760 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); 4761 } 4762 } 4763 4764 // Build a debug info table entry. 4765 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V)) 4766 V = BCI->getOperand(0); 4767 const AllocaInst *AI = dyn_cast<AllocaInst>(V); 4768 // Don't handle byval struct arguments or VLAs, for example. 4769 if (!AI) { 4770 DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n"); 4771 DEBUG(dbgs() << " Last seen at:\n " << *V << "\n"); 4772 return 0; 4773 } 4774 DenseMap<const AllocaInst*, int>::iterator SI = 4775 FuncInfo.StaticAllocaMap.find(AI); 4776 if (SI == FuncInfo.StaticAllocaMap.end()) 4777 return 0; // VLAs. 4778 int FI = SI->second; 4779 4780 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 4781 if (!DI.getDebugLoc().isUnknown() && MMI.hasDebugInfo()) 4782 MMI.setVariableDbgInfo(Variable, FI, DI.getDebugLoc()); 4783 return 0; 4784 } 4785 4786 case Intrinsic::eh_typeid_for: { 4787 // Find the type id for the given typeinfo. 4788 GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0)); 4789 unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV); 4790 Res = DAG.getConstant(TypeID, MVT::i32); 4791 setValue(&I, Res); 4792 return 0; 4793 } 4794 4795 case Intrinsic::eh_return_i32: 4796 case Intrinsic::eh_return_i64: 4797 DAG.getMachineFunction().getMMI().setCallsEHReturn(true); 4798 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, 4799 MVT::Other, 4800 getControlRoot(), 4801 getValue(I.getArgOperand(0)), 4802 getValue(I.getArgOperand(1)))); 4803 return 0; 4804 case Intrinsic::eh_unwind_init: 4805 DAG.getMachineFunction().getMMI().setCallsUnwindInit(true); 4806 return 0; 4807 case Intrinsic::eh_dwarf_cfa: { 4808 SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl, 4809 TLI->getPointerTy()); 4810 SDValue Offset = DAG.getNode(ISD::ADD, sdl, 4811 CfaArg.getValueType(), 4812 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl, 4813 CfaArg.getValueType()), 4814 CfaArg); 4815 SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, 4816 TLI->getPointerTy(), 4817 DAG.getConstant(0, TLI->getPointerTy())); 4818 setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(), 4819 FA, Offset)); 4820 return 0; 4821 } 4822 case Intrinsic::eh_sjlj_callsite: { 4823 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 4824 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0)); 4825 assert(CI && "Non-constant call site value in eh.sjlj.callsite!"); 4826 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"); 4827 4828 MMI.setCurrentCallSite(CI->getZExtValue()); 4829 return 0; 4830 } 4831 case Intrinsic::eh_sjlj_functioncontext: { 4832 // Get and store the index of the function context. 4833 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 4834 AllocaInst *FnCtx = 4835 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); 4836 int FI = FuncInfo.StaticAllocaMap[FnCtx]; 4837 MFI->setFunctionContextIndex(FI); 4838 return 0; 4839 } 4840 case Intrinsic::eh_sjlj_setjmp: { 4841 SDValue Ops[2]; 4842 Ops[0] = getRoot(); 4843 Ops[1] = getValue(I.getArgOperand(0)); 4844 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, 4845 DAG.getVTList(MVT::i32, MVT::Other), 4846 Ops, 2); 4847 setValue(&I, Op.getValue(0)); 4848 DAG.setRoot(Op.getValue(1)); 4849 return 0; 4850 } 4851 case Intrinsic::eh_sjlj_longjmp: { 4852 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, 4853 getRoot(), getValue(I.getArgOperand(0)))); 4854 return 0; 4855 } 4856 4857 case Intrinsic::x86_mmx_pslli_w: 4858 case Intrinsic::x86_mmx_pslli_d: 4859 case Intrinsic::x86_mmx_pslli_q: 4860 case Intrinsic::x86_mmx_psrli_w: 4861 case Intrinsic::x86_mmx_psrli_d: 4862 case Intrinsic::x86_mmx_psrli_q: 4863 case Intrinsic::x86_mmx_psrai_w: 4864 case Intrinsic::x86_mmx_psrai_d: { 4865 SDValue ShAmt = getValue(I.getArgOperand(1)); 4866 if (isa<ConstantSDNode>(ShAmt)) { 4867 visitTargetIntrinsic(I, Intrinsic); 4868 return 0; 4869 } 4870 unsigned NewIntrinsic = 0; 4871 EVT ShAmtVT = MVT::v2i32; 4872 switch (Intrinsic) { 4873 case Intrinsic::x86_mmx_pslli_w: 4874 NewIntrinsic = Intrinsic::x86_mmx_psll_w; 4875 break; 4876 case Intrinsic::x86_mmx_pslli_d: 4877 NewIntrinsic = Intrinsic::x86_mmx_psll_d; 4878 break; 4879 case Intrinsic::x86_mmx_pslli_q: 4880 NewIntrinsic = Intrinsic::x86_mmx_psll_q; 4881 break; 4882 case Intrinsic::x86_mmx_psrli_w: 4883 NewIntrinsic = Intrinsic::x86_mmx_psrl_w; 4884 break; 4885 case Intrinsic::x86_mmx_psrli_d: 4886 NewIntrinsic = Intrinsic::x86_mmx_psrl_d; 4887 break; 4888 case Intrinsic::x86_mmx_psrli_q: 4889 NewIntrinsic = Intrinsic::x86_mmx_psrl_q; 4890 break; 4891 case Intrinsic::x86_mmx_psrai_w: 4892 NewIntrinsic = Intrinsic::x86_mmx_psra_w; 4893 break; 4894 case Intrinsic::x86_mmx_psrai_d: 4895 NewIntrinsic = Intrinsic::x86_mmx_psra_d; 4896 break; 4897 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 4898 } 4899 4900 // The vector shift intrinsics with scalars uses 32b shift amounts but 4901 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits 4902 // to be zero. 4903 // We must do this early because v2i32 is not a legal type. 4904 SDValue ShOps[2]; 4905 ShOps[0] = ShAmt; 4906 ShOps[1] = DAG.getConstant(0, MVT::i32); 4907 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, &ShOps[0], 2); 4908 EVT DestVT = TLI->getValueType(I.getType()); 4909 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt); 4910 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT, 4911 DAG.getConstant(NewIntrinsic, MVT::i32), 4912 getValue(I.getArgOperand(0)), ShAmt); 4913 setValue(&I, Res); 4914 return 0; 4915 } 4916 case Intrinsic::x86_avx_vinsertf128_pd_256: 4917 case Intrinsic::x86_avx_vinsertf128_ps_256: 4918 case Intrinsic::x86_avx_vinsertf128_si_256: 4919 case Intrinsic::x86_avx2_vinserti128: { 4920 EVT DestVT = TLI->getValueType(I.getType()); 4921 EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType()); 4922 uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) * 4923 ElVT.getVectorNumElements(); 4924 Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT, 4925 getValue(I.getArgOperand(0)), 4926 getValue(I.getArgOperand(1)), 4927 DAG.getConstant(Idx, TLI->getVectorIdxTy())); 4928 setValue(&I, Res); 4929 return 0; 4930 } 4931 case Intrinsic::x86_avx_vextractf128_pd_256: 4932 case Intrinsic::x86_avx_vextractf128_ps_256: 4933 case Intrinsic::x86_avx_vextractf128_si_256: 4934 case Intrinsic::x86_avx2_vextracti128: { 4935 EVT DestVT = TLI->getValueType(I.getType()); 4936 uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) * 4937 DestVT.getVectorNumElements(); 4938 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT, 4939 getValue(I.getArgOperand(0)), 4940 DAG.getConstant(Idx, TLI->getVectorIdxTy())); 4941 setValue(&I, Res); 4942 return 0; 4943 } 4944 case Intrinsic::convertff: 4945 case Intrinsic::convertfsi: 4946 case Intrinsic::convertfui: 4947 case Intrinsic::convertsif: 4948 case Intrinsic::convertuif: 4949 case Intrinsic::convertss: 4950 case Intrinsic::convertsu: 4951 case Intrinsic::convertus: 4952 case Intrinsic::convertuu: { 4953 ISD::CvtCode Code = ISD::CVT_INVALID; 4954 switch (Intrinsic) { 4955 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 4956 case Intrinsic::convertff: Code = ISD::CVT_FF; break; 4957 case Intrinsic::convertfsi: Code = ISD::CVT_FS; break; 4958 case Intrinsic::convertfui: Code = ISD::CVT_FU; break; 4959 case Intrinsic::convertsif: Code = ISD::CVT_SF; break; 4960 case Intrinsic::convertuif: Code = ISD::CVT_UF; break; 4961 case Intrinsic::convertss: Code = ISD::CVT_SS; break; 4962 case Intrinsic::convertsu: Code = ISD::CVT_SU; break; 4963 case Intrinsic::convertus: Code = ISD::CVT_US; break; 4964 case Intrinsic::convertuu: Code = ISD::CVT_UU; break; 4965 } 4966 EVT DestVT = TLI->getValueType(I.getType()); 4967 const Value *Op1 = I.getArgOperand(0); 4968 Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1), 4969 DAG.getValueType(DestVT), 4970 DAG.getValueType(getValue(Op1).getValueType()), 4971 getValue(I.getArgOperand(1)), 4972 getValue(I.getArgOperand(2)), 4973 Code); 4974 setValue(&I, Res); 4975 return 0; 4976 } 4977 case Intrinsic::powi: 4978 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), 4979 getValue(I.getArgOperand(1)), DAG)); 4980 return 0; 4981 case Intrinsic::log: 4982 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4983 return 0; 4984 case Intrinsic::log2: 4985 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4986 return 0; 4987 case Intrinsic::log10: 4988 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4989 return 0; 4990 case Intrinsic::exp: 4991 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4992 return 0; 4993 case Intrinsic::exp2: 4994 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI)); 4995 return 0; 4996 case Intrinsic::pow: 4997 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), 4998 getValue(I.getArgOperand(1)), DAG, *TLI)); 4999 return 0; 5000 case Intrinsic::sqrt: 5001 case Intrinsic::fabs: 5002 case Intrinsic::sin: 5003 case Intrinsic::cos: 5004 case Intrinsic::floor: 5005 case Intrinsic::ceil: 5006 case Intrinsic::trunc: 5007 case Intrinsic::rint: 5008 case Intrinsic::nearbyint: 5009 case Intrinsic::round: { 5010 unsigned Opcode; 5011 switch (Intrinsic) { 5012 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5013 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 5014 case Intrinsic::fabs: Opcode = ISD::FABS; break; 5015 case Intrinsic::sin: Opcode = ISD::FSIN; break; 5016 case Intrinsic::cos: Opcode = ISD::FCOS; break; 5017 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 5018 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 5019 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 5020 case Intrinsic::rint: Opcode = ISD::FRINT; break; 5021 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 5022 case Intrinsic::round: Opcode = ISD::FROUND; break; 5023 } 5024 5025 setValue(&I, DAG.getNode(Opcode, sdl, 5026 getValue(I.getArgOperand(0)).getValueType(), 5027 getValue(I.getArgOperand(0)))); 5028 return 0; 5029 } 5030 case Intrinsic::copysign: 5031 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, 5032 getValue(I.getArgOperand(0)).getValueType(), 5033 getValue(I.getArgOperand(0)), 5034 getValue(I.getArgOperand(1)))); 5035 return 0; 5036 case Intrinsic::fma: 5037 setValue(&I, DAG.getNode(ISD::FMA, sdl, 5038 getValue(I.getArgOperand(0)).getValueType(), 5039 getValue(I.getArgOperand(0)), 5040 getValue(I.getArgOperand(1)), 5041 getValue(I.getArgOperand(2)))); 5042 return 0; 5043 case Intrinsic::fmuladd: { 5044 EVT VT = TLI->getValueType(I.getType()); 5045 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && 5046 TLI->isFMAFasterThanFMulAndFAdd(VT)) { 5047 setValue(&I, DAG.getNode(ISD::FMA, sdl, 5048 getValue(I.getArgOperand(0)).getValueType(), 5049 getValue(I.getArgOperand(0)), 5050 getValue(I.getArgOperand(1)), 5051 getValue(I.getArgOperand(2)))); 5052 } else { 5053 SDValue Mul = DAG.getNode(ISD::FMUL, sdl, 5054 getValue(I.getArgOperand(0)).getValueType(), 5055 getValue(I.getArgOperand(0)), 5056 getValue(I.getArgOperand(1))); 5057 SDValue Add = DAG.getNode(ISD::FADD, sdl, 5058 getValue(I.getArgOperand(0)).getValueType(), 5059 Mul, 5060 getValue(I.getArgOperand(2))); 5061 setValue(&I, Add); 5062 } 5063 return 0; 5064 } 5065 case Intrinsic::convert_to_fp16: 5066 setValue(&I, DAG.getNode(ISD::FP32_TO_FP16, sdl, 5067 MVT::i16, getValue(I.getArgOperand(0)))); 5068 return 0; 5069 case Intrinsic::convert_from_fp16: 5070 setValue(&I, DAG.getNode(ISD::FP16_TO_FP32, sdl, 5071 MVT::f32, getValue(I.getArgOperand(0)))); 5072 return 0; 5073 case Intrinsic::pcmarker: { 5074 SDValue Tmp = getValue(I.getArgOperand(0)); 5075 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); 5076 return 0; 5077 } 5078 case Intrinsic::readcyclecounter: { 5079 SDValue Op = getRoot(); 5080 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, 5081 DAG.getVTList(MVT::i64, MVT::Other), 5082 &Op, 1); 5083 setValue(&I, Res); 5084 DAG.setRoot(Res.getValue(1)); 5085 return 0; 5086 } 5087 case Intrinsic::bswap: 5088 setValue(&I, DAG.getNode(ISD::BSWAP, sdl, 5089 getValue(I.getArgOperand(0)).getValueType(), 5090 getValue(I.getArgOperand(0)))); 5091 return 0; 5092 case Intrinsic::cttz: { 5093 SDValue Arg = getValue(I.getArgOperand(0)); 5094 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 5095 EVT Ty = Arg.getValueType(); 5096 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, 5097 sdl, Ty, Arg)); 5098 return 0; 5099 } 5100 case Intrinsic::ctlz: { 5101 SDValue Arg = getValue(I.getArgOperand(0)); 5102 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 5103 EVT Ty = Arg.getValueType(); 5104 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, 5105 sdl, Ty, Arg)); 5106 return 0; 5107 } 5108 case Intrinsic::ctpop: { 5109 SDValue Arg = getValue(I.getArgOperand(0)); 5110 EVT Ty = Arg.getValueType(); 5111 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); 5112 return 0; 5113 } 5114 case Intrinsic::stacksave: { 5115 SDValue Op = getRoot(); 5116 Res = DAG.getNode(ISD::STACKSAVE, sdl, 5117 DAG.getVTList(TLI->getPointerTy(), MVT::Other), &Op, 1); 5118 setValue(&I, Res); 5119 DAG.setRoot(Res.getValue(1)); 5120 return 0; 5121 } 5122 case Intrinsic::stackrestore: { 5123 Res = getValue(I.getArgOperand(0)); 5124 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); 5125 return 0; 5126 } 5127 case Intrinsic::stackprotector: { 5128 // Emit code into the DAG to store the stack guard onto the stack. 5129 MachineFunction &MF = DAG.getMachineFunction(); 5130 MachineFrameInfo *MFI = MF.getFrameInfo(); 5131 EVT PtrTy = TLI->getPointerTy(); 5132 5133 SDValue Src = getValue(I.getArgOperand(0)); // The guard's value. 5134 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); 5135 5136 int FI = FuncInfo.StaticAllocaMap[Slot]; 5137 MFI->setStackProtectorIndex(FI); 5138 5139 SDValue FIN = DAG.getFrameIndex(FI, PtrTy); 5140 5141 // Store the stack protector onto the stack. 5142 Res = DAG.getStore(getRoot(), sdl, Src, FIN, 5143 MachinePointerInfo::getFixedStack(FI), 5144 true, false, 0); 5145 setValue(&I, Res); 5146 DAG.setRoot(Res); 5147 return 0; 5148 } 5149 case Intrinsic::objectsize: { 5150 // If we don't know by now, we're never going to know. 5151 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1)); 5152 5153 assert(CI && "Non-constant type in __builtin_object_size?"); 5154 5155 SDValue Arg = getValue(I.getCalledValue()); 5156 EVT Ty = Arg.getValueType(); 5157 5158 if (CI->isZero()) 5159 Res = DAG.getConstant(-1ULL, Ty); 5160 else 5161 Res = DAG.getConstant(0, Ty); 5162 5163 setValue(&I, Res); 5164 return 0; 5165 } 5166 case Intrinsic::annotation: 5167 case Intrinsic::ptr_annotation: 5168 // Drop the intrinsic, but forward the value 5169 setValue(&I, getValue(I.getOperand(0))); 5170 return 0; 5171 case Intrinsic::var_annotation: 5172 // Discard annotate attributes 5173 return 0; 5174 5175 case Intrinsic::init_trampoline: { 5176 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts()); 5177 5178 SDValue Ops[6]; 5179 Ops[0] = getRoot(); 5180 Ops[1] = getValue(I.getArgOperand(0)); 5181 Ops[2] = getValue(I.getArgOperand(1)); 5182 Ops[3] = getValue(I.getArgOperand(2)); 5183 Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); 5184 Ops[5] = DAG.getSrcValue(F); 5185 5186 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops, 6); 5187 5188 DAG.setRoot(Res); 5189 return 0; 5190 } 5191 case Intrinsic::adjust_trampoline: { 5192 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, 5193 TLI->getPointerTy(), 5194 getValue(I.getArgOperand(0)))); 5195 return 0; 5196 } 5197 case Intrinsic::gcroot: 5198 if (GFI) { 5199 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts(); 5200 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1)); 5201 5202 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); 5203 GFI->addStackRoot(FI->getIndex(), TypeMap); 5204 } 5205 return 0; 5206 case Intrinsic::gcread: 5207 case Intrinsic::gcwrite: 5208 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); 5209 case Intrinsic::flt_rounds: 5210 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32)); 5211 return 0; 5212 5213 case Intrinsic::expect: { 5214 // Just replace __builtin_expect(exp, c) with EXP. 5215 setValue(&I, getValue(I.getArgOperand(0))); 5216 return 0; 5217 } 5218 5219 case Intrinsic::debugtrap: 5220 case Intrinsic::trap: { 5221 StringRef TrapFuncName = TM.Options.getTrapFunctionName(); 5222 if (TrapFuncName.empty()) { 5223 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ? 5224 ISD::TRAP : ISD::DEBUGTRAP; 5225 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot())); 5226 return 0; 5227 } 5228 TargetLowering::ArgListTy Args; 5229 TargetLowering:: 5230 CallLoweringInfo CLI(getRoot(), I.getType(), 5231 false, false, false, false, 0, CallingConv::C, 5232 /*isTailCall=*/false, 5233 /*doesNotRet=*/false, /*isReturnValueUsed=*/true, 5234 DAG.getExternalSymbol(TrapFuncName.data(), 5235 TLI->getPointerTy()), 5236 Args, DAG, sdl); 5237 std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI); 5238 DAG.setRoot(Result.second); 5239 return 0; 5240 } 5241 5242 case Intrinsic::uadd_with_overflow: 5243 case Intrinsic::sadd_with_overflow: 5244 case Intrinsic::usub_with_overflow: 5245 case Intrinsic::ssub_with_overflow: 5246 case Intrinsic::umul_with_overflow: 5247 case Intrinsic::smul_with_overflow: { 5248 ISD::NodeType Op; 5249 switch (Intrinsic) { 5250 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 5251 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; 5252 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; 5253 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; 5254 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; 5255 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; 5256 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; 5257 } 5258 SDValue Op1 = getValue(I.getArgOperand(0)); 5259 SDValue Op2 = getValue(I.getArgOperand(1)); 5260 5261 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1); 5262 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); 5263 return 0; 5264 } 5265 case Intrinsic::prefetch: { 5266 SDValue Ops[5]; 5267 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 5268 Ops[0] = getRoot(); 5269 Ops[1] = getValue(I.getArgOperand(0)); 5270 Ops[2] = getValue(I.getArgOperand(1)); 5271 Ops[3] = getValue(I.getArgOperand(2)); 5272 Ops[4] = getValue(I.getArgOperand(3)); 5273 DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl, 5274 DAG.getVTList(MVT::Other), 5275 &Ops[0], 5, 5276 EVT::getIntegerVT(*Context, 8), 5277 MachinePointerInfo(I.getArgOperand(0)), 5278 0, /* align */ 5279 false, /* volatile */ 5280 rw==0, /* read */ 5281 rw==1)); /* write */ 5282 return 0; 5283 } 5284 case Intrinsic::lifetime_start: 5285 case Intrinsic::lifetime_end: { 5286 bool IsStart = (Intrinsic == Intrinsic::lifetime_start); 5287 // Stack coloring is not enabled in O0, discard region information. 5288 if (TM.getOptLevel() == CodeGenOpt::None) 5289 return 0; 5290 5291 SmallVector<Value *, 4> Allocas; 5292 GetUnderlyingObjects(I.getArgOperand(1), Allocas, TD); 5293 5294 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(), 5295 E = Allocas.end(); Object != E; ++Object) { 5296 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object); 5297 5298 // Could not find an Alloca. 5299 if (!LifetimeObject) 5300 continue; 5301 5302 int FI = FuncInfo.StaticAllocaMap[LifetimeObject]; 5303 5304 SDValue Ops[2]; 5305 Ops[0] = getRoot(); 5306 Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true); 5307 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END); 5308 5309 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops, 2); 5310 DAG.setRoot(Res); 5311 } 5312 return 0; 5313 } 5314 case Intrinsic::invariant_start: 5315 // Discard region information. 5316 setValue(&I, DAG.getUNDEF(TLI->getPointerTy())); 5317 return 0; 5318 case Intrinsic::invariant_end: 5319 // Discard region information. 5320 return 0; 5321 case Intrinsic::stackprotectorcheck: { 5322 // Do not actually emit anything for this basic block. Instead we initialize 5323 // the stack protector descriptor and export the guard variable so we can 5324 // access it in FinishBasicBlock. 5325 const BasicBlock *BB = I.getParent(); 5326 SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I); 5327 ExportFromCurrentBlock(SPDescriptor.getGuard()); 5328 5329 // Flush our exports since we are going to process a terminator. 5330 (void)getControlRoot(); 5331 return 0; 5332 } 5333 case Intrinsic::donothing: 5334 // ignore 5335 return 0; 5336 case Intrinsic::experimental_stackmap: { 5337 visitStackmap(I); 5338 return 0; 5339 } 5340 case Intrinsic::experimental_patchpoint_void: 5341 case Intrinsic::experimental_patchpoint_i64: { 5342 visitPatchpoint(I); 5343 return 0; 5344 } 5345 } 5346} 5347 5348void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, 5349 bool isTailCall, 5350 MachineBasicBlock *LandingPad) { 5351 const TargetLowering *TLI = TM.getTargetLowering(); 5352 PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); 5353 FunctionType *FTy = cast<FunctionType>(PT->getElementType()); 5354 Type *RetTy = FTy->getReturnType(); 5355 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 5356 MCSymbol *BeginLabel = 0; 5357 5358 TargetLowering::ArgListTy Args; 5359 TargetLowering::ArgListEntry Entry; 5360 Args.reserve(CS.arg_size()); 5361 5362 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); 5363 i != e; ++i) { 5364 const Value *V = *i; 5365 5366 // Skip empty types 5367 if (V->getType()->isEmptyTy()) 5368 continue; 5369 5370 SDValue ArgNode = getValue(V); 5371 Entry.Node = ArgNode; Entry.Ty = V->getType(); 5372 5373 // Skip the first return-type Attribute to get to params. 5374 Entry.setAttributes(&CS, i - CS.arg_begin() + 1); 5375 Args.push_back(Entry); 5376 } 5377 5378 if (LandingPad) { 5379 // Insert a label before the invoke call to mark the try range. This can be 5380 // used to detect deletion of the invoke via the MachineModuleInfo. 5381 BeginLabel = MMI.getContext().CreateTempSymbol(); 5382 5383 // For SjLj, keep track of which landing pads go with which invokes 5384 // so as to maintain the ordering of pads in the LSDA. 5385 unsigned CallSiteIndex = MMI.getCurrentCallSite(); 5386 if (CallSiteIndex) { 5387 MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); 5388 LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex); 5389 5390 // Now that the call site is handled, stop tracking it. 5391 MMI.setCurrentCallSite(0); 5392 } 5393 5394 // Both PendingLoads and PendingExports must be flushed here; 5395 // this call might not return. 5396 (void)getRoot(); 5397 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel)); 5398 } 5399 5400 // Check if target-independent constraints permit a tail call here. 5401 // Target-dependent constraints are checked within TLI->LowerCallTo. 5402 if (isTailCall && !isInTailCallPosition(CS, *TLI)) 5403 isTailCall = false; 5404 5405 TargetLowering:: 5406 CallLoweringInfo CLI(getRoot(), RetTy, FTy, isTailCall, Callee, Args, DAG, 5407 getCurSDLoc(), CS); 5408 std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI); 5409 assert((isTailCall || Result.second.getNode()) && 5410 "Non-null chain expected with non-tail call!"); 5411 assert((Result.second.getNode() || !Result.first.getNode()) && 5412 "Null value expected with tail call!"); 5413 if (Result.first.getNode()) 5414 setValue(CS.getInstruction(), Result.first); 5415 5416 if (!Result.second.getNode()) { 5417 // As a special case, a null chain means that a tail call has been emitted 5418 // and the DAG root is already updated. 5419 HasTailCall = true; 5420 5421 // Since there's no actual continuation from this block, nothing can be 5422 // relying on us setting vregs for them. 5423 PendingExports.clear(); 5424 } else { 5425 DAG.setRoot(Result.second); 5426 } 5427 5428 if (LandingPad) { 5429 // Insert a label at the end of the invoke call to mark the try range. This 5430 // can be used to detect deletion of the invoke via the MachineModuleInfo. 5431 MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol(); 5432 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel)); 5433 5434 // Inform MachineModuleInfo of range. 5435 MMI.addInvoke(LandingPad, BeginLabel, EndLabel); 5436 } 5437} 5438 5439/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the 5440/// value is equal or not-equal to zero. 5441static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) { 5442 for (Value::const_use_iterator UI = V->use_begin(), E = V->use_end(); 5443 UI != E; ++UI) { 5444 if (const ICmpInst *IC = dyn_cast<ICmpInst>(*UI)) 5445 if (IC->isEquality()) 5446 if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1))) 5447 if (C->isNullValue()) 5448 continue; 5449 // Unknown instruction. 5450 return false; 5451 } 5452 return true; 5453} 5454 5455static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, 5456 Type *LoadTy, 5457 SelectionDAGBuilder &Builder) { 5458 5459 // Check to see if this load can be trivially constant folded, e.g. if the 5460 // input is from a string literal. 5461 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { 5462 // Cast pointer to the type we really want to load. 5463 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), 5464 PointerType::getUnqual(LoadTy)); 5465 5466 if (const Constant *LoadCst = 5467 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput), 5468 Builder.TD)) 5469 return Builder.getValue(LoadCst); 5470 } 5471 5472 // Otherwise, we have to emit the load. If the pointer is to unfoldable but 5473 // still constant memory, the input chain can be the entry node. 5474 SDValue Root; 5475 bool ConstantMemory = false; 5476 5477 // Do not serialize (non-volatile) loads of constant memory with anything. 5478 if (Builder.AA->pointsToConstantMemory(PtrVal)) { 5479 Root = Builder.DAG.getEntryNode(); 5480 ConstantMemory = true; 5481 } else { 5482 // Do not serialize non-volatile loads against each other. 5483 Root = Builder.DAG.getRoot(); 5484 } 5485 5486 SDValue Ptr = Builder.getValue(PtrVal); 5487 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, 5488 Ptr, MachinePointerInfo(PtrVal), 5489 false /*volatile*/, 5490 false /*nontemporal*/, 5491 false /*isinvariant*/, 1 /* align=1 */); 5492 5493 if (!ConstantMemory) 5494 Builder.PendingLoads.push_back(LoadVal.getValue(1)); 5495 return LoadVal; 5496} 5497 5498/// processIntegerCallValue - Record the value for an instruction that 5499/// produces an integer result, converting the type where necessary. 5500void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, 5501 SDValue Value, 5502 bool IsSigned) { 5503 EVT VT = TM.getTargetLowering()->getValueType(I.getType(), true); 5504 if (IsSigned) 5505 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT); 5506 else 5507 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT); 5508 setValue(&I, Value); 5509} 5510 5511/// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form. 5512/// If so, return true and lower it, otherwise return false and it will be 5513/// lowered like a normal call. 5514bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) { 5515 // Verify that the prototype makes sense. int memcmp(void*,void*,size_t) 5516 if (I.getNumArgOperands() != 3) 5517 return false; 5518 5519 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); 5520 if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() || 5521 !I.getArgOperand(2)->getType()->isIntegerTy() || 5522 !I.getType()->isIntegerTy()) 5523 return false; 5524 5525 const Value *Size = I.getArgOperand(2); 5526 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size); 5527 if (CSize && CSize->getZExtValue() == 0) { 5528 EVT CallVT = TM.getTargetLowering()->getValueType(I.getType(), true); 5529 setValue(&I, DAG.getConstant(0, CallVT)); 5530 return true; 5531 } 5532 5533 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5534 std::pair<SDValue, SDValue> Res = 5535 TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(), 5536 getValue(LHS), getValue(RHS), getValue(Size), 5537 MachinePointerInfo(LHS), 5538 MachinePointerInfo(RHS)); 5539 if (Res.first.getNode()) { 5540 processIntegerCallValue(I, Res.first, true); 5541 PendingLoads.push_back(Res.second); 5542 return true; 5543 } 5544 5545 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 5546 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 5547 if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) { 5548 bool ActuallyDoIt = true; 5549 MVT LoadVT; 5550 Type *LoadTy; 5551 switch (CSize->getZExtValue()) { 5552 default: 5553 LoadVT = MVT::Other; 5554 LoadTy = 0; 5555 ActuallyDoIt = false; 5556 break; 5557 case 2: 5558 LoadVT = MVT::i16; 5559 LoadTy = Type::getInt16Ty(CSize->getContext()); 5560 break; 5561 case 4: 5562 LoadVT = MVT::i32; 5563 LoadTy = Type::getInt32Ty(CSize->getContext()); 5564 break; 5565 case 8: 5566 LoadVT = MVT::i64; 5567 LoadTy = Type::getInt64Ty(CSize->getContext()); 5568 break; 5569 /* 5570 case 16: 5571 LoadVT = MVT::v4i32; 5572 LoadTy = Type::getInt32Ty(CSize->getContext()); 5573 LoadTy = VectorType::get(LoadTy, 4); 5574 break; 5575 */ 5576 } 5577 5578 // This turns into unaligned loads. We only do this if the target natively 5579 // supports the MVT we'll be loading or if it is small enough (<= 4) that 5580 // we'll only produce a small number of byte loads. 5581 5582 // Require that we can find a legal MVT, and only do this if the target 5583 // supports unaligned loads of that type. Expanding into byte loads would 5584 // bloat the code. 5585 const TargetLowering *TLI = TM.getTargetLowering(); 5586 if (ActuallyDoIt && CSize->getZExtValue() > 4) { 5587 // TODO: Handle 5 byte compare as 4-byte + 1 byte. 5588 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. 5589 if (!TLI->isTypeLegal(LoadVT) ||!TLI->allowsUnalignedMemoryAccesses(LoadVT)) 5590 ActuallyDoIt = false; 5591 } 5592 5593 if (ActuallyDoIt) { 5594 SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this); 5595 SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this); 5596 5597 SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal, 5598 ISD::SETNE); 5599 processIntegerCallValue(I, Res, false); 5600 return true; 5601 } 5602 } 5603 5604 5605 return false; 5606} 5607 5608/// visitMemChrCall -- See if we can lower a memchr call into an optimized 5609/// form. If so, return true and lower it, otherwise return false and it 5610/// will be lowered like a normal call. 5611bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { 5612 // Verify that the prototype makes sense. void *memchr(void *, int, size_t) 5613 if (I.getNumArgOperands() != 3) 5614 return false; 5615 5616 const Value *Src = I.getArgOperand(0); 5617 const Value *Char = I.getArgOperand(1); 5618 const Value *Length = I.getArgOperand(2); 5619 if (!Src->getType()->isPointerTy() || 5620 !Char->getType()->isIntegerTy() || 5621 !Length->getType()->isIntegerTy() || 5622 !I.getType()->isPointerTy()) 5623 return false; 5624 5625 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5626 std::pair<SDValue, SDValue> Res = 5627 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), 5628 getValue(Src), getValue(Char), getValue(Length), 5629 MachinePointerInfo(Src)); 5630 if (Res.first.getNode()) { 5631 setValue(&I, Res.first); 5632 PendingLoads.push_back(Res.second); 5633 return true; 5634 } 5635 5636 return false; 5637} 5638 5639/// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an 5640/// optimized form. If so, return true and lower it, otherwise return false 5641/// and it will be lowered like a normal call. 5642bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { 5643 // Verify that the prototype makes sense. char *strcpy(char *, char *) 5644 if (I.getNumArgOperands() != 2) 5645 return false; 5646 5647 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5648 if (!Arg0->getType()->isPointerTy() || 5649 !Arg1->getType()->isPointerTy() || 5650 !I.getType()->isPointerTy()) 5651 return false; 5652 5653 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5654 std::pair<SDValue, SDValue> Res = 5655 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), 5656 getValue(Arg0), getValue(Arg1), 5657 MachinePointerInfo(Arg0), 5658 MachinePointerInfo(Arg1), isStpcpy); 5659 if (Res.first.getNode()) { 5660 setValue(&I, Res.first); 5661 DAG.setRoot(Res.second); 5662 return true; 5663 } 5664 5665 return false; 5666} 5667 5668/// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form. 5669/// If so, return true and lower it, otherwise return false and it will be 5670/// lowered like a normal call. 5671bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { 5672 // Verify that the prototype makes sense. int strcmp(void*,void*) 5673 if (I.getNumArgOperands() != 2) 5674 return false; 5675 5676 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5677 if (!Arg0->getType()->isPointerTy() || 5678 !Arg1->getType()->isPointerTy() || 5679 !I.getType()->isIntegerTy()) 5680 return false; 5681 5682 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5683 std::pair<SDValue, SDValue> Res = 5684 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), 5685 getValue(Arg0), getValue(Arg1), 5686 MachinePointerInfo(Arg0), 5687 MachinePointerInfo(Arg1)); 5688 if (Res.first.getNode()) { 5689 processIntegerCallValue(I, Res.first, true); 5690 PendingLoads.push_back(Res.second); 5691 return true; 5692 } 5693 5694 return false; 5695} 5696 5697/// visitStrLenCall -- See if we can lower a strlen call into an optimized 5698/// form. If so, return true and lower it, otherwise return false and it 5699/// will be lowered like a normal call. 5700bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { 5701 // Verify that the prototype makes sense. size_t strlen(char *) 5702 if (I.getNumArgOperands() != 1) 5703 return false; 5704 5705 const Value *Arg0 = I.getArgOperand(0); 5706 if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy()) 5707 return false; 5708 5709 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5710 std::pair<SDValue, SDValue> Res = 5711 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), 5712 getValue(Arg0), MachinePointerInfo(Arg0)); 5713 if (Res.first.getNode()) { 5714 processIntegerCallValue(I, Res.first, false); 5715 PendingLoads.push_back(Res.second); 5716 return true; 5717 } 5718 5719 return false; 5720} 5721 5722/// visitStrNLenCall -- See if we can lower a strnlen call into an optimized 5723/// form. If so, return true and lower it, otherwise return false and it 5724/// will be lowered like a normal call. 5725bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { 5726 // Verify that the prototype makes sense. size_t strnlen(char *, size_t) 5727 if (I.getNumArgOperands() != 2) 5728 return false; 5729 5730 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 5731 if (!Arg0->getType()->isPointerTy() || 5732 !Arg1->getType()->isIntegerTy() || 5733 !I.getType()->isIntegerTy()) 5734 return false; 5735 5736 const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo(); 5737 std::pair<SDValue, SDValue> Res = 5738 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), 5739 getValue(Arg0), getValue(Arg1), 5740 MachinePointerInfo(Arg0)); 5741 if (Res.first.getNode()) { 5742 processIntegerCallValue(I, Res.first, false); 5743 PendingLoads.push_back(Res.second); 5744 return true; 5745 } 5746 5747 return false; 5748} 5749 5750/// visitUnaryFloatCall - If a call instruction is a unary floating-point 5751/// operation (as expected), translate it to an SDNode with the specified opcode 5752/// and return true. 5753bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, 5754 unsigned Opcode) { 5755 // Sanity check that it really is a unary floating-point call. 5756 if (I.getNumArgOperands() != 1 || 5757 !I.getArgOperand(0)->getType()->isFloatingPointTy() || 5758 I.getType() != I.getArgOperand(0)->getType() || 5759 !I.onlyReadsMemory()) 5760 return false; 5761 5762 SDValue Tmp = getValue(I.getArgOperand(0)); 5763 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp)); 5764 return true; 5765} 5766 5767void SelectionDAGBuilder::visitCall(const CallInst &I) { 5768 // Handle inline assembly differently. 5769 if (isa<InlineAsm>(I.getCalledValue())) { 5770 visitInlineAsm(&I); 5771 return; 5772 } 5773 5774 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 5775 ComputeUsesVAFloatArgument(I, &MMI); 5776 5777 const char *RenameFn = 0; 5778 if (Function *F = I.getCalledFunction()) { 5779 if (F->isDeclaration()) { 5780 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) { 5781 if (unsigned IID = II->getIntrinsicID(F)) { 5782 RenameFn = visitIntrinsicCall(I, IID); 5783 if (!RenameFn) 5784 return; 5785 } 5786 } 5787 if (unsigned IID = F->getIntrinsicID()) { 5788 RenameFn = visitIntrinsicCall(I, IID); 5789 if (!RenameFn) 5790 return; 5791 } 5792 } 5793 5794 // Check for well-known libc/libm calls. If the function is internal, it 5795 // can't be a library call. 5796 LibFunc::Func Func; 5797 if (!F->hasLocalLinkage() && F->hasName() && 5798 LibInfo->getLibFunc(F->getName(), Func) && 5799 LibInfo->hasOptimizedCodeGen(Func)) { 5800 switch (Func) { 5801 default: break; 5802 case LibFunc::copysign: 5803 case LibFunc::copysignf: 5804 case LibFunc::copysignl: 5805 if (I.getNumArgOperands() == 2 && // Basic sanity checks. 5806 I.getArgOperand(0)->getType()->isFloatingPointTy() && 5807 I.getType() == I.getArgOperand(0)->getType() && 5808 I.getType() == I.getArgOperand(1)->getType() && 5809 I.onlyReadsMemory()) { 5810 SDValue LHS = getValue(I.getArgOperand(0)); 5811 SDValue RHS = getValue(I.getArgOperand(1)); 5812 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), 5813 LHS.getValueType(), LHS, RHS)); 5814 return; 5815 } 5816 break; 5817 case LibFunc::fabs: 5818 case LibFunc::fabsf: 5819 case LibFunc::fabsl: 5820 if (visitUnaryFloatCall(I, ISD::FABS)) 5821 return; 5822 break; 5823 case LibFunc::sin: 5824 case LibFunc::sinf: 5825 case LibFunc::sinl: 5826 if (visitUnaryFloatCall(I, ISD::FSIN)) 5827 return; 5828 break; 5829 case LibFunc::cos: 5830 case LibFunc::cosf: 5831 case LibFunc::cosl: 5832 if (visitUnaryFloatCall(I, ISD::FCOS)) 5833 return; 5834 break; 5835 case LibFunc::sqrt: 5836 case LibFunc::sqrtf: 5837 case LibFunc::sqrtl: 5838 case LibFunc::sqrt_finite: 5839 case LibFunc::sqrtf_finite: 5840 case LibFunc::sqrtl_finite: 5841 if (visitUnaryFloatCall(I, ISD::FSQRT)) 5842 return; 5843 break; 5844 case LibFunc::floor: 5845 case LibFunc::floorf: 5846 case LibFunc::floorl: 5847 if (visitUnaryFloatCall(I, ISD::FFLOOR)) 5848 return; 5849 break; 5850 case LibFunc::nearbyint: 5851 case LibFunc::nearbyintf: 5852 case LibFunc::nearbyintl: 5853 if (visitUnaryFloatCall(I, ISD::FNEARBYINT)) 5854 return; 5855 break; 5856 case LibFunc::ceil: 5857 case LibFunc::ceilf: 5858 case LibFunc::ceill: 5859 if (visitUnaryFloatCall(I, ISD::FCEIL)) 5860 return; 5861 break; 5862 case LibFunc::rint: 5863 case LibFunc::rintf: 5864 case LibFunc::rintl: 5865 if (visitUnaryFloatCall(I, ISD::FRINT)) 5866 return; 5867 break; 5868 case LibFunc::round: 5869 case LibFunc::roundf: 5870 case LibFunc::roundl: 5871 if (visitUnaryFloatCall(I, ISD::FROUND)) 5872 return; 5873 break; 5874 case LibFunc::trunc: 5875 case LibFunc::truncf: 5876 case LibFunc::truncl: 5877 if (visitUnaryFloatCall(I, ISD::FTRUNC)) 5878 return; 5879 break; 5880 case LibFunc::log2: 5881 case LibFunc::log2f: 5882 case LibFunc::log2l: 5883 if (visitUnaryFloatCall(I, ISD::FLOG2)) 5884 return; 5885 break; 5886 case LibFunc::exp2: 5887 case LibFunc::exp2f: 5888 case LibFunc::exp2l: 5889 if (visitUnaryFloatCall(I, ISD::FEXP2)) 5890 return; 5891 break; 5892 case LibFunc::memcmp: 5893 if (visitMemCmpCall(I)) 5894 return; 5895 break; 5896 case LibFunc::memchr: 5897 if (visitMemChrCall(I)) 5898 return; 5899 break; 5900 case LibFunc::strcpy: 5901 if (visitStrCpyCall(I, false)) 5902 return; 5903 break; 5904 case LibFunc::stpcpy: 5905 if (visitStrCpyCall(I, true)) 5906 return; 5907 break; 5908 case LibFunc::strcmp: 5909 if (visitStrCmpCall(I)) 5910 return; 5911 break; 5912 case LibFunc::strlen: 5913 if (visitStrLenCall(I)) 5914 return; 5915 break; 5916 case LibFunc::strnlen: 5917 if (visitStrNLenCall(I)) 5918 return; 5919 break; 5920 } 5921 } 5922 } 5923 5924 SDValue Callee; 5925 if (!RenameFn) 5926 Callee = getValue(I.getCalledValue()); 5927 else 5928 Callee = DAG.getExternalSymbol(RenameFn, 5929 TM.getTargetLowering()->getPointerTy()); 5930 5931 // Check if we can potentially perform a tail call. More detailed checking is 5932 // be done within LowerCallTo, after more information about the call is known. 5933 LowerCallTo(&I, Callee, I.isTailCall()); 5934} 5935 5936namespace { 5937 5938/// AsmOperandInfo - This contains information for each constraint that we are 5939/// lowering. 5940class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { 5941public: 5942 /// CallOperand - If this is the result output operand or a clobber 5943 /// this is null, otherwise it is the incoming operand to the CallInst. 5944 /// This gets modified as the asm is processed. 5945 SDValue CallOperand; 5946 5947 /// AssignedRegs - If this is a register or register class operand, this 5948 /// contains the set of register corresponding to the operand. 5949 RegsForValue AssignedRegs; 5950 5951 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) 5952 : TargetLowering::AsmOperandInfo(info), CallOperand(0,0) { 5953 } 5954 5955 /// getCallOperandValEVT - Return the EVT of the Value* that this operand 5956 /// corresponds to. If there is no Value* for this operand, it returns 5957 /// MVT::Other. 5958 EVT getCallOperandValEVT(LLVMContext &Context, 5959 const TargetLowering &TLI, 5960 const DataLayout *TD) const { 5961 if (CallOperandVal == 0) return MVT::Other; 5962 5963 if (isa<BasicBlock>(CallOperandVal)) 5964 return TLI.getPointerTy(); 5965 5966 llvm::Type *OpTy = CallOperandVal->getType(); 5967 5968 // FIXME: code duplicated from TargetLowering::ParseConstraints(). 5969 // If this is an indirect operand, the operand is a pointer to the 5970 // accessed type. 5971 if (isIndirect) { 5972 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); 5973 if (!PtrTy) 5974 report_fatal_error("Indirect operand for inline asm not a pointer!"); 5975 OpTy = PtrTy->getElementType(); 5976 } 5977 5978 // Look for vector wrapped in a struct. e.g. { <16 x i8> }. 5979 if (StructType *STy = dyn_cast<StructType>(OpTy)) 5980 if (STy->getNumElements() == 1) 5981 OpTy = STy->getElementType(0); 5982 5983 // If OpTy is not a single value, it may be a struct/union that we 5984 // can tile with integers. 5985 if (!OpTy->isSingleValueType() && OpTy->isSized()) { 5986 unsigned BitSize = TD->getTypeSizeInBits(OpTy); 5987 switch (BitSize) { 5988 default: break; 5989 case 1: 5990 case 8: 5991 case 16: 5992 case 32: 5993 case 64: 5994 case 128: 5995 OpTy = IntegerType::get(Context, BitSize); 5996 break; 5997 } 5998 } 5999 6000 return TLI.getValueType(OpTy, true); 6001 } 6002}; 6003 6004typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector; 6005 6006} // end anonymous namespace 6007 6008/// GetRegistersForValue - Assign registers (virtual or physical) for the 6009/// specified operand. We prefer to assign virtual registers, to allow the 6010/// register allocator to handle the assignment process. However, if the asm 6011/// uses features that we can't model on machineinstrs, we have SDISel do the 6012/// allocation. This produces generally horrible, but correct, code. 6013/// 6014/// OpInfo describes the operand. 6015/// 6016static void GetRegistersForValue(SelectionDAG &DAG, 6017 const TargetLowering &TLI, 6018 SDLoc DL, 6019 SDISelAsmOperandInfo &OpInfo) { 6020 LLVMContext &Context = *DAG.getContext(); 6021 6022 MachineFunction &MF = DAG.getMachineFunction(); 6023 SmallVector<unsigned, 4> Regs; 6024 6025 // If this is a constraint for a single physreg, or a constraint for a 6026 // register class, find it. 6027 std::pair<unsigned, const TargetRegisterClass*> PhysReg = 6028 TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, 6029 OpInfo.ConstraintVT); 6030 6031 unsigned NumRegs = 1; 6032 if (OpInfo.ConstraintVT != MVT::Other) { 6033 // If this is a FP input in an integer register (or visa versa) insert a bit 6034 // cast of the input value. More generally, handle any case where the input 6035 // value disagrees with the register class we plan to stick this in. 6036 if (OpInfo.Type == InlineAsm::isInput && 6037 PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) { 6038 // Try to convert to the first EVT that the reg class contains. If the 6039 // types are identical size, use a bitcast to convert (e.g. two differing 6040 // vector types). 6041 MVT RegVT = *PhysReg.second->vt_begin(); 6042 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) { 6043 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, 6044 RegVT, OpInfo.CallOperand); 6045 OpInfo.ConstraintVT = RegVT; 6046 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) { 6047 // If the input is a FP value and we want it in FP registers, do a 6048 // bitcast to the corresponding integer type. This turns an f64 value 6049 // into i64, which can be passed with two i32 values on a 32-bit 6050 // machine. 6051 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); 6052 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, 6053 RegVT, OpInfo.CallOperand); 6054 OpInfo.ConstraintVT = RegVT; 6055 } 6056 } 6057 6058 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT); 6059 } 6060 6061 MVT RegVT; 6062 EVT ValueVT = OpInfo.ConstraintVT; 6063 6064 // If this is a constraint for a specific physical register, like {r17}, 6065 // assign it now. 6066 if (unsigned AssignedReg = PhysReg.first) { 6067 const TargetRegisterClass *RC = PhysReg.second; 6068 if (OpInfo.ConstraintVT == MVT::Other) 6069 ValueVT = *RC->vt_begin(); 6070 6071 // Get the actual register value type. This is important, because the user 6072 // may have asked for (e.g.) the AX register in i32 type. We need to 6073 // remember that AX is actually i16 to get the right extension. 6074 RegVT = *RC->vt_begin(); 6075 6076 // This is a explicit reference to a physical register. 6077 Regs.push_back(AssignedReg); 6078 6079 // If this is an expanded reference, add the rest of the regs to Regs. 6080 if (NumRegs != 1) { 6081 TargetRegisterClass::iterator I = RC->begin(); 6082 for (; *I != AssignedReg; ++I) 6083 assert(I != RC->end() && "Didn't find reg!"); 6084 6085 // Already added the first reg. 6086 --NumRegs; ++I; 6087 for (; NumRegs; --NumRegs, ++I) { 6088 assert(I != RC->end() && "Ran out of registers to allocate!"); 6089 Regs.push_back(*I); 6090 } 6091 } 6092 6093 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 6094 return; 6095 } 6096 6097 // Otherwise, if this was a reference to an LLVM register class, create vregs 6098 // for this reference. 6099 if (const TargetRegisterClass *RC = PhysReg.second) { 6100 RegVT = *RC->vt_begin(); 6101 if (OpInfo.ConstraintVT == MVT::Other) 6102 ValueVT = RegVT; 6103 6104 // Create the appropriate number of virtual registers. 6105 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 6106 for (; NumRegs; --NumRegs) 6107 Regs.push_back(RegInfo.createVirtualRegister(RC)); 6108 6109 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 6110 return; 6111 } 6112 6113 // Otherwise, we couldn't allocate enough registers for this. 6114} 6115 6116/// visitInlineAsm - Handle a call to an InlineAsm object. 6117/// 6118void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { 6119 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue()); 6120 6121 /// ConstraintOperands - Information about all of the constraints. 6122 SDISelAsmOperandInfoVector ConstraintOperands; 6123 6124 const TargetLowering *TLI = TM.getTargetLowering(); 6125 TargetLowering::AsmOperandInfoVector 6126 TargetConstraints = TLI->ParseConstraints(CS); 6127 6128 bool hasMemory = false; 6129 6130 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. 6131 unsigned ResNo = 0; // ResNo - The result number of the next output. 6132 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 6133 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i])); 6134 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); 6135 6136 MVT OpVT = MVT::Other; 6137 6138 // Compute the value type for each operand. 6139 switch (OpInfo.Type) { 6140 case InlineAsm::isOutput: 6141 // Indirect outputs just consume an argument. 6142 if (OpInfo.isIndirect) { 6143 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); 6144 break; 6145 } 6146 6147 // The return value of the call is this value. As such, there is no 6148 // corresponding argument. 6149 assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); 6150 if (StructType *STy = dyn_cast<StructType>(CS.getType())) { 6151 OpVT = TLI->getSimpleValueType(STy->getElementType(ResNo)); 6152 } else { 6153 assert(ResNo == 0 && "Asm only has one result!"); 6154 OpVT = TLI->getSimpleValueType(CS.getType()); 6155 } 6156 ++ResNo; 6157 break; 6158 case InlineAsm::isInput: 6159 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); 6160 break; 6161 case InlineAsm::isClobber: 6162 // Nothing to do. 6163 break; 6164 } 6165 6166 // If this is an input or an indirect output, process the call argument. 6167 // BasicBlocks are labels, currently appearing only in asm's. 6168 if (OpInfo.CallOperandVal) { 6169 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) { 6170 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); 6171 } else { 6172 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); 6173 } 6174 6175 OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), *TLI, TD). 6176 getSimpleVT(); 6177 } 6178 6179 OpInfo.ConstraintVT = OpVT; 6180 6181 // Indirect operand accesses access memory. 6182 if (OpInfo.isIndirect) 6183 hasMemory = true; 6184 else { 6185 for (unsigned j = 0, ee = OpInfo.Codes.size(); j != ee; ++j) { 6186 TargetLowering::ConstraintType 6187 CType = TLI->getConstraintType(OpInfo.Codes[j]); 6188 if (CType == TargetLowering::C_Memory) { 6189 hasMemory = true; 6190 break; 6191 } 6192 } 6193 } 6194 } 6195 6196 SDValue Chain, Flag; 6197 6198 // We won't need to flush pending loads if this asm doesn't touch 6199 // memory and is nonvolatile. 6200 if (hasMemory || IA->hasSideEffects()) 6201 Chain = getRoot(); 6202 else 6203 Chain = DAG.getRoot(); 6204 6205 // Second pass over the constraints: compute which constraint option to use 6206 // and assign registers to constraints that want a specific physreg. 6207 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6208 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6209 6210 // If this is an output operand with a matching input operand, look up the 6211 // matching input. If their types mismatch, e.g. one is an integer, the 6212 // other is floating point, or their sizes are different, flag it as an 6213 // error. 6214 if (OpInfo.hasMatchingInput()) { 6215 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; 6216 6217 if (OpInfo.ConstraintVT != Input.ConstraintVT) { 6218 std::pair<unsigned, const TargetRegisterClass*> MatchRC = 6219 TLI->getRegForInlineAsmConstraint(OpInfo.ConstraintCode, 6220 OpInfo.ConstraintVT); 6221 std::pair<unsigned, const TargetRegisterClass*> InputRC = 6222 TLI->getRegForInlineAsmConstraint(Input.ConstraintCode, 6223 Input.ConstraintVT); 6224 if ((OpInfo.ConstraintVT.isInteger() != 6225 Input.ConstraintVT.isInteger()) || 6226 (MatchRC.second != InputRC.second)) { 6227 report_fatal_error("Unsupported asm: input constraint" 6228 " with a matching output constraint of" 6229 " incompatible type!"); 6230 } 6231 Input.ConstraintVT = OpInfo.ConstraintVT; 6232 } 6233 } 6234 6235 // Compute the constraint code and ConstraintType to use. 6236 TLI->ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); 6237 6238 if (OpInfo.ConstraintType == TargetLowering::C_Memory && 6239 OpInfo.Type == InlineAsm::isClobber) 6240 continue; 6241 6242 // If this is a memory input, and if the operand is not indirect, do what we 6243 // need to to provide an address for the memory input. 6244 if (OpInfo.ConstraintType == TargetLowering::C_Memory && 6245 !OpInfo.isIndirect) { 6246 assert((OpInfo.isMultipleAlternative || 6247 (OpInfo.Type == InlineAsm::isInput)) && 6248 "Can only indirectify direct input operands!"); 6249 6250 // Memory operands really want the address of the value. If we don't have 6251 // an indirect input, put it in the constpool if we can, otherwise spill 6252 // it to a stack slot. 6253 // TODO: This isn't quite right. We need to handle these according to 6254 // the addressing mode that the constraint wants. Also, this may take 6255 // an additional register for the computation and we don't want that 6256 // either. 6257 6258 // If the operand is a float, integer, or vector constant, spill to a 6259 // constant pool entry to get its address. 6260 const Value *OpVal = OpInfo.CallOperandVal; 6261 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || 6262 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) { 6263 OpInfo.CallOperand = DAG.getConstantPool(cast<Constant>(OpVal), 6264 TLI->getPointerTy()); 6265 } else { 6266 // Otherwise, create a stack slot and emit a store to it before the 6267 // asm. 6268 Type *Ty = OpVal->getType(); 6269 uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty); 6270 unsigned Align = TLI->getDataLayout()->getPrefTypeAlignment(Ty); 6271 MachineFunction &MF = DAG.getMachineFunction(); 6272 int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); 6273 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI->getPointerTy()); 6274 Chain = DAG.getStore(Chain, getCurSDLoc(), 6275 OpInfo.CallOperand, StackSlot, 6276 MachinePointerInfo::getFixedStack(SSFI), 6277 false, false, 0); 6278 OpInfo.CallOperand = StackSlot; 6279 } 6280 6281 // There is no longer a Value* corresponding to this operand. 6282 OpInfo.CallOperandVal = 0; 6283 6284 // It is now an indirect operand. 6285 OpInfo.isIndirect = true; 6286 } 6287 6288 // If this constraint is for a specific register, allocate it before 6289 // anything else. 6290 if (OpInfo.ConstraintType == TargetLowering::C_Register) 6291 GetRegistersForValue(DAG, *TLI, getCurSDLoc(), OpInfo); 6292 } 6293 6294 // Second pass - Loop over all of the operands, assigning virtual or physregs 6295 // to register class operands. 6296 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6297 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6298 6299 // C_Register operands have already been allocated, Other/Memory don't need 6300 // to be. 6301 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass) 6302 GetRegistersForValue(DAG, *TLI, getCurSDLoc(), OpInfo); 6303 } 6304 6305 // AsmNodeOperands - The operands for the ISD::INLINEASM node. 6306 std::vector<SDValue> AsmNodeOperands; 6307 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain 6308 AsmNodeOperands.push_back( 6309 DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), 6310 TLI->getPointerTy())); 6311 6312 // If we have a !srcloc metadata node associated with it, we want to attach 6313 // this to the ultimately generated inline asm machineinstr. To do this, we 6314 // pass in the third operand as this (potentially null) inline asm MDNode. 6315 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc"); 6316 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); 6317 6318 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore 6319 // bits as operand 3. 6320 unsigned ExtraInfo = 0; 6321 if (IA->hasSideEffects()) 6322 ExtraInfo |= InlineAsm::Extra_HasSideEffects; 6323 if (IA->isAlignStack()) 6324 ExtraInfo |= InlineAsm::Extra_IsAlignStack; 6325 // Set the asm dialect. 6326 ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect; 6327 6328 // Determine if this InlineAsm MayLoad or MayStore based on the constraints. 6329 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 6330 TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 6331 6332 // Compute the constraint code and ConstraintType to use. 6333 TLI->ComputeConstraintToUse(OpInfo, SDValue()); 6334 6335 // Ideally, we would only check against memory constraints. However, the 6336 // meaning of an other constraint can be target-specific and we can't easily 6337 // reason about it. Therefore, be conservative and set MayLoad/MayStore 6338 // for other constriants as well. 6339 if (OpInfo.ConstraintType == TargetLowering::C_Memory || 6340 OpInfo.ConstraintType == TargetLowering::C_Other) { 6341 if (OpInfo.Type == InlineAsm::isInput) 6342 ExtraInfo |= InlineAsm::Extra_MayLoad; 6343 else if (OpInfo.Type == InlineAsm::isOutput) 6344 ExtraInfo |= InlineAsm::Extra_MayStore; 6345 else if (OpInfo.Type == InlineAsm::isClobber) 6346 ExtraInfo |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); 6347 } 6348 } 6349 6350 AsmNodeOperands.push_back(DAG.getTargetConstant(ExtraInfo, 6351 TLI->getPointerTy())); 6352 6353 // Loop over all of the inputs, copying the operand values into the 6354 // appropriate registers and processing the output regs. 6355 RegsForValue RetValRegs; 6356 6357 // IndirectStoresToEmit - The set of stores to emit after the inline asm node. 6358 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit; 6359 6360 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { 6361 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; 6362 6363 switch (OpInfo.Type) { 6364 case InlineAsm::isOutput: { 6365 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass && 6366 OpInfo.ConstraintType != TargetLowering::C_Register) { 6367 // Memory output, or 'other' output (e.g. 'X' constraint). 6368 assert(OpInfo.isIndirect && "Memory output must be indirect operand"); 6369 6370 // Add information to the INLINEASM node to know about this output. 6371 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 6372 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, 6373 TLI->getPointerTy())); 6374 AsmNodeOperands.push_back(OpInfo.CallOperand); 6375 break; 6376 } 6377 6378 // Otherwise, this is a register or register class output. 6379 6380 // Copy the output from the appropriate register. Find a register that 6381 // we can use. 6382 if (OpInfo.AssignedRegs.Regs.empty()) { 6383 LLVMContext &Ctx = *DAG.getContext(); 6384 Ctx.emitError(CS.getInstruction(), 6385 "couldn't allocate output register for constraint '" + 6386 Twine(OpInfo.ConstraintCode) + "'"); 6387 return; 6388 } 6389 6390 // If this is an indirect operand, store through the pointer after the 6391 // asm. 6392 if (OpInfo.isIndirect) { 6393 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, 6394 OpInfo.CallOperandVal)); 6395 } else { 6396 // This is the result value of the call. 6397 assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); 6398 // Concatenate this output onto the outputs list. 6399 RetValRegs.append(OpInfo.AssignedRegs); 6400 } 6401 6402 // Add information to the INLINEASM node to know that this register is 6403 // set. 6404 OpInfo.AssignedRegs 6405 .AddInlineAsmOperands(OpInfo.isEarlyClobber 6406 ? InlineAsm::Kind_RegDefEarlyClobber 6407 : InlineAsm::Kind_RegDef, 6408 false, 0, DAG, AsmNodeOperands); 6409 break; 6410 } 6411 case InlineAsm::isInput: { 6412 SDValue InOperandVal = OpInfo.CallOperand; 6413 6414 if (OpInfo.isMatchingInputConstraint()) { // Matching constraint? 6415 // If this is required to match an output register we have already set, 6416 // just use its register. 6417 unsigned OperandNo = OpInfo.getMatchedOperand(); 6418 6419 // Scan until we find the definition we already emitted of this operand. 6420 // When we find it, create a RegsForValue operand. 6421 unsigned CurOp = InlineAsm::Op_FirstOperand; 6422 for (; OperandNo; --OperandNo) { 6423 // Advance to the next operand. 6424 unsigned OpFlag = 6425 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 6426 assert((InlineAsm::isRegDefKind(OpFlag) || 6427 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || 6428 InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"); 6429 CurOp += InlineAsm::getNumOperandRegisters(OpFlag)+1; 6430 } 6431 6432 unsigned OpFlag = 6433 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 6434 if (InlineAsm::isRegDefKind(OpFlag) || 6435 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { 6436 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. 6437 if (OpInfo.isIndirect) { 6438 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c 6439 LLVMContext &Ctx = *DAG.getContext(); 6440 Ctx.emitError(CS.getInstruction(), "inline asm not supported yet:" 6441 " don't know how to handle tied " 6442 "indirect register inputs"); 6443 return; 6444 } 6445 6446 RegsForValue MatchedRegs; 6447 MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); 6448 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType(); 6449 MatchedRegs.RegVTs.push_back(RegVT); 6450 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo(); 6451 for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag); 6452 i != e; ++i) { 6453 if (const TargetRegisterClass *RC = TLI->getRegClassFor(RegVT)) 6454 MatchedRegs.Regs.push_back(RegInfo.createVirtualRegister(RC)); 6455 else { 6456 LLVMContext &Ctx = *DAG.getContext(); 6457 Ctx.emitError(CS.getInstruction(), 6458 "inline asm error: This value" 6459 " type register class is not natively supported!"); 6460 return; 6461 } 6462 } 6463 // Use the produced MatchedRegs object to 6464 MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(), 6465 Chain, &Flag, CS.getInstruction()); 6466 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, 6467 true, OpInfo.getMatchedOperand(), 6468 DAG, AsmNodeOperands); 6469 break; 6470 } 6471 6472 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"); 6473 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 && 6474 "Unexpected number of operands"); 6475 // Add information to the INLINEASM node to know about this input. 6476 // See InlineAsm.h isUseOperandTiedToDef. 6477 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, 6478 OpInfo.getMatchedOperand()); 6479 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag, 6480 TLI->getPointerTy())); 6481 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); 6482 break; 6483 } 6484 6485 // Treat indirect 'X' constraint as memory. 6486 if (OpInfo.ConstraintType == TargetLowering::C_Other && 6487 OpInfo.isIndirect) 6488 OpInfo.ConstraintType = TargetLowering::C_Memory; 6489 6490 if (OpInfo.ConstraintType == TargetLowering::C_Other) { 6491 std::vector<SDValue> Ops; 6492 TLI->LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, 6493 Ops, DAG); 6494 if (Ops.empty()) { 6495 LLVMContext &Ctx = *DAG.getContext(); 6496 Ctx.emitError(CS.getInstruction(), 6497 "invalid operand for inline asm constraint '" + 6498 Twine(OpInfo.ConstraintCode) + "'"); 6499 return; 6500 } 6501 6502 // Add information to the INLINEASM node to know about this input. 6503 unsigned ResOpType = 6504 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); 6505 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 6506 TLI->getPointerTy())); 6507 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end()); 6508 break; 6509 } 6510 6511 if (OpInfo.ConstraintType == TargetLowering::C_Memory) { 6512 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!"); 6513 assert(InOperandVal.getValueType() == TLI->getPointerTy() && 6514 "Memory operands expect pointer values"); 6515 6516 // Add information to the INLINEASM node to know about this input. 6517 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 6518 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 6519 TLI->getPointerTy())); 6520 AsmNodeOperands.push_back(InOperandVal); 6521 break; 6522 } 6523 6524 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass || 6525 OpInfo.ConstraintType == TargetLowering::C_Register) && 6526 "Unknown constraint type!"); 6527 6528 // TODO: Support this. 6529 if (OpInfo.isIndirect) { 6530 LLVMContext &Ctx = *DAG.getContext(); 6531 Ctx.emitError(CS.getInstruction(), 6532 "Don't know how to handle indirect register inputs yet " 6533 "for constraint '" + 6534 Twine(OpInfo.ConstraintCode) + "'"); 6535 return; 6536 } 6537 6538 // Copy the input into the appropriate registers. 6539 if (OpInfo.AssignedRegs.Regs.empty()) { 6540 LLVMContext &Ctx = *DAG.getContext(); 6541 Ctx.emitError(CS.getInstruction(), 6542 "couldn't allocate input reg for constraint '" + 6543 Twine(OpInfo.ConstraintCode) + "'"); 6544 return; 6545 } 6546 6547 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurSDLoc(), 6548 Chain, &Flag, CS.getInstruction()); 6549 6550 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, 6551 DAG, AsmNodeOperands); 6552 break; 6553 } 6554 case InlineAsm::isClobber: { 6555 // Add the clobbered value to the operand list, so that the register 6556 // allocator is aware that the physreg got clobbered. 6557 if (!OpInfo.AssignedRegs.Regs.empty()) 6558 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, 6559 false, 0, DAG, 6560 AsmNodeOperands); 6561 break; 6562 } 6563 } 6564 } 6565 6566 // Finish up input operands. Set the input chain and add the flag last. 6567 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 6568 if (Flag.getNode()) AsmNodeOperands.push_back(Flag); 6569 6570 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(), 6571 DAG.getVTList(MVT::Other, MVT::Glue), 6572 &AsmNodeOperands[0], AsmNodeOperands.size()); 6573 Flag = Chain.getValue(1); 6574 6575 // If this asm returns a register value, copy the result from that register 6576 // and set it as the value of the call. 6577 if (!RetValRegs.Regs.empty()) { 6578 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 6579 Chain, &Flag, CS.getInstruction()); 6580 6581 // FIXME: Why don't we do this for inline asms with MRVs? 6582 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) { 6583 EVT ResultType = TLI->getValueType(CS.getType()); 6584 6585 // If any of the results of the inline asm is a vector, it may have the 6586 // wrong width/num elts. This can happen for register classes that can 6587 // contain multiple different value types. The preg or vreg allocated may 6588 // not have the same VT as was expected. Convert it to the right type 6589 // with bit_convert. 6590 if (ResultType != Val.getValueType() && Val.getValueType().isVector()) { 6591 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), 6592 ResultType, Val); 6593 6594 } else if (ResultType != Val.getValueType() && 6595 ResultType.isInteger() && Val.getValueType().isInteger()) { 6596 // If a result value was tied to an input value, the computed result may 6597 // have a wider width than the expected result. Extract the relevant 6598 // portion. 6599 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val); 6600 } 6601 6602 assert(ResultType == Val.getValueType() && "Asm result value mismatch!"); 6603 } 6604 6605 setValue(CS.getInstruction(), Val); 6606 // Don't need to use this as a chain in this case. 6607 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty()) 6608 return; 6609 } 6610 6611 std::vector<std::pair<SDValue, const Value *> > StoresToEmit; 6612 6613 // Process indirect outputs, first output all of the flagged copies out of 6614 // physregs. 6615 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { 6616 RegsForValue &OutRegs = IndirectStoresToEmit[i].first; 6617 const Value *Ptr = IndirectStoresToEmit[i].second; 6618 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 6619 Chain, &Flag, IA); 6620 StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); 6621 } 6622 6623 // Emit the non-flagged stores from the physregs. 6624 SmallVector<SDValue, 8> OutChains; 6625 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) { 6626 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), 6627 StoresToEmit[i].first, 6628 getValue(StoresToEmit[i].second), 6629 MachinePointerInfo(StoresToEmit[i].second), 6630 false, false, 0); 6631 OutChains.push_back(Val); 6632 } 6633 6634 if (!OutChains.empty()) 6635 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, 6636 &OutChains[0], OutChains.size()); 6637 6638 DAG.setRoot(Chain); 6639} 6640 6641void SelectionDAGBuilder::visitVAStart(const CallInst &I) { 6642 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), 6643 MVT::Other, getRoot(), 6644 getValue(I.getArgOperand(0)), 6645 DAG.getSrcValue(I.getArgOperand(0)))); 6646} 6647 6648void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { 6649 const TargetLowering *TLI = TM.getTargetLowering(); 6650 const DataLayout &TD = *TLI->getDataLayout(); 6651 SDValue V = DAG.getVAArg(TLI->getValueType(I.getType()), getCurSDLoc(), 6652 getRoot(), getValue(I.getOperand(0)), 6653 DAG.getSrcValue(I.getOperand(0)), 6654 TD.getABITypeAlignment(I.getType())); 6655 setValue(&I, V); 6656 DAG.setRoot(V.getValue(1)); 6657} 6658 6659void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { 6660 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), 6661 MVT::Other, getRoot(), 6662 getValue(I.getArgOperand(0)), 6663 DAG.getSrcValue(I.getArgOperand(0)))); 6664} 6665 6666void SelectionDAGBuilder::visitVACopy(const CallInst &I) { 6667 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), 6668 MVT::Other, getRoot(), 6669 getValue(I.getArgOperand(0)), 6670 getValue(I.getArgOperand(1)), 6671 DAG.getSrcValue(I.getArgOperand(0)), 6672 DAG.getSrcValue(I.getArgOperand(1)))); 6673} 6674 6675/// \brief Lower an argument list according to the target calling convention. 6676/// 6677/// \return A tuple of <return-value, token-chain> 6678/// 6679/// This is a helper for lowering intrinsics that follow a target calling 6680/// convention or require stack pointer adjustment. Only a subset of the 6681/// intrinsic's operands need to participate in the calling convention. 6682std::pair<SDValue, SDValue> 6683SelectionDAGBuilder::LowerCallOperands(const CallInst &CI, unsigned ArgIdx, 6684 unsigned NumArgs, SDValue Callee, 6685 bool useVoidTy) { 6686 TargetLowering::ArgListTy Args; 6687 Args.reserve(NumArgs); 6688 6689 // Populate the argument list. 6690 // Attributes for args start at offset 1, after the return attribute. 6691 ImmutableCallSite CS(&CI); 6692 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1; 6693 ArgI != ArgE; ++ArgI) { 6694 const Value *V = CI.getOperand(ArgI); 6695 6696 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); 6697 6698 TargetLowering::ArgListEntry Entry; 6699 Entry.Node = getValue(V); 6700 Entry.Ty = V->getType(); 6701 Entry.setAttributes(&CS, AttrI); 6702 Args.push_back(Entry); 6703 } 6704 6705 Type *retTy = useVoidTy ? Type::getVoidTy(*DAG.getContext()) : CI.getType(); 6706 TargetLowering::CallLoweringInfo CLI(getRoot(), retTy, /*retSExt*/ false, 6707 /*retZExt*/ false, /*isVarArg*/ false, /*isInReg*/ false, NumArgs, 6708 CI.getCallingConv(), /*isTailCall*/ false, /*doesNotReturn*/ false, 6709 /*isReturnValueUsed*/ CI.use_empty(), Callee, Args, DAG, getCurSDLoc()); 6710 6711 const TargetLowering *TLI = TM.getTargetLowering(); 6712 return TLI->LowerCallTo(CLI); 6713} 6714 6715/// \brief Lower llvm.experimental.stackmap directly to its target opcode. 6716void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { 6717 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>, 6718 // [live variables...]) 6719 6720 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value."); 6721 6722 SDValue Callee = getValue(CI.getCalledValue()); 6723 6724 // Lower into a call sequence with no args and no return value. 6725 std::pair<SDValue, SDValue> Result = LowerCallOperands(CI, 0, 0, Callee); 6726 // Set the root to the target-lowered call chain. 6727 SDValue Chain = Result.second; 6728 DAG.setRoot(Chain); 6729 6730 /// Get a call instruction from the call sequence chain. 6731 /// Tail calls are not allowed. 6732 SDNode *CallEnd = Chain.getNode(); 6733 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && 6734 "Expected a callseq node."); 6735 SDNode *Call = CallEnd->getOperand(0).getNode(); 6736 bool hasGlue = Call->getGluedNode(); 6737 6738 // Replace the target specific call node with the stackmap intrinsic. 6739 SmallVector<SDValue, 8> Ops; 6740 6741 // Add the <id> and <numShadowBytes> constants. 6742 for (unsigned i = 0; i < 2; ++i) { 6743 SDValue tmp = getValue(CI.getOperand(i)); 6744 Ops.push_back(DAG.getTargetConstant( 6745 cast<ConstantSDNode>(tmp)->getZExtValue(), MVT::i32)); 6746 } 6747 // Push live variables for the stack map. 6748 for (unsigned i = 2, e = CI.getNumArgOperands(); i != e; ++i) 6749 Ops.push_back(getValue(CI.getArgOperand(i))); 6750 6751 // Push the chain (this is originally the first operand of the call, but 6752 // becomes now the last or second to last operand). 6753 Ops.push_back(*(Call->op_begin())); 6754 6755 // Push the glue flag (last operand). 6756 if (hasGlue) 6757 Ops.push_back(*(Call->op_end()-1)); 6758 6759 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6760 6761 // Replace the target specific call node with a STACKMAP node. 6762 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::STACKMAP, getCurSDLoc(), 6763 NodeTys, Ops); 6764 6765 // StackMap generates no value, so nothing goes in the NodeMap. 6766 6767 // Fixup the consumers of the intrinsic. The chain and glue may be used in the 6768 // call sequence. 6769 DAG.ReplaceAllUsesWith(Call, MN); 6770 6771 DAG.DeleteNode(Call); 6772} 6773 6774/// \brief Lower llvm.experimental.patchpoint directly to its target opcode. 6775void SelectionDAGBuilder::visitPatchpoint(const CallInst &CI) { 6776 // void|i64 @llvm.experimental.patchpoint.void|i64(i32 <id>, 6777 // i32 <numBytes>, 6778 // i8* <target>, 6779 // i32 <numArgs>, 6780 // [Args...], 6781 // [live variables...]) 6782 6783 CallingConv::ID CC = CI.getCallingConv(); 6784 bool isAnyRegCC = CC == CallingConv::AnyReg; 6785 bool hasDef = !CI.getType()->isVoidTy(); 6786 SDValue Callee = getValue(CI.getOperand(2)); // <target> 6787 6788 // Get the real number of arguments participating in the call <numArgs> 6789 unsigned NumArgs = 6790 cast<ConstantSDNode>(getValue(CI.getArgOperand(3)))->getZExtValue(); 6791 6792 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> 6793 assert(CI.getNumArgOperands() >= NumArgs + 4 && 6794 "Not enough arguments provided to the patchpoint intrinsic"); 6795 6796 // For AnyRegCC the arguments are lowered later on manually. 6797 unsigned NumCallArgs = isAnyRegCC ? 0 : NumArgs; 6798 std::pair<SDValue, SDValue> Result = 6799 LowerCallOperands(CI, 4, NumCallArgs, Callee, isAnyRegCC); 6800 6801 // Set the root to the target-lowered call chain. 6802 SDValue Chain = Result.second; 6803 DAG.setRoot(Chain); 6804 6805 SDNode *CallEnd = Chain.getNode(); 6806 if (hasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) 6807 CallEnd = CallEnd->getOperand(0).getNode(); 6808 6809 /// Get a call instruction from the call sequence chain. 6810 /// Tail calls are not allowed. 6811 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && 6812 "Expected a callseq node."); 6813 SDNode *Call = CallEnd->getOperand(0).getNode(); 6814 bool hasGlue = Call->getGluedNode(); 6815 6816 // Replace the target specific call node with the patchable intrinsic. 6817 SmallVector<SDValue, 8> Ops; 6818 6819 // Add the <id> and <numNopBytes> constants. 6820 for (unsigned i = 0; i < 2; ++i) { 6821 SDValue tmp = getValue(CI.getOperand(i)); 6822 Ops.push_back(DAG.getTargetConstant( 6823 cast<ConstantSDNode>(tmp)->getZExtValue(), MVT::i32)); 6824 } 6825 // Assume that the Callee is a constant address. 6826 Ops.push_back( 6827 DAG.getIntPtrConstant(cast<ConstantSDNode>(Callee)->getZExtValue(), 6828 /*isTarget=*/true)); 6829 6830 // Adjust <numArgs> to account for any arguments that have been passed on the 6831 // stack instead. 6832 // Call Node: Chain, Target, {Args}, RegMask, [Glue] 6833 unsigned NumCallRegArgs = Call->getNumOperands() - (hasGlue ? 4 : 3); 6834 NumCallRegArgs = isAnyRegCC ? NumArgs : NumCallRegArgs; 6835 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32)); 6836 6837 // Add the calling convention 6838 Ops.push_back(DAG.getTargetConstant((unsigned)CC, MVT::i32)); 6839 6840 // Add the arguments we omitted previously. The register allocator should 6841 // place these in any free register. 6842 if (isAnyRegCC) 6843 for (unsigned i = 4, e = NumArgs + 4; i != e; ++i) 6844 Ops.push_back(getValue(CI.getArgOperand(i))); 6845 6846 // Push the arguments from the call instruction. 6847 SDNode::op_iterator e = hasGlue ? Call->op_end()-2 : Call->op_end()-1; 6848 for (SDNode::op_iterator i = Call->op_begin()+2; i != e; ++i) 6849 Ops.push_back(*i); 6850 6851 // Push live variables for the stack map. 6852 for (unsigned i = NumArgs + 4, e = CI.getNumArgOperands(); i != e; ++i) { 6853 SDValue OpVal = getValue(CI.getArgOperand(i)); 6854 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) { 6855 Ops.push_back( 6856 DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64)); 6857 Ops.push_back( 6858 DAG.getTargetConstant(C->getSExtValue(), MVT::i64)); 6859 } else 6860 Ops.push_back(OpVal); 6861 } 6862 6863 // Push the register mask info. 6864 if (hasGlue) 6865 Ops.push_back(*(Call->op_end()-2)); 6866 else 6867 Ops.push_back(*(Call->op_end()-1)); 6868 6869 // Push the chain (this is originally the first operand of the call, but 6870 // becomes now the last or second to last operand). 6871 Ops.push_back(*(Call->op_begin())); 6872 6873 // Push the glue flag (last operand). 6874 if (hasGlue) 6875 Ops.push_back(*(Call->op_end()-1)); 6876 6877 SDVTList NodeTys; 6878 if (isAnyRegCC && hasDef) { 6879 // Create the return types based on the intrinsic definition 6880 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6881 SmallVector<EVT, 3> ValueVTs; 6882 ComputeValueVTs(TLI, CI.getType(), ValueVTs); 6883 assert(ValueVTs.size() == 1 && "Expected only one return value type."); 6884 6885 // There is always a chain and a glue type at the end 6886 ValueVTs.push_back(MVT::Other); 6887 ValueVTs.push_back(MVT::Glue); 6888 NodeTys = DAG.getVTList(ValueVTs.data(), ValueVTs.size()); 6889 } else 6890 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 6891 6892 // Replace the target specific call node with a PATCHPOINT node. 6893 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT, 6894 getCurSDLoc(), NodeTys, Ops); 6895 6896 // Update the NodeMap. 6897 if (hasDef) { 6898 if (isAnyRegCC) 6899 setValue(&CI, SDValue(MN, 0)); 6900 else 6901 setValue(&CI, Result.first); 6902 } 6903 6904 // Fixup the consumers of the intrinsic. The chain and glue may be used in the 6905 // call sequence. Furthermore the location of the chain and glue can change 6906 // when the AnyReg calling convention is used and the intrinsic returns a 6907 // value. 6908 if (isAnyRegCC && hasDef) { 6909 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; 6910 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)}; 6911 DAG.ReplaceAllUsesOfValuesWith(From, To, 2); 6912 } else 6913 DAG.ReplaceAllUsesWith(Call, MN); 6914 DAG.DeleteNode(Call); 6915} 6916 6917/// Returns an AttributeSet representing the attributes applied to the return 6918/// value of the given call. 6919static AttributeSet getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) { 6920 SmallVector<Attribute::AttrKind, 2> Attrs; 6921 if (CLI.RetSExt) 6922 Attrs.push_back(Attribute::SExt); 6923 if (CLI.RetZExt) 6924 Attrs.push_back(Attribute::ZExt); 6925 if (CLI.IsInReg) 6926 Attrs.push_back(Attribute::InReg); 6927 6928 return AttributeSet::get(CLI.RetTy->getContext(), AttributeSet::ReturnIndex, 6929 Attrs); 6930} 6931 6932/// TargetLowering::LowerCallTo - This is the default LowerCallTo 6933/// implementation, which just calls LowerCall. 6934/// FIXME: When all targets are 6935/// migrated to using LowerCall, this hook should be integrated into SDISel. 6936std::pair<SDValue, SDValue> 6937TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { 6938 // Handle the incoming return values from the call. 6939 CLI.Ins.clear(); 6940 Type *OrigRetTy = CLI.RetTy; 6941 SmallVector<EVT, 4> RetTys; 6942 SmallVector<uint64_t, 4> Offsets; 6943 ComputeValueVTs(*this, CLI.RetTy, RetTys, &Offsets); 6944 6945 SmallVector<ISD::OutputArg, 4> Outs; 6946 GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this); 6947 6948 bool CanLowerReturn = 6949 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(), 6950 CLI.IsVarArg, Outs, CLI.RetTy->getContext()); 6951 6952 SDValue DemoteStackSlot; 6953 int DemoteStackIdx = -100; 6954 if (!CanLowerReturn) { 6955 // FIXME: equivalent assert? 6956 // assert(!CS.hasInAllocaArgument() && 6957 // "sret demotion is incompatible with inalloca"); 6958 uint64_t TySize = getDataLayout()->getTypeAllocSize(CLI.RetTy); 6959 unsigned Align = getDataLayout()->getPrefTypeAlignment(CLI.RetTy); 6960 MachineFunction &MF = CLI.DAG.getMachineFunction(); 6961 DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); 6962 Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy); 6963 6964 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getPointerTy()); 6965 ArgListEntry Entry; 6966 Entry.Node = DemoteStackSlot; 6967 Entry.Ty = StackSlotPtrType; 6968 Entry.isSExt = false; 6969 Entry.isZExt = false; 6970 Entry.isInReg = false; 6971 Entry.isSRet = true; 6972 Entry.isNest = false; 6973 Entry.isByVal = false; 6974 Entry.isReturned = false; 6975 Entry.Alignment = Align; 6976 CLI.Args.insert(CLI.Args.begin(), Entry); 6977 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext()); 6978 } else { 6979 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 6980 EVT VT = RetTys[I]; 6981 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); 6982 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); 6983 for (unsigned i = 0; i != NumRegs; ++i) { 6984 ISD::InputArg MyFlags; 6985 MyFlags.VT = RegisterVT; 6986 MyFlags.ArgVT = VT; 6987 MyFlags.Used = CLI.IsReturnValueUsed; 6988 if (CLI.RetSExt) 6989 MyFlags.Flags.setSExt(); 6990 if (CLI.RetZExt) 6991 MyFlags.Flags.setZExt(); 6992 if (CLI.IsInReg) 6993 MyFlags.Flags.setInReg(); 6994 CLI.Ins.push_back(MyFlags); 6995 } 6996 } 6997 } 6998 6999 // Handle all of the outgoing arguments. 7000 CLI.Outs.clear(); 7001 CLI.OutVals.clear(); 7002 ArgListTy &Args = CLI.Args; 7003 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 7004 SmallVector<EVT, 4> ValueVTs; 7005 ComputeValueVTs(*this, Args[i].Ty, ValueVTs); 7006 for (unsigned Value = 0, NumValues = ValueVTs.size(); 7007 Value != NumValues; ++Value) { 7008 EVT VT = ValueVTs[Value]; 7009 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext()); 7010 SDValue Op = SDValue(Args[i].Node.getNode(), 7011 Args[i].Node.getResNo() + Value); 7012 ISD::ArgFlagsTy Flags; 7013 unsigned OriginalAlignment = 7014 getDataLayout()->getABITypeAlignment(ArgTy); 7015 7016 if (Args[i].isZExt) 7017 Flags.setZExt(); 7018 if (Args[i].isSExt) 7019 Flags.setSExt(); 7020 if (Args[i].isInReg) 7021 Flags.setInReg(); 7022 if (Args[i].isSRet) 7023 Flags.setSRet(); 7024 if (Args[i].isByVal) { 7025 Flags.setByVal(); 7026 PointerType *Ty = cast<PointerType>(Args[i].Ty); 7027 Type *ElementTy = Ty->getElementType(); 7028 Flags.setByValSize(getDataLayout()->getTypeAllocSize(ElementTy)); 7029 // For ByVal, alignment should come from FE. BE will guess if this 7030 // info is not there but there are cases it cannot get right. 7031 unsigned FrameAlign; 7032 if (Args[i].Alignment) 7033 FrameAlign = Args[i].Alignment; 7034 else 7035 FrameAlign = getByValTypeAlignment(ElementTy); 7036 Flags.setByValAlign(FrameAlign); 7037 } 7038 if (Args[i].isNest) 7039 Flags.setNest(); 7040 Flags.setOrigAlign(OriginalAlignment); 7041 7042 MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT); 7043 unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT); 7044 SmallVector<SDValue, 4> Parts(NumParts); 7045 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 7046 7047 if (Args[i].isSExt) 7048 ExtendKind = ISD::SIGN_EXTEND; 7049 else if (Args[i].isZExt) 7050 ExtendKind = ISD::ZERO_EXTEND; 7051 7052 // Conservatively only handle 'returned' on non-vectors for now 7053 if (Args[i].isReturned && !Op.getValueType().isVector()) { 7054 assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues && 7055 "unexpected use of 'returned'"); 7056 // Before passing 'returned' to the target lowering code, ensure that 7057 // either the register MVT and the actual EVT are the same size or that 7058 // the return value and argument are extended in the same way; in these 7059 // cases it's safe to pass the argument register value unchanged as the 7060 // return register value (although it's at the target's option whether 7061 // to do so) 7062 // TODO: allow code generation to take advantage of partially preserved 7063 // registers rather than clobbering the entire register when the 7064 // parameter extension method is not compatible with the return 7065 // extension method 7066 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) || 7067 (ExtendKind != ISD::ANY_EXTEND && 7068 CLI.RetSExt == Args[i].isSExt && CLI.RetZExt == Args[i].isZExt)) 7069 Flags.setReturned(); 7070 } 7071 7072 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, 7073 PartVT, CLI.CS ? CLI.CS->getInstruction() : 0, ExtendKind); 7074 7075 for (unsigned j = 0; j != NumParts; ++j) { 7076 // if it isn't first piece, alignment must be 1 7077 ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT, 7078 i < CLI.NumFixedArgs, 7079 i, j*Parts[j].getValueType().getStoreSize()); 7080 if (NumParts > 1 && j == 0) 7081 MyFlags.Flags.setSplit(); 7082 else if (j != 0) 7083 MyFlags.Flags.setOrigAlign(1); 7084 7085 CLI.Outs.push_back(MyFlags); 7086 CLI.OutVals.push_back(Parts[j]); 7087 } 7088 } 7089 } 7090 7091 SmallVector<SDValue, 4> InVals; 7092 CLI.Chain = LowerCall(CLI, InVals); 7093 7094 // Verify that the target's LowerCall behaved as expected. 7095 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && 7096 "LowerCall didn't return a valid chain!"); 7097 assert((!CLI.IsTailCall || InVals.empty()) && 7098 "LowerCall emitted a return value for a tail call!"); 7099 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && 7100 "LowerCall didn't emit the correct number of values!"); 7101 7102 // For a tail call, the return value is merely live-out and there aren't 7103 // any nodes in the DAG representing it. Return a special value to 7104 // indicate that a tail call has been emitted and no more Instructions 7105 // should be processed in the current block. 7106 if (CLI.IsTailCall) { 7107 CLI.DAG.setRoot(CLI.Chain); 7108 return std::make_pair(SDValue(), SDValue()); 7109 } 7110 7111 DEBUG(for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) { 7112 assert(InVals[i].getNode() && 7113 "LowerCall emitted a null value!"); 7114 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && 7115 "LowerCall emitted a value with the wrong type!"); 7116 }); 7117 7118 SmallVector<SDValue, 4> ReturnValues; 7119 if (!CanLowerReturn) { 7120 // The instruction result is the result of loading from the 7121 // hidden sret parameter. 7122 SmallVector<EVT, 1> PVTs; 7123 Type *PtrRetTy = PointerType::getUnqual(OrigRetTy); 7124 7125 ComputeValueVTs(*this, PtrRetTy, PVTs); 7126 assert(PVTs.size() == 1 && "Pointers should fit in one register"); 7127 EVT PtrVT = PVTs[0]; 7128 7129 unsigned NumValues = RetTys.size(); 7130 ReturnValues.resize(NumValues); 7131 SmallVector<SDValue, 4> Chains(NumValues); 7132 7133 for (unsigned i = 0; i < NumValues; ++i) { 7134 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, 7135 CLI.DAG.getConstant(Offsets[i], PtrVT)); 7136 SDValue L = CLI.DAG.getLoad( 7137 RetTys[i], CLI.DL, CLI.Chain, Add, 7138 MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]), false, 7139 false, false, 1); 7140 ReturnValues[i] = L; 7141 Chains[i] = L.getValue(1); 7142 } 7143 7144 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, 7145 &Chains[0], NumValues); 7146 } else { 7147 // Collect the legal value parts into potentially illegal values 7148 // that correspond to the original function's return values. 7149 ISD::NodeType AssertOp = ISD::DELETED_NODE; 7150 if (CLI.RetSExt) 7151 AssertOp = ISD::AssertSext; 7152 else if (CLI.RetZExt) 7153 AssertOp = ISD::AssertZext; 7154 unsigned CurReg = 0; 7155 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 7156 EVT VT = RetTys[I]; 7157 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); 7158 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); 7159 7160 ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], 7161 NumRegs, RegisterVT, VT, NULL, 7162 AssertOp)); 7163 CurReg += NumRegs; 7164 } 7165 7166 // For a function returning void, there is no return value. We can't create 7167 // such a node, so we just return a null return value in that case. In 7168 // that case, nothing will actually look at the value. 7169 if (ReturnValues.empty()) 7170 return std::make_pair(SDValue(), CLI.Chain); 7171 } 7172 7173 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, 7174 CLI.DAG.getVTList(&RetTys[0], RetTys.size()), 7175 &ReturnValues[0], ReturnValues.size()); 7176 return std::make_pair(Res, CLI.Chain); 7177} 7178 7179void TargetLowering::LowerOperationWrapper(SDNode *N, 7180 SmallVectorImpl<SDValue> &Results, 7181 SelectionDAG &DAG) const { 7182 SDValue Res = LowerOperation(SDValue(N, 0), DAG); 7183 if (Res.getNode()) 7184 Results.push_back(Res); 7185} 7186 7187SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 7188 llvm_unreachable("LowerOperation not implemented for this target!"); 7189} 7190 7191void 7192SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) { 7193 SDValue Op = getNonRegisterValue(V); 7194 assert((Op.getOpcode() != ISD::CopyFromReg || 7195 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && 7196 "Copy from a reg to the same reg!"); 7197 assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg"); 7198 7199 const TargetLowering *TLI = TM.getTargetLowering(); 7200 RegsForValue RFV(V->getContext(), *TLI, Reg, V->getType()); 7201 SDValue Chain = DAG.getEntryNode(); 7202 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, 0, V); 7203 PendingExports.push_back(Chain); 7204} 7205 7206#include "llvm/CodeGen/SelectionDAGISel.h" 7207 7208/// isOnlyUsedInEntryBlock - If the specified argument is only used in the 7209/// entry block, return true. This includes arguments used by switches, since 7210/// the switch may expand into multiple basic blocks. 7211static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { 7212 // With FastISel active, we may be splitting blocks, so force creation 7213 // of virtual registers for all non-dead arguments. 7214 if (FastISel) 7215 return A->use_empty(); 7216 7217 const BasicBlock *Entry = A->getParent()->begin(); 7218 for (Value::const_use_iterator UI = A->use_begin(), E = A->use_end(); 7219 UI != E; ++UI) { 7220 const User *U = *UI; 7221 if (cast<Instruction>(U)->getParent() != Entry || isa<SwitchInst>(U)) 7222 return false; // Use not in entry block. 7223 } 7224 return true; 7225} 7226 7227void SelectionDAGISel::LowerArguments(const Function &F) { 7228 SelectionDAG &DAG = SDB->DAG; 7229 SDLoc dl = SDB->getCurSDLoc(); 7230 const TargetLowering *TLI = getTargetLowering(); 7231 const DataLayout *TD = TLI->getDataLayout(); 7232 SmallVector<ISD::InputArg, 16> Ins; 7233 7234 if (!FuncInfo->CanLowerReturn) { 7235 // Put in an sret pointer parameter before all the other parameters. 7236 SmallVector<EVT, 1> ValueVTs; 7237 ComputeValueVTs(*getTargetLowering(), 7238 PointerType::getUnqual(F.getReturnType()), ValueVTs); 7239 7240 // NOTE: Assuming that a pointer will never break down to more than one VT 7241 // or one register. 7242 ISD::ArgFlagsTy Flags; 7243 Flags.setSRet(); 7244 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); 7245 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true, 0, 0); 7246 Ins.push_back(RetArg); 7247 } 7248 7249 // Set up the incoming argument description vector. 7250 unsigned Idx = 1; 7251 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); 7252 I != E; ++I, ++Idx) { 7253 SmallVector<EVT, 4> ValueVTs; 7254 ComputeValueVTs(*TLI, I->getType(), ValueVTs); 7255 bool isArgValueUsed = !I->use_empty(); 7256 unsigned PartBase = 0; 7257 for (unsigned Value = 0, NumValues = ValueVTs.size(); 7258 Value != NumValues; ++Value) { 7259 EVT VT = ValueVTs[Value]; 7260 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); 7261 ISD::ArgFlagsTy Flags; 7262 unsigned OriginalAlignment = 7263 TD->getABITypeAlignment(ArgTy); 7264 7265 if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) 7266 Flags.setZExt(); 7267 if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) 7268 Flags.setSExt(); 7269 if (F.getAttributes().hasAttribute(Idx, Attribute::InReg)) 7270 Flags.setInReg(); 7271 if (F.getAttributes().hasAttribute(Idx, Attribute::StructRet)) 7272 Flags.setSRet(); 7273 if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal)) { 7274 Flags.setByVal(); 7275 PointerType *Ty = cast<PointerType>(I->getType()); 7276 Type *ElementTy = Ty->getElementType(); 7277 Flags.setByValSize(TD->getTypeAllocSize(ElementTy)); 7278 // For ByVal, alignment should be passed from FE. BE will guess if 7279 // this info is not there but there are cases it cannot get right. 7280 unsigned FrameAlign; 7281 if (F.getParamAlignment(Idx)) 7282 FrameAlign = F.getParamAlignment(Idx); 7283 else 7284 FrameAlign = TLI->getByValTypeAlignment(ElementTy); 7285 Flags.setByValAlign(FrameAlign); 7286 } 7287 if (F.getAttributes().hasAttribute(Idx, Attribute::Nest)) 7288 Flags.setNest(); 7289 Flags.setOrigAlign(OriginalAlignment); 7290 7291 MVT RegisterVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7292 unsigned NumRegs = TLI->getNumRegisters(*CurDAG->getContext(), VT); 7293 for (unsigned i = 0; i != NumRegs; ++i) { 7294 ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed, 7295 Idx-1, PartBase+i*RegisterVT.getStoreSize()); 7296 if (NumRegs > 1 && i == 0) 7297 MyFlags.Flags.setSplit(); 7298 // if it isn't first piece, alignment must be 1 7299 else if (i > 0) 7300 MyFlags.Flags.setOrigAlign(1); 7301 Ins.push_back(MyFlags); 7302 } 7303 PartBase += VT.getStoreSize(); 7304 } 7305 } 7306 7307 // Call the target to set up the argument values. 7308 SmallVector<SDValue, 8> InVals; 7309 SDValue NewRoot = TLI->LowerFormalArguments(DAG.getRoot(), F.getCallingConv(), 7310 F.isVarArg(), Ins, 7311 dl, DAG, InVals); 7312 7313 // Verify that the target's LowerFormalArguments behaved as expected. 7314 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && 7315 "LowerFormalArguments didn't return a valid chain!"); 7316 assert(InVals.size() == Ins.size() && 7317 "LowerFormalArguments didn't emit the correct number of values!"); 7318 DEBUG({ 7319 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 7320 assert(InVals[i].getNode() && 7321 "LowerFormalArguments emitted a null value!"); 7322 assert(EVT(Ins[i].VT) == InVals[i].getValueType() && 7323 "LowerFormalArguments emitted a value with the wrong type!"); 7324 } 7325 }); 7326 7327 // Update the DAG with the new chain value resulting from argument lowering. 7328 DAG.setRoot(NewRoot); 7329 7330 // Set up the argument values. 7331 unsigned i = 0; 7332 Idx = 1; 7333 if (!FuncInfo->CanLowerReturn) { 7334 // Create a virtual register for the sret pointer, and put in a copy 7335 // from the sret argument into it. 7336 SmallVector<EVT, 1> ValueVTs; 7337 ComputeValueVTs(*TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs); 7338 MVT VT = ValueVTs[0].getSimpleVT(); 7339 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7340 ISD::NodeType AssertOp = ISD::DELETED_NODE; 7341 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, 7342 RegVT, VT, NULL, AssertOp); 7343 7344 MachineFunction& MF = SDB->DAG.getMachineFunction(); 7345 MachineRegisterInfo& RegInfo = MF.getRegInfo(); 7346 unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); 7347 FuncInfo->DemoteRegister = SRetReg; 7348 NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), 7349 SRetReg, ArgValue); 7350 DAG.setRoot(NewRoot); 7351 7352 // i indexes lowered arguments. Bump it past the hidden sret argument. 7353 // Idx indexes LLVM arguments. Don't touch it. 7354 ++i; 7355 } 7356 7357 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; 7358 ++I, ++Idx) { 7359 SmallVector<SDValue, 4> ArgValues; 7360 SmallVector<EVT, 4> ValueVTs; 7361 ComputeValueVTs(*TLI, I->getType(), ValueVTs); 7362 unsigned NumValues = ValueVTs.size(); 7363 7364 // If this argument is unused then remember its value. It is used to generate 7365 // debugging information. 7366 if (I->use_empty() && NumValues) { 7367 SDB->setUnusedArgValue(I, InVals[i]); 7368 7369 // Also remember any frame index for use in FastISel. 7370 if (FrameIndexSDNode *FI = 7371 dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) 7372 FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7373 } 7374 7375 for (unsigned Val = 0; Val != NumValues; ++Val) { 7376 EVT VT = ValueVTs[Val]; 7377 MVT PartVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 7378 unsigned NumParts = TLI->getNumRegisters(*CurDAG->getContext(), VT); 7379 7380 if (!I->use_empty()) { 7381 ISD::NodeType AssertOp = ISD::DELETED_NODE; 7382 if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) 7383 AssertOp = ISD::AssertSext; 7384 else if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) 7385 AssertOp = ISD::AssertZext; 7386 7387 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], 7388 NumParts, PartVT, VT, 7389 NULL, AssertOp)); 7390 } 7391 7392 i += NumParts; 7393 } 7394 7395 // We don't need to do anything else for unused arguments. 7396 if (ArgValues.empty()) 7397 continue; 7398 7399 // Note down frame index. 7400 if (FrameIndexSDNode *FI = 7401 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) 7402 FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7403 7404 SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues, 7405 SDB->getCurSDLoc()); 7406 7407 SDB->setValue(I, Res); 7408 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { 7409 if (LoadSDNode *LNode = 7410 dyn_cast<LoadSDNode>(Res.getOperand(0).getNode())) 7411 if (FrameIndexSDNode *FI = 7412 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 7413 FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); 7414 } 7415 7416 // If this argument is live outside of the entry block, insert a copy from 7417 // wherever we got it to the vreg that other BB's will reference it as. 7418 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) { 7419 // If we can, though, try to skip creating an unnecessary vreg. 7420 // FIXME: This isn't very clean... it would be nice to make this more 7421 // general. It's also subtly incompatible with the hacks FastISel 7422 // uses with vregs. 7423 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); 7424 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 7425 FuncInfo->ValueMap[I] = Reg; 7426 continue; 7427 } 7428 } 7429 if (!isOnlyUsedInEntryBlock(I, TM.Options.EnableFastISel)) { 7430 FuncInfo->InitializeRegForValue(I); 7431 SDB->CopyToExportRegsIfNeeded(I); 7432 } 7433 } 7434 7435 assert(i == InVals.size() && "Argument register count mismatch!"); 7436 7437 // Finally, if the target has anything special to do, allow it to do so. 7438 // FIXME: this should insert code into the DAG! 7439 EmitFunctionEntryCode(); 7440} 7441 7442/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to 7443/// ensure constants are generated when needed. Remember the virtual registers 7444/// that need to be added to the Machine PHI nodes as input. We cannot just 7445/// directly add them, because expansion might result in multiple MBB's for one 7446/// BB. As such, the start of the BB might correspond to a different MBB than 7447/// the end. 7448/// 7449void 7450SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { 7451 const TerminatorInst *TI = LLVMBB->getTerminator(); 7452 7453 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; 7454 7455 // Check successor nodes' PHI nodes that expect a constant to be available 7456 // from this block. 7457 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { 7458 const BasicBlock *SuccBB = TI->getSuccessor(succ); 7459 if (!isa<PHINode>(SuccBB->begin())) continue; 7460 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; 7461 7462 // If this terminator has multiple identical successors (common for 7463 // switches), only handle each succ once. 7464 if (!SuccsHandled.insert(SuccMBB)) continue; 7465 7466 MachineBasicBlock::iterator MBBI = SuccMBB->begin(); 7467 7468 // At this point we know that there is a 1-1 correspondence between LLVM PHI 7469 // nodes and Machine PHI nodes, but the incoming operands have not been 7470 // emitted yet. 7471 for (BasicBlock::const_iterator I = SuccBB->begin(); 7472 const PHINode *PN = dyn_cast<PHINode>(I); ++I) { 7473 // Ignore dead phi's. 7474 if (PN->use_empty()) continue; 7475 7476 // Skip empty types 7477 if (PN->getType()->isEmptyTy()) 7478 continue; 7479 7480 unsigned Reg; 7481 const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); 7482 7483 if (const Constant *C = dyn_cast<Constant>(PHIOp)) { 7484 unsigned &RegOut = ConstantsOut[C]; 7485 if (RegOut == 0) { 7486 RegOut = FuncInfo.CreateRegs(C->getType()); 7487 CopyValueToVirtualRegister(C, RegOut); 7488 } 7489 Reg = RegOut; 7490 } else { 7491 DenseMap<const Value *, unsigned>::iterator I = 7492 FuncInfo.ValueMap.find(PHIOp); 7493 if (I != FuncInfo.ValueMap.end()) 7494 Reg = I->second; 7495 else { 7496 assert(isa<AllocaInst>(PHIOp) && 7497 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && 7498 "Didn't codegen value into a register!??"); 7499 Reg = FuncInfo.CreateRegs(PHIOp->getType()); 7500 CopyValueToVirtualRegister(PHIOp, Reg); 7501 } 7502 } 7503 7504 // Remember that this register needs to added to the machine PHI node as 7505 // the input for this MBB. 7506 SmallVector<EVT, 4> ValueVTs; 7507 const TargetLowering *TLI = TM.getTargetLowering(); 7508 ComputeValueVTs(*TLI, PN->getType(), ValueVTs); 7509 for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { 7510 EVT VT = ValueVTs[vti]; 7511 unsigned NumRegisters = TLI->getNumRegisters(*DAG.getContext(), VT); 7512 for (unsigned i = 0, e = NumRegisters; i != e; ++i) 7513 FuncInfo.PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); 7514 Reg += NumRegisters; 7515 } 7516 } 7517 } 7518 7519 ConstantsOut.clear(); 7520} 7521 7522/// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB 7523/// is 0. 7524MachineBasicBlock * 7525SelectionDAGBuilder::StackProtectorDescriptor:: 7526AddSuccessorMBB(const BasicBlock *BB, 7527 MachineBasicBlock *ParentMBB, 7528 MachineBasicBlock *SuccMBB) { 7529 // If SuccBB has not been created yet, create it. 7530 if (!SuccMBB) { 7531 MachineFunction *MF = ParentMBB->getParent(); 7532 MachineFunction::iterator BBI = ParentMBB; 7533 SuccMBB = MF->CreateMachineBasicBlock(BB); 7534 MF->insert(++BBI, SuccMBB); 7535 } 7536 // Add it as a successor of ParentMBB. 7537 ParentMBB->addSuccessor(SuccMBB); 7538 return SuccMBB; 7539} 7540