1//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements the SelectionDAG class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "SDNodeDbgValue.h" 16#include "llvm/ADT/SetVector.h" 17#include "llvm/ADT/SmallPtrSet.h" 18#include "llvm/ADT/SmallSet.h" 19#include "llvm/ADT/SmallVector.h" 20#include "llvm/ADT/StringExtras.h" 21#include "llvm/Analysis/TargetTransformInfo.h" 22#include "llvm/Analysis/ValueTracking.h" 23#include "llvm/Assembly/Writer.h" 24#include "llvm/CodeGen/MachineBasicBlock.h" 25#include "llvm/CodeGen/MachineConstantPool.h" 26#include "llvm/CodeGen/MachineFrameInfo.h" 27#include "llvm/CodeGen/MachineModuleInfo.h" 28#include "llvm/DebugInfo.h" 29#include "llvm/IR/CallingConv.h" 30#include "llvm/IR/Constants.h" 31#include "llvm/IR/DataLayout.h" 32#include "llvm/IR/DerivedTypes.h" 33#include "llvm/IR/Function.h" 34#include "llvm/IR/GlobalAlias.h" 35#include "llvm/IR/GlobalVariable.h" 36#include "llvm/IR/Intrinsics.h" 37#include "llvm/Support/CommandLine.h" 38#include "llvm/Support/Debug.h" 39#include "llvm/Support/ErrorHandling.h" 40#include "llvm/Support/ManagedStatic.h" 41#include "llvm/Support/MathExtras.h" 42#include "llvm/Support/Mutex.h" 43#include "llvm/Support/raw_ostream.h" 44#include "llvm/Target/TargetInstrInfo.h" 45#include "llvm/Target/TargetIntrinsicInfo.h" 46#include "llvm/Target/TargetLowering.h" 47#include "llvm/Target/TargetMachine.h" 48#include "llvm/Target/TargetOptions.h" 49#include "llvm/Target/TargetRegisterInfo.h" 50#include "llvm/Target/TargetSelectionDAGInfo.h" 51#include <algorithm> 52#include <cmath> 53using namespace llvm; 54 55/// makeVTList - Return an instance of the SDVTList struct initialized with the 56/// specified members. 57static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58 SDVTList Res = {VTs, NumVTs}; 59 return Res; 60} 61 62// Default null implementations of the callbacks. 63void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 64void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 65 66//===----------------------------------------------------------------------===// 67// ConstantFPSDNode Class 68//===----------------------------------------------------------------------===// 69 70/// isExactlyValue - We don't rely on operator== working on double values, as 71/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 72/// As such, this method can be used to do an exact bit-for-bit comparison of 73/// two floating point values. 74bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 75 return getValueAPF().bitwiseIsEqual(V); 76} 77 78bool ConstantFPSDNode::isValueValidForType(EVT VT, 79 const APFloat& Val) { 80 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 81 82 // convert modifies in place, so make a copy. 83 APFloat Val2 = APFloat(Val); 84 bool losesInfo; 85 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 86 APFloat::rmNearestTiesToEven, 87 &losesInfo); 88 return !losesInfo; 89} 90 91//===----------------------------------------------------------------------===// 92// ISD Namespace 93//===----------------------------------------------------------------------===// 94 95/// isBuildVectorAllOnes - Return true if the specified node is a 96/// BUILD_VECTOR where all of the elements are ~0 or undef. 97bool ISD::isBuildVectorAllOnes(const SDNode *N) { 98 // Look through a bit convert. 99 if (N->getOpcode() == ISD::BITCAST) 100 N = N->getOperand(0).getNode(); 101 102 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 103 104 unsigned i = 0, e = N->getNumOperands(); 105 106 // Skip over all of the undef values. 107 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 108 ++i; 109 110 // Do not accept an all-undef vector. 111 if (i == e) return false; 112 113 // Do not accept build_vectors that aren't all constants or which have non-~0 114 // elements. We have to be a bit careful here, as the type of the constant 115 // may not be the same as the type of the vector elements due to type 116 // legalization (the elements are promoted to a legal type for the target and 117 // a vector of a type may be legal when the base element type is not). 118 // We only want to check enough bits to cover the vector elements, because 119 // we care if the resultant vector is all ones, not whether the individual 120 // constants are. 121 SDValue NotZero = N->getOperand(i); 122 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 123 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 124 if (CN->getAPIntValue().countTrailingOnes() < EltSize) 125 return false; 126 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 127 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 128 return false; 129 } else 130 return false; 131 132 // Okay, we have at least one ~0 value, check to see if the rest match or are 133 // undefs. Even with the above element type twiddling, this should be OK, as 134 // the same type legalization should have applied to all the elements. 135 for (++i; i != e; ++i) 136 if (N->getOperand(i) != NotZero && 137 N->getOperand(i).getOpcode() != ISD::UNDEF) 138 return false; 139 return true; 140} 141 142 143/// isBuildVectorAllZeros - Return true if the specified node is a 144/// BUILD_VECTOR where all of the elements are 0 or undef. 145bool ISD::isBuildVectorAllZeros(const SDNode *N) { 146 // Look through a bit convert. 147 if (N->getOpcode() == ISD::BITCAST) 148 N = N->getOperand(0).getNode(); 149 150 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 151 152 unsigned i = 0, e = N->getNumOperands(); 153 154 // Skip over all of the undef values. 155 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 156 ++i; 157 158 // Do not accept an all-undef vector. 159 if (i == e) return false; 160 161 // Do not accept build_vectors that aren't all constants or which have non-0 162 // elements. 163 SDValue Zero = N->getOperand(i); 164 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 165 if (!CN->isNullValue()) 166 return false; 167 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 168 if (!CFPN->getValueAPF().isPosZero()) 169 return false; 170 } else 171 return false; 172 173 // Okay, we have at least one 0 value, check to see if the rest match or are 174 // undefs. 175 for (++i; i != e; ++i) 176 if (N->getOperand(i) != Zero && 177 N->getOperand(i).getOpcode() != ISD::UNDEF) 178 return false; 179 return true; 180} 181 182/// isScalarToVector - Return true if the specified node is a 183/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 184/// element is not an undef. 185bool ISD::isScalarToVector(const SDNode *N) { 186 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 187 return true; 188 189 if (N->getOpcode() != ISD::BUILD_VECTOR) 190 return false; 191 if (N->getOperand(0).getOpcode() == ISD::UNDEF) 192 return false; 193 unsigned NumElems = N->getNumOperands(); 194 if (NumElems == 1) 195 return false; 196 for (unsigned i = 1; i < NumElems; ++i) { 197 SDValue V = N->getOperand(i); 198 if (V.getOpcode() != ISD::UNDEF) 199 return false; 200 } 201 return true; 202} 203 204/// allOperandsUndef - Return true if the node has at least one operand 205/// and all operands of the specified node are ISD::UNDEF. 206bool ISD::allOperandsUndef(const SDNode *N) { 207 // Return false if the node has no operands. 208 // This is "logically inconsistent" with the definition of "all" but 209 // is probably the desired behavior. 210 if (N->getNumOperands() == 0) 211 return false; 212 213 for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 214 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 215 return false; 216 217 return true; 218} 219 220/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 221/// when given the operation for (X op Y). 222ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 223 // To perform this operation, we just need to swap the L and G bits of the 224 // operation. 225 unsigned OldL = (Operation >> 2) & 1; 226 unsigned OldG = (Operation >> 1) & 1; 227 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 228 (OldL << 1) | // New G bit 229 (OldG << 2)); // New L bit. 230} 231 232/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 233/// 'op' is a valid SetCC operation. 234ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 235 unsigned Operation = Op; 236 if (isInteger) 237 Operation ^= 7; // Flip L, G, E bits, but not U. 238 else 239 Operation ^= 15; // Flip all of the condition bits. 240 241 if (Operation > ISD::SETTRUE2) 242 Operation &= ~8; // Don't let N and U bits get set. 243 244 return ISD::CondCode(Operation); 245} 246 247 248/// isSignedOp - For an integer comparison, return 1 if the comparison is a 249/// signed operation and 2 if the result is an unsigned comparison. Return zero 250/// if the operation does not depend on the sign of the input (setne and seteq). 251static int isSignedOp(ISD::CondCode Opcode) { 252 switch (Opcode) { 253 default: llvm_unreachable("Illegal integer setcc operation!"); 254 case ISD::SETEQ: 255 case ISD::SETNE: return 0; 256 case ISD::SETLT: 257 case ISD::SETLE: 258 case ISD::SETGT: 259 case ISD::SETGE: return 1; 260 case ISD::SETULT: 261 case ISD::SETULE: 262 case ISD::SETUGT: 263 case ISD::SETUGE: return 2; 264 } 265} 266 267/// getSetCCOrOperation - Return the result of a logical OR between different 268/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 269/// returns SETCC_INVALID if it is not possible to represent the resultant 270/// comparison. 271ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 272 bool isInteger) { 273 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 274 // Cannot fold a signed integer setcc with an unsigned integer setcc. 275 return ISD::SETCC_INVALID; 276 277 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 278 279 // If the N and U bits get set then the resultant comparison DOES suddenly 280 // care about orderedness, and is true when ordered. 281 if (Op > ISD::SETTRUE2) 282 Op &= ~16; // Clear the U bit if the N bit is set. 283 284 // Canonicalize illegal integer setcc's. 285 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 286 Op = ISD::SETNE; 287 288 return ISD::CondCode(Op); 289} 290 291/// getSetCCAndOperation - Return the result of a logical AND between different 292/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 293/// function returns zero if it is not possible to represent the resultant 294/// comparison. 295ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 296 bool isInteger) { 297 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 298 // Cannot fold a signed setcc with an unsigned setcc. 299 return ISD::SETCC_INVALID; 300 301 // Combine all of the condition bits. 302 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 303 304 // Canonicalize illegal integer setcc's. 305 if (isInteger) { 306 switch (Result) { 307 default: break; 308 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 309 case ISD::SETOEQ: // SETEQ & SETU[LG]E 310 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 311 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 312 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 313 } 314 } 315 316 return Result; 317} 318 319//===----------------------------------------------------------------------===// 320// SDNode Profile Support 321//===----------------------------------------------------------------------===// 322 323/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 324/// 325static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 326 ID.AddInteger(OpC); 327} 328 329/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 330/// solely with their pointer. 331static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 332 ID.AddPointer(VTList.VTs); 333} 334 335/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336/// 337static void AddNodeIDOperands(FoldingSetNodeID &ID, 338 const SDValue *Ops, unsigned NumOps) { 339 for (; NumOps; --NumOps, ++Ops) { 340 ID.AddPointer(Ops->getNode()); 341 ID.AddInteger(Ops->getResNo()); 342 } 343} 344 345/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 346/// 347static void AddNodeIDOperands(FoldingSetNodeID &ID, 348 const SDUse *Ops, unsigned NumOps) { 349 for (; NumOps; --NumOps, ++Ops) { 350 ID.AddPointer(Ops->getNode()); 351 ID.AddInteger(Ops->getResNo()); 352 } 353} 354 355static void AddNodeIDNode(FoldingSetNodeID &ID, 356 unsigned short OpC, SDVTList VTList, 357 const SDValue *OpList, unsigned N) { 358 AddNodeIDOpcode(ID, OpC); 359 AddNodeIDValueTypes(ID, VTList); 360 AddNodeIDOperands(ID, OpList, N); 361} 362 363/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 364/// the NodeID data. 365static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 366 switch (N->getOpcode()) { 367 case ISD::TargetExternalSymbol: 368 case ISD::ExternalSymbol: 369 llvm_unreachable("Should only be used on nodes with operands"); 370 default: break; // Normal nodes don't need extra info. 371 case ISD::TargetConstant: 372 case ISD::Constant: 373 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 374 break; 375 case ISD::TargetConstantFP: 376 case ISD::ConstantFP: { 377 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 378 break; 379 } 380 case ISD::TargetGlobalAddress: 381 case ISD::GlobalAddress: 382 case ISD::TargetGlobalTLSAddress: 383 case ISD::GlobalTLSAddress: { 384 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 385 ID.AddPointer(GA->getGlobal()); 386 ID.AddInteger(GA->getOffset()); 387 ID.AddInteger(GA->getTargetFlags()); 388 ID.AddInteger(GA->getAddressSpace()); 389 break; 390 } 391 case ISD::BasicBlock: 392 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 393 break; 394 case ISD::Register: 395 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 396 break; 397 case ISD::RegisterMask: 398 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 399 break; 400 case ISD::SRCVALUE: 401 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 402 break; 403 case ISD::FrameIndex: 404 case ISD::TargetFrameIndex: 405 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 406 break; 407 case ISD::JumpTable: 408 case ISD::TargetJumpTable: 409 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 410 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 411 break; 412 case ISD::ConstantPool: 413 case ISD::TargetConstantPool: { 414 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 415 ID.AddInteger(CP->getAlignment()); 416 ID.AddInteger(CP->getOffset()); 417 if (CP->isMachineConstantPoolEntry()) 418 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 419 else 420 ID.AddPointer(CP->getConstVal()); 421 ID.AddInteger(CP->getTargetFlags()); 422 break; 423 } 424 case ISD::TargetIndex: { 425 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 426 ID.AddInteger(TI->getIndex()); 427 ID.AddInteger(TI->getOffset()); 428 ID.AddInteger(TI->getTargetFlags()); 429 break; 430 } 431 case ISD::LOAD: { 432 const LoadSDNode *LD = cast<LoadSDNode>(N); 433 ID.AddInteger(LD->getMemoryVT().getRawBits()); 434 ID.AddInteger(LD->getRawSubclassData()); 435 ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 436 break; 437 } 438 case ISD::STORE: { 439 const StoreSDNode *ST = cast<StoreSDNode>(N); 440 ID.AddInteger(ST->getMemoryVT().getRawBits()); 441 ID.AddInteger(ST->getRawSubclassData()); 442 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 443 break; 444 } 445 case ISD::ATOMIC_CMP_SWAP: 446 case ISD::ATOMIC_SWAP: 447 case ISD::ATOMIC_LOAD_ADD: 448 case ISD::ATOMIC_LOAD_SUB: 449 case ISD::ATOMIC_LOAD_AND: 450 case ISD::ATOMIC_LOAD_OR: 451 case ISD::ATOMIC_LOAD_XOR: 452 case ISD::ATOMIC_LOAD_NAND: 453 case ISD::ATOMIC_LOAD_MIN: 454 case ISD::ATOMIC_LOAD_MAX: 455 case ISD::ATOMIC_LOAD_UMIN: 456 case ISD::ATOMIC_LOAD_UMAX: 457 case ISD::ATOMIC_LOAD: 458 case ISD::ATOMIC_STORE: { 459 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 460 ID.AddInteger(AT->getMemoryVT().getRawBits()); 461 ID.AddInteger(AT->getRawSubclassData()); 462 ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 463 break; 464 } 465 case ISD::PREFETCH: { 466 const MemSDNode *PF = cast<MemSDNode>(N); 467 ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 468 break; 469 } 470 case ISD::VECTOR_SHUFFLE: { 471 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 472 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 473 i != e; ++i) 474 ID.AddInteger(SVN->getMaskElt(i)); 475 break; 476 } 477 case ISD::TargetBlockAddress: 478 case ISD::BlockAddress: { 479 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 480 ID.AddPointer(BA->getBlockAddress()); 481 ID.AddInteger(BA->getOffset()); 482 ID.AddInteger(BA->getTargetFlags()); 483 break; 484 } 485 } // end switch (N->getOpcode()) 486 487 // Target specific memory nodes could also have address spaces to check. 488 if (N->isTargetMemoryOpcode()) 489 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 490} 491 492/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 493/// data. 494static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 495 AddNodeIDOpcode(ID, N->getOpcode()); 496 // Add the return value info. 497 AddNodeIDValueTypes(ID, N->getVTList()); 498 // Add the operand info. 499 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 500 501 // Handle SDNode leafs with special info. 502 AddNodeIDCustom(ID, N); 503} 504 505/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 506/// the CSE map that carries volatility, temporalness, indexing mode, and 507/// extension/truncation information. 508/// 509static inline unsigned 510encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 511 bool isNonTemporal, bool isInvariant) { 512 assert((ConvType & 3) == ConvType && 513 "ConvType may not require more than 2 bits!"); 514 assert((AM & 7) == AM && 515 "AM may not require more than 3 bits!"); 516 return ConvType | 517 (AM << 2) | 518 (isVolatile << 5) | 519 (isNonTemporal << 6) | 520 (isInvariant << 7); 521} 522 523//===----------------------------------------------------------------------===// 524// SelectionDAG Class 525//===----------------------------------------------------------------------===// 526 527/// doNotCSE - Return true if CSE should not be performed for this node. 528static bool doNotCSE(SDNode *N) { 529 if (N->getValueType(0) == MVT::Glue) 530 return true; // Never CSE anything that produces a flag. 531 532 switch (N->getOpcode()) { 533 default: break; 534 case ISD::HANDLENODE: 535 case ISD::EH_LABEL: 536 return true; // Never CSE these nodes. 537 } 538 539 // Check that remaining values produced are not flags. 540 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 541 if (N->getValueType(i) == MVT::Glue) 542 return true; // Never CSE anything that produces a flag. 543 544 return false; 545} 546 547/// RemoveDeadNodes - This method deletes all unreachable nodes in the 548/// SelectionDAG. 549void SelectionDAG::RemoveDeadNodes() { 550 // Create a dummy node (which is not added to allnodes), that adds a reference 551 // to the root node, preventing it from being deleted. 552 HandleSDNode Dummy(getRoot()); 553 554 SmallVector<SDNode*, 128> DeadNodes; 555 556 // Add all obviously-dead nodes to the DeadNodes worklist. 557 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 558 if (I->use_empty()) 559 DeadNodes.push_back(I); 560 561 RemoveDeadNodes(DeadNodes); 562 563 // If the root changed (e.g. it was a dead load, update the root). 564 setRoot(Dummy.getValue()); 565} 566 567/// RemoveDeadNodes - This method deletes the unreachable nodes in the 568/// given list, and any nodes that become unreachable as a result. 569void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 570 571 // Process the worklist, deleting the nodes and adding their uses to the 572 // worklist. 573 while (!DeadNodes.empty()) { 574 SDNode *N = DeadNodes.pop_back_val(); 575 576 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 577 DUL->NodeDeleted(N, 0); 578 579 // Take the node out of the appropriate CSE map. 580 RemoveNodeFromCSEMaps(N); 581 582 // Next, brutally remove the operand list. This is safe to do, as there are 583 // no cycles in the graph. 584 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 585 SDUse &Use = *I++; 586 SDNode *Operand = Use.getNode(); 587 Use.set(SDValue()); 588 589 // Now that we removed this operand, see if there are no uses of it left. 590 if (Operand->use_empty()) 591 DeadNodes.push_back(Operand); 592 } 593 594 DeallocateNode(N); 595 } 596} 597 598void SelectionDAG::RemoveDeadNode(SDNode *N){ 599 SmallVector<SDNode*, 16> DeadNodes(1, N); 600 601 // Create a dummy node that adds a reference to the root node, preventing 602 // it from being deleted. (This matters if the root is an operand of the 603 // dead node.) 604 HandleSDNode Dummy(getRoot()); 605 606 RemoveDeadNodes(DeadNodes); 607} 608 609void SelectionDAG::DeleteNode(SDNode *N) { 610 // First take this out of the appropriate CSE map. 611 RemoveNodeFromCSEMaps(N); 612 613 // Finally, remove uses due to operands of this node, remove from the 614 // AllNodes list, and delete the node. 615 DeleteNodeNotInCSEMaps(N); 616} 617 618void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 619 assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 620 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 621 622 // Drop all of the operands and decrement used node's use counts. 623 N->DropOperands(); 624 625 DeallocateNode(N); 626} 627 628void SDDbgInfo::erase(const SDNode *Node) { 629 DbgValMapType::iterator I = DbgValMap.find(Node); 630 if (I == DbgValMap.end()) 631 return; 632 for (unsigned J = 0, N = I->second.size(); J != N; ++J) 633 I->second[J]->setIsInvalidated(); 634 DbgValMap.erase(I); 635} 636 637void SelectionDAG::DeallocateNode(SDNode *N) { 638 if (N->OperandsNeedDelete) 639 delete[] N->OperandList; 640 641 // Set the opcode to DELETED_NODE to help catch bugs when node 642 // memory is reallocated. 643 N->NodeType = ISD::DELETED_NODE; 644 645 NodeAllocator.Deallocate(AllNodes.remove(N)); 646 647 // If any of the SDDbgValue nodes refer to this SDNode, invalidate 648 // them and forget about that node. 649 DbgInfo->erase(N); 650} 651 652/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 653/// correspond to it. This is useful when we're about to delete or repurpose 654/// the node. We don't want future request for structurally identical nodes 655/// to return N anymore. 656bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 657 bool Erased = false; 658 switch (N->getOpcode()) { 659 case ISD::HANDLENODE: return false; // noop. 660 case ISD::CONDCODE: 661 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 662 "Cond code doesn't exist!"); 663 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 664 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 665 break; 666 case ISD::ExternalSymbol: 667 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 668 break; 669 case ISD::TargetExternalSymbol: { 670 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 671 Erased = TargetExternalSymbols.erase( 672 std::pair<std::string,unsigned char>(ESN->getSymbol(), 673 ESN->getTargetFlags())); 674 break; 675 } 676 case ISD::VALUETYPE: { 677 EVT VT = cast<VTSDNode>(N)->getVT(); 678 if (VT.isExtended()) { 679 Erased = ExtendedValueTypeNodes.erase(VT); 680 } else { 681 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 682 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 683 } 684 break; 685 } 686 default: 687 // Remove it from the CSE Map. 688 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 689 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 690 Erased = CSEMap.RemoveNode(N); 691 break; 692 } 693#ifndef NDEBUG 694 // Verify that the node was actually in one of the CSE maps, unless it has a 695 // flag result (which cannot be CSE'd) or is one of the special cases that are 696 // not subject to CSE. 697 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 698 !N->isMachineOpcode() && !doNotCSE(N)) { 699 N->dump(this); 700 dbgs() << "\n"; 701 llvm_unreachable("Node is not in map!"); 702 } 703#endif 704 return Erased; 705} 706 707/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 708/// maps and modified in place. Add it back to the CSE maps, unless an identical 709/// node already exists, in which case transfer all its users to the existing 710/// node. This transfer can potentially trigger recursive merging. 711/// 712void 713SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 714 // For node types that aren't CSE'd, just act as if no identical node 715 // already exists. 716 if (!doNotCSE(N)) { 717 SDNode *Existing = CSEMap.GetOrInsertNode(N); 718 if (Existing != N) { 719 // If there was already an existing matching node, use ReplaceAllUsesWith 720 // to replace the dead one with the existing one. This can cause 721 // recursive merging of other unrelated nodes down the line. 722 ReplaceAllUsesWith(N, Existing); 723 724 // N is now dead. Inform the listeners and delete it. 725 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 726 DUL->NodeDeleted(N, Existing); 727 DeleteNodeNotInCSEMaps(N); 728 return; 729 } 730 } 731 732 // If the node doesn't already exist, we updated it. Inform listeners. 733 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 734 DUL->NodeUpdated(N); 735} 736 737/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 738/// were replaced with those specified. If this node is never memoized, 739/// return null, otherwise return a pointer to the slot it would take. If a 740/// node already exists with these operands, the slot will be non-null. 741SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 742 void *&InsertPos) { 743 if (doNotCSE(N)) 744 return 0; 745 746 SDValue Ops[] = { Op }; 747 FoldingSetNodeID ID; 748 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 749 AddNodeIDCustom(ID, N); 750 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 751 return Node; 752} 753 754/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 755/// were replaced with those specified. If this node is never memoized, 756/// return null, otherwise return a pointer to the slot it would take. If a 757/// node already exists with these operands, the slot will be non-null. 758SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 759 SDValue Op1, SDValue Op2, 760 void *&InsertPos) { 761 if (doNotCSE(N)) 762 return 0; 763 764 SDValue Ops[] = { Op1, Op2 }; 765 FoldingSetNodeID ID; 766 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 767 AddNodeIDCustom(ID, N); 768 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 769 return Node; 770} 771 772 773/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 774/// were replaced with those specified. If this node is never memoized, 775/// return null, otherwise return a pointer to the slot it would take. If a 776/// node already exists with these operands, the slot will be non-null. 777SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 778 const SDValue *Ops,unsigned NumOps, 779 void *&InsertPos) { 780 if (doNotCSE(N)) 781 return 0; 782 783 FoldingSetNodeID ID; 784 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 785 AddNodeIDCustom(ID, N); 786 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 787 return Node; 788} 789 790#ifndef NDEBUG 791/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 792static void VerifyNodeCommon(SDNode *N) { 793 switch (N->getOpcode()) { 794 default: 795 break; 796 case ISD::BUILD_PAIR: { 797 EVT VT = N->getValueType(0); 798 assert(N->getNumValues() == 1 && "Too many results!"); 799 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 800 "Wrong return type!"); 801 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 802 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 803 "Mismatched operand types!"); 804 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 805 "Wrong operand type!"); 806 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 807 "Wrong return type size"); 808 break; 809 } 810 case ISD::BUILD_VECTOR: { 811 assert(N->getNumValues() == 1 && "Too many results!"); 812 assert(N->getValueType(0).isVector() && "Wrong return type!"); 813 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 814 "Wrong number of operands!"); 815 EVT EltVT = N->getValueType(0).getVectorElementType(); 816 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 817 assert((I->getValueType() == EltVT || 818 (EltVT.isInteger() && I->getValueType().isInteger() && 819 EltVT.bitsLE(I->getValueType()))) && 820 "Wrong operand type!"); 821 assert(I->getValueType() == N->getOperand(0).getValueType() && 822 "Operands must all have the same type"); 823 } 824 break; 825 } 826 } 827} 828 829/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 830static void VerifySDNode(SDNode *N) { 831 // The SDNode allocators cannot be used to allocate nodes with fields that are 832 // not present in an SDNode! 833 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 834 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 835 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 836 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 837 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 838 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 839 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 840 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 841 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 842 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 843 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 844 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 845 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 846 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 847 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 848 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 849 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 850 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 851 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 852 853 VerifyNodeCommon(N); 854} 855 856/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 857/// invalid. 858static void VerifyMachineNode(SDNode *N) { 859 // The MachineNode allocators cannot be used to allocate nodes with fields 860 // that are not present in a MachineNode! 861 // Currently there are no such nodes. 862 863 VerifyNodeCommon(N); 864} 865#endif // NDEBUG 866 867/// getEVTAlignment - Compute the default alignment value for the 868/// given type. 869/// 870unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 871 Type *Ty = VT == MVT::iPTR ? 872 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 873 VT.getTypeForEVT(*getContext()); 874 875 return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty); 876} 877 878// EntryNode could meaningfully have debug info if we can find it... 879SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 880 : TM(tm), TSI(*tm.getSelectionDAGInfo()), TTI(0), TLI(0), OptLevel(OL), 881 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 882 Root(getEntryNode()), NewNodesMustHaveLegalTypes(false), 883 UpdateListeners(0) { 884 AllNodes.push_back(&EntryNode); 885 DbgInfo = new SDDbgInfo(); 886} 887 888void SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti, 889 const TargetLowering *tli) { 890 MF = &mf; 891 TTI = tti; 892 TLI = tli; 893 Context = &mf.getFunction()->getContext(); 894} 895 896SelectionDAG::~SelectionDAG() { 897 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 898 allnodes_clear(); 899 delete DbgInfo; 900} 901 902void SelectionDAG::allnodes_clear() { 903 assert(&*AllNodes.begin() == &EntryNode); 904 AllNodes.remove(AllNodes.begin()); 905 while (!AllNodes.empty()) 906 DeallocateNode(AllNodes.begin()); 907} 908 909void SelectionDAG::clear() { 910 allnodes_clear(); 911 OperandAllocator.Reset(); 912 CSEMap.clear(); 913 914 ExtendedValueTypeNodes.clear(); 915 ExternalSymbols.clear(); 916 TargetExternalSymbols.clear(); 917 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 918 static_cast<CondCodeSDNode*>(0)); 919 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 920 static_cast<SDNode*>(0)); 921 922 EntryNode.UseList = 0; 923 AllNodes.push_back(&EntryNode); 924 Root = getEntryNode(); 925 DbgInfo->clear(); 926} 927 928SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 929 return VT.bitsGT(Op.getValueType()) ? 930 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 931 getNode(ISD::TRUNCATE, DL, VT, Op); 932} 933 934SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 935 return VT.bitsGT(Op.getValueType()) ? 936 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 937 getNode(ISD::TRUNCATE, DL, VT, Op); 938} 939 940SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 941 return VT.bitsGT(Op.getValueType()) ? 942 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 943 getNode(ISD::TRUNCATE, DL, VT, Op); 944} 945 946SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) { 947 assert(!VT.isVector() && 948 "getZeroExtendInReg should use the vector element type instead of " 949 "the vector type!"); 950 if (Op.getValueType() == VT) return Op; 951 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 952 APInt Imm = APInt::getLowBitsSet(BitWidth, 953 VT.getSizeInBits()); 954 return getNode(ISD::AND, DL, Op.getValueType(), Op, 955 getConstant(Imm, Op.getValueType())); 956} 957 958/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 959/// 960SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) { 961 EVT EltVT = VT.getScalarType(); 962 SDValue NegOne = 963 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 964 return getNode(ISD::XOR, DL, VT, Val, NegOne); 965} 966 967SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 968 EVT EltVT = VT.getScalarType(); 969 assert((EltVT.getSizeInBits() >= 64 || 970 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 971 "getConstant with a uint64_t value that doesn't fit in the type!"); 972 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 973} 974 975SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 976 return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 977} 978 979SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 980 assert(VT.isInteger() && "Cannot create FP integer constant!"); 981 982 EVT EltVT = VT.getScalarType(); 983 const ConstantInt *Elt = &Val; 984 985 const TargetLowering *TLI = TM.getTargetLowering(); 986 987 // In some cases the vector type is legal but the element type is illegal and 988 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 989 // inserted value (the type does not need to match the vector element type). 990 // Any extra bits introduced will be truncated away. 991 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 992 TargetLowering::TypePromoteInteger) { 993 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 994 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 995 Elt = ConstantInt::get(*getContext(), NewVal); 996 } 997 // In other cases the element type is illegal and needs to be expanded, for 998 // example v2i64 on MIPS32. In this case, find the nearest legal type, split 999 // the value into n parts and use a vector type with n-times the elements. 1000 // Then bitcast to the type requested. 1001 // Legalizing constants too early makes the DAGCombiner's job harder so we 1002 // only legalize if the DAG tells us we must produce legal types. 1003 else if (NewNodesMustHaveLegalTypes && VT.isVector() && 1004 TLI->getTypeAction(*getContext(), EltVT) == 1005 TargetLowering::TypeExpandInteger) { 1006 APInt NewVal = Elt->getValue(); 1007 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1008 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1009 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1010 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1011 1012 // Check the temporary vector is the correct size. If this fails then 1013 // getTypeToTransformTo() probably returned a type whose size (in bits) 1014 // isn't a power-of-2 factor of the requested type size. 1015 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1016 1017 SmallVector<SDValue, 2> EltParts; 1018 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { 1019 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) 1020 .trunc(ViaEltSizeInBits), 1021 ViaEltVT, isT)); 1022 } 1023 1024 // EltParts is currently in little endian order. If we actually want 1025 // big-endian order then reverse it now. 1026 if (TLI->isBigEndian()) 1027 std::reverse(EltParts.begin(), EltParts.end()); 1028 1029 // The elements must be reversed when the element order is different 1030 // to the endianness of the elements (because the BITCAST is itself a 1031 // vector shuffle in this situation). However, we do not need any code to 1032 // perform this reversal because getConstant() is producing a vector 1033 // splat. 1034 // This situation occurs in MIPS MSA. 1035 1036 SmallVector<SDValue, 8> Ops; 1037 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) 1038 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); 1039 1040 SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT, 1041 getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT, 1042 &Ops[0], Ops.size())); 1043 return Result; 1044 } 1045 1046 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1047 "APInt size does not match type size!"); 1048 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1049 FoldingSetNodeID ID; 1050 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1051 ID.AddPointer(Elt); 1052 void *IP = 0; 1053 SDNode *N = NULL; 1054 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1055 if (!VT.isVector()) 1056 return SDValue(N, 0); 1057 1058 if (!N) { 1059 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 1060 CSEMap.InsertNode(N, IP); 1061 AllNodes.push_back(N); 1062 } 1063 1064 SDValue Result(N, 0); 1065 if (VT.isVector()) { 1066 SmallVector<SDValue, 8> Ops; 1067 Ops.assign(VT.getVectorNumElements(), Result); 1068 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1069 } 1070 return Result; 1071} 1072 1073SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 1074 return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget); 1075} 1076 1077 1078SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 1079 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 1080} 1081 1082SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 1083 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1084 1085 EVT EltVT = VT.getScalarType(); 1086 1087 // Do the map lookup using the actual bit pattern for the floating point 1088 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1089 // we don't have issues with SNANs. 1090 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1091 FoldingSetNodeID ID; 1092 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1093 ID.AddPointer(&V); 1094 void *IP = 0; 1095 SDNode *N = NULL; 1096 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1097 if (!VT.isVector()) 1098 return SDValue(N, 0); 1099 1100 if (!N) { 1101 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1102 CSEMap.InsertNode(N, IP); 1103 AllNodes.push_back(N); 1104 } 1105 1106 SDValue Result(N, 0); 1107 if (VT.isVector()) { 1108 SmallVector<SDValue, 8> Ops; 1109 Ops.assign(VT.getVectorNumElements(), Result); 1110 // FIXME SDLoc info might be appropriate here 1111 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1112 } 1113 return Result; 1114} 1115 1116SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1117 EVT EltVT = VT.getScalarType(); 1118 if (EltVT==MVT::f32) 1119 return getConstantFP(APFloat((float)Val), VT, isTarget); 1120 else if (EltVT==MVT::f64) 1121 return getConstantFP(APFloat(Val), VT, isTarget); 1122 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 1123 EltVT==MVT::f16) { 1124 bool ignored; 1125 APFloat apf = APFloat(Val); 1126 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1127 &ignored); 1128 return getConstantFP(apf, VT, isTarget); 1129 } else 1130 llvm_unreachable("Unsupported type in getConstantFP"); 1131} 1132 1133SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL, 1134 EVT VT, int64_t Offset, 1135 bool isTargetGA, 1136 unsigned char TargetFlags) { 1137 assert((TargetFlags == 0 || isTargetGA) && 1138 "Cannot set target flags on target-independent globals"); 1139 const TargetLowering *TLI = TM.getTargetLowering(); 1140 1141 // Truncate (with sign-extension) the offset value to the pointer size. 1142 unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 1143 if (BitWidth < 64) 1144 Offset = SignExtend64(Offset, BitWidth); 1145 1146 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1147 if (!GVar) { 1148 // If GV is an alias then use the aliasee for determining thread-localness. 1149 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1150 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1151 } 1152 1153 unsigned Opc; 1154 if (GVar && GVar->isThreadLocal()) 1155 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1156 else 1157 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1158 1159 FoldingSetNodeID ID; 1160 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1161 ID.AddPointer(GV); 1162 ID.AddInteger(Offset); 1163 ID.AddInteger(TargetFlags); 1164 ID.AddInteger(GV->getType()->getAddressSpace()); 1165 void *IP = 0; 1166 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1167 return SDValue(E, 0); 1168 1169 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(), 1170 DL.getDebugLoc(), GV, VT, 1171 Offset, TargetFlags); 1172 CSEMap.InsertNode(N, IP); 1173 AllNodes.push_back(N); 1174 return SDValue(N, 0); 1175} 1176 1177SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1178 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1179 FoldingSetNodeID ID; 1180 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1181 ID.AddInteger(FI); 1182 void *IP = 0; 1183 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1184 return SDValue(E, 0); 1185 1186 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1187 CSEMap.InsertNode(N, IP); 1188 AllNodes.push_back(N); 1189 return SDValue(N, 0); 1190} 1191 1192SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1193 unsigned char TargetFlags) { 1194 assert((TargetFlags == 0 || isTarget) && 1195 "Cannot set target flags on target-independent jump tables"); 1196 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1197 FoldingSetNodeID ID; 1198 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1199 ID.AddInteger(JTI); 1200 ID.AddInteger(TargetFlags); 1201 void *IP = 0; 1202 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1203 return SDValue(E, 0); 1204 1205 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1206 TargetFlags); 1207 CSEMap.InsertNode(N, IP); 1208 AllNodes.push_back(N); 1209 return SDValue(N, 0); 1210} 1211 1212SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1213 unsigned Alignment, int Offset, 1214 bool isTarget, 1215 unsigned char TargetFlags) { 1216 assert((TargetFlags == 0 || isTarget) && 1217 "Cannot set target flags on target-independent globals"); 1218 if (Alignment == 0) 1219 Alignment = 1220 TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1221 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1222 FoldingSetNodeID ID; 1223 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1224 ID.AddInteger(Alignment); 1225 ID.AddInteger(Offset); 1226 ID.AddPointer(C); 1227 ID.AddInteger(TargetFlags); 1228 void *IP = 0; 1229 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1230 return SDValue(E, 0); 1231 1232 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1233 Alignment, TargetFlags); 1234 CSEMap.InsertNode(N, IP); 1235 AllNodes.push_back(N); 1236 return SDValue(N, 0); 1237} 1238 1239 1240SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1241 unsigned Alignment, int Offset, 1242 bool isTarget, 1243 unsigned char TargetFlags) { 1244 assert((TargetFlags == 0 || isTarget) && 1245 "Cannot set target flags on target-independent globals"); 1246 if (Alignment == 0) 1247 Alignment = 1248 TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1249 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1250 FoldingSetNodeID ID; 1251 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1252 ID.AddInteger(Alignment); 1253 ID.AddInteger(Offset); 1254 C->addSelectionDAGCSEId(ID); 1255 ID.AddInteger(TargetFlags); 1256 void *IP = 0; 1257 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1258 return SDValue(E, 0); 1259 1260 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1261 Alignment, TargetFlags); 1262 CSEMap.InsertNode(N, IP); 1263 AllNodes.push_back(N); 1264 return SDValue(N, 0); 1265} 1266 1267SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1268 unsigned char TargetFlags) { 1269 FoldingSetNodeID ID; 1270 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 1271 ID.AddInteger(Index); 1272 ID.AddInteger(Offset); 1273 ID.AddInteger(TargetFlags); 1274 void *IP = 0; 1275 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1276 return SDValue(E, 0); 1277 1278 SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 1279 TargetFlags); 1280 CSEMap.InsertNode(N, IP); 1281 AllNodes.push_back(N); 1282 return SDValue(N, 0); 1283} 1284 1285SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1286 FoldingSetNodeID ID; 1287 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1288 ID.AddPointer(MBB); 1289 void *IP = 0; 1290 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1291 return SDValue(E, 0); 1292 1293 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1294 CSEMap.InsertNode(N, IP); 1295 AllNodes.push_back(N); 1296 return SDValue(N, 0); 1297} 1298 1299SDValue SelectionDAG::getValueType(EVT VT) { 1300 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1301 ValueTypeNodes.size()) 1302 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1303 1304 SDNode *&N = VT.isExtended() ? 1305 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1306 1307 if (N) return SDValue(N, 0); 1308 N = new (NodeAllocator) VTSDNode(VT); 1309 AllNodes.push_back(N); 1310 return SDValue(N, 0); 1311} 1312 1313SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1314 SDNode *&N = ExternalSymbols[Sym]; 1315 if (N) return SDValue(N, 0); 1316 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1317 AllNodes.push_back(N); 1318 return SDValue(N, 0); 1319} 1320 1321SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1322 unsigned char TargetFlags) { 1323 SDNode *&N = 1324 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1325 TargetFlags)]; 1326 if (N) return SDValue(N, 0); 1327 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1328 AllNodes.push_back(N); 1329 return SDValue(N, 0); 1330} 1331 1332SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1333 if ((unsigned)Cond >= CondCodeNodes.size()) 1334 CondCodeNodes.resize(Cond+1); 1335 1336 if (CondCodeNodes[Cond] == 0) { 1337 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1338 CondCodeNodes[Cond] = N; 1339 AllNodes.push_back(N); 1340 } 1341 1342 return SDValue(CondCodeNodes[Cond], 0); 1343} 1344 1345// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1346// the shuffle mask M that point at N1 to point at N2, and indices that point 1347// N2 to point at N1. 1348static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1349 std::swap(N1, N2); 1350 int NElts = M.size(); 1351 for (int i = 0; i != NElts; ++i) { 1352 if (M[i] >= NElts) 1353 M[i] -= NElts; 1354 else if (M[i] >= 0) 1355 M[i] += NElts; 1356 } 1357} 1358 1359SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, 1360 SDValue N2, const int *Mask) { 1361 assert(VT == N1.getValueType() && VT == N2.getValueType() && 1362 "Invalid VECTOR_SHUFFLE"); 1363 1364 // Canonicalize shuffle undef, undef -> undef 1365 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1366 return getUNDEF(VT); 1367 1368 // Validate that all indices in Mask are within the range of the elements 1369 // input to the shuffle. 1370 unsigned NElts = VT.getVectorNumElements(); 1371 SmallVector<int, 8> MaskVec; 1372 for (unsigned i = 0; i != NElts; ++i) { 1373 assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1374 MaskVec.push_back(Mask[i]); 1375 } 1376 1377 // Canonicalize shuffle v, v -> v, undef 1378 if (N1 == N2) { 1379 N2 = getUNDEF(VT); 1380 for (unsigned i = 0; i != NElts; ++i) 1381 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1382 } 1383 1384 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1385 if (N1.getOpcode() == ISD::UNDEF) 1386 commuteShuffle(N1, N2, MaskVec); 1387 1388 // Canonicalize all index into lhs, -> shuffle lhs, undef 1389 // Canonicalize all index into rhs, -> shuffle rhs, undef 1390 bool AllLHS = true, AllRHS = true; 1391 bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1392 for (unsigned i = 0; i != NElts; ++i) { 1393 if (MaskVec[i] >= (int)NElts) { 1394 if (N2Undef) 1395 MaskVec[i] = -1; 1396 else 1397 AllLHS = false; 1398 } else if (MaskVec[i] >= 0) { 1399 AllRHS = false; 1400 } 1401 } 1402 if (AllLHS && AllRHS) 1403 return getUNDEF(VT); 1404 if (AllLHS && !N2Undef) 1405 N2 = getUNDEF(VT); 1406 if (AllRHS) { 1407 N1 = getUNDEF(VT); 1408 commuteShuffle(N1, N2, MaskVec); 1409 } 1410 1411 // If Identity shuffle return that node. 1412 bool Identity = true; 1413 for (unsigned i = 0; i != NElts; ++i) { 1414 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1415 } 1416 if (Identity && NElts) 1417 return N1; 1418 1419 FoldingSetNodeID ID; 1420 SDValue Ops[2] = { N1, N2 }; 1421 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1422 for (unsigned i = 0; i != NElts; ++i) 1423 ID.AddInteger(MaskVec[i]); 1424 1425 void* IP = 0; 1426 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1427 return SDValue(E, 0); 1428 1429 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1430 // SDNode doesn't have access to it. This memory will be "leaked" when 1431 // the node is deallocated, but recovered when the NodeAllocator is released. 1432 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1433 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1434 1435 ShuffleVectorSDNode *N = 1436 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), 1437 dl.getDebugLoc(), N1, N2, 1438 MaskAlloc); 1439 CSEMap.InsertNode(N, IP); 1440 AllNodes.push_back(N); 1441 return SDValue(N, 0); 1442} 1443 1444SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, 1445 SDValue Val, SDValue DTy, 1446 SDValue STy, SDValue Rnd, SDValue Sat, 1447 ISD::CvtCode Code) { 1448 // If the src and dest types are the same and the conversion is between 1449 // integer types of the same sign or two floats, no conversion is necessary. 1450 if (DTy == STy && 1451 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1452 return Val; 1453 1454 FoldingSetNodeID ID; 1455 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1456 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1457 void* IP = 0; 1458 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1459 return SDValue(E, 0); 1460 1461 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), 1462 dl.getDebugLoc(), 1463 Ops, 5, Code); 1464 CSEMap.InsertNode(N, IP); 1465 AllNodes.push_back(N); 1466 return SDValue(N, 0); 1467} 1468 1469SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1470 FoldingSetNodeID ID; 1471 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1472 ID.AddInteger(RegNo); 1473 void *IP = 0; 1474 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1475 return SDValue(E, 0); 1476 1477 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1478 CSEMap.InsertNode(N, IP); 1479 AllNodes.push_back(N); 1480 return SDValue(N, 0); 1481} 1482 1483SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1484 FoldingSetNodeID ID; 1485 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1486 ID.AddPointer(RegMask); 1487 void *IP = 0; 1488 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1489 return SDValue(E, 0); 1490 1491 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1492 CSEMap.InsertNode(N, IP); 1493 AllNodes.push_back(N); 1494 return SDValue(N, 0); 1495} 1496 1497SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { 1498 FoldingSetNodeID ID; 1499 SDValue Ops[] = { Root }; 1500 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1501 ID.AddPointer(Label); 1502 void *IP = 0; 1503 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1504 return SDValue(E, 0); 1505 1506 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), 1507 dl.getDebugLoc(), Root, Label); 1508 CSEMap.InsertNode(N, IP); 1509 AllNodes.push_back(N); 1510 return SDValue(N, 0); 1511} 1512 1513 1514SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1515 int64_t Offset, 1516 bool isTarget, 1517 unsigned char TargetFlags) { 1518 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1519 1520 FoldingSetNodeID ID; 1521 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1522 ID.AddPointer(BA); 1523 ID.AddInteger(Offset); 1524 ID.AddInteger(TargetFlags); 1525 void *IP = 0; 1526 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1527 return SDValue(E, 0); 1528 1529 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 1530 TargetFlags); 1531 CSEMap.InsertNode(N, IP); 1532 AllNodes.push_back(N); 1533 return SDValue(N, 0); 1534} 1535 1536SDValue SelectionDAG::getSrcValue(const Value *V) { 1537 assert((!V || V->getType()->isPointerTy()) && 1538 "SrcValue is not a pointer?"); 1539 1540 FoldingSetNodeID ID; 1541 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1542 ID.AddPointer(V); 1543 1544 void *IP = 0; 1545 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1546 return SDValue(E, 0); 1547 1548 SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1549 CSEMap.InsertNode(N, IP); 1550 AllNodes.push_back(N); 1551 return SDValue(N, 0); 1552} 1553 1554/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1555SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1556 FoldingSetNodeID ID; 1557 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1558 ID.AddPointer(MD); 1559 1560 void *IP = 0; 1561 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1562 return SDValue(E, 0); 1563 1564 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1565 CSEMap.InsertNode(N, IP); 1566 AllNodes.push_back(N); 1567 return SDValue(N, 0); 1568} 1569 1570/// getAddrSpaceCast - Return an AddrSpaceCastSDNode. 1571SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, 1572 unsigned SrcAS, unsigned DestAS) { 1573 SDValue Ops[] = {Ptr}; 1574 FoldingSetNodeID ID; 1575 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), &Ops[0], 1); 1576 ID.AddInteger(SrcAS); 1577 ID.AddInteger(DestAS); 1578 1579 void *IP = 0; 1580 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1581 return SDValue(E, 0); 1582 1583 SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(), 1584 dl.getDebugLoc(), 1585 VT, Ptr, SrcAS, DestAS); 1586 CSEMap.InsertNode(N, IP); 1587 AllNodes.push_back(N); 1588 return SDValue(N, 0); 1589} 1590 1591/// getShiftAmountOperand - Return the specified value casted to 1592/// the target's desired shift amount type. 1593SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1594 EVT OpTy = Op.getValueType(); 1595 EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy); 1596 if (OpTy == ShTy || OpTy.isVector()) return Op; 1597 1598 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1599 return getNode(Opcode, SDLoc(Op), ShTy, Op); 1600} 1601 1602/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1603/// specified value type. 1604SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1605 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1606 unsigned ByteSize = VT.getStoreSize(); 1607 Type *Ty = VT.getTypeForEVT(*getContext()); 1608 const TargetLowering *TLI = TM.getTargetLowering(); 1609 unsigned StackAlign = 1610 std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 1611 1612 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1613 return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1614} 1615 1616/// CreateStackTemporary - Create a stack temporary suitable for holding 1617/// either of the specified value types. 1618SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1619 unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1620 VT2.getStoreSizeInBits())/8; 1621 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1622 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1623 const TargetLowering *TLI = TM.getTargetLowering(); 1624 const DataLayout *TD = TLI->getDataLayout(); 1625 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1626 TD->getPrefTypeAlignment(Ty2)); 1627 1628 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1629 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1630 return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1631} 1632 1633SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1634 SDValue N2, ISD::CondCode Cond, SDLoc dl) { 1635 // These setcc operations always fold. 1636 switch (Cond) { 1637 default: break; 1638 case ISD::SETFALSE: 1639 case ISD::SETFALSE2: return getConstant(0, VT); 1640 case ISD::SETTRUE: 1641 case ISD::SETTRUE2: { 1642 const TargetLowering *TLI = TM.getTargetLowering(); 1643 TargetLowering::BooleanContent Cnt = TLI->getBooleanContents(VT.isVector()); 1644 return getConstant( 1645 Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT); 1646 } 1647 1648 case ISD::SETOEQ: 1649 case ISD::SETOGT: 1650 case ISD::SETOGE: 1651 case ISD::SETOLT: 1652 case ISD::SETOLE: 1653 case ISD::SETONE: 1654 case ISD::SETO: 1655 case ISD::SETUO: 1656 case ISD::SETUEQ: 1657 case ISD::SETUNE: 1658 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1659 break; 1660 } 1661 1662 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1663 const APInt &C2 = N2C->getAPIntValue(); 1664 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1665 const APInt &C1 = N1C->getAPIntValue(); 1666 1667 switch (Cond) { 1668 default: llvm_unreachable("Unknown integer setcc!"); 1669 case ISD::SETEQ: return getConstant(C1 == C2, VT); 1670 case ISD::SETNE: return getConstant(C1 != C2, VT); 1671 case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1672 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1673 case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1674 case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1675 case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1676 case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1677 case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1678 case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1679 } 1680 } 1681 } 1682 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1683 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1684 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1685 switch (Cond) { 1686 default: break; 1687 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1688 return getUNDEF(VT); 1689 // fall through 1690 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1691 case ISD::SETNE: if (R==APFloat::cmpUnordered) 1692 return getUNDEF(VT); 1693 // fall through 1694 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1695 R==APFloat::cmpLessThan, VT); 1696 case ISD::SETLT: if (R==APFloat::cmpUnordered) 1697 return getUNDEF(VT); 1698 // fall through 1699 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1700 case ISD::SETGT: if (R==APFloat::cmpUnordered) 1701 return getUNDEF(VT); 1702 // fall through 1703 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1704 case ISD::SETLE: if (R==APFloat::cmpUnordered) 1705 return getUNDEF(VT); 1706 // fall through 1707 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1708 R==APFloat::cmpEqual, VT); 1709 case ISD::SETGE: if (R==APFloat::cmpUnordered) 1710 return getUNDEF(VT); 1711 // fall through 1712 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1713 R==APFloat::cmpEqual, VT); 1714 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1715 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1716 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1717 R==APFloat::cmpEqual, VT); 1718 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1719 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1720 R==APFloat::cmpLessThan, VT); 1721 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1722 R==APFloat::cmpUnordered, VT); 1723 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1724 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1725 } 1726 } else { 1727 // Ensure that the constant occurs on the RHS. 1728 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 1729 MVT CompVT = N1.getValueType().getSimpleVT(); 1730 if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT)) 1731 return SDValue(); 1732 1733 return getSetCC(dl, VT, N2, N1, SwappedCond); 1734 } 1735 } 1736 1737 // Could not fold it. 1738 return SDValue(); 1739} 1740 1741/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1742/// use this predicate to simplify operations downstream. 1743bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1744 // This predicate is not safe for vector operations. 1745 if (Op.getValueType().isVector()) 1746 return false; 1747 1748 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1749 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1750} 1751 1752/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1753/// this predicate to simplify operations downstream. Mask is known to be zero 1754/// for bits that V cannot have. 1755bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1756 unsigned Depth) const { 1757 APInt KnownZero, KnownOne; 1758 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 1759 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1760 return (KnownZero & Mask) == Mask; 1761} 1762 1763/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1764/// known to be either zero or one and return them in the KnownZero/KnownOne 1765/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1766/// processing. 1767void SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 1768 APInt &KnownOne, unsigned Depth) const { 1769 const TargetLowering *TLI = TM.getTargetLowering(); 1770 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1771 1772 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1773 if (Depth == 6) 1774 return; // Limit search depth. 1775 1776 APInt KnownZero2, KnownOne2; 1777 1778 switch (Op.getOpcode()) { 1779 case ISD::Constant: 1780 // We know all of the bits for a constant! 1781 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 1782 KnownZero = ~KnownOne; 1783 return; 1784 case ISD::AND: 1785 // If either the LHS or the RHS are Zero, the result is zero. 1786 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1787 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1788 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1789 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1790 1791 // Output known-1 bits are only known if set in both the LHS & RHS. 1792 KnownOne &= KnownOne2; 1793 // Output known-0 are known to be clear if zero in either the LHS | RHS. 1794 KnownZero |= KnownZero2; 1795 return; 1796 case ISD::OR: 1797 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1798 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1799 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1800 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1801 1802 // Output known-0 bits are only known if clear in both the LHS & RHS. 1803 KnownZero &= KnownZero2; 1804 // Output known-1 are known to be set if set in either the LHS | RHS. 1805 KnownOne |= KnownOne2; 1806 return; 1807 case ISD::XOR: { 1808 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1809 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1810 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1811 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1812 1813 // Output known-0 bits are known if clear or set in both the LHS & RHS. 1814 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1815 // Output known-1 are known to be set if set in only one of the LHS, RHS. 1816 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1817 KnownZero = KnownZeroOut; 1818 return; 1819 } 1820 case ISD::MUL: { 1821 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1822 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1823 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1824 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1825 1826 // If low bits are zero in either operand, output low known-0 bits. 1827 // Also compute a conserative estimate for high known-0 bits. 1828 // More trickiness is possible, but this is sufficient for the 1829 // interesting case of alignment computation. 1830 KnownOne.clearAllBits(); 1831 unsigned TrailZ = KnownZero.countTrailingOnes() + 1832 KnownZero2.countTrailingOnes(); 1833 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1834 KnownZero2.countLeadingOnes(), 1835 BitWidth) - BitWidth; 1836 1837 TrailZ = std::min(TrailZ, BitWidth); 1838 LeadZ = std::min(LeadZ, BitWidth); 1839 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1840 APInt::getHighBitsSet(BitWidth, LeadZ); 1841 return; 1842 } 1843 case ISD::UDIV: { 1844 // For the purposes of computing leading zeros we can conservatively 1845 // treat a udiv as a logical right shift by the power of 2 known to 1846 // be less than the denominator. 1847 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1848 unsigned LeadZ = KnownZero2.countLeadingOnes(); 1849 1850 KnownOne2.clearAllBits(); 1851 KnownZero2.clearAllBits(); 1852 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1853 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1854 if (RHSUnknownLeadingOnes != BitWidth) 1855 LeadZ = std::min(BitWidth, 1856 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1857 1858 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 1859 return; 1860 } 1861 case ISD::SELECT: 1862 ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 1863 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1864 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1865 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1866 1867 // Only known if known in both the LHS and RHS. 1868 KnownOne &= KnownOne2; 1869 KnownZero &= KnownZero2; 1870 return; 1871 case ISD::SELECT_CC: 1872 ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 1873 ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 1874 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1875 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1876 1877 // Only known if known in both the LHS and RHS. 1878 KnownOne &= KnownOne2; 1879 KnownZero &= KnownZero2; 1880 return; 1881 case ISD::SADDO: 1882 case ISD::UADDO: 1883 case ISD::SSUBO: 1884 case ISD::USUBO: 1885 case ISD::SMULO: 1886 case ISD::UMULO: 1887 if (Op.getResNo() != 1) 1888 return; 1889 // The boolean result conforms to getBooleanContents. Fall through. 1890 case ISD::SETCC: 1891 // If we know the result of a setcc has the top bits zero, use this info. 1892 if (TLI->getBooleanContents(Op.getValueType().isVector()) == 1893 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1894 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1895 return; 1896 case ISD::SHL: 1897 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1898 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1899 unsigned ShAmt = SA->getZExtValue(); 1900 1901 // If the shift count is an invalid immediate, don't do anything. 1902 if (ShAmt >= BitWidth) 1903 return; 1904 1905 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1906 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1907 KnownZero <<= ShAmt; 1908 KnownOne <<= ShAmt; 1909 // low bits known zero. 1910 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1911 } 1912 return; 1913 case ISD::SRL: 1914 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1915 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1916 unsigned ShAmt = SA->getZExtValue(); 1917 1918 // If the shift count is an invalid immediate, don't do anything. 1919 if (ShAmt >= BitWidth) 1920 return; 1921 1922 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1923 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1924 KnownZero = KnownZero.lshr(ShAmt); 1925 KnownOne = KnownOne.lshr(ShAmt); 1926 1927 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1928 KnownZero |= HighBits; // High bits known zero. 1929 } 1930 return; 1931 case ISD::SRA: 1932 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1933 unsigned ShAmt = SA->getZExtValue(); 1934 1935 // If the shift count is an invalid immediate, don't do anything. 1936 if (ShAmt >= BitWidth) 1937 return; 1938 1939 // If any of the demanded bits are produced by the sign extension, we also 1940 // demand the input sign bit. 1941 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1942 1943 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1944 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1945 KnownZero = KnownZero.lshr(ShAmt); 1946 KnownOne = KnownOne.lshr(ShAmt); 1947 1948 // Handle the sign bits. 1949 APInt SignBit = APInt::getSignBit(BitWidth); 1950 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1951 1952 if (KnownZero.intersects(SignBit)) { 1953 KnownZero |= HighBits; // New bits are known zero. 1954 } else if (KnownOne.intersects(SignBit)) { 1955 KnownOne |= HighBits; // New bits are known one. 1956 } 1957 } 1958 return; 1959 case ISD::SIGN_EXTEND_INREG: { 1960 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1961 unsigned EBits = EVT.getScalarType().getSizeInBits(); 1962 1963 // Sign extension. Compute the demanded bits in the result that are not 1964 // present in the input. 1965 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 1966 1967 APInt InSignBit = APInt::getSignBit(EBits); 1968 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 1969 1970 // If the sign extended bits are demanded, we know that the sign 1971 // bit is demanded. 1972 InSignBit = InSignBit.zext(BitWidth); 1973 if (NewBits.getBoolValue()) 1974 InputDemandedBits |= InSignBit; 1975 1976 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1977 KnownOne &= InputDemandedBits; 1978 KnownZero &= InputDemandedBits; 1979 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1980 1981 // If the sign bit of the input is known set or clear, then we know the 1982 // top bits of the result. 1983 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1984 KnownZero |= NewBits; 1985 KnownOne &= ~NewBits; 1986 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1987 KnownOne |= NewBits; 1988 KnownZero &= ~NewBits; 1989 } else { // Input sign bit unknown 1990 KnownZero &= ~NewBits; 1991 KnownOne &= ~NewBits; 1992 } 1993 return; 1994 } 1995 case ISD::CTTZ: 1996 case ISD::CTTZ_ZERO_UNDEF: 1997 case ISD::CTLZ: 1998 case ISD::CTLZ_ZERO_UNDEF: 1999 case ISD::CTPOP: { 2000 unsigned LowBits = Log2_32(BitWidth)+1; 2001 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 2002 KnownOne.clearAllBits(); 2003 return; 2004 } 2005 case ISD::LOAD: { 2006 LoadSDNode *LD = cast<LoadSDNode>(Op); 2007 // If this is a ZEXTLoad and we are looking at the loaded value. 2008 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 2009 EVT VT = LD->getMemoryVT(); 2010 unsigned MemBits = VT.getScalarType().getSizeInBits(); 2011 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 2012 } else if (const MDNode *Ranges = LD->getRanges()) { 2013 computeMaskedBitsLoad(*Ranges, KnownZero); 2014 } 2015 return; 2016 } 2017 case ISD::ZERO_EXTEND: { 2018 EVT InVT = Op.getOperand(0).getValueType(); 2019 unsigned InBits = InVT.getScalarType().getSizeInBits(); 2020 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2021 KnownZero = KnownZero.trunc(InBits); 2022 KnownOne = KnownOne.trunc(InBits); 2023 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2024 KnownZero = KnownZero.zext(BitWidth); 2025 KnownOne = KnownOne.zext(BitWidth); 2026 KnownZero |= NewBits; 2027 return; 2028 } 2029 case ISD::SIGN_EXTEND: { 2030 EVT InVT = Op.getOperand(0).getValueType(); 2031 unsigned InBits = InVT.getScalarType().getSizeInBits(); 2032 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2033 2034 KnownZero = KnownZero.trunc(InBits); 2035 KnownOne = KnownOne.trunc(InBits); 2036 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2037 2038 // Note if the sign bit is known to be zero or one. 2039 bool SignBitKnownZero = KnownZero.isNegative(); 2040 bool SignBitKnownOne = KnownOne.isNegative(); 2041 assert(!(SignBitKnownZero && SignBitKnownOne) && 2042 "Sign bit can't be known to be both zero and one!"); 2043 2044 KnownZero = KnownZero.zext(BitWidth); 2045 KnownOne = KnownOne.zext(BitWidth); 2046 2047 // If the sign bit is known zero or one, the top bits match. 2048 if (SignBitKnownZero) 2049 KnownZero |= NewBits; 2050 else if (SignBitKnownOne) 2051 KnownOne |= NewBits; 2052 return; 2053 } 2054 case ISD::ANY_EXTEND: { 2055 EVT InVT = Op.getOperand(0).getValueType(); 2056 unsigned InBits = InVT.getScalarType().getSizeInBits(); 2057 KnownZero = KnownZero.trunc(InBits); 2058 KnownOne = KnownOne.trunc(InBits); 2059 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2060 KnownZero = KnownZero.zext(BitWidth); 2061 KnownOne = KnownOne.zext(BitWidth); 2062 return; 2063 } 2064 case ISD::TRUNCATE: { 2065 EVT InVT = Op.getOperand(0).getValueType(); 2066 unsigned InBits = InVT.getScalarType().getSizeInBits(); 2067 KnownZero = KnownZero.zext(InBits); 2068 KnownOne = KnownOne.zext(InBits); 2069 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2070 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 2071 KnownZero = KnownZero.trunc(BitWidth); 2072 KnownOne = KnownOne.trunc(BitWidth); 2073 break; 2074 } 2075 case ISD::AssertZext: { 2076 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2077 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 2078 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2079 KnownZero |= (~InMask); 2080 KnownOne &= (~KnownZero); 2081 return; 2082 } 2083 case ISD::FGETSIGN: 2084 // All bits are zero except the low bit. 2085 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 2086 return; 2087 2088 case ISD::SUB: { 2089 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 2090 // We know that the top bits of C-X are clear if X contains less bits 2091 // than C (i.e. no wrap-around can happen). For example, 20-X is 2092 // positive if we can prove that X is >= 0 and < 16. 2093 if (CLHS->getAPIntValue().isNonNegative()) { 2094 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2095 // NLZ can't be BitWidth with no sign bit 2096 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2097 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2098 2099 // If all of the MaskV bits are known to be zero, then we know the 2100 // output top bits are zero, because we now know that the output is 2101 // from [0-C]. 2102 if ((KnownZero2 & MaskV) == MaskV) { 2103 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2104 // Top bits known zero. 2105 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 2106 } 2107 } 2108 } 2109 } 2110 // fall through 2111 case ISD::ADD: 2112 case ISD::ADDE: { 2113 // Output known-0 bits are known if clear or set in both the low clear bits 2114 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2115 // low 3 bits clear. 2116 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 2117 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2118 unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2119 2120 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2121 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2122 KnownZeroOut = std::min(KnownZeroOut, 2123 KnownZero2.countTrailingOnes()); 2124 2125 if (Op.getOpcode() == ISD::ADD) { 2126 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2127 return; 2128 } 2129 2130 // With ADDE, a carry bit may be added in, so we can only use this 2131 // information if we know (at least) that the low two bits are clear. We 2132 // then return to the caller that the low bit is unknown but that other bits 2133 // are known zero. 2134 if (KnownZeroOut >= 2) // ADDE 2135 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2136 return; 2137 } 2138 case ISD::SREM: 2139 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2140 const APInt &RA = Rem->getAPIntValue().abs(); 2141 if (RA.isPowerOf2()) { 2142 APInt LowBits = RA - 1; 2143 ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 2144 2145 // The low bits of the first operand are unchanged by the srem. 2146 KnownZero = KnownZero2 & LowBits; 2147 KnownOne = KnownOne2 & LowBits; 2148 2149 // If the first operand is non-negative or has all low bits zero, then 2150 // the upper bits are all zero. 2151 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2152 KnownZero |= ~LowBits; 2153 2154 // If the first operand is negative and not all low bits are zero, then 2155 // the upper bits are all one. 2156 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2157 KnownOne |= ~LowBits; 2158 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2159 } 2160 } 2161 return; 2162 case ISD::UREM: { 2163 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2164 const APInt &RA = Rem->getAPIntValue(); 2165 if (RA.isPowerOf2()) { 2166 APInt LowBits = (RA - 1); 2167 KnownZero |= ~LowBits; 2168 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 2169 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2170 break; 2171 } 2172 } 2173 2174 // Since the result is less than or equal to either operand, any leading 2175 // zero bits in either operand must also exist in the result. 2176 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2177 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2178 2179 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2180 KnownZero2.countLeadingOnes()); 2181 KnownOne.clearAllBits(); 2182 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 2183 return; 2184 } 2185 case ISD::FrameIndex: 2186 case ISD::TargetFrameIndex: 2187 if (unsigned Align = InferPtrAlignment(Op)) { 2188 // The low bits are known zero if the pointer is aligned. 2189 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2190 return; 2191 } 2192 break; 2193 2194 default: 2195 if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2196 break; 2197 // Fallthrough 2198 case ISD::INTRINSIC_WO_CHAIN: 2199 case ISD::INTRINSIC_W_CHAIN: 2200 case ISD::INTRINSIC_VOID: 2201 // Allow the target to implement this method for its nodes. 2202 TLI->computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 2203 return; 2204 } 2205} 2206 2207/// ComputeNumSignBits - Return the number of times the sign bit of the 2208/// register is replicated into the other bits. We know that at least 1 bit 2209/// is always equal to the sign bit (itself), but other cases can give us 2210/// information. For example, immediately after an "SRA X, 2", we know that 2211/// the top 3 bits are all equal to each other, so we return 3. 2212unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2213 const TargetLowering *TLI = TM.getTargetLowering(); 2214 EVT VT = Op.getValueType(); 2215 assert(VT.isInteger() && "Invalid VT!"); 2216 unsigned VTBits = VT.getScalarType().getSizeInBits(); 2217 unsigned Tmp, Tmp2; 2218 unsigned FirstAnswer = 1; 2219 2220 if (Depth == 6) 2221 return 1; // Limit search depth. 2222 2223 switch (Op.getOpcode()) { 2224 default: break; 2225 case ISD::AssertSext: 2226 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2227 return VTBits-Tmp+1; 2228 case ISD::AssertZext: 2229 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2230 return VTBits-Tmp; 2231 2232 case ISD::Constant: { 2233 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2234 return Val.getNumSignBits(); 2235 } 2236 2237 case ISD::SIGN_EXTEND: 2238 Tmp = 2239 VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2240 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2241 2242 case ISD::SIGN_EXTEND_INREG: 2243 // Max of the input and what this extends. 2244 Tmp = 2245 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2246 Tmp = VTBits-Tmp+1; 2247 2248 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2249 return std::max(Tmp, Tmp2); 2250 2251 case ISD::SRA: 2252 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2253 // SRA X, C -> adds C sign bits. 2254 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2255 Tmp += C->getZExtValue(); 2256 if (Tmp > VTBits) Tmp = VTBits; 2257 } 2258 return Tmp; 2259 case ISD::SHL: 2260 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2261 // shl destroys sign bits. 2262 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2263 if (C->getZExtValue() >= VTBits || // Bad shift. 2264 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2265 return Tmp - C->getZExtValue(); 2266 } 2267 break; 2268 case ISD::AND: 2269 case ISD::OR: 2270 case ISD::XOR: // NOT is handled here. 2271 // Logical binary ops preserve the number of sign bits at the worst. 2272 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2273 if (Tmp != 1) { 2274 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2275 FirstAnswer = std::min(Tmp, Tmp2); 2276 // We computed what we know about the sign bits as our first 2277 // answer. Now proceed to the generic code that uses 2278 // ComputeMaskedBits, and pick whichever answer is better. 2279 } 2280 break; 2281 2282 case ISD::SELECT: 2283 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2284 if (Tmp == 1) return 1; // Early out. 2285 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2286 return std::min(Tmp, Tmp2); 2287 2288 case ISD::SADDO: 2289 case ISD::UADDO: 2290 case ISD::SSUBO: 2291 case ISD::USUBO: 2292 case ISD::SMULO: 2293 case ISD::UMULO: 2294 if (Op.getResNo() != 1) 2295 break; 2296 // The boolean result conforms to getBooleanContents. Fall through. 2297 case ISD::SETCC: 2298 // If setcc returns 0/-1, all bits are sign bits. 2299 if (TLI->getBooleanContents(Op.getValueType().isVector()) == 2300 TargetLowering::ZeroOrNegativeOneBooleanContent) 2301 return VTBits; 2302 break; 2303 case ISD::ROTL: 2304 case ISD::ROTR: 2305 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2306 unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2307 2308 // Handle rotate right by N like a rotate left by 32-N. 2309 if (Op.getOpcode() == ISD::ROTR) 2310 RotAmt = (VTBits-RotAmt) & (VTBits-1); 2311 2312 // If we aren't rotating out all of the known-in sign bits, return the 2313 // number that are left. This handles rotl(sext(x), 1) for example. 2314 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2315 if (Tmp > RotAmt+1) return Tmp-RotAmt; 2316 } 2317 break; 2318 case ISD::ADD: 2319 // Add can have at most one carry bit. Thus we know that the output 2320 // is, at worst, one more bit than the inputs. 2321 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2322 if (Tmp == 1) return 1; // Early out. 2323 2324 // Special case decrementing a value (ADD X, -1): 2325 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2326 if (CRHS->isAllOnesValue()) { 2327 APInt KnownZero, KnownOne; 2328 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2329 2330 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2331 // sign bits set. 2332 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2333 return VTBits; 2334 2335 // If we are subtracting one from a positive number, there is no carry 2336 // out of the result. 2337 if (KnownZero.isNegative()) 2338 return Tmp; 2339 } 2340 2341 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2342 if (Tmp2 == 1) return 1; 2343 return std::min(Tmp, Tmp2)-1; 2344 2345 case ISD::SUB: 2346 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2347 if (Tmp2 == 1) return 1; 2348 2349 // Handle NEG. 2350 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2351 if (CLHS->isNullValue()) { 2352 APInt KnownZero, KnownOne; 2353 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 2354 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2355 // sign bits set. 2356 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2357 return VTBits; 2358 2359 // If the input is known to be positive (the sign bit is known clear), 2360 // the output of the NEG has the same number of sign bits as the input. 2361 if (KnownZero.isNegative()) 2362 return Tmp2; 2363 2364 // Otherwise, we treat this like a SUB. 2365 } 2366 2367 // Sub can have at most one carry bit. Thus we know that the output 2368 // is, at worst, one more bit than the inputs. 2369 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2370 if (Tmp == 1) return 1; // Early out. 2371 return std::min(Tmp, Tmp2)-1; 2372 case ISD::TRUNCATE: 2373 // FIXME: it's tricky to do anything useful for this, but it is an important 2374 // case for targets like X86. 2375 break; 2376 } 2377 2378 // If we are looking at the loaded value of the SDNode. 2379 if (Op.getResNo() == 0) { 2380 // Handle LOADX separately here. EXTLOAD case will fallthrough. 2381 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 2382 unsigned ExtType = LD->getExtensionType(); 2383 switch (ExtType) { 2384 default: break; 2385 case ISD::SEXTLOAD: // '17' bits known 2386 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2387 return VTBits-Tmp+1; 2388 case ISD::ZEXTLOAD: // '16' bits known 2389 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2390 return VTBits-Tmp; 2391 } 2392 } 2393 } 2394 2395 // Allow the target to implement this method for its nodes. 2396 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2397 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2398 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2399 Op.getOpcode() == ISD::INTRINSIC_VOID) { 2400 unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, Depth); 2401 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2402 } 2403 2404 // Finally, if we can prove that the top bits of the result are 0's or 1's, 2405 // use this information. 2406 APInt KnownZero, KnownOne; 2407 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 2408 2409 APInt Mask; 2410 if (KnownZero.isNegative()) { // sign bit is 0 2411 Mask = KnownZero; 2412 } else if (KnownOne.isNegative()) { // sign bit is 1; 2413 Mask = KnownOne; 2414 } else { 2415 // Nothing known. 2416 return FirstAnswer; 2417 } 2418 2419 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2420 // the number of identical bits in the top of the input value. 2421 Mask = ~Mask; 2422 Mask <<= Mask.getBitWidth()-VTBits; 2423 // Return # leading zeros. We use 'min' here in case Val was zero before 2424 // shifting. We don't want to return '64' as for an i32 "0". 2425 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2426} 2427 2428/// isBaseWithConstantOffset - Return true if the specified operand is an 2429/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2430/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2431/// semantics as an ADD. This handles the equivalence: 2432/// X|Cst == X+Cst iff X&Cst = 0. 2433bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2434 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2435 !isa<ConstantSDNode>(Op.getOperand(1))) 2436 return false; 2437 2438 if (Op.getOpcode() == ISD::OR && 2439 !MaskedValueIsZero(Op.getOperand(0), 2440 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2441 return false; 2442 2443 return true; 2444} 2445 2446 2447bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2448 // If we're told that NaNs won't happen, assume they won't. 2449 if (getTarget().Options.NoNaNsFPMath) 2450 return true; 2451 2452 // If the value is a constant, we can obviously see if it is a NaN or not. 2453 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2454 return !C->getValueAPF().isNaN(); 2455 2456 // TODO: Recognize more cases here. 2457 2458 return false; 2459} 2460 2461bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2462 // If the value is a constant, we can obviously see if it is a zero or not. 2463 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2464 return !C->isZero(); 2465 2466 // TODO: Recognize more cases here. 2467 switch (Op.getOpcode()) { 2468 default: break; 2469 case ISD::OR: 2470 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2471 return !C->isNullValue(); 2472 break; 2473 } 2474 2475 return false; 2476} 2477 2478bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2479 // Check the obvious case. 2480 if (A == B) return true; 2481 2482 // For for negative and positive zero. 2483 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2484 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2485 if (CA->isZero() && CB->isZero()) return true; 2486 2487 // Otherwise they may not be equal. 2488 return false; 2489} 2490 2491/// getNode - Gets or creates the specified node. 2492/// 2493SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) { 2494 FoldingSetNodeID ID; 2495 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2496 void *IP = 0; 2497 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2498 return SDValue(E, 0); 2499 2500 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), 2501 DL.getDebugLoc(), getVTList(VT)); 2502 CSEMap.InsertNode(N, IP); 2503 2504 AllNodes.push_back(N); 2505#ifndef NDEBUG 2506 VerifySDNode(N); 2507#endif 2508 return SDValue(N, 0); 2509} 2510 2511SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 2512 EVT VT, SDValue Operand) { 2513 // Constant fold unary operations with an integer constant operand. 2514 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2515 const APInt &Val = C->getAPIntValue(); 2516 switch (Opcode) { 2517 default: break; 2518 case ISD::SIGN_EXTEND: 2519 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2520 case ISD::ANY_EXTEND: 2521 case ISD::ZERO_EXTEND: 2522 case ISD::TRUNCATE: 2523 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2524 case ISD::UINT_TO_FP: 2525 case ISD::SINT_TO_FP: { 2526 APFloat apf(EVTToAPFloatSemantics(VT), 2527 APInt::getNullValue(VT.getSizeInBits())); 2528 (void)apf.convertFromAPInt(Val, 2529 Opcode==ISD::SINT_TO_FP, 2530 APFloat::rmNearestTiesToEven); 2531 return getConstantFP(apf, VT); 2532 } 2533 case ISD::BITCAST: 2534 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2535 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 2536 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2537 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 2538 break; 2539 case ISD::BSWAP: 2540 return getConstant(Val.byteSwap(), VT); 2541 case ISD::CTPOP: 2542 return getConstant(Val.countPopulation(), VT); 2543 case ISD::CTLZ: 2544 case ISD::CTLZ_ZERO_UNDEF: 2545 return getConstant(Val.countLeadingZeros(), VT); 2546 case ISD::CTTZ: 2547 case ISD::CTTZ_ZERO_UNDEF: 2548 return getConstant(Val.countTrailingZeros(), VT); 2549 } 2550 } 2551 2552 // Constant fold unary operations with a floating point constant operand. 2553 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2554 APFloat V = C->getValueAPF(); // make copy 2555 switch (Opcode) { 2556 case ISD::FNEG: 2557 V.changeSign(); 2558 return getConstantFP(V, VT); 2559 case ISD::FABS: 2560 V.clearSign(); 2561 return getConstantFP(V, VT); 2562 case ISD::FCEIL: { 2563 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 2564 if (fs == APFloat::opOK || fs == APFloat::opInexact) 2565 return getConstantFP(V, VT); 2566 break; 2567 } 2568 case ISD::FTRUNC: { 2569 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 2570 if (fs == APFloat::opOK || fs == APFloat::opInexact) 2571 return getConstantFP(V, VT); 2572 break; 2573 } 2574 case ISD::FFLOOR: { 2575 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 2576 if (fs == APFloat::opOK || fs == APFloat::opInexact) 2577 return getConstantFP(V, VT); 2578 break; 2579 } 2580 case ISD::FP_EXTEND: { 2581 bool ignored; 2582 // This can return overflow, underflow, or inexact; we don't care. 2583 // FIXME need to be more flexible about rounding mode. 2584 (void)V.convert(EVTToAPFloatSemantics(VT), 2585 APFloat::rmNearestTiesToEven, &ignored); 2586 return getConstantFP(V, VT); 2587 } 2588 case ISD::FP_TO_SINT: 2589 case ISD::FP_TO_UINT: { 2590 integerPart x[2]; 2591 bool ignored; 2592 assert(integerPartWidth >= 64); 2593 // FIXME need to be more flexible about rounding mode. 2594 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2595 Opcode==ISD::FP_TO_SINT, 2596 APFloat::rmTowardZero, &ignored); 2597 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2598 break; 2599 APInt api(VT.getSizeInBits(), x); 2600 return getConstant(api, VT); 2601 } 2602 case ISD::BITCAST: 2603 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2604 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2605 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2606 return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2607 break; 2608 } 2609 } 2610 2611 unsigned OpOpcode = Operand.getNode()->getOpcode(); 2612 switch (Opcode) { 2613 case ISD::TokenFactor: 2614 case ISD::MERGE_VALUES: 2615 case ISD::CONCAT_VECTORS: 2616 return Operand; // Factor, merge or concat of one node? No need. 2617 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2618 case ISD::FP_EXTEND: 2619 assert(VT.isFloatingPoint() && 2620 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2621 if (Operand.getValueType() == VT) return Operand; // noop conversion. 2622 assert((!VT.isVector() || 2623 VT.getVectorNumElements() == 2624 Operand.getValueType().getVectorNumElements()) && 2625 "Vector element count mismatch!"); 2626 if (Operand.getOpcode() == ISD::UNDEF) 2627 return getUNDEF(VT); 2628 break; 2629 case ISD::SIGN_EXTEND: 2630 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2631 "Invalid SIGN_EXTEND!"); 2632 if (Operand.getValueType() == VT) return Operand; // noop extension 2633 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2634 "Invalid sext node, dst < src!"); 2635 assert((!VT.isVector() || 2636 VT.getVectorNumElements() == 2637 Operand.getValueType().getVectorNumElements()) && 2638 "Vector element count mismatch!"); 2639 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2640 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2641 else if (OpOpcode == ISD::UNDEF) 2642 // sext(undef) = 0, because the top bits will all be the same. 2643 return getConstant(0, VT); 2644 break; 2645 case ISD::ZERO_EXTEND: 2646 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2647 "Invalid ZERO_EXTEND!"); 2648 if (Operand.getValueType() == VT) return Operand; // noop extension 2649 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2650 "Invalid zext node, dst < src!"); 2651 assert((!VT.isVector() || 2652 VT.getVectorNumElements() == 2653 Operand.getValueType().getVectorNumElements()) && 2654 "Vector element count mismatch!"); 2655 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2656 return getNode(ISD::ZERO_EXTEND, DL, VT, 2657 Operand.getNode()->getOperand(0)); 2658 else if (OpOpcode == ISD::UNDEF) 2659 // zext(undef) = 0, because the top bits will be zero. 2660 return getConstant(0, VT); 2661 break; 2662 case ISD::ANY_EXTEND: 2663 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2664 "Invalid ANY_EXTEND!"); 2665 if (Operand.getValueType() == VT) return Operand; // noop extension 2666 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2667 "Invalid anyext node, dst < src!"); 2668 assert((!VT.isVector() || 2669 VT.getVectorNumElements() == 2670 Operand.getValueType().getVectorNumElements()) && 2671 "Vector element count mismatch!"); 2672 2673 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2674 OpOpcode == ISD::ANY_EXTEND) 2675 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2676 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2677 else if (OpOpcode == ISD::UNDEF) 2678 return getUNDEF(VT); 2679 2680 // (ext (trunx x)) -> x 2681 if (OpOpcode == ISD::TRUNCATE) { 2682 SDValue OpOp = Operand.getNode()->getOperand(0); 2683 if (OpOp.getValueType() == VT) 2684 return OpOp; 2685 } 2686 break; 2687 case ISD::TRUNCATE: 2688 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2689 "Invalid TRUNCATE!"); 2690 if (Operand.getValueType() == VT) return Operand; // noop truncate 2691 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2692 "Invalid truncate node, src < dst!"); 2693 assert((!VT.isVector() || 2694 VT.getVectorNumElements() == 2695 Operand.getValueType().getVectorNumElements()) && 2696 "Vector element count mismatch!"); 2697 if (OpOpcode == ISD::TRUNCATE) 2698 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2699 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2700 OpOpcode == ISD::ANY_EXTEND) { 2701 // If the source is smaller than the dest, we still need an extend. 2702 if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2703 .bitsLT(VT.getScalarType())) 2704 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2705 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2706 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2707 return Operand.getNode()->getOperand(0); 2708 } 2709 if (OpOpcode == ISD::UNDEF) 2710 return getUNDEF(VT); 2711 break; 2712 case ISD::BITCAST: 2713 // Basic sanity checking. 2714 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2715 && "Cannot BITCAST between types of different sizes!"); 2716 if (VT == Operand.getValueType()) return Operand; // noop conversion. 2717 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2718 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2719 if (OpOpcode == ISD::UNDEF) 2720 return getUNDEF(VT); 2721 break; 2722 case ISD::SCALAR_TO_VECTOR: 2723 assert(VT.isVector() && !Operand.getValueType().isVector() && 2724 (VT.getVectorElementType() == Operand.getValueType() || 2725 (VT.getVectorElementType().isInteger() && 2726 Operand.getValueType().isInteger() && 2727 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2728 "Illegal SCALAR_TO_VECTOR node!"); 2729 if (OpOpcode == ISD::UNDEF) 2730 return getUNDEF(VT); 2731 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2732 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2733 isa<ConstantSDNode>(Operand.getOperand(1)) && 2734 Operand.getConstantOperandVal(1) == 0 && 2735 Operand.getOperand(0).getValueType() == VT) 2736 return Operand.getOperand(0); 2737 break; 2738 case ISD::FNEG: 2739 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2740 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2741 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2742 Operand.getNode()->getOperand(0)); 2743 if (OpOpcode == ISD::FNEG) // --X -> X 2744 return Operand.getNode()->getOperand(0); 2745 break; 2746 case ISD::FABS: 2747 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2748 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2749 break; 2750 } 2751 2752 SDNode *N; 2753 SDVTList VTs = getVTList(VT); 2754 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2755 FoldingSetNodeID ID; 2756 SDValue Ops[1] = { Operand }; 2757 AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2758 void *IP = 0; 2759 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2760 return SDValue(E, 0); 2761 2762 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2763 DL.getDebugLoc(), VTs, Operand); 2764 CSEMap.InsertNode(N, IP); 2765 } else { 2766 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2767 DL.getDebugLoc(), VTs, Operand); 2768 } 2769 2770 AllNodes.push_back(N); 2771#ifndef NDEBUG 2772 VerifySDNode(N); 2773#endif 2774 return SDValue(N, 0); 2775} 2776 2777SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 2778 SDNode *Cst1, SDNode *Cst2) { 2779 SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 2780 SmallVector<SDValue, 4> Outputs; 2781 EVT SVT = VT.getScalarType(); 2782 2783 ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 2784 ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 2785 if (Scalar1 && Scalar2) { 2786 // Scalar instruction. 2787 Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 2788 } else { 2789 // For vectors extract each constant element into Inputs so we can constant 2790 // fold them individually. 2791 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 2792 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 2793 if (!BV1 || !BV2) 2794 return SDValue(); 2795 2796 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 2797 2798 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 2799 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 2800 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 2801 if (!V1 || !V2) // Not a constant, bail. 2802 return SDValue(); 2803 2804 // Avoid BUILD_VECTOR nodes that perform implicit truncation. 2805 // FIXME: This is valid and could be handled by truncating the APInts. 2806 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 2807 return SDValue(); 2808 2809 Inputs.push_back(std::make_pair(V1, V2)); 2810 } 2811 } 2812 2813 // We have a number of constant values, constant fold them element by element. 2814 for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 2815 const APInt &C1 = Inputs[I].first->getAPIntValue(); 2816 const APInt &C2 = Inputs[I].second->getAPIntValue(); 2817 2818 switch (Opcode) { 2819 case ISD::ADD: 2820 Outputs.push_back(getConstant(C1 + C2, SVT)); 2821 break; 2822 case ISD::SUB: 2823 Outputs.push_back(getConstant(C1 - C2, SVT)); 2824 break; 2825 case ISD::MUL: 2826 Outputs.push_back(getConstant(C1 * C2, SVT)); 2827 break; 2828 case ISD::UDIV: 2829 if (!C2.getBoolValue()) 2830 return SDValue(); 2831 Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 2832 break; 2833 case ISD::UREM: 2834 if (!C2.getBoolValue()) 2835 return SDValue(); 2836 Outputs.push_back(getConstant(C1.urem(C2), SVT)); 2837 break; 2838 case ISD::SDIV: 2839 if (!C2.getBoolValue()) 2840 return SDValue(); 2841 Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 2842 break; 2843 case ISD::SREM: 2844 if (!C2.getBoolValue()) 2845 return SDValue(); 2846 Outputs.push_back(getConstant(C1.srem(C2), SVT)); 2847 break; 2848 case ISD::AND: 2849 Outputs.push_back(getConstant(C1 & C2, SVT)); 2850 break; 2851 case ISD::OR: 2852 Outputs.push_back(getConstant(C1 | C2, SVT)); 2853 break; 2854 case ISD::XOR: 2855 Outputs.push_back(getConstant(C1 ^ C2, SVT)); 2856 break; 2857 case ISD::SHL: 2858 Outputs.push_back(getConstant(C1 << C2, SVT)); 2859 break; 2860 case ISD::SRL: 2861 Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 2862 break; 2863 case ISD::SRA: 2864 Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 2865 break; 2866 case ISD::ROTL: 2867 Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 2868 break; 2869 case ISD::ROTR: 2870 Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 2871 break; 2872 default: 2873 return SDValue(); 2874 } 2875 } 2876 2877 // Handle the scalar case first. 2878 if (Scalar1 && Scalar2) 2879 return Outputs.back(); 2880 2881 // Otherwise build a big vector out of the scalar elements we generated. 2882 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(), 2883 Outputs.size()); 2884} 2885 2886SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, 2887 SDValue N2) { 2888 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2889 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2890 switch (Opcode) { 2891 default: break; 2892 case ISD::TokenFactor: 2893 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2894 N2.getValueType() == MVT::Other && "Invalid token factor!"); 2895 // Fold trivial token factors. 2896 if (N1.getOpcode() == ISD::EntryToken) return N2; 2897 if (N2.getOpcode() == ISD::EntryToken) return N1; 2898 if (N1 == N2) return N1; 2899 break; 2900 case ISD::CONCAT_VECTORS: 2901 // Concat of UNDEFs is UNDEF. 2902 if (N1.getOpcode() == ISD::UNDEF && 2903 N2.getOpcode() == ISD::UNDEF) 2904 return getUNDEF(VT); 2905 2906 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2907 // one big BUILD_VECTOR. 2908 if (N1.getOpcode() == ISD::BUILD_VECTOR && 2909 N2.getOpcode() == ISD::BUILD_VECTOR) { 2910 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2911 N1.getNode()->op_end()); 2912 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2913 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2914 } 2915 break; 2916 case ISD::AND: 2917 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2918 assert(N1.getValueType() == N2.getValueType() && 2919 N1.getValueType() == VT && "Binary operator types must match!"); 2920 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2921 // worth handling here. 2922 if (N2C && N2C->isNullValue()) 2923 return N2; 2924 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2925 return N1; 2926 break; 2927 case ISD::OR: 2928 case ISD::XOR: 2929 case ISD::ADD: 2930 case ISD::SUB: 2931 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2932 assert(N1.getValueType() == N2.getValueType() && 2933 N1.getValueType() == VT && "Binary operator types must match!"); 2934 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2935 // it's worth handling here. 2936 if (N2C && N2C->isNullValue()) 2937 return N1; 2938 break; 2939 case ISD::UDIV: 2940 case ISD::UREM: 2941 case ISD::MULHU: 2942 case ISD::MULHS: 2943 case ISD::MUL: 2944 case ISD::SDIV: 2945 case ISD::SREM: 2946 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2947 assert(N1.getValueType() == N2.getValueType() && 2948 N1.getValueType() == VT && "Binary operator types must match!"); 2949 break; 2950 case ISD::FADD: 2951 case ISD::FSUB: 2952 case ISD::FMUL: 2953 case ISD::FDIV: 2954 case ISD::FREM: 2955 if (getTarget().Options.UnsafeFPMath) { 2956 if (Opcode == ISD::FADD) { 2957 // 0+x --> x 2958 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2959 if (CFP->getValueAPF().isZero()) 2960 return N2; 2961 // x+0 --> x 2962 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2963 if (CFP->getValueAPF().isZero()) 2964 return N1; 2965 } else if (Opcode == ISD::FSUB) { 2966 // x-0 --> x 2967 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2968 if (CFP->getValueAPF().isZero()) 2969 return N1; 2970 } else if (Opcode == ISD::FMUL) { 2971 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 2972 SDValue V = N2; 2973 2974 // If the first operand isn't the constant, try the second 2975 if (!CFP) { 2976 CFP = dyn_cast<ConstantFPSDNode>(N2); 2977 V = N1; 2978 } 2979 2980 if (CFP) { 2981 // 0*x --> 0 2982 if (CFP->isZero()) 2983 return SDValue(CFP,0); 2984 // 1*x --> x 2985 if (CFP->isExactlyValue(1.0)) 2986 return V; 2987 } 2988 } 2989 } 2990 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2991 assert(N1.getValueType() == N2.getValueType() && 2992 N1.getValueType() == VT && "Binary operator types must match!"); 2993 break; 2994 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2995 assert(N1.getValueType() == VT && 2996 N1.getValueType().isFloatingPoint() && 2997 N2.getValueType().isFloatingPoint() && 2998 "Invalid FCOPYSIGN!"); 2999 break; 3000 case ISD::SHL: 3001 case ISD::SRA: 3002 case ISD::SRL: 3003 case ISD::ROTL: 3004 case ISD::ROTR: 3005 assert(VT == N1.getValueType() && 3006 "Shift operators return type must be the same as their first arg"); 3007 assert(VT.isInteger() && N2.getValueType().isInteger() && 3008 "Shifts only work on integers"); 3009 assert((!VT.isVector() || VT == N2.getValueType()) && 3010 "Vector shift amounts must be in the same as their first arg"); 3011 // Verify that the shift amount VT is bit enough to hold valid shift 3012 // amounts. This catches things like trying to shift an i1024 value by an 3013 // i8, which is easy to fall into in generic code that uses 3014 // TLI.getShiftAmount(). 3015 assert(N2.getValueType().getSizeInBits() >= 3016 Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 3017 "Invalid use of small shift amount with oversized value!"); 3018 3019 // Always fold shifts of i1 values so the code generator doesn't need to 3020 // handle them. Since we know the size of the shift has to be less than the 3021 // size of the value, the shift/rotate count is guaranteed to be zero. 3022 if (VT == MVT::i1) 3023 return N1; 3024 if (N2C && N2C->isNullValue()) 3025 return N1; 3026 break; 3027 case ISD::FP_ROUND_INREG: { 3028 EVT EVT = cast<VTSDNode>(N2)->getVT(); 3029 assert(VT == N1.getValueType() && "Not an inreg round!"); 3030 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 3031 "Cannot FP_ROUND_INREG integer types"); 3032 assert(EVT.isVector() == VT.isVector() && 3033 "FP_ROUND_INREG type should be vector iff the operand " 3034 "type is vector!"); 3035 assert((!EVT.isVector() || 3036 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3037 "Vector element counts must match in FP_ROUND_INREG"); 3038 assert(EVT.bitsLE(VT) && "Not rounding down!"); 3039 (void)EVT; 3040 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 3041 break; 3042 } 3043 case ISD::FP_ROUND: 3044 assert(VT.isFloatingPoint() && 3045 N1.getValueType().isFloatingPoint() && 3046 VT.bitsLE(N1.getValueType()) && 3047 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 3048 if (N1.getValueType() == VT) return N1; // noop conversion. 3049 break; 3050 case ISD::AssertSext: 3051 case ISD::AssertZext: { 3052 EVT EVT = cast<VTSDNode>(N2)->getVT(); 3053 assert(VT == N1.getValueType() && "Not an inreg extend!"); 3054 assert(VT.isInteger() && EVT.isInteger() && 3055 "Cannot *_EXTEND_INREG FP types"); 3056 assert(!EVT.isVector() && 3057 "AssertSExt/AssertZExt type should be the vector element type " 3058 "rather than the vector type!"); 3059 assert(EVT.bitsLE(VT) && "Not extending!"); 3060 if (VT == EVT) return N1; // noop assertion. 3061 break; 3062 } 3063 case ISD::SIGN_EXTEND_INREG: { 3064 EVT EVT = cast<VTSDNode>(N2)->getVT(); 3065 assert(VT == N1.getValueType() && "Not an inreg extend!"); 3066 assert(VT.isInteger() && EVT.isInteger() && 3067 "Cannot *_EXTEND_INREG FP types"); 3068 assert(EVT.isVector() == VT.isVector() && 3069 "SIGN_EXTEND_INREG type should be vector iff the operand " 3070 "type is vector!"); 3071 assert((!EVT.isVector() || 3072 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3073 "Vector element counts must match in SIGN_EXTEND_INREG"); 3074 assert(EVT.bitsLE(VT) && "Not extending!"); 3075 if (EVT == VT) return N1; // Not actually extending 3076 3077 if (N1C) { 3078 APInt Val = N1C->getAPIntValue(); 3079 unsigned FromBits = EVT.getScalarType().getSizeInBits(); 3080 Val <<= Val.getBitWidth()-FromBits; 3081 Val = Val.ashr(Val.getBitWidth()-FromBits); 3082 return getConstant(Val, VT); 3083 } 3084 break; 3085 } 3086 case ISD::EXTRACT_VECTOR_ELT: 3087 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 3088 if (N1.getOpcode() == ISD::UNDEF) 3089 return getUNDEF(VT); 3090 3091 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 3092 // expanding copies of large vectors from registers. 3093 if (N2C && 3094 N1.getOpcode() == ISD::CONCAT_VECTORS && 3095 N1.getNumOperands() > 0) { 3096 unsigned Factor = 3097 N1.getOperand(0).getValueType().getVectorNumElements(); 3098 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 3099 N1.getOperand(N2C->getZExtValue() / Factor), 3100 getConstant(N2C->getZExtValue() % Factor, 3101 N2.getValueType())); 3102 } 3103 3104 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 3105 // expanding large vector constants. 3106 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 3107 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 3108 3109 if (VT != Elt.getValueType()) 3110 // If the vector element type is not legal, the BUILD_VECTOR operands 3111 // are promoted and implicitly truncated, and the result implicitly 3112 // extended. Make that explicit here. 3113 Elt = getAnyExtOrTrunc(Elt, DL, VT); 3114 3115 return Elt; 3116 } 3117 3118 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 3119 // operations are lowered to scalars. 3120 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 3121 // If the indices are the same, return the inserted element else 3122 // if the indices are known different, extract the element from 3123 // the original vector. 3124 SDValue N1Op2 = N1.getOperand(2); 3125 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 3126 3127 if (N1Op2C && N2C) { 3128 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 3129 if (VT == N1.getOperand(1).getValueType()) 3130 return N1.getOperand(1); 3131 else 3132 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 3133 } 3134 3135 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 3136 } 3137 } 3138 break; 3139 case ISD::EXTRACT_ELEMENT: 3140 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 3141 assert(!N1.getValueType().isVector() && !VT.isVector() && 3142 (N1.getValueType().isInteger() == VT.isInteger()) && 3143 N1.getValueType() != VT && 3144 "Wrong types for EXTRACT_ELEMENT!"); 3145 3146 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 3147 // 64-bit integers into 32-bit parts. Instead of building the extract of 3148 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 3149 if (N1.getOpcode() == ISD::BUILD_PAIR) 3150 return N1.getOperand(N2C->getZExtValue()); 3151 3152 // EXTRACT_ELEMENT of a constant int is also very common. 3153 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 3154 unsigned ElementSize = VT.getSizeInBits(); 3155 unsigned Shift = ElementSize * N2C->getZExtValue(); 3156 APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 3157 return getConstant(ShiftedVal.trunc(ElementSize), VT); 3158 } 3159 break; 3160 case ISD::EXTRACT_SUBVECTOR: { 3161 SDValue Index = N2; 3162 if (VT.isSimple() && N1.getValueType().isSimple()) { 3163 assert(VT.isVector() && N1.getValueType().isVector() && 3164 "Extract subvector VTs must be a vectors!"); 3165 assert(VT.getVectorElementType() == 3166 N1.getValueType().getVectorElementType() && 3167 "Extract subvector VTs must have the same element type!"); 3168 assert(VT.getSimpleVT() <= N1.getSimpleValueType() && 3169 "Extract subvector must be from larger vector to smaller vector!"); 3170 3171 if (isa<ConstantSDNode>(Index.getNode())) { 3172 assert((VT.getVectorNumElements() + 3173 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3174 <= N1.getValueType().getVectorNumElements()) 3175 && "Extract subvector overflow!"); 3176 } 3177 3178 // Trivial extraction. 3179 if (VT.getSimpleVT() == N1.getSimpleValueType()) 3180 return N1; 3181 } 3182 break; 3183 } 3184 } 3185 3186 // Perform trivial constant folding. 3187 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 3188 if (SV.getNode()) return SV; 3189 3190 // Canonicalize constant to RHS if commutative. 3191 if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 3192 std::swap(N1C, N2C); 3193 std::swap(N1, N2); 3194 } 3195 3196 // Constant fold FP operations. 3197 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 3198 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 3199 if (N1CFP) { 3200 if (!N2CFP && isCommutativeBinOp(Opcode)) { 3201 // Canonicalize constant to RHS if commutative. 3202 std::swap(N1CFP, N2CFP); 3203 std::swap(N1, N2); 3204 } else if (N2CFP) { 3205 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3206 APFloat::opStatus s; 3207 switch (Opcode) { 3208 case ISD::FADD: 3209 s = V1.add(V2, APFloat::rmNearestTiesToEven); 3210 if (s != APFloat::opInvalidOp) 3211 return getConstantFP(V1, VT); 3212 break; 3213 case ISD::FSUB: 3214 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3215 if (s!=APFloat::opInvalidOp) 3216 return getConstantFP(V1, VT); 3217 break; 3218 case ISD::FMUL: 3219 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3220 if (s!=APFloat::opInvalidOp) 3221 return getConstantFP(V1, VT); 3222 break; 3223 case ISD::FDIV: 3224 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3225 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3226 return getConstantFP(V1, VT); 3227 break; 3228 case ISD::FREM : 3229 s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3230 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3231 return getConstantFP(V1, VT); 3232 break; 3233 case ISD::FCOPYSIGN: 3234 V1.copySign(V2); 3235 return getConstantFP(V1, VT); 3236 default: break; 3237 } 3238 } 3239 3240 if (Opcode == ISD::FP_ROUND) { 3241 APFloat V = N1CFP->getValueAPF(); // make copy 3242 bool ignored; 3243 // This can return overflow, underflow, or inexact; we don't care. 3244 // FIXME need to be more flexible about rounding mode. 3245 (void)V.convert(EVTToAPFloatSemantics(VT), 3246 APFloat::rmNearestTiesToEven, &ignored); 3247 return getConstantFP(V, VT); 3248 } 3249 } 3250 3251 // Canonicalize an UNDEF to the RHS, even over a constant. 3252 if (N1.getOpcode() == ISD::UNDEF) { 3253 if (isCommutativeBinOp(Opcode)) { 3254 std::swap(N1, N2); 3255 } else { 3256 switch (Opcode) { 3257 case ISD::FP_ROUND_INREG: 3258 case ISD::SIGN_EXTEND_INREG: 3259 case ISD::SUB: 3260 case ISD::FSUB: 3261 case ISD::FDIV: 3262 case ISD::FREM: 3263 case ISD::SRA: 3264 return N1; // fold op(undef, arg2) -> undef 3265 case ISD::UDIV: 3266 case ISD::SDIV: 3267 case ISD::UREM: 3268 case ISD::SREM: 3269 case ISD::SRL: 3270 case ISD::SHL: 3271 if (!VT.isVector()) 3272 return getConstant(0, VT); // fold op(undef, arg2) -> 0 3273 // For vectors, we can't easily build an all zero vector, just return 3274 // the LHS. 3275 return N2; 3276 } 3277 } 3278 } 3279 3280 // Fold a bunch of operators when the RHS is undef. 3281 if (N2.getOpcode() == ISD::UNDEF) { 3282 switch (Opcode) { 3283 case ISD::XOR: 3284 if (N1.getOpcode() == ISD::UNDEF) 3285 // Handle undef ^ undef -> 0 special case. This is a common 3286 // idiom (misuse). 3287 return getConstant(0, VT); 3288 // fallthrough 3289 case ISD::ADD: 3290 case ISD::ADDC: 3291 case ISD::ADDE: 3292 case ISD::SUB: 3293 case ISD::UDIV: 3294 case ISD::SDIV: 3295 case ISD::UREM: 3296 case ISD::SREM: 3297 return N2; // fold op(arg1, undef) -> undef 3298 case ISD::FADD: 3299 case ISD::FSUB: 3300 case ISD::FMUL: 3301 case ISD::FDIV: 3302 case ISD::FREM: 3303 if (getTarget().Options.UnsafeFPMath) 3304 return N2; 3305 break; 3306 case ISD::MUL: 3307 case ISD::AND: 3308 case ISD::SRL: 3309 case ISD::SHL: 3310 if (!VT.isVector()) 3311 return getConstant(0, VT); // fold op(arg1, undef) -> 0 3312 // For vectors, we can't easily build an all zero vector, just return 3313 // the LHS. 3314 return N1; 3315 case ISD::OR: 3316 if (!VT.isVector()) 3317 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3318 // For vectors, we can't easily build an all one vector, just return 3319 // the LHS. 3320 return N1; 3321 case ISD::SRA: 3322 return N1; 3323 } 3324 } 3325 3326 // Memoize this node if possible. 3327 SDNode *N; 3328 SDVTList VTs = getVTList(VT); 3329 if (VT != MVT::Glue) { 3330 SDValue Ops[] = { N1, N2 }; 3331 FoldingSetNodeID ID; 3332 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3333 void *IP = 0; 3334 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3335 return SDValue(E, 0); 3336 3337 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3338 DL.getDebugLoc(), VTs, N1, N2); 3339 CSEMap.InsertNode(N, IP); 3340 } else { 3341 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3342 DL.getDebugLoc(), VTs, N1, N2); 3343 } 3344 3345 AllNodes.push_back(N); 3346#ifndef NDEBUG 3347 VerifySDNode(N); 3348#endif 3349 return SDValue(N, 0); 3350} 3351 3352SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3353 SDValue N1, SDValue N2, SDValue N3) { 3354 // Perform various simplifications. 3355 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3356 switch (Opcode) { 3357 case ISD::FMA: { 3358 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 3359 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 3360 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 3361 if (N1CFP && N2CFP && N3CFP) { 3362 APFloat V1 = N1CFP->getValueAPF(); 3363 const APFloat &V2 = N2CFP->getValueAPF(); 3364 const APFloat &V3 = N3CFP->getValueAPF(); 3365 APFloat::opStatus s = 3366 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 3367 if (s != APFloat::opInvalidOp) 3368 return getConstantFP(V1, VT); 3369 } 3370 break; 3371 } 3372 case ISD::CONCAT_VECTORS: 3373 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3374 // one big BUILD_VECTOR. 3375 if (N1.getOpcode() == ISD::BUILD_VECTOR && 3376 N2.getOpcode() == ISD::BUILD_VECTOR && 3377 N3.getOpcode() == ISD::BUILD_VECTOR) { 3378 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3379 N1.getNode()->op_end()); 3380 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3381 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3382 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3383 } 3384 break; 3385 case ISD::SETCC: { 3386 // Use FoldSetCC to simplify SETCC's. 3387 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3388 if (Simp.getNode()) return Simp; 3389 break; 3390 } 3391 case ISD::SELECT: 3392 if (N1C) { 3393 if (N1C->getZExtValue()) 3394 return N2; // select true, X, Y -> X 3395 return N3; // select false, X, Y -> Y 3396 } 3397 3398 if (N2 == N3) return N2; // select C, X, X -> X 3399 break; 3400 case ISD::VECTOR_SHUFFLE: 3401 llvm_unreachable("should use getVectorShuffle constructor!"); 3402 case ISD::INSERT_SUBVECTOR: { 3403 SDValue Index = N3; 3404 if (VT.isSimple() && N1.getValueType().isSimple() 3405 && N2.getValueType().isSimple()) { 3406 assert(VT.isVector() && N1.getValueType().isVector() && 3407 N2.getValueType().isVector() && 3408 "Insert subvector VTs must be a vectors"); 3409 assert(VT == N1.getValueType() && 3410 "Dest and insert subvector source types must match!"); 3411 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && 3412 "Insert subvector must be from smaller vector to larger vector!"); 3413 if (isa<ConstantSDNode>(Index.getNode())) { 3414 assert((N2.getValueType().getVectorNumElements() + 3415 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3416 <= VT.getVectorNumElements()) 3417 && "Insert subvector overflow!"); 3418 } 3419 3420 // Trivial insertion. 3421 if (VT.getSimpleVT() == N2.getSimpleValueType()) 3422 return N2; 3423 } 3424 break; 3425 } 3426 case ISD::BITCAST: 3427 // Fold bit_convert nodes from a type to themselves. 3428 if (N1.getValueType() == VT) 3429 return N1; 3430 break; 3431 } 3432 3433 // Memoize node if it doesn't produce a flag. 3434 SDNode *N; 3435 SDVTList VTs = getVTList(VT); 3436 if (VT != MVT::Glue) { 3437 SDValue Ops[] = { N1, N2, N3 }; 3438 FoldingSetNodeID ID; 3439 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3440 void *IP = 0; 3441 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3442 return SDValue(E, 0); 3443 3444 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3445 DL.getDebugLoc(), VTs, N1, N2, N3); 3446 CSEMap.InsertNode(N, IP); 3447 } else { 3448 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3449 DL.getDebugLoc(), VTs, N1, N2, N3); 3450 } 3451 3452 AllNodes.push_back(N); 3453#ifndef NDEBUG 3454 VerifySDNode(N); 3455#endif 3456 return SDValue(N, 0); 3457} 3458 3459SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3460 SDValue N1, SDValue N2, SDValue N3, 3461 SDValue N4) { 3462 SDValue Ops[] = { N1, N2, N3, N4 }; 3463 return getNode(Opcode, DL, VT, Ops, 4); 3464} 3465 3466SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3467 SDValue N1, SDValue N2, SDValue N3, 3468 SDValue N4, SDValue N5) { 3469 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3470 return getNode(Opcode, DL, VT, Ops, 5); 3471} 3472 3473/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3474/// the incoming stack arguments to be loaded from the stack. 3475SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3476 SmallVector<SDValue, 8> ArgChains; 3477 3478 // Include the original chain at the beginning of the list. When this is 3479 // used by target LowerCall hooks, this helps legalize find the 3480 // CALLSEQ_BEGIN node. 3481 ArgChains.push_back(Chain); 3482 3483 // Add a chain value for each stack argument. 3484 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3485 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3486 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3487 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3488 if (FI->getIndex() < 0) 3489 ArgChains.push_back(SDValue(L, 1)); 3490 3491 // Build a tokenfactor for all the chains. 3492 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, 3493 &ArgChains[0], ArgChains.size()); 3494} 3495 3496/// getMemsetValue - Vectorized representation of the memset value 3497/// operand. 3498static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3499 SDLoc dl) { 3500 assert(Value.getOpcode() != ISD::UNDEF); 3501 3502 unsigned NumBits = VT.getScalarType().getSizeInBits(); 3503 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3504 assert(C->getAPIntValue().getBitWidth() == 8); 3505 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 3506 if (VT.isInteger()) 3507 return DAG.getConstant(Val, VT); 3508 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 3509 } 3510 3511 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3512 if (NumBits > 8) { 3513 // Use a multiplication with 0x010101... to extend the input to the 3514 // required length. 3515 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 3516 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3517 } 3518 3519 return Value; 3520} 3521 3522/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3523/// used when a memcpy is turned into a memset when the source is a constant 3524/// string ptr. 3525static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, 3526 const TargetLowering &TLI, StringRef Str) { 3527 // Handle vector with all elements zero. 3528 if (Str.empty()) { 3529 if (VT.isInteger()) 3530 return DAG.getConstant(0, VT); 3531 else if (VT == MVT::f32 || VT == MVT::f64) 3532 return DAG.getConstantFP(0.0, VT); 3533 else if (VT.isVector()) { 3534 unsigned NumElts = VT.getVectorNumElements(); 3535 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3536 return DAG.getNode(ISD::BITCAST, dl, VT, 3537 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3538 EltVT, NumElts))); 3539 } else 3540 llvm_unreachable("Expected type!"); 3541 } 3542 3543 assert(!VT.isVector() && "Can't handle vector type here!"); 3544 unsigned NumVTBits = VT.getSizeInBits(); 3545 unsigned NumVTBytes = NumVTBits / 8; 3546 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 3547 3548 APInt Val(NumVTBits, 0); 3549 if (TLI.isLittleEndian()) { 3550 for (unsigned i = 0; i != NumBytes; ++i) 3551 Val |= (uint64_t)(unsigned char)Str[i] << i*8; 3552 } else { 3553 for (unsigned i = 0; i != NumBytes; ++i) 3554 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 3555 } 3556 3557 // If the "cost" of materializing the integer immediate is 1 or free, then 3558 // it is cost effective to turn the load into the immediate. 3559 const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 3560 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 3561 return DAG.getConstant(Val, VT); 3562 return SDValue(0, 0); 3563} 3564 3565/// getMemBasePlusOffset - Returns base and offset node for the 3566/// 3567static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, 3568 SelectionDAG &DAG) { 3569 EVT VT = Base.getValueType(); 3570 return DAG.getNode(ISD::ADD, dl, 3571 VT, Base, DAG.getConstant(Offset, VT)); 3572} 3573 3574/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3575/// 3576static bool isMemSrcFromString(SDValue Src, StringRef &Str) { 3577 unsigned SrcDelta = 0; 3578 GlobalAddressSDNode *G = NULL; 3579 if (Src.getOpcode() == ISD::GlobalAddress) 3580 G = cast<GlobalAddressSDNode>(Src); 3581 else if (Src.getOpcode() == ISD::ADD && 3582 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3583 Src.getOperand(1).getOpcode() == ISD::Constant) { 3584 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3585 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3586 } 3587 if (!G) 3588 return false; 3589 3590 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 3591} 3592 3593/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3594/// to replace the memset / memcpy. Return true if the number of memory ops 3595/// is below the threshold. It returns the types of the sequence of 3596/// memory ops to perform memset / memcpy by reference. 3597static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3598 unsigned Limit, uint64_t Size, 3599 unsigned DstAlign, unsigned SrcAlign, 3600 bool IsMemset, 3601 bool ZeroMemset, 3602 bool MemcpyStrSrc, 3603 bool AllowOverlap, 3604 SelectionDAG &DAG, 3605 const TargetLowering &TLI) { 3606 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3607 "Expecting memcpy / memset source to meet alignment requirement!"); 3608 // If 'SrcAlign' is zero, that means the memory operation does not need to 3609 // load the value, i.e. memset or memcpy from constant string. Otherwise, 3610 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3611 // is the specified alignment of the memory operation. If it is zero, that 3612 // means it's possible to change the alignment of the destination. 3613 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3614 // not need to be loaded. 3615 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3616 IsMemset, ZeroMemset, MemcpyStrSrc, 3617 DAG.getMachineFunction()); 3618 3619 if (VT == MVT::Other) { 3620 if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 3621 TLI.allowsUnalignedMemoryAccesses(VT)) { 3622 VT = TLI.getPointerTy(); 3623 } else { 3624 switch (DstAlign & 7) { 3625 case 0: VT = MVT::i64; break; 3626 case 4: VT = MVT::i32; break; 3627 case 2: VT = MVT::i16; break; 3628 default: VT = MVT::i8; break; 3629 } 3630 } 3631 3632 MVT LVT = MVT::i64; 3633 while (!TLI.isTypeLegal(LVT)) 3634 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3635 assert(LVT.isInteger()); 3636 3637 if (VT.bitsGT(LVT)) 3638 VT = LVT; 3639 } 3640 3641 unsigned NumMemOps = 0; 3642 while (Size != 0) { 3643 unsigned VTSize = VT.getSizeInBits() / 8; 3644 while (VTSize > Size) { 3645 // For now, only use non-vector load / store's for the left-over pieces. 3646 EVT NewVT = VT; 3647 unsigned NewVTSize; 3648 3649 bool Found = false; 3650 if (VT.isVector() || VT.isFloatingPoint()) { 3651 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 3652 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 3653 TLI.isSafeMemOpType(NewVT.getSimpleVT())) 3654 Found = true; 3655 else if (NewVT == MVT::i64 && 3656 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 3657 TLI.isSafeMemOpType(MVT::f64)) { 3658 // i64 is usually not legal on 32-bit targets, but f64 may be. 3659 NewVT = MVT::f64; 3660 Found = true; 3661 } 3662 } 3663 3664 if (!Found) { 3665 do { 3666 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 3667 if (NewVT == MVT::i8) 3668 break; 3669 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 3670 } 3671 NewVTSize = NewVT.getSizeInBits() / 8; 3672 3673 // If the new VT cannot cover all of the remaining bits, then consider 3674 // issuing a (or a pair of) unaligned and overlapping load / store. 3675 // FIXME: Only does this for 64-bit or more since we don't have proper 3676 // cost model for unaligned load / store. 3677 bool Fast; 3678 if (NumMemOps && AllowOverlap && 3679 VTSize >= 8 && NewVTSize < Size && 3680 TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 3681 VTSize = Size; 3682 else { 3683 VT = NewVT; 3684 VTSize = NewVTSize; 3685 } 3686 } 3687 3688 if (++NumMemOps > Limit) 3689 return false; 3690 3691 MemOps.push_back(VT); 3692 Size -= VTSize; 3693 } 3694 3695 return true; 3696} 3697 3698static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3699 SDValue Chain, SDValue Dst, 3700 SDValue Src, uint64_t Size, 3701 unsigned Align, bool isVol, 3702 bool AlwaysInline, 3703 MachinePointerInfo DstPtrInfo, 3704 MachinePointerInfo SrcPtrInfo) { 3705 // Turn a memcpy of undef to nop. 3706 if (Src.getOpcode() == ISD::UNDEF) 3707 return Chain; 3708 3709 // Expand memcpy to a series of load and store ops if the size operand falls 3710 // below a certain threshold. 3711 // TODO: In the AlwaysInline case, if the size is big then generate a loop 3712 // rather than maybe a humongous number of loads and stores. 3713 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3714 std::vector<EVT> MemOps; 3715 bool DstAlignCanChange = false; 3716 MachineFunction &MF = DAG.getMachineFunction(); 3717 MachineFrameInfo *MFI = MF.getFrameInfo(); 3718 bool OptSize = 3719 MF.getFunction()->getAttributes(). 3720 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3721 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3722 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3723 DstAlignCanChange = true; 3724 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3725 if (Align > SrcAlign) 3726 SrcAlign = Align; 3727 StringRef Str; 3728 bool CopyFromStr = isMemSrcFromString(Src, Str); 3729 bool isZeroStr = CopyFromStr && Str.empty(); 3730 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3731 3732 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3733 (DstAlignCanChange ? 0 : Align), 3734 (isZeroStr ? 0 : SrcAlign), 3735 false, false, CopyFromStr, true, DAG, TLI)) 3736 return SDValue(); 3737 3738 if (DstAlignCanChange) { 3739 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3740 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3741 3742 // Don't promote to an alignment that would require dynamic stack 3743 // realignment. 3744 const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 3745 if (!TRI->needsStackRealignment(MF)) 3746 while (NewAlign > Align && 3747 TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 3748 NewAlign /= 2; 3749 3750 if (NewAlign > Align) { 3751 // Give the stack frame object a larger alignment if needed. 3752 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3753 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3754 Align = NewAlign; 3755 } 3756 } 3757 3758 SmallVector<SDValue, 8> OutChains; 3759 unsigned NumMemOps = MemOps.size(); 3760 uint64_t SrcOff = 0, DstOff = 0; 3761 for (unsigned i = 0; i != NumMemOps; ++i) { 3762 EVT VT = MemOps[i]; 3763 unsigned VTSize = VT.getSizeInBits() / 8; 3764 SDValue Value, Store; 3765 3766 if (VTSize > Size) { 3767 // Issuing an unaligned load / store pair that overlaps with the previous 3768 // pair. Adjust the offset accordingly. 3769 assert(i == NumMemOps-1 && i != 0); 3770 SrcOff -= VTSize - Size; 3771 DstOff -= VTSize - Size; 3772 } 3773 3774 if (CopyFromStr && 3775 (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3776 // It's unlikely a store of a vector immediate can be done in a single 3777 // instruction. It would require a load from a constantpool first. 3778 // We only handle zero vectors here. 3779 // FIXME: Handle other cases where store of vector immediate is done in 3780 // a single instruction. 3781 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 3782 if (Value.getNode()) 3783 Store = DAG.getStore(Chain, dl, Value, 3784 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3785 DstPtrInfo.getWithOffset(DstOff), isVol, 3786 false, Align); 3787 } 3788 3789 if (!Store.getNode()) { 3790 // The type might not be legal for the target. This should only happen 3791 // if the type is smaller than a legal type, as on PPC, so the right 3792 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3793 // to Load/Store if NVT==VT. 3794 // FIXME does the case above also need this? 3795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3796 assert(NVT.bitsGE(VT)); 3797 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3798 getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3799 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3800 MinAlign(SrcAlign, SrcOff)); 3801 Store = DAG.getTruncStore(Chain, dl, Value, 3802 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3803 DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3804 false, Align); 3805 } 3806 OutChains.push_back(Store); 3807 SrcOff += VTSize; 3808 DstOff += VTSize; 3809 Size -= VTSize; 3810 } 3811 3812 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3813 &OutChains[0], OutChains.size()); 3814} 3815 3816static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3817 SDValue Chain, SDValue Dst, 3818 SDValue Src, uint64_t Size, 3819 unsigned Align, bool isVol, 3820 bool AlwaysInline, 3821 MachinePointerInfo DstPtrInfo, 3822 MachinePointerInfo SrcPtrInfo) { 3823 // Turn a memmove of undef to nop. 3824 if (Src.getOpcode() == ISD::UNDEF) 3825 return Chain; 3826 3827 // Expand memmove to a series of load and store ops if the size operand falls 3828 // below a certain threshold. 3829 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3830 std::vector<EVT> MemOps; 3831 bool DstAlignCanChange = false; 3832 MachineFunction &MF = DAG.getMachineFunction(); 3833 MachineFrameInfo *MFI = MF.getFrameInfo(); 3834 bool OptSize = MF.getFunction()->getAttributes(). 3835 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3836 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3837 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3838 DstAlignCanChange = true; 3839 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3840 if (Align > SrcAlign) 3841 SrcAlign = Align; 3842 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3843 3844 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3845 (DstAlignCanChange ? 0 : Align), SrcAlign, 3846 false, false, false, false, DAG, TLI)) 3847 return SDValue(); 3848 3849 if (DstAlignCanChange) { 3850 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3851 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3852 if (NewAlign > Align) { 3853 // Give the stack frame object a larger alignment if needed. 3854 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3855 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3856 Align = NewAlign; 3857 } 3858 } 3859 3860 uint64_t SrcOff = 0, DstOff = 0; 3861 SmallVector<SDValue, 8> LoadValues; 3862 SmallVector<SDValue, 8> LoadChains; 3863 SmallVector<SDValue, 8> OutChains; 3864 unsigned NumMemOps = MemOps.size(); 3865 for (unsigned i = 0; i < NumMemOps; i++) { 3866 EVT VT = MemOps[i]; 3867 unsigned VTSize = VT.getSizeInBits() / 8; 3868 SDValue Value; 3869 3870 Value = DAG.getLoad(VT, dl, Chain, 3871 getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3872 SrcPtrInfo.getWithOffset(SrcOff), isVol, 3873 false, false, SrcAlign); 3874 LoadValues.push_back(Value); 3875 LoadChains.push_back(Value.getValue(1)); 3876 SrcOff += VTSize; 3877 } 3878 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3879 &LoadChains[0], LoadChains.size()); 3880 OutChains.clear(); 3881 for (unsigned i = 0; i < NumMemOps; i++) { 3882 EVT VT = MemOps[i]; 3883 unsigned VTSize = VT.getSizeInBits() / 8; 3884 SDValue Store; 3885 3886 Store = DAG.getStore(Chain, dl, LoadValues[i], 3887 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3888 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3889 OutChains.push_back(Store); 3890 DstOff += VTSize; 3891 } 3892 3893 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3894 &OutChains[0], OutChains.size()); 3895} 3896 3897/// \brief Lower the call to 'memset' intrinsic function into a series of store 3898/// operations. 3899/// 3900/// \param DAG Selection DAG where lowered code is placed. 3901/// \param dl Link to corresponding IR location. 3902/// \param Chain Control flow dependency. 3903/// \param Dst Pointer to destination memory location. 3904/// \param Src Value of byte to write into the memory. 3905/// \param Size Number of bytes to write. 3906/// \param Align Alignment of the destination in bytes. 3907/// \param isVol True if destination is volatile. 3908/// \param DstPtrInfo IR information on the memory pointer. 3909/// \returns New head in the control flow, if lowering was successful, empty 3910/// SDValue otherwise. 3911/// 3912/// The function tries to replace 'llvm.memset' intrinsic with several store 3913/// operations and value calculation code. This is usually profitable for small 3914/// memory size. 3915static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, 3916 SDValue Chain, SDValue Dst, 3917 SDValue Src, uint64_t Size, 3918 unsigned Align, bool isVol, 3919 MachinePointerInfo DstPtrInfo) { 3920 // Turn a memset of undef to nop. 3921 if (Src.getOpcode() == ISD::UNDEF) 3922 return Chain; 3923 3924 // Expand memset to a series of load/store ops if the size operand 3925 // falls below a certain threshold. 3926 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3927 std::vector<EVT> MemOps; 3928 bool DstAlignCanChange = false; 3929 MachineFunction &MF = DAG.getMachineFunction(); 3930 MachineFrameInfo *MFI = MF.getFrameInfo(); 3931 bool OptSize = MF.getFunction()->getAttributes(). 3932 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3933 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3934 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3935 DstAlignCanChange = true; 3936 bool IsZeroVal = 3937 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3938 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3939 Size, (DstAlignCanChange ? 0 : Align), 0, 3940 true, IsZeroVal, false, true, DAG, TLI)) 3941 return SDValue(); 3942 3943 if (DstAlignCanChange) { 3944 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3945 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3946 if (NewAlign > Align) { 3947 // Give the stack frame object a larger alignment if needed. 3948 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3949 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3950 Align = NewAlign; 3951 } 3952 } 3953 3954 SmallVector<SDValue, 8> OutChains; 3955 uint64_t DstOff = 0; 3956 unsigned NumMemOps = MemOps.size(); 3957 3958 // Find the largest store and generate the bit pattern for it. 3959 EVT LargestVT = MemOps[0]; 3960 for (unsigned i = 1; i < NumMemOps; i++) 3961 if (MemOps[i].bitsGT(LargestVT)) 3962 LargestVT = MemOps[i]; 3963 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3964 3965 for (unsigned i = 0; i < NumMemOps; i++) { 3966 EVT VT = MemOps[i]; 3967 unsigned VTSize = VT.getSizeInBits() / 8; 3968 if (VTSize > Size) { 3969 // Issuing an unaligned load / store pair that overlaps with the previous 3970 // pair. Adjust the offset accordingly. 3971 assert(i == NumMemOps-1 && i != 0); 3972 DstOff -= VTSize - Size; 3973 } 3974 3975 // If this store is smaller than the largest store see whether we can get 3976 // the smaller value for free with a truncate. 3977 SDValue Value = MemSetValue; 3978 if (VT.bitsLT(LargestVT)) { 3979 if (!LargestVT.isVector() && !VT.isVector() && 3980 TLI.isTruncateFree(LargestVT, VT)) 3981 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3982 else 3983 Value = getMemsetValue(Src, VT, DAG, dl); 3984 } 3985 assert(Value.getValueType() == VT && "Value with wrong type."); 3986 SDValue Store = DAG.getStore(Chain, dl, Value, 3987 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3988 DstPtrInfo.getWithOffset(DstOff), 3989 isVol, false, Align); 3990 OutChains.push_back(Store); 3991 DstOff += VT.getSizeInBits() / 8; 3992 Size -= VTSize; 3993 } 3994 3995 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3996 &OutChains[0], OutChains.size()); 3997} 3998 3999SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, 4000 SDValue Src, SDValue Size, 4001 unsigned Align, bool isVol, bool AlwaysInline, 4002 MachinePointerInfo DstPtrInfo, 4003 MachinePointerInfo SrcPtrInfo) { 4004 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4005 4006 // Check to see if we should lower the memcpy to loads and stores first. 4007 // For cases within the target-specified limits, this is the best choice. 4008 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4009 if (ConstantSize) { 4010 // Memcpy with size zero? Just return the original chain. 4011 if (ConstantSize->isNullValue()) 4012 return Chain; 4013 4014 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4015 ConstantSize->getZExtValue(),Align, 4016 isVol, false, DstPtrInfo, SrcPtrInfo); 4017 if (Result.getNode()) 4018 return Result; 4019 } 4020 4021 // Then check to see if we should lower the memcpy with target-specific 4022 // code. If the target chooses to do this, this is the next best. 4023 SDValue Result = 4024 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 4025 isVol, AlwaysInline, 4026 DstPtrInfo, SrcPtrInfo); 4027 if (Result.getNode()) 4028 return Result; 4029 4030 // If we really need inline code and the target declined to provide it, 4031 // use a (potentially long) sequence of loads and stores. 4032 if (AlwaysInline) { 4033 assert(ConstantSize && "AlwaysInline requires a constant size!"); 4034 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4035 ConstantSize->getZExtValue(), Align, isVol, 4036 true, DstPtrInfo, SrcPtrInfo); 4037 } 4038 4039 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 4040 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 4041 // respect volatile, so they may do things like read or write memory 4042 // beyond the given memory regions. But fixing this isn't easy, and most 4043 // people don't care. 4044 4045 const TargetLowering *TLI = TM.getTargetLowering(); 4046 4047 // Emit a library call. 4048 TargetLowering::ArgListTy Args; 4049 TargetLowering::ArgListEntry Entry; 4050 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4051 Entry.Node = Dst; Args.push_back(Entry); 4052 Entry.Node = Src; Args.push_back(Entry); 4053 Entry.Node = Size; Args.push_back(Entry); 4054 // FIXME: pass in SDLoc 4055 TargetLowering:: 4056 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4057 false, false, false, false, 0, 4058 TLI->getLibcallCallingConv(RTLIB::MEMCPY), 4059 /*isTailCall=*/false, 4060 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4061 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 4062 TLI->getPointerTy()), 4063 Args, *this, dl); 4064 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4065 4066 return CallResult.second; 4067} 4068 4069SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, 4070 SDValue Src, SDValue Size, 4071 unsigned Align, bool isVol, 4072 MachinePointerInfo DstPtrInfo, 4073 MachinePointerInfo SrcPtrInfo) { 4074 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4075 4076 // Check to see if we should lower the memmove to loads and stores first. 4077 // For cases within the target-specified limits, this is the best choice. 4078 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4079 if (ConstantSize) { 4080 // Memmove with size zero? Just return the original chain. 4081 if (ConstantSize->isNullValue()) 4082 return Chain; 4083 4084 SDValue Result = 4085 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 4086 ConstantSize->getZExtValue(), Align, isVol, 4087 false, DstPtrInfo, SrcPtrInfo); 4088 if (Result.getNode()) 4089 return Result; 4090 } 4091 4092 // Then check to see if we should lower the memmove with target-specific 4093 // code. If the target chooses to do this, this is the next best. 4094 SDValue Result = 4095 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4096 DstPtrInfo, SrcPtrInfo); 4097 if (Result.getNode()) 4098 return Result; 4099 4100 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 4101 // not be safe. See memcpy above for more details. 4102 4103 const TargetLowering *TLI = TM.getTargetLowering(); 4104 4105 // Emit a library call. 4106 TargetLowering::ArgListTy Args; 4107 TargetLowering::ArgListEntry Entry; 4108 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4109 Entry.Node = Dst; Args.push_back(Entry); 4110 Entry.Node = Src; Args.push_back(Entry); 4111 Entry.Node = Size; Args.push_back(Entry); 4112 // FIXME: pass in SDLoc 4113 TargetLowering:: 4114 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4115 false, false, false, false, 0, 4116 TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 4117 /*isTailCall=*/false, 4118 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4119 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 4120 TLI->getPointerTy()), 4121 Args, *this, dl); 4122 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4123 4124 return CallResult.second; 4125} 4126 4127SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, 4128 SDValue Src, SDValue Size, 4129 unsigned Align, bool isVol, 4130 MachinePointerInfo DstPtrInfo) { 4131 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4132 4133 // Check to see if we should lower the memset to stores first. 4134 // For cases within the target-specified limits, this is the best choice. 4135 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4136 if (ConstantSize) { 4137 // Memset with size zero? Just return the original chain. 4138 if (ConstantSize->isNullValue()) 4139 return Chain; 4140 4141 SDValue Result = 4142 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 4143 Align, isVol, DstPtrInfo); 4144 4145 if (Result.getNode()) 4146 return Result; 4147 } 4148 4149 // Then check to see if we should lower the memset with target-specific 4150 // code. If the target chooses to do this, this is the next best. 4151 SDValue Result = 4152 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4153 DstPtrInfo); 4154 if (Result.getNode()) 4155 return Result; 4156 4157 // Emit a library call. 4158 const TargetLowering *TLI = TM.getTargetLowering(); 4159 Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext()); 4160 TargetLowering::ArgListTy Args; 4161 TargetLowering::ArgListEntry Entry; 4162 Entry.Node = Dst; Entry.Ty = IntPtrTy; 4163 Args.push_back(Entry); 4164 // Extend or truncate the argument to be an i32 value for the call. 4165 if (Src.getValueType().bitsGT(MVT::i32)) 4166 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 4167 else 4168 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 4169 Entry.Node = Src; 4170 Entry.Ty = Type::getInt32Ty(*getContext()); 4171 Entry.isSExt = true; 4172 Args.push_back(Entry); 4173 Entry.Node = Size; 4174 Entry.Ty = IntPtrTy; 4175 Entry.isSExt = false; 4176 Args.push_back(Entry); 4177 // FIXME: pass in SDLoc 4178 TargetLowering:: 4179 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4180 false, false, false, false, 0, 4181 TLI->getLibcallCallingConv(RTLIB::MEMSET), 4182 /*isTailCall=*/false, 4183 /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 4184 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 4185 TLI->getPointerTy()), 4186 Args, *this, dl); 4187 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4188 4189 return CallResult.second; 4190} 4191 4192SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4193 SDVTList VTList, SDValue* Ops, unsigned NumOps, 4194 MachineMemOperand *MMO, 4195 AtomicOrdering Ordering, 4196 SynchronizationScope SynchScope) { 4197 FoldingSetNodeID ID; 4198 ID.AddInteger(MemVT.getRawBits()); 4199 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4200 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4201 void* IP = 0; 4202 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4203 cast<AtomicSDNode>(E)->refineAlignment(MMO); 4204 return SDValue(E, 0); 4205 } 4206 4207 // Allocate the operands array for the node out of the BumpPtrAllocator, since 4208 // SDNode doesn't have access to it. This memory will be "leaked" when 4209 // the node is deallocated, but recovered when the allocator is released. 4210 // If the number of operands is less than 5 we use AtomicSDNode's internal 4211 // storage. 4212 SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) : 0; 4213 4214 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), 4215 dl.getDebugLoc(), VTList, MemVT, 4216 Ops, DynOps, NumOps, MMO, 4217 Ordering, SynchScope); 4218 CSEMap.InsertNode(N, IP); 4219 AllNodes.push_back(N); 4220 return SDValue(N, 0); 4221} 4222 4223SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4224 SDValue Chain, SDValue Ptr, SDValue Cmp, 4225 SDValue Swp, MachinePointerInfo PtrInfo, 4226 unsigned Alignment, 4227 AtomicOrdering Ordering, 4228 SynchronizationScope SynchScope) { 4229 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4230 Alignment = getEVTAlignment(MemVT); 4231 4232 MachineFunction &MF = getMachineFunction(); 4233 4234 // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 4235 // For now, atomics are considered to be volatile always. 4236 // FIXME: Volatile isn't really correct; we should keep track of atomic 4237 // orderings in the memoperand. 4238 unsigned Flags = MachineMemOperand::MOVolatile; 4239 if (Opcode != ISD::ATOMIC_STORE) 4240 Flags |= MachineMemOperand::MOLoad; 4241 if (Opcode != ISD::ATOMIC_LOAD) 4242 Flags |= MachineMemOperand::MOStore; 4243 4244 MachineMemOperand *MMO = 4245 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 4246 4247 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 4248 Ordering, SynchScope); 4249} 4250 4251SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4252 SDValue Chain, 4253 SDValue Ptr, SDValue Cmp, 4254 SDValue Swp, MachineMemOperand *MMO, 4255 AtomicOrdering Ordering, 4256 SynchronizationScope SynchScope) { 4257 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 4258 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 4259 4260 EVT VT = Cmp.getValueType(); 4261 4262 SDVTList VTs = getVTList(VT, MVT::Other); 4263 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 4264 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, Ordering, SynchScope); 4265} 4266 4267SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4268 SDValue Chain, 4269 SDValue Ptr, SDValue Val, 4270 const Value* PtrVal, 4271 unsigned Alignment, 4272 AtomicOrdering Ordering, 4273 SynchronizationScope SynchScope) { 4274 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4275 Alignment = getEVTAlignment(MemVT); 4276 4277 MachineFunction &MF = getMachineFunction(); 4278 // An atomic store does not load. An atomic load does not store. 4279 // (An atomicrmw obviously both loads and stores.) 4280 // For now, atomics are considered to be volatile always, and they are 4281 // chained as such. 4282 // FIXME: Volatile isn't really correct; we should keep track of atomic 4283 // orderings in the memoperand. 4284 unsigned Flags = MachineMemOperand::MOVolatile; 4285 if (Opcode != ISD::ATOMIC_STORE) 4286 Flags |= MachineMemOperand::MOLoad; 4287 if (Opcode != ISD::ATOMIC_LOAD) 4288 Flags |= MachineMemOperand::MOStore; 4289 4290 MachineMemOperand *MMO = 4291 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4292 MemVT.getStoreSize(), Alignment); 4293 4294 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 4295 Ordering, SynchScope); 4296} 4297 4298SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4299 SDValue Chain, 4300 SDValue Ptr, SDValue Val, 4301 MachineMemOperand *MMO, 4302 AtomicOrdering Ordering, 4303 SynchronizationScope SynchScope) { 4304 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 4305 Opcode == ISD::ATOMIC_LOAD_SUB || 4306 Opcode == ISD::ATOMIC_LOAD_AND || 4307 Opcode == ISD::ATOMIC_LOAD_OR || 4308 Opcode == ISD::ATOMIC_LOAD_XOR || 4309 Opcode == ISD::ATOMIC_LOAD_NAND || 4310 Opcode == ISD::ATOMIC_LOAD_MIN || 4311 Opcode == ISD::ATOMIC_LOAD_MAX || 4312 Opcode == ISD::ATOMIC_LOAD_UMIN || 4313 Opcode == ISD::ATOMIC_LOAD_UMAX || 4314 Opcode == ISD::ATOMIC_SWAP || 4315 Opcode == ISD::ATOMIC_STORE) && 4316 "Invalid Atomic Op"); 4317 4318 EVT VT = Val.getValueType(); 4319 4320 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 4321 getVTList(VT, MVT::Other); 4322 SDValue Ops[] = {Chain, Ptr, Val}; 4323 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope); 4324} 4325 4326SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4327 EVT VT, SDValue Chain, 4328 SDValue Ptr, 4329 const Value* PtrVal, 4330 unsigned Alignment, 4331 AtomicOrdering Ordering, 4332 SynchronizationScope SynchScope) { 4333 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4334 Alignment = getEVTAlignment(MemVT); 4335 4336 MachineFunction &MF = getMachineFunction(); 4337 // An atomic store does not load. An atomic load does not store. 4338 // (An atomicrmw obviously both loads and stores.) 4339 // For now, atomics are considered to be volatile always, and they are 4340 // chained as such. 4341 // FIXME: Volatile isn't really correct; we should keep track of atomic 4342 // orderings in the memoperand. 4343 unsigned Flags = MachineMemOperand::MOVolatile; 4344 if (Opcode != ISD::ATOMIC_STORE) 4345 Flags |= MachineMemOperand::MOLoad; 4346 if (Opcode != ISD::ATOMIC_LOAD) 4347 Flags |= MachineMemOperand::MOStore; 4348 4349 MachineMemOperand *MMO = 4350 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4351 MemVT.getStoreSize(), Alignment); 4352 4353 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4354 Ordering, SynchScope); 4355} 4356 4357SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4358 EVT VT, SDValue Chain, 4359 SDValue Ptr, 4360 MachineMemOperand *MMO, 4361 AtomicOrdering Ordering, 4362 SynchronizationScope SynchScope) { 4363 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4364 4365 SDVTList VTs = getVTList(VT, MVT::Other); 4366 SDValue Ops[] = {Chain, Ptr}; 4367 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 2, MMO, Ordering, SynchScope); 4368} 4369 4370/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4371SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4372 SDLoc dl) { 4373 if (NumOps == 1) 4374 return Ops[0]; 4375 4376 SmallVector<EVT, 4> VTs; 4377 VTs.reserve(NumOps); 4378 for (unsigned i = 0; i < NumOps; ++i) 4379 VTs.push_back(Ops[i].getValueType()); 4380 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4381 Ops, NumOps); 4382} 4383 4384SDValue 4385SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, 4386 const EVT *VTs, unsigned NumVTs, 4387 const SDValue *Ops, unsigned NumOps, 4388 EVT MemVT, MachinePointerInfo PtrInfo, 4389 unsigned Align, bool Vol, 4390 bool ReadMem, bool WriteMem) { 4391 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4392 MemVT, PtrInfo, Align, Vol, 4393 ReadMem, WriteMem); 4394} 4395 4396SDValue 4397SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4398 const SDValue *Ops, unsigned NumOps, 4399 EVT MemVT, MachinePointerInfo PtrInfo, 4400 unsigned Align, bool Vol, 4401 bool ReadMem, bool WriteMem) { 4402 if (Align == 0) // Ensure that codegen never sees alignment 0 4403 Align = getEVTAlignment(MemVT); 4404 4405 MachineFunction &MF = getMachineFunction(); 4406 unsigned Flags = 0; 4407 if (WriteMem) 4408 Flags |= MachineMemOperand::MOStore; 4409 if (ReadMem) 4410 Flags |= MachineMemOperand::MOLoad; 4411 if (Vol) 4412 Flags |= MachineMemOperand::MOVolatile; 4413 MachineMemOperand *MMO = 4414 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4415 4416 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4417} 4418 4419SDValue 4420SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4421 const SDValue *Ops, unsigned NumOps, 4422 EVT MemVT, MachineMemOperand *MMO) { 4423 assert((Opcode == ISD::INTRINSIC_VOID || 4424 Opcode == ISD::INTRINSIC_W_CHAIN || 4425 Opcode == ISD::PREFETCH || 4426 Opcode == ISD::LIFETIME_START || 4427 Opcode == ISD::LIFETIME_END || 4428 (Opcode <= INT_MAX && 4429 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4430 "Opcode is not a memory-accessing opcode!"); 4431 4432 // Memoize the node unless it returns a flag. 4433 MemIntrinsicSDNode *N; 4434 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4435 FoldingSetNodeID ID; 4436 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4437 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4438 void *IP = 0; 4439 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4440 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4441 return SDValue(E, 0); 4442 } 4443 4444 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4445 dl.getDebugLoc(), VTList, Ops, 4446 NumOps, MemVT, MMO); 4447 CSEMap.InsertNode(N, IP); 4448 } else { 4449 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4450 dl.getDebugLoc(), VTList, Ops, 4451 NumOps, MemVT, MMO); 4452 } 4453 AllNodes.push_back(N); 4454 return SDValue(N, 0); 4455} 4456 4457/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4458/// MachinePointerInfo record from it. This is particularly useful because the 4459/// code generator has many cases where it doesn't bother passing in a 4460/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4461static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4462 // If this is FI+Offset, we can model it. 4463 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4464 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4465 4466 // If this is (FI+Offset1)+Offset2, we can model it. 4467 if (Ptr.getOpcode() != ISD::ADD || 4468 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4469 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4470 return MachinePointerInfo(); 4471 4472 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4473 return MachinePointerInfo::getFixedStack(FI, Offset+ 4474 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4475} 4476 4477/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4478/// MachinePointerInfo record from it. This is particularly useful because the 4479/// code generator has many cases where it doesn't bother passing in a 4480/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4481static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4482 // If the 'Offset' value isn't a constant, we can't handle this. 4483 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4484 return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4485 if (OffsetOp.getOpcode() == ISD::UNDEF) 4486 return InferPointerInfo(Ptr); 4487 return MachinePointerInfo(); 4488} 4489 4490 4491SDValue 4492SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4493 EVT VT, SDLoc dl, SDValue Chain, 4494 SDValue Ptr, SDValue Offset, 4495 MachinePointerInfo PtrInfo, EVT MemVT, 4496 bool isVolatile, bool isNonTemporal, bool isInvariant, 4497 unsigned Alignment, const MDNode *TBAAInfo, 4498 const MDNode *Ranges) { 4499 assert(Chain.getValueType() == MVT::Other && 4500 "Invalid chain type"); 4501 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4502 Alignment = getEVTAlignment(VT); 4503 4504 unsigned Flags = MachineMemOperand::MOLoad; 4505 if (isVolatile) 4506 Flags |= MachineMemOperand::MOVolatile; 4507 if (isNonTemporal) 4508 Flags |= MachineMemOperand::MONonTemporal; 4509 if (isInvariant) 4510 Flags |= MachineMemOperand::MOInvariant; 4511 4512 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4513 // clients. 4514 if (PtrInfo.V == 0) 4515 PtrInfo = InferPointerInfo(Ptr, Offset); 4516 4517 MachineFunction &MF = getMachineFunction(); 4518 MachineMemOperand *MMO = 4519 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4520 TBAAInfo, Ranges); 4521 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4522} 4523 4524SDValue 4525SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4526 EVT VT, SDLoc dl, SDValue Chain, 4527 SDValue Ptr, SDValue Offset, EVT MemVT, 4528 MachineMemOperand *MMO) { 4529 if (VT == MemVT) { 4530 ExtType = ISD::NON_EXTLOAD; 4531 } else if (ExtType == ISD::NON_EXTLOAD) { 4532 assert(VT == MemVT && "Non-extending load from different memory type!"); 4533 } else { 4534 // Extending load. 4535 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4536 "Should only be an extending load, not truncating!"); 4537 assert(VT.isInteger() == MemVT.isInteger() && 4538 "Cannot convert from FP to Int or Int -> FP!"); 4539 assert(VT.isVector() == MemVT.isVector() && 4540 "Cannot use trunc store to convert to or from a vector!"); 4541 assert((!VT.isVector() || 4542 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4543 "Cannot use trunc store to change the number of vector elements!"); 4544 } 4545 4546 bool Indexed = AM != ISD::UNINDEXED; 4547 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4548 "Unindexed load with an offset!"); 4549 4550 SDVTList VTs = Indexed ? 4551 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4552 SDValue Ops[] = { Chain, Ptr, Offset }; 4553 FoldingSetNodeID ID; 4554 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4555 ID.AddInteger(MemVT.getRawBits()); 4556 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4557 MMO->isNonTemporal(), 4558 MMO->isInvariant())); 4559 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4560 void *IP = 0; 4561 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4562 cast<LoadSDNode>(E)->refineAlignment(MMO); 4563 return SDValue(E, 0); 4564 } 4565 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), 4566 dl.getDebugLoc(), VTs, AM, ExtType, 4567 MemVT, MMO); 4568 CSEMap.InsertNode(N, IP); 4569 AllNodes.push_back(N); 4570 return SDValue(N, 0); 4571} 4572 4573SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4574 SDValue Chain, SDValue Ptr, 4575 MachinePointerInfo PtrInfo, 4576 bool isVolatile, bool isNonTemporal, 4577 bool isInvariant, unsigned Alignment, 4578 const MDNode *TBAAInfo, 4579 const MDNode *Ranges) { 4580 SDValue Undef = getUNDEF(Ptr.getValueType()); 4581 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4582 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4583 TBAAInfo, Ranges); 4584} 4585 4586SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4587 SDValue Chain, SDValue Ptr, 4588 MachineMemOperand *MMO) { 4589 SDValue Undef = getUNDEF(Ptr.getValueType()); 4590 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4591 VT, MMO); 4592} 4593 4594SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4595 SDValue Chain, SDValue Ptr, 4596 MachinePointerInfo PtrInfo, EVT MemVT, 4597 bool isVolatile, bool isNonTemporal, 4598 unsigned Alignment, const MDNode *TBAAInfo) { 4599 SDValue Undef = getUNDEF(Ptr.getValueType()); 4600 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4601 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4602 TBAAInfo); 4603} 4604 4605 4606SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4607 SDValue Chain, SDValue Ptr, EVT MemVT, 4608 MachineMemOperand *MMO) { 4609 SDValue Undef = getUNDEF(Ptr.getValueType()); 4610 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4611 MemVT, MMO); 4612} 4613 4614SDValue 4615SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, 4616 SDValue Offset, ISD::MemIndexedMode AM) { 4617 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4618 assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4619 "Load is already a indexed load!"); 4620 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4621 LD->getChain(), Base, Offset, LD->getPointerInfo(), 4622 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4623 false, LD->getAlignment()); 4624} 4625 4626SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4627 SDValue Ptr, MachinePointerInfo PtrInfo, 4628 bool isVolatile, bool isNonTemporal, 4629 unsigned Alignment, const MDNode *TBAAInfo) { 4630 assert(Chain.getValueType() == MVT::Other && 4631 "Invalid chain type"); 4632 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4633 Alignment = getEVTAlignment(Val.getValueType()); 4634 4635 unsigned Flags = MachineMemOperand::MOStore; 4636 if (isVolatile) 4637 Flags |= MachineMemOperand::MOVolatile; 4638 if (isNonTemporal) 4639 Flags |= MachineMemOperand::MONonTemporal; 4640 4641 if (PtrInfo.V == 0) 4642 PtrInfo = InferPointerInfo(Ptr); 4643 4644 MachineFunction &MF = getMachineFunction(); 4645 MachineMemOperand *MMO = 4646 MF.getMachineMemOperand(PtrInfo, Flags, 4647 Val.getValueType().getStoreSize(), Alignment, 4648 TBAAInfo); 4649 4650 return getStore(Chain, dl, Val, Ptr, MMO); 4651} 4652 4653SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4654 SDValue Ptr, MachineMemOperand *MMO) { 4655 assert(Chain.getValueType() == MVT::Other && 4656 "Invalid chain type"); 4657 EVT VT = Val.getValueType(); 4658 SDVTList VTs = getVTList(MVT::Other); 4659 SDValue Undef = getUNDEF(Ptr.getValueType()); 4660 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4661 FoldingSetNodeID ID; 4662 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4663 ID.AddInteger(VT.getRawBits()); 4664 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4665 MMO->isNonTemporal(), MMO->isInvariant())); 4666 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4667 void *IP = 0; 4668 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4669 cast<StoreSDNode>(E)->refineAlignment(MMO); 4670 return SDValue(E, 0); 4671 } 4672 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4673 dl.getDebugLoc(), VTs, 4674 ISD::UNINDEXED, false, VT, MMO); 4675 CSEMap.InsertNode(N, IP); 4676 AllNodes.push_back(N); 4677 return SDValue(N, 0); 4678} 4679 4680SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4681 SDValue Ptr, MachinePointerInfo PtrInfo, 4682 EVT SVT,bool isVolatile, bool isNonTemporal, 4683 unsigned Alignment, 4684 const MDNode *TBAAInfo) { 4685 assert(Chain.getValueType() == MVT::Other && 4686 "Invalid chain type"); 4687 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4688 Alignment = getEVTAlignment(SVT); 4689 4690 unsigned Flags = MachineMemOperand::MOStore; 4691 if (isVolatile) 4692 Flags |= MachineMemOperand::MOVolatile; 4693 if (isNonTemporal) 4694 Flags |= MachineMemOperand::MONonTemporal; 4695 4696 if (PtrInfo.V == 0) 4697 PtrInfo = InferPointerInfo(Ptr); 4698 4699 MachineFunction &MF = getMachineFunction(); 4700 MachineMemOperand *MMO = 4701 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4702 TBAAInfo); 4703 4704 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4705} 4706 4707SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4708 SDValue Ptr, EVT SVT, 4709 MachineMemOperand *MMO) { 4710 EVT VT = Val.getValueType(); 4711 4712 assert(Chain.getValueType() == MVT::Other && 4713 "Invalid chain type"); 4714 if (VT == SVT) 4715 return getStore(Chain, dl, Val, Ptr, MMO); 4716 4717 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4718 "Should only be a truncating store, not extending!"); 4719 assert(VT.isInteger() == SVT.isInteger() && 4720 "Can't do FP-INT conversion!"); 4721 assert(VT.isVector() == SVT.isVector() && 4722 "Cannot use trunc store to convert to or from a vector!"); 4723 assert((!VT.isVector() || 4724 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4725 "Cannot use trunc store to change the number of vector elements!"); 4726 4727 SDVTList VTs = getVTList(MVT::Other); 4728 SDValue Undef = getUNDEF(Ptr.getValueType()); 4729 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4730 FoldingSetNodeID ID; 4731 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4732 ID.AddInteger(SVT.getRawBits()); 4733 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4734 MMO->isNonTemporal(), MMO->isInvariant())); 4735 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4736 void *IP = 0; 4737 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4738 cast<StoreSDNode>(E)->refineAlignment(MMO); 4739 return SDValue(E, 0); 4740 } 4741 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4742 dl.getDebugLoc(), VTs, 4743 ISD::UNINDEXED, true, SVT, MMO); 4744 CSEMap.InsertNode(N, IP); 4745 AllNodes.push_back(N); 4746 return SDValue(N, 0); 4747} 4748 4749SDValue 4750SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, 4751 SDValue Offset, ISD::MemIndexedMode AM) { 4752 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4753 assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4754 "Store is already a indexed store!"); 4755 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4756 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4757 FoldingSetNodeID ID; 4758 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4759 ID.AddInteger(ST->getMemoryVT().getRawBits()); 4760 ID.AddInteger(ST->getRawSubclassData()); 4761 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 4762 void *IP = 0; 4763 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4764 return SDValue(E, 0); 4765 4766 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4767 dl.getDebugLoc(), VTs, AM, 4768 ST->isTruncatingStore(), 4769 ST->getMemoryVT(), 4770 ST->getMemOperand()); 4771 CSEMap.InsertNode(N, IP); 4772 AllNodes.push_back(N); 4773 return SDValue(N, 0); 4774} 4775 4776SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl, 4777 SDValue Chain, SDValue Ptr, 4778 SDValue SV, 4779 unsigned Align) { 4780 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4781 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4782} 4783 4784SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4785 const SDUse *Ops, unsigned NumOps) { 4786 switch (NumOps) { 4787 case 0: return getNode(Opcode, DL, VT); 4788 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4789 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4790 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4791 default: break; 4792 } 4793 4794 // Copy from an SDUse array into an SDValue array for use with 4795 // the regular getNode logic. 4796 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4797 return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4798} 4799 4800SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4801 const SDValue *Ops, unsigned NumOps) { 4802 switch (NumOps) { 4803 case 0: return getNode(Opcode, DL, VT); 4804 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4805 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4806 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4807 default: break; 4808 } 4809 4810 switch (Opcode) { 4811 default: break; 4812 case ISD::SELECT_CC: { 4813 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4814 assert(Ops[0].getValueType() == Ops[1].getValueType() && 4815 "LHS and RHS of condition must have same type!"); 4816 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4817 "True and False arms of SelectCC must have same type!"); 4818 assert(Ops[2].getValueType() == VT && 4819 "select_cc node must be of same type as true and false value!"); 4820 break; 4821 } 4822 case ISD::BR_CC: { 4823 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4824 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4825 "LHS/RHS of comparison should match types!"); 4826 break; 4827 } 4828 } 4829 4830 // Memoize nodes. 4831 SDNode *N; 4832 SDVTList VTs = getVTList(VT); 4833 4834 if (VT != MVT::Glue) { 4835 FoldingSetNodeID ID; 4836 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4837 void *IP = 0; 4838 4839 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4840 return SDValue(E, 0); 4841 4842 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4843 VTs, Ops, NumOps); 4844 CSEMap.InsertNode(N, IP); 4845 } else { 4846 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4847 VTs, Ops, NumOps); 4848 } 4849 4850 AllNodes.push_back(N); 4851#ifndef NDEBUG 4852 VerifySDNode(N); 4853#endif 4854 return SDValue(N, 0); 4855} 4856 4857SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4858 ArrayRef<EVT> ResultTys, 4859 const SDValue *Ops, unsigned NumOps) { 4860 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4861 Ops, NumOps); 4862} 4863 4864SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4865 const EVT *VTs, unsigned NumVTs, 4866 const SDValue *Ops, unsigned NumOps) { 4867 if (NumVTs == 1) 4868 return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4869 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4870} 4871 4872SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4873 const SDValue *Ops, unsigned NumOps) { 4874 if (VTList.NumVTs == 1) 4875 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4876 4877#if 0 4878 switch (Opcode) { 4879 // FIXME: figure out how to safely handle things like 4880 // int foo(int x) { return 1 << (x & 255); } 4881 // int bar() { return foo(256); } 4882 case ISD::SRA_PARTS: 4883 case ISD::SRL_PARTS: 4884 case ISD::SHL_PARTS: 4885 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4886 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4887 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4888 else if (N3.getOpcode() == ISD::AND) 4889 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4890 // If the and is only masking out bits that cannot effect the shift, 4891 // eliminate the and. 4892 unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4893 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4894 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4895 } 4896 break; 4897 } 4898#endif 4899 4900 // Memoize the node unless it returns a flag. 4901 SDNode *N; 4902 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4903 FoldingSetNodeID ID; 4904 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4905 void *IP = 0; 4906 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4907 return SDValue(E, 0); 4908 4909 if (NumOps == 1) { 4910 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4911 DL.getDebugLoc(), VTList, Ops[0]); 4912 } else if (NumOps == 2) { 4913 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4914 DL.getDebugLoc(), VTList, Ops[0], 4915 Ops[1]); 4916 } else if (NumOps == 3) { 4917 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4918 DL.getDebugLoc(), VTList, Ops[0], 4919 Ops[1], Ops[2]); 4920 } else { 4921 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4922 VTList, Ops, NumOps); 4923 } 4924 CSEMap.InsertNode(N, IP); 4925 } else { 4926 if (NumOps == 1) { 4927 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4928 DL.getDebugLoc(), VTList, Ops[0]); 4929 } else if (NumOps == 2) { 4930 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4931 DL.getDebugLoc(), VTList, Ops[0], 4932 Ops[1]); 4933 } else if (NumOps == 3) { 4934 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4935 DL.getDebugLoc(), VTList, Ops[0], 4936 Ops[1], Ops[2]); 4937 } else { 4938 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4939 VTList, Ops, NumOps); 4940 } 4941 } 4942 AllNodes.push_back(N); 4943#ifndef NDEBUG 4944 VerifySDNode(N); 4945#endif 4946 return SDValue(N, 0); 4947} 4948 4949SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) { 4950 return getNode(Opcode, DL, VTList, 0, 0); 4951} 4952 4953SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4954 SDValue N1) { 4955 SDValue Ops[] = { N1 }; 4956 return getNode(Opcode, DL, VTList, Ops, 1); 4957} 4958 4959SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4960 SDValue N1, SDValue N2) { 4961 SDValue Ops[] = { N1, N2 }; 4962 return getNode(Opcode, DL, VTList, Ops, 2); 4963} 4964 4965SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4966 SDValue N1, SDValue N2, SDValue N3) { 4967 SDValue Ops[] = { N1, N2, N3 }; 4968 return getNode(Opcode, DL, VTList, Ops, 3); 4969} 4970 4971SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4972 SDValue N1, SDValue N2, SDValue N3, 4973 SDValue N4) { 4974 SDValue Ops[] = { N1, N2, N3, N4 }; 4975 return getNode(Opcode, DL, VTList, Ops, 4); 4976} 4977 4978SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4979 SDValue N1, SDValue N2, SDValue N3, 4980 SDValue N4, SDValue N5) { 4981 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4982 return getNode(Opcode, DL, VTList, Ops, 5); 4983} 4984 4985SDVTList SelectionDAG::getVTList(EVT VT) { 4986 return makeVTList(SDNode::getValueTypeList(VT), 1); 4987} 4988 4989SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4990 FoldingSetNodeID ID; 4991 ID.AddInteger(2U); 4992 ID.AddInteger(VT1.getRawBits()); 4993 ID.AddInteger(VT2.getRawBits()); 4994 4995 void *IP = 0; 4996 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4997 if (Result == NULL) { 4998 EVT *Array = Allocator.Allocate<EVT>(2); 4999 Array[0] = VT1; 5000 Array[1] = VT2; 5001 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 5002 VTListMap.InsertNode(Result, IP); 5003 } 5004 return Result->getSDVTList(); 5005} 5006 5007SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 5008 FoldingSetNodeID ID; 5009 ID.AddInteger(3U); 5010 ID.AddInteger(VT1.getRawBits()); 5011 ID.AddInteger(VT2.getRawBits()); 5012 ID.AddInteger(VT3.getRawBits()); 5013 5014 void *IP = 0; 5015 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5016 if (Result == NULL) { 5017 EVT *Array = Allocator.Allocate<EVT>(3); 5018 Array[0] = VT1; 5019 Array[1] = VT2; 5020 Array[2] = VT3; 5021 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 5022 VTListMap.InsertNode(Result, IP); 5023 } 5024 return Result->getSDVTList(); 5025} 5026 5027SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 5028 FoldingSetNodeID ID; 5029 ID.AddInteger(4U); 5030 ID.AddInteger(VT1.getRawBits()); 5031 ID.AddInteger(VT2.getRawBits()); 5032 ID.AddInteger(VT3.getRawBits()); 5033 ID.AddInteger(VT4.getRawBits()); 5034 5035 void *IP = 0; 5036 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5037 if (Result == NULL) { 5038 EVT *Array = Allocator.Allocate<EVT>(4); 5039 Array[0] = VT1; 5040 Array[1] = VT2; 5041 Array[2] = VT3; 5042 Array[3] = VT4; 5043 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 5044 VTListMap.InsertNode(Result, IP); 5045 } 5046 return Result->getSDVTList(); 5047} 5048 5049SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 5050 FoldingSetNodeID ID; 5051 ID.AddInteger(NumVTs); 5052 for (unsigned index = 0; index < NumVTs; index++) { 5053 ID.AddInteger(VTs[index].getRawBits()); 5054 } 5055 5056 void *IP = 0; 5057 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5058 if (Result == NULL) { 5059 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 5060 std::copy(VTs, VTs + NumVTs, Array); 5061 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 5062 VTListMap.InsertNode(Result, IP); 5063 } 5064 return Result->getSDVTList(); 5065} 5066 5067 5068/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 5069/// specified operands. If the resultant node already exists in the DAG, 5070/// this does not modify the specified node, instead it returns the node that 5071/// already exists. If the resultant node does not exist in the DAG, the 5072/// input node is returned. As a degenerate case, if you specify the same 5073/// input operands as the node already has, the input node is returned. 5074SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 5075 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 5076 5077 // Check to see if there is no change. 5078 if (Op == N->getOperand(0)) return N; 5079 5080 // See if the modified node already exists. 5081 void *InsertPos = 0; 5082 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 5083 return Existing; 5084 5085 // Nope it doesn't. Remove the node from its current place in the maps. 5086 if (InsertPos) 5087 if (!RemoveNodeFromCSEMaps(N)) 5088 InsertPos = 0; 5089 5090 // Now we update the operands. 5091 N->OperandList[0].set(Op); 5092 5093 // If this gets put into a CSE map, add it. 5094 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5095 return N; 5096} 5097 5098SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 5099 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 5100 5101 // Check to see if there is no change. 5102 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 5103 return N; // No operands changed, just return the input node. 5104 5105 // See if the modified node already exists. 5106 void *InsertPos = 0; 5107 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 5108 return Existing; 5109 5110 // Nope it doesn't. Remove the node from its current place in the maps. 5111 if (InsertPos) 5112 if (!RemoveNodeFromCSEMaps(N)) 5113 InsertPos = 0; 5114 5115 // Now we update the operands. 5116 if (N->OperandList[0] != Op1) 5117 N->OperandList[0].set(Op1); 5118 if (N->OperandList[1] != Op2) 5119 N->OperandList[1].set(Op2); 5120 5121 // If this gets put into a CSE map, add it. 5122 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5123 return N; 5124} 5125 5126SDNode *SelectionDAG:: 5127UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 5128 SDValue Ops[] = { Op1, Op2, Op3 }; 5129 return UpdateNodeOperands(N, Ops, 3); 5130} 5131 5132SDNode *SelectionDAG:: 5133UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5134 SDValue Op3, SDValue Op4) { 5135 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 5136 return UpdateNodeOperands(N, Ops, 4); 5137} 5138 5139SDNode *SelectionDAG:: 5140UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5141 SDValue Op3, SDValue Op4, SDValue Op5) { 5142 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 5143 return UpdateNodeOperands(N, Ops, 5); 5144} 5145 5146SDNode *SelectionDAG:: 5147UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 5148 assert(N->getNumOperands() == NumOps && 5149 "Update with wrong number of operands"); 5150 5151 // Check to see if there is no change. 5152 bool AnyChange = false; 5153 for (unsigned i = 0; i != NumOps; ++i) { 5154 if (Ops[i] != N->getOperand(i)) { 5155 AnyChange = true; 5156 break; 5157 } 5158 } 5159 5160 // No operands changed, just return the input node. 5161 if (!AnyChange) return N; 5162 5163 // See if the modified node already exists. 5164 void *InsertPos = 0; 5165 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 5166 return Existing; 5167 5168 // Nope it doesn't. Remove the node from its current place in the maps. 5169 if (InsertPos) 5170 if (!RemoveNodeFromCSEMaps(N)) 5171 InsertPos = 0; 5172 5173 // Now we update the operands. 5174 for (unsigned i = 0; i != NumOps; ++i) 5175 if (N->OperandList[i] != Ops[i]) 5176 N->OperandList[i].set(Ops[i]); 5177 5178 // If this gets put into a CSE map, add it. 5179 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5180 return N; 5181} 5182 5183/// DropOperands - Release the operands and set this node to have 5184/// zero operands. 5185void SDNode::DropOperands() { 5186 // Unlike the code in MorphNodeTo that does this, we don't need to 5187 // watch for dead nodes here. 5188 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 5189 SDUse &Use = *I++; 5190 Use.set(SDValue()); 5191 } 5192} 5193 5194/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 5195/// machine opcode. 5196/// 5197SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5198 EVT VT) { 5199 SDVTList VTs = getVTList(VT); 5200 return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 5201} 5202 5203SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5204 EVT VT, SDValue Op1) { 5205 SDVTList VTs = getVTList(VT); 5206 SDValue Ops[] = { Op1 }; 5207 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5208} 5209 5210SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5211 EVT VT, SDValue Op1, 5212 SDValue Op2) { 5213 SDVTList VTs = getVTList(VT); 5214 SDValue Ops[] = { Op1, Op2 }; 5215 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5216} 5217 5218SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5219 EVT VT, SDValue Op1, 5220 SDValue Op2, SDValue Op3) { 5221 SDVTList VTs = getVTList(VT); 5222 SDValue Ops[] = { Op1, Op2, Op3 }; 5223 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5224} 5225 5226SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5227 EVT VT, const SDValue *Ops, 5228 unsigned NumOps) { 5229 SDVTList VTs = getVTList(VT); 5230 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5231} 5232 5233SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5234 EVT VT1, EVT VT2, const SDValue *Ops, 5235 unsigned NumOps) { 5236 SDVTList VTs = getVTList(VT1, VT2); 5237 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5238} 5239 5240SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5241 EVT VT1, EVT VT2) { 5242 SDVTList VTs = getVTList(VT1, VT2); 5243 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 5244} 5245 5246SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5247 EVT VT1, EVT VT2, EVT VT3, 5248 const SDValue *Ops, unsigned NumOps) { 5249 SDVTList VTs = getVTList(VT1, VT2, VT3); 5250 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5251} 5252 5253SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5254 EVT VT1, EVT VT2, EVT VT3, EVT VT4, 5255 const SDValue *Ops, unsigned NumOps) { 5256 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5257 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5258} 5259 5260SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5261 EVT VT1, EVT VT2, 5262 SDValue Op1) { 5263 SDVTList VTs = getVTList(VT1, VT2); 5264 SDValue Ops[] = { Op1 }; 5265 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5266} 5267 5268SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5269 EVT VT1, EVT VT2, 5270 SDValue Op1, SDValue Op2) { 5271 SDVTList VTs = getVTList(VT1, VT2); 5272 SDValue Ops[] = { Op1, Op2 }; 5273 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5274} 5275 5276SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5277 EVT VT1, EVT VT2, 5278 SDValue Op1, SDValue Op2, 5279 SDValue Op3) { 5280 SDVTList VTs = getVTList(VT1, VT2); 5281 SDValue Ops[] = { Op1, Op2, Op3 }; 5282 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5283} 5284 5285SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5286 EVT VT1, EVT VT2, EVT VT3, 5287 SDValue Op1, SDValue Op2, 5288 SDValue Op3) { 5289 SDVTList VTs = getVTList(VT1, VT2, VT3); 5290 SDValue Ops[] = { Op1, Op2, Op3 }; 5291 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5292} 5293 5294SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5295 SDVTList VTs, const SDValue *Ops, 5296 unsigned NumOps) { 5297 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 5298 // Reset the NodeID to -1. 5299 N->setNodeId(-1); 5300 return N; 5301} 5302 5303/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away 5304/// the line number information on the merged node since it is not possible to 5305/// preserve the information that operation is associated with multiple lines. 5306/// This will make the debugger working better at -O0, were there is a higher 5307/// probability having other instructions associated with that line. 5308/// 5309/// For IROrder, we keep the smaller of the two 5310SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { 5311 DebugLoc NLoc = N->getDebugLoc(); 5312 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && 5313 (OLoc.getDebugLoc() != NLoc)) { 5314 N->setDebugLoc(DebugLoc()); 5315 } 5316 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 5317 N->setIROrder(Order); 5318 return N; 5319} 5320 5321/// MorphNodeTo - This *mutates* the specified node to have the specified 5322/// return type, opcode, and operands. 5323/// 5324/// Note that MorphNodeTo returns the resultant node. If there is already a 5325/// node of the specified opcode and operands, it returns that node instead of 5326/// the current one. Note that the SDLoc need not be the same. 5327/// 5328/// Using MorphNodeTo is faster than creating a new node and swapping it in 5329/// with ReplaceAllUsesWith both because it often avoids allocating a new 5330/// node, and because it doesn't require CSE recalculation for any of 5331/// the node's users. 5332/// 5333SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 5334 SDVTList VTs, const SDValue *Ops, 5335 unsigned NumOps) { 5336 // If an identical node already exists, use it. 5337 void *IP = 0; 5338 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 5339 FoldingSetNodeID ID; 5340 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 5341 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 5342 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N)); 5343 } 5344 5345 if (!RemoveNodeFromCSEMaps(N)) 5346 IP = 0; 5347 5348 // Start the morphing. 5349 N->NodeType = Opc; 5350 N->ValueList = VTs.VTs; 5351 N->NumValues = VTs.NumVTs; 5352 5353 // Clear the operands list, updating used nodes to remove this from their 5354 // use list. Keep track of any operands that become dead as a result. 5355 SmallPtrSet<SDNode*, 16> DeadNodeSet; 5356 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 5357 SDUse &Use = *I++; 5358 SDNode *Used = Use.getNode(); 5359 Use.set(SDValue()); 5360 if (Used->use_empty()) 5361 DeadNodeSet.insert(Used); 5362 } 5363 5364 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 5365 // Initialize the memory references information. 5366 MN->setMemRefs(0, 0); 5367 // If NumOps is larger than the # of operands we can have in a 5368 // MachineSDNode, reallocate the operand list. 5369 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5370 if (MN->OperandsNeedDelete) 5371 delete[] MN->OperandList; 5372 if (NumOps > array_lengthof(MN->LocalOperands)) 5373 // We're creating a final node that will live unmorphed for the 5374 // remainder of the current SelectionDAG iteration, so we can allocate 5375 // the operands directly out of a pool with no recycling metadata. 5376 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5377 Ops, NumOps); 5378 else 5379 MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5380 MN->OperandsNeedDelete = false; 5381 } else 5382 MN->InitOperands(MN->OperandList, Ops, NumOps); 5383 } else { 5384 // If NumOps is larger than the # of operands we currently have, reallocate 5385 // the operand list. 5386 if (NumOps > N->NumOperands) { 5387 if (N->OperandsNeedDelete) 5388 delete[] N->OperandList; 5389 N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5390 N->OperandsNeedDelete = true; 5391 } else 5392 N->InitOperands(N->OperandList, Ops, NumOps); 5393 } 5394 5395 // Delete any nodes that are still dead after adding the uses for the 5396 // new operands. 5397 if (!DeadNodeSet.empty()) { 5398 SmallVector<SDNode *, 16> DeadNodes; 5399 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5400 E = DeadNodeSet.end(); I != E; ++I) 5401 if ((*I)->use_empty()) 5402 DeadNodes.push_back(*I); 5403 RemoveDeadNodes(DeadNodes); 5404 } 5405 5406 if (IP) 5407 CSEMap.InsertNode(N, IP); // Memoize the new node. 5408 return N; 5409} 5410 5411 5412/// getMachineNode - These are used for target selectors to create a new node 5413/// with specified return type(s), MachineInstr opcode, and operands. 5414/// 5415/// Note that getMachineNode returns the resultant node. If there is already a 5416/// node of the specified opcode and operands, it returns that node instead of 5417/// the current one. 5418MachineSDNode * 5419SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) { 5420 SDVTList VTs = getVTList(VT); 5421 return getMachineNode(Opcode, dl, VTs, None); 5422} 5423 5424MachineSDNode * 5425SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) { 5426 SDVTList VTs = getVTList(VT); 5427 SDValue Ops[] = { Op1 }; 5428 return getMachineNode(Opcode, dl, VTs, Ops); 5429} 5430 5431MachineSDNode * 5432SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5433 SDValue Op1, SDValue Op2) { 5434 SDVTList VTs = getVTList(VT); 5435 SDValue Ops[] = { Op1, Op2 }; 5436 return getMachineNode(Opcode, dl, VTs, Ops); 5437} 5438 5439MachineSDNode * 5440SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5441 SDValue Op1, SDValue Op2, SDValue Op3) { 5442 SDVTList VTs = getVTList(VT); 5443 SDValue Ops[] = { Op1, Op2, Op3 }; 5444 return getMachineNode(Opcode, dl, VTs, Ops); 5445} 5446 5447MachineSDNode * 5448SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5449 ArrayRef<SDValue> Ops) { 5450 SDVTList VTs = getVTList(VT); 5451 return getMachineNode(Opcode, dl, VTs, Ops); 5452} 5453 5454MachineSDNode * 5455SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) { 5456 SDVTList VTs = getVTList(VT1, VT2); 5457 return getMachineNode(Opcode, dl, VTs, None); 5458} 5459 5460MachineSDNode * 5461SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5462 EVT VT1, EVT VT2, SDValue Op1) { 5463 SDVTList VTs = getVTList(VT1, VT2); 5464 SDValue Ops[] = { Op1 }; 5465 return getMachineNode(Opcode, dl, VTs, Ops); 5466} 5467 5468MachineSDNode * 5469SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5470 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5471 SDVTList VTs = getVTList(VT1, VT2); 5472 SDValue Ops[] = { Op1, Op2 }; 5473 return getMachineNode(Opcode, dl, VTs, Ops); 5474} 5475 5476MachineSDNode * 5477SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5478 EVT VT1, EVT VT2, SDValue Op1, 5479 SDValue Op2, SDValue Op3) { 5480 SDVTList VTs = getVTList(VT1, VT2); 5481 SDValue Ops[] = { Op1, Op2, Op3 }; 5482 return getMachineNode(Opcode, dl, VTs, Ops); 5483} 5484 5485MachineSDNode * 5486SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5487 EVT VT1, EVT VT2, 5488 ArrayRef<SDValue> Ops) { 5489 SDVTList VTs = getVTList(VT1, VT2); 5490 return getMachineNode(Opcode, dl, VTs, Ops); 5491} 5492 5493MachineSDNode * 5494SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5495 EVT VT1, EVT VT2, EVT VT3, 5496 SDValue Op1, SDValue Op2) { 5497 SDVTList VTs = getVTList(VT1, VT2, VT3); 5498 SDValue Ops[] = { Op1, Op2 }; 5499 return getMachineNode(Opcode, dl, VTs, Ops); 5500} 5501 5502MachineSDNode * 5503SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5504 EVT VT1, EVT VT2, EVT VT3, 5505 SDValue Op1, SDValue Op2, SDValue Op3) { 5506 SDVTList VTs = getVTList(VT1, VT2, VT3); 5507 SDValue Ops[] = { Op1, Op2, Op3 }; 5508 return getMachineNode(Opcode, dl, VTs, Ops); 5509} 5510 5511MachineSDNode * 5512SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5513 EVT VT1, EVT VT2, EVT VT3, 5514 ArrayRef<SDValue> Ops) { 5515 SDVTList VTs = getVTList(VT1, VT2, VT3); 5516 return getMachineNode(Opcode, dl, VTs, Ops); 5517} 5518 5519MachineSDNode * 5520SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, 5521 EVT VT2, EVT VT3, EVT VT4, 5522 ArrayRef<SDValue> Ops) { 5523 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5524 return getMachineNode(Opcode, dl, VTs, Ops); 5525} 5526 5527MachineSDNode * 5528SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5529 ArrayRef<EVT> ResultTys, 5530 ArrayRef<SDValue> Ops) { 5531 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5532 return getMachineNode(Opcode, dl, VTs, Ops); 5533} 5534 5535MachineSDNode * 5536SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, 5537 ArrayRef<SDValue> OpsArray) { 5538 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5539 MachineSDNode *N; 5540 void *IP = 0; 5541 const SDValue *Ops = OpsArray.data(); 5542 unsigned NumOps = OpsArray.size(); 5543 5544 if (DoCSE) { 5545 FoldingSetNodeID ID; 5546 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5547 IP = 0; 5548 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5549 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); 5550 } 5551 } 5552 5553 // Allocate a new MachineSDNode. 5554 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), 5555 DL.getDebugLoc(), VTs); 5556 5557 // Initialize the operands list. 5558 if (NumOps > array_lengthof(N->LocalOperands)) 5559 // We're creating a final node that will live unmorphed for the 5560 // remainder of the current SelectionDAG iteration, so we can allocate 5561 // the operands directly out of a pool with no recycling metadata. 5562 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5563 Ops, NumOps); 5564 else 5565 N->InitOperands(N->LocalOperands, Ops, NumOps); 5566 N->OperandsNeedDelete = false; 5567 5568 if (DoCSE) 5569 CSEMap.InsertNode(N, IP); 5570 5571 AllNodes.push_back(N); 5572#ifndef NDEBUG 5573 VerifyMachineNode(N); 5574#endif 5575 return N; 5576} 5577 5578/// getTargetExtractSubreg - A convenience function for creating 5579/// TargetOpcode::EXTRACT_SUBREG nodes. 5580SDValue 5581SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, 5582 SDValue Operand) { 5583 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5584 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5585 VT, Operand, SRIdxVal); 5586 return SDValue(Subreg, 0); 5587} 5588 5589/// getTargetInsertSubreg - A convenience function for creating 5590/// TargetOpcode::INSERT_SUBREG nodes. 5591SDValue 5592SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, 5593 SDValue Operand, SDValue Subreg) { 5594 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5595 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5596 VT, Operand, Subreg, SRIdxVal); 5597 return SDValue(Result, 0); 5598} 5599 5600/// getNodeIfExists - Get the specified node if it's already available, or 5601/// else return NULL. 5602SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5603 const SDValue *Ops, unsigned NumOps) { 5604 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5605 FoldingSetNodeID ID; 5606 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5607 void *IP = 0; 5608 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5609 return E; 5610 } 5611 return NULL; 5612} 5613 5614/// getDbgValue - Creates a SDDbgValue node. 5615/// 5616SDDbgValue * 5617SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5618 DebugLoc DL, unsigned O) { 5619 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5620} 5621 5622SDDbgValue * 5623SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5624 DebugLoc DL, unsigned O) { 5625 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5626} 5627 5628SDDbgValue * 5629SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5630 DebugLoc DL, unsigned O) { 5631 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5632} 5633 5634namespace { 5635 5636/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5637/// pointed to by a use iterator is deleted, increment the use iterator 5638/// so that it doesn't dangle. 5639/// 5640class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5641 SDNode::use_iterator &UI; 5642 SDNode::use_iterator &UE; 5643 5644 virtual void NodeDeleted(SDNode *N, SDNode *E) { 5645 // Increment the iterator as needed. 5646 while (UI != UE && N == *UI) 5647 ++UI; 5648 } 5649 5650public: 5651 RAUWUpdateListener(SelectionDAG &d, 5652 SDNode::use_iterator &ui, 5653 SDNode::use_iterator &ue) 5654 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 5655}; 5656 5657} 5658 5659/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5660/// This can cause recursive merging of nodes in the DAG. 5661/// 5662/// This version assumes From has a single result value. 5663/// 5664void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 5665 SDNode *From = FromN.getNode(); 5666 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5667 "Cannot replace with this method!"); 5668 assert(From != To.getNode() && "Cannot replace uses of with self"); 5669 5670 // Iterate over all the existing uses of From. New uses will be added 5671 // to the beginning of the use list, which we avoid visiting. 5672 // This specifically avoids visiting uses of From that arise while the 5673 // replacement is happening, because any such uses would be the result 5674 // of CSE: If an existing node looks like From after one of its operands 5675 // is replaced by To, we don't want to replace of all its users with To 5676 // too. See PR3018 for more info. 5677 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5678 RAUWUpdateListener Listener(*this, UI, UE); 5679 while (UI != UE) { 5680 SDNode *User = *UI; 5681 5682 // This node is about to morph, remove its old self from the CSE maps. 5683 RemoveNodeFromCSEMaps(User); 5684 5685 // A user can appear in a use list multiple times, and when this 5686 // happens the uses are usually next to each other in the list. 5687 // To help reduce the number of CSE recomputations, process all 5688 // the uses of this user that we can find this way. 5689 do { 5690 SDUse &Use = UI.getUse(); 5691 ++UI; 5692 Use.set(To); 5693 } while (UI != UE && *UI == User); 5694 5695 // Now that we have modified User, add it back to the CSE maps. If it 5696 // already exists there, recursively merge the results together. 5697 AddModifiedNodeToCSEMaps(User); 5698 } 5699 5700 // If we just RAUW'd the root, take note. 5701 if (FromN == getRoot()) 5702 setRoot(To); 5703} 5704 5705/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5706/// This can cause recursive merging of nodes in the DAG. 5707/// 5708/// This version assumes that for each value of From, there is a 5709/// corresponding value in To in the same position with the same type. 5710/// 5711void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 5712#ifndef NDEBUG 5713 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5714 assert((!From->hasAnyUseOfValue(i) || 5715 From->getValueType(i) == To->getValueType(i)) && 5716 "Cannot use this version of ReplaceAllUsesWith!"); 5717#endif 5718 5719 // Handle the trivial case. 5720 if (From == To) 5721 return; 5722 5723 // Iterate over just the existing users of From. See the comments in 5724 // the ReplaceAllUsesWith above. 5725 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5726 RAUWUpdateListener Listener(*this, UI, UE); 5727 while (UI != UE) { 5728 SDNode *User = *UI; 5729 5730 // This node is about to morph, remove its old self from the CSE maps. 5731 RemoveNodeFromCSEMaps(User); 5732 5733 // A user can appear in a use list multiple times, and when this 5734 // happens the uses are usually next to each other in the list. 5735 // To help reduce the number of CSE recomputations, process all 5736 // the uses of this user that we can find this way. 5737 do { 5738 SDUse &Use = UI.getUse(); 5739 ++UI; 5740 Use.setNode(To); 5741 } while (UI != UE && *UI == User); 5742 5743 // Now that we have modified User, add it back to the CSE maps. If it 5744 // already exists there, recursively merge the results together. 5745 AddModifiedNodeToCSEMaps(User); 5746 } 5747 5748 // If we just RAUW'd the root, take note. 5749 if (From == getRoot().getNode()) 5750 setRoot(SDValue(To, getRoot().getResNo())); 5751} 5752 5753/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5754/// This can cause recursive merging of nodes in the DAG. 5755/// 5756/// This version can replace From with any result values. To must match the 5757/// number and types of values returned by From. 5758void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 5759 if (From->getNumValues() == 1) // Handle the simple case efficiently. 5760 return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 5761 5762 // Iterate over just the existing users of From. See the comments in 5763 // the ReplaceAllUsesWith above. 5764 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5765 RAUWUpdateListener Listener(*this, UI, UE); 5766 while (UI != UE) { 5767 SDNode *User = *UI; 5768 5769 // This node is about to morph, remove its old self from the CSE maps. 5770 RemoveNodeFromCSEMaps(User); 5771 5772 // A user can appear in a use list multiple times, and when this 5773 // happens the uses are usually next to each other in the list. 5774 // To help reduce the number of CSE recomputations, process all 5775 // the uses of this user that we can find this way. 5776 do { 5777 SDUse &Use = UI.getUse(); 5778 const SDValue &ToOp = To[Use.getResNo()]; 5779 ++UI; 5780 Use.set(ToOp); 5781 } while (UI != UE && *UI == User); 5782 5783 // Now that we have modified User, add it back to the CSE maps. If it 5784 // already exists there, recursively merge the results together. 5785 AddModifiedNodeToCSEMaps(User); 5786 } 5787 5788 // If we just RAUW'd the root, take note. 5789 if (From == getRoot().getNode()) 5790 setRoot(SDValue(To[getRoot().getResNo()])); 5791} 5792 5793/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5794/// uses of other values produced by From.getNode() alone. The Deleted 5795/// vector is handled the same way as for ReplaceAllUsesWith. 5796void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 5797 // Handle the really simple, really trivial case efficiently. 5798 if (From == To) return; 5799 5800 // Handle the simple, trivial, case efficiently. 5801 if (From.getNode()->getNumValues() == 1) { 5802 ReplaceAllUsesWith(From, To); 5803 return; 5804 } 5805 5806 // Iterate over just the existing users of From. See the comments in 5807 // the ReplaceAllUsesWith above. 5808 SDNode::use_iterator UI = From.getNode()->use_begin(), 5809 UE = From.getNode()->use_end(); 5810 RAUWUpdateListener Listener(*this, UI, UE); 5811 while (UI != UE) { 5812 SDNode *User = *UI; 5813 bool UserRemovedFromCSEMaps = false; 5814 5815 // A user can appear in a use list multiple times, and when this 5816 // happens the uses are usually next to each other in the list. 5817 // To help reduce the number of CSE recomputations, process all 5818 // the uses of this user that we can find this way. 5819 do { 5820 SDUse &Use = UI.getUse(); 5821 5822 // Skip uses of different values from the same node. 5823 if (Use.getResNo() != From.getResNo()) { 5824 ++UI; 5825 continue; 5826 } 5827 5828 // If this node hasn't been modified yet, it's still in the CSE maps, 5829 // so remove its old self from the CSE maps. 5830 if (!UserRemovedFromCSEMaps) { 5831 RemoveNodeFromCSEMaps(User); 5832 UserRemovedFromCSEMaps = true; 5833 } 5834 5835 ++UI; 5836 Use.set(To); 5837 } while (UI != UE && *UI == User); 5838 5839 // We are iterating over all uses of the From node, so if a use 5840 // doesn't use the specific value, no changes are made. 5841 if (!UserRemovedFromCSEMaps) 5842 continue; 5843 5844 // Now that we have modified User, add it back to the CSE maps. If it 5845 // already exists there, recursively merge the results together. 5846 AddModifiedNodeToCSEMaps(User); 5847 } 5848 5849 // If we just RAUW'd the root, take note. 5850 if (From == getRoot()) 5851 setRoot(To); 5852} 5853 5854namespace { 5855 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5856 /// to record information about a use. 5857 struct UseMemo { 5858 SDNode *User; 5859 unsigned Index; 5860 SDUse *Use; 5861 }; 5862 5863 /// operator< - Sort Memos by User. 5864 bool operator<(const UseMemo &L, const UseMemo &R) { 5865 return (intptr_t)L.User < (intptr_t)R.User; 5866 } 5867} 5868 5869/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5870/// uses of other values produced by From.getNode() alone. The same value 5871/// may appear in both the From and To list. The Deleted vector is 5872/// handled the same way as for ReplaceAllUsesWith. 5873void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5874 const SDValue *To, 5875 unsigned Num){ 5876 // Handle the simple, trivial case efficiently. 5877 if (Num == 1) 5878 return ReplaceAllUsesOfValueWith(*From, *To); 5879 5880 // Read up all the uses and make records of them. This helps 5881 // processing new uses that are introduced during the 5882 // replacement process. 5883 SmallVector<UseMemo, 4> Uses; 5884 for (unsigned i = 0; i != Num; ++i) { 5885 unsigned FromResNo = From[i].getResNo(); 5886 SDNode *FromNode = From[i].getNode(); 5887 for (SDNode::use_iterator UI = FromNode->use_begin(), 5888 E = FromNode->use_end(); UI != E; ++UI) { 5889 SDUse &Use = UI.getUse(); 5890 if (Use.getResNo() == FromResNo) { 5891 UseMemo Memo = { *UI, i, &Use }; 5892 Uses.push_back(Memo); 5893 } 5894 } 5895 } 5896 5897 // Sort the uses, so that all the uses from a given User are together. 5898 std::sort(Uses.begin(), Uses.end()); 5899 5900 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5901 UseIndex != UseIndexEnd; ) { 5902 // We know that this user uses some value of From. If it is the right 5903 // value, update it. 5904 SDNode *User = Uses[UseIndex].User; 5905 5906 // This node is about to morph, remove its old self from the CSE maps. 5907 RemoveNodeFromCSEMaps(User); 5908 5909 // The Uses array is sorted, so all the uses for a given User 5910 // are next to each other in the list. 5911 // To help reduce the number of CSE recomputations, process all 5912 // the uses of this user that we can find this way. 5913 do { 5914 unsigned i = Uses[UseIndex].Index; 5915 SDUse &Use = *Uses[UseIndex].Use; 5916 ++UseIndex; 5917 5918 Use.set(To[i]); 5919 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5920 5921 // Now that we have modified User, add it back to the CSE maps. If it 5922 // already exists there, recursively merge the results together. 5923 AddModifiedNodeToCSEMaps(User); 5924 } 5925} 5926 5927/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5928/// based on their topological order. It returns the maximum id and a vector 5929/// of the SDNodes* in assigned order by reference. 5930unsigned SelectionDAG::AssignTopologicalOrder() { 5931 5932 unsigned DAGSize = 0; 5933 5934 // SortedPos tracks the progress of the algorithm. Nodes before it are 5935 // sorted, nodes after it are unsorted. When the algorithm completes 5936 // it is at the end of the list. 5937 allnodes_iterator SortedPos = allnodes_begin(); 5938 5939 // Visit all the nodes. Move nodes with no operands to the front of 5940 // the list immediately. Annotate nodes that do have operands with their 5941 // operand count. Before we do this, the Node Id fields of the nodes 5942 // may contain arbitrary values. After, the Node Id fields for nodes 5943 // before SortedPos will contain the topological sort index, and the 5944 // Node Id fields for nodes At SortedPos and after will contain the 5945 // count of outstanding operands. 5946 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5947 SDNode *N = I++; 5948 checkForCycles(N); 5949 unsigned Degree = N->getNumOperands(); 5950 if (Degree == 0) { 5951 // A node with no uses, add it to the result array immediately. 5952 N->setNodeId(DAGSize++); 5953 allnodes_iterator Q = N; 5954 if (Q != SortedPos) 5955 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5956 assert(SortedPos != AllNodes.end() && "Overran node list"); 5957 ++SortedPos; 5958 } else { 5959 // Temporarily use the Node Id as scratch space for the degree count. 5960 N->setNodeId(Degree); 5961 } 5962 } 5963 5964 // Visit all the nodes. As we iterate, move nodes into sorted order, 5965 // such that by the time the end is reached all nodes will be sorted. 5966 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5967 SDNode *N = I; 5968 checkForCycles(N); 5969 // N is in sorted position, so all its uses have one less operand 5970 // that needs to be sorted. 5971 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5972 UI != UE; ++UI) { 5973 SDNode *P = *UI; 5974 unsigned Degree = P->getNodeId(); 5975 assert(Degree != 0 && "Invalid node degree"); 5976 --Degree; 5977 if (Degree == 0) { 5978 // All of P's operands are sorted, so P may sorted now. 5979 P->setNodeId(DAGSize++); 5980 if (P != SortedPos) 5981 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5982 assert(SortedPos != AllNodes.end() && "Overran node list"); 5983 ++SortedPos; 5984 } else { 5985 // Update P's outstanding operand count. 5986 P->setNodeId(Degree); 5987 } 5988 } 5989 if (I == SortedPos) { 5990#ifndef NDEBUG 5991 SDNode *S = ++I; 5992 dbgs() << "Overran sorted position:\n"; 5993 S->dumprFull(); 5994#endif 5995 llvm_unreachable(0); 5996 } 5997 } 5998 5999 assert(SortedPos == AllNodes.end() && 6000 "Topological sort incomplete!"); 6001 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 6002 "First node in topological sort is not the entry token!"); 6003 assert(AllNodes.front().getNodeId() == 0 && 6004 "First node in topological sort has non-zero id!"); 6005 assert(AllNodes.front().getNumOperands() == 0 && 6006 "First node in topological sort has operands!"); 6007 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 6008 "Last node in topologic sort has unexpected id!"); 6009 assert(AllNodes.back().use_empty() && 6010 "Last node in topologic sort has users!"); 6011 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 6012 return DAGSize; 6013} 6014 6015/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 6016/// value is produced by SD. 6017void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 6018 DbgInfo->add(DB, SD, isParameter); 6019 if (SD) 6020 SD->setHasDebugValue(true); 6021} 6022 6023/// TransferDbgValues - Transfer SDDbgValues. 6024void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 6025 if (From == To || !From.getNode()->getHasDebugValue()) 6026 return; 6027 SDNode *FromNode = From.getNode(); 6028 SDNode *ToNode = To.getNode(); 6029 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 6030 SmallVector<SDDbgValue *, 2> ClonedDVs; 6031 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 6032 I != E; ++I) { 6033 SDDbgValue *Dbg = *I; 6034 if (Dbg->getKind() == SDDbgValue::SDNODE) { 6035 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 6036 Dbg->getOffset(), Dbg->getDebugLoc(), 6037 Dbg->getOrder()); 6038 ClonedDVs.push_back(Clone); 6039 } 6040 } 6041 for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(), 6042 E = ClonedDVs.end(); I != E; ++I) 6043 AddDbgValue(*I, ToNode, false); 6044} 6045 6046//===----------------------------------------------------------------------===// 6047// SDNode Class 6048//===----------------------------------------------------------------------===// 6049 6050HandleSDNode::~HandleSDNode() { 6051 DropOperands(); 6052} 6053 6054GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 6055 DebugLoc DL, const GlobalValue *GA, 6056 EVT VT, int64_t o, unsigned char TF) 6057 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 6058 TheGlobal = GA; 6059} 6060 6061AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, 6062 SDValue X, unsigned SrcAS, 6063 unsigned DestAS) 6064 : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), 6065 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 6066 6067MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6068 EVT memvt, MachineMemOperand *mmo) 6069 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 6070 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6071 MMO->isNonTemporal(), MMO->isInvariant()); 6072 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6073 assert(isNonTemporal() == MMO->isNonTemporal() && 6074 "Non-temporal encoding error!"); 6075 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6076} 6077 6078MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6079 const SDValue *Ops, unsigned NumOps, EVT memvt, 6080 MachineMemOperand *mmo) 6081 : SDNode(Opc, Order, dl, VTs, Ops, NumOps), 6082 MemoryVT(memvt), MMO(mmo) { 6083 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6084 MMO->isNonTemporal(), MMO->isInvariant()); 6085 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6086 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6087} 6088 6089/// Profile - Gather unique data for the node. 6090/// 6091void SDNode::Profile(FoldingSetNodeID &ID) const { 6092 AddNodeIDNode(ID, this); 6093} 6094 6095namespace { 6096 struct EVTArray { 6097 std::vector<EVT> VTs; 6098 6099 EVTArray() { 6100 VTs.reserve(MVT::LAST_VALUETYPE); 6101 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 6102 VTs.push_back(MVT((MVT::SimpleValueType)i)); 6103 } 6104 }; 6105} 6106 6107static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 6108static ManagedStatic<EVTArray> SimpleVTArray; 6109static ManagedStatic<sys::SmartMutex<true> > VTMutex; 6110 6111/// getValueTypeList - Return a pointer to the specified value type. 6112/// 6113const EVT *SDNode::getValueTypeList(EVT VT) { 6114 if (VT.isExtended()) { 6115 sys::SmartScopedLock<true> Lock(*VTMutex); 6116 return &(*EVTs->insert(VT).first); 6117 } else { 6118 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 6119 "Value type out of range!"); 6120 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 6121 } 6122} 6123 6124/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 6125/// indicated value. This method ignores uses of other values defined by this 6126/// operation. 6127bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 6128 assert(Value < getNumValues() && "Bad value!"); 6129 6130 // TODO: Only iterate over uses of a given value of the node 6131 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 6132 if (UI.getUse().getResNo() == Value) { 6133 if (NUses == 0) 6134 return false; 6135 --NUses; 6136 } 6137 } 6138 6139 // Found exactly the right number of uses? 6140 return NUses == 0; 6141} 6142 6143 6144/// hasAnyUseOfValue - Return true if there are any use of the indicated 6145/// value. This method ignores uses of other values defined by this operation. 6146bool SDNode::hasAnyUseOfValue(unsigned Value) const { 6147 assert(Value < getNumValues() && "Bad value!"); 6148 6149 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 6150 if (UI.getUse().getResNo() == Value) 6151 return true; 6152 6153 return false; 6154} 6155 6156 6157/// isOnlyUserOf - Return true if this node is the only use of N. 6158/// 6159bool SDNode::isOnlyUserOf(SDNode *N) const { 6160 bool Seen = false; 6161 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 6162 SDNode *User = *I; 6163 if (User == this) 6164 Seen = true; 6165 else 6166 return false; 6167 } 6168 6169 return Seen; 6170} 6171 6172/// isOperand - Return true if this node is an operand of N. 6173/// 6174bool SDValue::isOperandOf(SDNode *N) const { 6175 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6176 if (*this == N->getOperand(i)) 6177 return true; 6178 return false; 6179} 6180 6181bool SDNode::isOperandOf(SDNode *N) const { 6182 for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 6183 if (this == N->OperandList[i].getNode()) 6184 return true; 6185 return false; 6186} 6187 6188/// reachesChainWithoutSideEffects - Return true if this operand (which must 6189/// be a chain) reaches the specified operand without crossing any 6190/// side-effecting instructions on any chain path. In practice, this looks 6191/// through token factors and non-volatile loads. In order to remain efficient, 6192/// this only looks a couple of nodes in, it does not do an exhaustive search. 6193bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 6194 unsigned Depth) const { 6195 if (*this == Dest) return true; 6196 6197 // Don't search too deeply, we just want to be able to see through 6198 // TokenFactor's etc. 6199 if (Depth == 0) return false; 6200 6201 // If this is a token factor, all inputs to the TF happen in parallel. If any 6202 // of the operands of the TF does not reach dest, then we cannot do the xform. 6203 if (getOpcode() == ISD::TokenFactor) { 6204 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6205 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 6206 return false; 6207 return true; 6208 } 6209 6210 // Loads don't have side effects, look through them. 6211 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 6212 if (!Ld->isVolatile()) 6213 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 6214 } 6215 return false; 6216} 6217 6218/// hasPredecessor - Return true if N is a predecessor of this node. 6219/// N is either an operand of this node, or can be reached by recursively 6220/// traversing up the operands. 6221/// NOTE: This is an expensive method. Use it carefully. 6222bool SDNode::hasPredecessor(const SDNode *N) const { 6223 SmallPtrSet<const SDNode *, 32> Visited; 6224 SmallVector<const SDNode *, 16> Worklist; 6225 return hasPredecessorHelper(N, Visited, Worklist); 6226} 6227 6228bool 6229SDNode::hasPredecessorHelper(const SDNode *N, 6230 SmallPtrSet<const SDNode *, 32> &Visited, 6231 SmallVectorImpl<const SDNode *> &Worklist) const { 6232 if (Visited.empty()) { 6233 Worklist.push_back(this); 6234 } else { 6235 // Take a look in the visited set. If we've already encountered this node 6236 // we needn't search further. 6237 if (Visited.count(N)) 6238 return true; 6239 } 6240 6241 // Haven't visited N yet. Continue the search. 6242 while (!Worklist.empty()) { 6243 const SDNode *M = Worklist.pop_back_val(); 6244 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 6245 SDNode *Op = M->getOperand(i).getNode(); 6246 if (Visited.insert(Op)) 6247 Worklist.push_back(Op); 6248 if (Op == N) 6249 return true; 6250 } 6251 } 6252 6253 return false; 6254} 6255 6256uint64_t SDNode::getConstantOperandVal(unsigned Num) const { 6257 assert(Num < NumOperands && "Invalid child # of SDNode!"); 6258 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 6259} 6260 6261SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6262 assert(N->getNumValues() == 1 && 6263 "Can't unroll a vector with multiple results!"); 6264 6265 EVT VT = N->getValueType(0); 6266 unsigned NE = VT.getVectorNumElements(); 6267 EVT EltVT = VT.getVectorElementType(); 6268 SDLoc dl(N); 6269 6270 SmallVector<SDValue, 8> Scalars; 6271 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6272 6273 // If ResNE is 0, fully unroll the vector op. 6274 if (ResNE == 0) 6275 ResNE = NE; 6276 else if (NE > ResNE) 6277 NE = ResNE; 6278 6279 unsigned i; 6280 for (i= 0; i != NE; ++i) { 6281 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6282 SDValue Operand = N->getOperand(j); 6283 EVT OperandVT = Operand.getValueType(); 6284 if (OperandVT.isVector()) { 6285 // A vector operand; extract a single element. 6286 const TargetLowering *TLI = TM.getTargetLowering(); 6287 EVT OperandEltVT = OperandVT.getVectorElementType(); 6288 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6289 OperandEltVT, 6290 Operand, 6291 getConstant(i, TLI->getVectorIdxTy())); 6292 } else { 6293 // A scalar operand; just use it as is. 6294 Operands[j] = Operand; 6295 } 6296 } 6297 6298 switch (N->getOpcode()) { 6299 default: 6300 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6301 &Operands[0], Operands.size())); 6302 break; 6303 case ISD::VSELECT: 6304 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6305 &Operands[0], Operands.size())); 6306 break; 6307 case ISD::SHL: 6308 case ISD::SRA: 6309 case ISD::SRL: 6310 case ISD::ROTL: 6311 case ISD::ROTR: 6312 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6313 getShiftAmountOperand(Operands[0].getValueType(), 6314 Operands[1]))); 6315 break; 6316 case ISD::SIGN_EXTEND_INREG: 6317 case ISD::FP_ROUND_INREG: { 6318 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6319 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6320 Operands[0], 6321 getValueType(ExtVT))); 6322 } 6323 } 6324 } 6325 6326 for (; i < ResNE; ++i) 6327 Scalars.push_back(getUNDEF(EltVT)); 6328 6329 return getNode(ISD::BUILD_VECTOR, dl, 6330 EVT::getVectorVT(*getContext(), EltVT, ResNE), 6331 &Scalars[0], Scalars.size()); 6332} 6333 6334 6335/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6336/// location that is 'Dist' units away from the location that the 'Base' load 6337/// is loading from. 6338bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6339 unsigned Bytes, int Dist) const { 6340 if (LD->getChain() != Base->getChain()) 6341 return false; 6342 EVT VT = LD->getValueType(0); 6343 if (VT.getSizeInBits() / 8 != Bytes) 6344 return false; 6345 6346 SDValue Loc = LD->getOperand(1); 6347 SDValue BaseLoc = Base->getOperand(1); 6348 if (Loc.getOpcode() == ISD::FrameIndex) { 6349 if (BaseLoc.getOpcode() != ISD::FrameIndex) 6350 return false; 6351 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6352 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6353 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6354 int FS = MFI->getObjectSize(FI); 6355 int BFS = MFI->getObjectSize(BFI); 6356 if (FS != BFS || FS != (int)Bytes) return false; 6357 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6358 } 6359 6360 // Handle X+C 6361 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6362 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6363 return true; 6364 6365 const GlobalValue *GV1 = NULL; 6366 const GlobalValue *GV2 = NULL; 6367 int64_t Offset1 = 0; 6368 int64_t Offset2 = 0; 6369 const TargetLowering *TLI = TM.getTargetLowering(); 6370 bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6371 bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6372 if (isGA1 && isGA2 && GV1 == GV2) 6373 return Offset1 == (Offset2 + Dist*Bytes); 6374 return false; 6375} 6376 6377 6378/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6379/// it cannot be inferred. 6380unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6381 // If this is a GlobalAddress + cst, return the alignment. 6382 const GlobalValue *GV; 6383 int64_t GVOffset = 0; 6384 const TargetLowering *TLI = TM.getTargetLowering(); 6385 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6386 unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 6387 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6388 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 6389 TLI->getDataLayout()); 6390 unsigned AlignBits = KnownZero.countTrailingOnes(); 6391 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6392 if (Align) 6393 return MinAlign(Align, GVOffset); 6394 } 6395 6396 // If this is a direct reference to a stack slot, use information about the 6397 // stack slot's alignment. 6398 int FrameIdx = 1 << 31; 6399 int64_t FrameOffset = 0; 6400 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6401 FrameIdx = FI->getIndex(); 6402 } else if (isBaseWithConstantOffset(Ptr) && 6403 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6404 // Handle FI+Cst 6405 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6406 FrameOffset = Ptr.getConstantOperandVal(1); 6407 } 6408 6409 if (FrameIdx != (1 << 31)) { 6410 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6411 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6412 FrameOffset); 6413 return FIInfoAlign; 6414 } 6415 6416 return 0; 6417} 6418 6419/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 6420/// which is split (or expanded) into two not necessarily identical pieces. 6421std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 6422 // Currently all types are split in half. 6423 EVT LoVT, HiVT; 6424 if (!VT.isVector()) { 6425 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 6426 } else { 6427 unsigned NumElements = VT.getVectorNumElements(); 6428 assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 6429 LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), 6430 NumElements/2); 6431 } 6432 return std::make_pair(LoVT, HiVT); 6433} 6434 6435/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 6436/// low/high part. 6437std::pair<SDValue, SDValue> 6438SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 6439 const EVT &HiVT) { 6440 assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <= 6441 N.getValueType().getVectorNumElements() && 6442 "More vector elements requested than available!"); 6443 SDValue Lo, Hi; 6444 Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, 6445 getConstant(0, TLI->getVectorIdxTy())); 6446 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 6447 getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy())); 6448 return std::make_pair(Lo, Hi); 6449} 6450 6451// getAddressSpace - Return the address space this GlobalAddress belongs to. 6452unsigned GlobalAddressSDNode::getAddressSpace() const { 6453 return getGlobal()->getType()->getAddressSpace(); 6454} 6455 6456 6457Type *ConstantPoolSDNode::getType() const { 6458 if (isMachineConstantPoolEntry()) 6459 return Val.MachineCPVal->getType(); 6460 return Val.ConstVal->getType(); 6461} 6462 6463bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6464 APInt &SplatUndef, 6465 unsigned &SplatBitSize, 6466 bool &HasAnyUndefs, 6467 unsigned MinSplatBits, 6468 bool isBigEndian) { 6469 EVT VT = getValueType(0); 6470 assert(VT.isVector() && "Expected a vector type"); 6471 unsigned sz = VT.getSizeInBits(); 6472 if (MinSplatBits > sz) 6473 return false; 6474 6475 SplatValue = APInt(sz, 0); 6476 SplatUndef = APInt(sz, 0); 6477 6478 // Get the bits. Bits with undefined values (when the corresponding element 6479 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6480 // in SplatValue. If any of the values are not constant, give up and return 6481 // false. 6482 unsigned int nOps = getNumOperands(); 6483 assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6484 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6485 6486 for (unsigned j = 0; j < nOps; ++j) { 6487 unsigned i = isBigEndian ? nOps-1-j : j; 6488 SDValue OpVal = getOperand(i); 6489 unsigned BitPos = j * EltBitSize; 6490 6491 if (OpVal.getOpcode() == ISD::UNDEF) 6492 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6493 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6494 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6495 zextOrTrunc(sz) << BitPos; 6496 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6497 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6498 else 6499 return false; 6500 } 6501 6502 // The build_vector is all constants or undefs. Find the smallest element 6503 // size that splats the vector. 6504 6505 HasAnyUndefs = (SplatUndef != 0); 6506 while (sz > 8) { 6507 6508 unsigned HalfSize = sz / 2; 6509 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6510 APInt LowValue = SplatValue.trunc(HalfSize); 6511 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6512 APInt LowUndef = SplatUndef.trunc(HalfSize); 6513 6514 // If the two halves do not match (ignoring undef bits), stop here. 6515 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6516 MinSplatBits > HalfSize) 6517 break; 6518 6519 SplatValue = HighValue | LowValue; 6520 SplatUndef = HighUndef & LowUndef; 6521 6522 sz = HalfSize; 6523 } 6524 6525 SplatBitSize = sz; 6526 return true; 6527} 6528 6529bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6530 // Find the first non-undef value in the shuffle mask. 6531 unsigned i, e; 6532 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6533 /* search */; 6534 6535 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6536 6537 // Make sure all remaining elements are either undef or the same as the first 6538 // non-undef value. 6539 for (int Idx = Mask[i]; i != e; ++i) 6540 if (Mask[i] >= 0 && Mask[i] != Idx) 6541 return false; 6542 return true; 6543} 6544 6545#ifdef XDEBUG 6546static void checkForCyclesHelper(const SDNode *N, 6547 SmallPtrSet<const SDNode*, 32> &Visited, 6548 SmallPtrSet<const SDNode*, 32> &Checked) { 6549 // If this node has already been checked, don't check it again. 6550 if (Checked.count(N)) 6551 return; 6552 6553 // If a node has already been visited on this depth-first walk, reject it as 6554 // a cycle. 6555 if (!Visited.insert(N)) { 6556 dbgs() << "Offending node:\n"; 6557 N->dumprFull(); 6558 errs() << "Detected cycle in SelectionDAG\n"; 6559 abort(); 6560 } 6561 6562 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6563 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6564 6565 Checked.insert(N); 6566 Visited.erase(N); 6567} 6568#endif 6569 6570void llvm::checkForCycles(const llvm::SDNode *N) { 6571#ifdef XDEBUG 6572 assert(N && "Checking nonexistent SDNode"); 6573 SmallPtrSet<const SDNode*, 32> visited; 6574 SmallPtrSet<const SDNode*, 32> checked; 6575 checkForCyclesHelper(N, visited, checked); 6576#endif 6577} 6578 6579void llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6580 checkForCycles(DAG->getRoot().getNode()); 6581} 6582