1193323Sed//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This implements the SelectionDAG class. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13201360Srdivacky 14193323Sed#include "llvm/CodeGen/SelectionDAG.h" 15249423Sdim#include "SDNodeDbgValue.h" 16249423Sdim#include "llvm/ADT/SetVector.h" 17249423Sdim#include "llvm/ADT/SmallPtrSet.h" 18249423Sdim#include "llvm/ADT/SmallSet.h" 19249423Sdim#include "llvm/ADT/SmallVector.h" 20249423Sdim#include "llvm/ADT/StringExtras.h" 21249423Sdim#include "llvm/Analysis/TargetTransformInfo.h" 22239462Sdim#include "llvm/Analysis/ValueTracking.h" 23193323Sed#include "llvm/Assembly/Writer.h" 24193323Sed#include "llvm/CodeGen/MachineBasicBlock.h" 25193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 26193323Sed#include "llvm/CodeGen/MachineFrameInfo.h" 27193323Sed#include "llvm/CodeGen/MachineModuleInfo.h" 28249423Sdim#include "llvm/DebugInfo.h" 29249423Sdim#include "llvm/IR/CallingConv.h" 30249423Sdim#include "llvm/IR/Constants.h" 31249423Sdim#include "llvm/IR/DataLayout.h" 32249423Sdim#include "llvm/IR/DerivedTypes.h" 33249423Sdim#include "llvm/IR/Function.h" 34249423Sdim#include "llvm/IR/GlobalAlias.h" 35249423Sdim#include "llvm/IR/GlobalVariable.h" 36249423Sdim#include "llvm/IR/Intrinsics.h" 37193323Sed#include "llvm/Support/CommandLine.h" 38202375Srdivacky#include "llvm/Support/Debug.h" 39198090Srdivacky#include "llvm/Support/ErrorHandling.h" 40195098Sed#include "llvm/Support/ManagedStatic.h" 41193323Sed#include "llvm/Support/MathExtras.h" 42249423Sdim#include "llvm/Support/Mutex.h" 43193323Sed#include "llvm/Support/raw_ostream.h" 44249423Sdim#include "llvm/Target/TargetInstrInfo.h" 45249423Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 46249423Sdim#include "llvm/Target/TargetLowering.h" 47249423Sdim#include "llvm/Target/TargetMachine.h" 48249423Sdim#include "llvm/Target/TargetOptions.h" 49249423Sdim#include "llvm/Target/TargetRegisterInfo.h" 50249423Sdim#include "llvm/Target/TargetSelectionDAGInfo.h" 51193323Sed#include <algorithm> 52193323Sed#include <cmath> 53193323Sedusing namespace llvm; 54193323Sed 55193323Sed/// makeVTList - Return an instance of the SDVTList struct initialized with the 56193323Sed/// specified members. 57198090Srdivackystatic SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58193323Sed SDVTList Res = {VTs, NumVTs}; 59193323Sed return Res; 60193323Sed} 61193323Sed 62239462Sdim// Default null implementations of the callbacks. 63239462Sdimvoid SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 64239462Sdimvoid SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 65193323Sed 66193323Sed//===----------------------------------------------------------------------===// 67193323Sed// ConstantFPSDNode Class 68193323Sed//===----------------------------------------------------------------------===// 69193323Sed 70193323Sed/// isExactlyValue - We don't rely on operator== working on double values, as 71193323Sed/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 72193323Sed/// As such, this method can be used to do an exact bit-for-bit comparison of 73193323Sed/// two floating point values. 74193323Sedbool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 75193323Sed return getValueAPF().bitwiseIsEqual(V); 76193323Sed} 77193323Sed 78198090Srdivackybool ConstantFPSDNode::isValueValidForType(EVT VT, 79193323Sed const APFloat& Val) { 80193323Sed assert(VT.isFloatingPoint() && "Can only convert between FP types"); 81193323Sed 82193323Sed // convert modifies in place, so make a copy. 83193323Sed APFloat Val2 = APFloat(Val); 84193323Sed bool losesInfo; 85249423Sdim (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 86249423Sdim APFloat::rmNearestTiesToEven, 87193323Sed &losesInfo); 88193323Sed return !losesInfo; 89193323Sed} 90193323Sed 91193323Sed//===----------------------------------------------------------------------===// 92193323Sed// ISD Namespace 93193323Sed//===----------------------------------------------------------------------===// 94193323Sed 95193323Sed/// isBuildVectorAllOnes - Return true if the specified node is a 96193323Sed/// BUILD_VECTOR where all of the elements are ~0 or undef. 97193323Sedbool ISD::isBuildVectorAllOnes(const SDNode *N) { 98193323Sed // Look through a bit convert. 99218893Sdim if (N->getOpcode() == ISD::BITCAST) 100193323Sed N = N->getOperand(0).getNode(); 101193323Sed 102193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 103193323Sed 104193323Sed unsigned i = 0, e = N->getNumOperands(); 105193323Sed 106193323Sed // Skip over all of the undef values. 107193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 108193323Sed ++i; 109193323Sed 110193323Sed // Do not accept an all-undef vector. 111193323Sed if (i == e) return false; 112193323Sed 113193323Sed // Do not accept build_vectors that aren't all constants or which have non-~0 114234353Sdim // elements. We have to be a bit careful here, as the type of the constant 115234353Sdim // may not be the same as the type of the vector elements due to type 116234353Sdim // legalization (the elements are promoted to a legal type for the target and 117234353Sdim // a vector of a type may be legal when the base element type is not). 118234353Sdim // We only want to check enough bits to cover the vector elements, because 119234353Sdim // we care if the resultant vector is all ones, not whether the individual 120234353Sdim // constants are. 121193323Sed SDValue NotZero = N->getOperand(i); 122234353Sdim unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 123243830Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 124243830Sdim if (CN->getAPIntValue().countTrailingOnes() < EltSize) 125193323Sed return false; 126243830Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 127243830Sdim if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 128193323Sed return false; 129193323Sed } else 130193323Sed return false; 131193323Sed 132193323Sed // Okay, we have at least one ~0 value, check to see if the rest match or are 133234353Sdim // undefs. Even with the above element type twiddling, this should be OK, as 134234353Sdim // the same type legalization should have applied to all the elements. 135193323Sed for (++i; i != e; ++i) 136193323Sed if (N->getOperand(i) != NotZero && 137193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 138193323Sed return false; 139193323Sed return true; 140193323Sed} 141193323Sed 142193323Sed 143193323Sed/// isBuildVectorAllZeros - Return true if the specified node is a 144193323Sed/// BUILD_VECTOR where all of the elements are 0 or undef. 145193323Sedbool ISD::isBuildVectorAllZeros(const SDNode *N) { 146193323Sed // Look through a bit convert. 147218893Sdim if (N->getOpcode() == ISD::BITCAST) 148193323Sed N = N->getOperand(0).getNode(); 149193323Sed 150193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 151193323Sed 152193323Sed unsigned i = 0, e = N->getNumOperands(); 153193323Sed 154193323Sed // Skip over all of the undef values. 155193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 156193323Sed ++i; 157193323Sed 158193323Sed // Do not accept an all-undef vector. 159193323Sed if (i == e) return false; 160193323Sed 161193574Sed // Do not accept build_vectors that aren't all constants or which have non-0 162193323Sed // elements. 163193323Sed SDValue Zero = N->getOperand(i); 164243830Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 165243830Sdim if (!CN->isNullValue()) 166193323Sed return false; 167243830Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 168243830Sdim if (!CFPN->getValueAPF().isPosZero()) 169193323Sed return false; 170193323Sed } else 171193323Sed return false; 172193323Sed 173193574Sed // Okay, we have at least one 0 value, check to see if the rest match or are 174193323Sed // undefs. 175193323Sed for (++i; i != e; ++i) 176193323Sed if (N->getOperand(i) != Zero && 177193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 178193323Sed return false; 179193323Sed return true; 180193323Sed} 181193323Sed 182193323Sed/// isScalarToVector - Return true if the specified node is a 183193323Sed/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 184193323Sed/// element is not an undef. 185193323Sedbool ISD::isScalarToVector(const SDNode *N) { 186193323Sed if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 187193323Sed return true; 188193323Sed 189193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) 190193323Sed return false; 191193323Sed if (N->getOperand(0).getOpcode() == ISD::UNDEF) 192193323Sed return false; 193193323Sed unsigned NumElems = N->getNumOperands(); 194218893Sdim if (NumElems == 1) 195218893Sdim return false; 196193323Sed for (unsigned i = 1; i < NumElems; ++i) { 197193323Sed SDValue V = N->getOperand(i); 198193323Sed if (V.getOpcode() != ISD::UNDEF) 199193323Sed return false; 200193323Sed } 201193323Sed return true; 202193323Sed} 203193323Sed 204239462Sdim/// allOperandsUndef - Return true if the node has at least one operand 205239462Sdim/// and all operands of the specified node are ISD::UNDEF. 206239462Sdimbool ISD::allOperandsUndef(const SDNode *N) { 207239462Sdim // Return false if the node has no operands. 208239462Sdim // This is "logically inconsistent" with the definition of "all" but 209239462Sdim // is probably the desired behavior. 210239462Sdim if (N->getNumOperands() == 0) 211239462Sdim return false; 212239462Sdim 213239462Sdim for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 214239462Sdim if (N->getOperand(i).getOpcode() != ISD::UNDEF) 215239462Sdim return false; 216239462Sdim 217239462Sdim return true; 218239462Sdim} 219239462Sdim 220193323Sed/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 221193323Sed/// when given the operation for (X op Y). 222193323SedISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 223193323Sed // To perform this operation, we just need to swap the L and G bits of the 224193323Sed // operation. 225193323Sed unsigned OldL = (Operation >> 2) & 1; 226193323Sed unsigned OldG = (Operation >> 1) & 1; 227193323Sed return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 228193323Sed (OldL << 1) | // New G bit 229193323Sed (OldG << 2)); // New L bit. 230193323Sed} 231193323Sed 232193323Sed/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 233193323Sed/// 'op' is a valid SetCC operation. 234193323SedISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 235193323Sed unsigned Operation = Op; 236193323Sed if (isInteger) 237193323Sed Operation ^= 7; // Flip L, G, E bits, but not U. 238193323Sed else 239193323Sed Operation ^= 15; // Flip all of the condition bits. 240193323Sed 241193323Sed if (Operation > ISD::SETTRUE2) 242193323Sed Operation &= ~8; // Don't let N and U bits get set. 243193323Sed 244193323Sed return ISD::CondCode(Operation); 245193323Sed} 246193323Sed 247193323Sed 248193323Sed/// isSignedOp - For an integer comparison, return 1 if the comparison is a 249193323Sed/// signed operation and 2 if the result is an unsigned comparison. Return zero 250193323Sed/// if the operation does not depend on the sign of the input (setne and seteq). 251193323Sedstatic int isSignedOp(ISD::CondCode Opcode) { 252193323Sed switch (Opcode) { 253198090Srdivacky default: llvm_unreachable("Illegal integer setcc operation!"); 254193323Sed case ISD::SETEQ: 255193323Sed case ISD::SETNE: return 0; 256193323Sed case ISD::SETLT: 257193323Sed case ISD::SETLE: 258193323Sed case ISD::SETGT: 259193323Sed case ISD::SETGE: return 1; 260193323Sed case ISD::SETULT: 261193323Sed case ISD::SETULE: 262193323Sed case ISD::SETUGT: 263193323Sed case ISD::SETUGE: return 2; 264193323Sed } 265193323Sed} 266193323Sed 267193323Sed/// getSetCCOrOperation - Return the result of a logical OR between different 268193323Sed/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 269193323Sed/// returns SETCC_INVALID if it is not possible to represent the resultant 270193323Sed/// comparison. 271193323SedISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 272193323Sed bool isInteger) { 273193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 274193323Sed // Cannot fold a signed integer setcc with an unsigned integer setcc. 275193323Sed return ISD::SETCC_INVALID; 276193323Sed 277193323Sed unsigned Op = Op1 | Op2; // Combine all of the condition bits. 278193323Sed 279193323Sed // If the N and U bits get set then the resultant comparison DOES suddenly 280193323Sed // care about orderedness, and is true when ordered. 281193323Sed if (Op > ISD::SETTRUE2) 282193323Sed Op &= ~16; // Clear the U bit if the N bit is set. 283193323Sed 284193323Sed // Canonicalize illegal integer setcc's. 285193323Sed if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 286193323Sed Op = ISD::SETNE; 287193323Sed 288193323Sed return ISD::CondCode(Op); 289193323Sed} 290193323Sed 291193323Sed/// getSetCCAndOperation - Return the result of a logical AND between different 292193323Sed/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 293193323Sed/// function returns zero if it is not possible to represent the resultant 294193323Sed/// comparison. 295193323SedISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 296193323Sed bool isInteger) { 297193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 298193323Sed // Cannot fold a signed setcc with an unsigned setcc. 299193323Sed return ISD::SETCC_INVALID; 300193323Sed 301193323Sed // Combine all of the condition bits. 302193323Sed ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 303193323Sed 304193323Sed // Canonicalize illegal integer setcc's. 305193323Sed if (isInteger) { 306193323Sed switch (Result) { 307193323Sed default: break; 308193323Sed case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 309193323Sed case ISD::SETOEQ: // SETEQ & SETU[LG]E 310193323Sed case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 311193323Sed case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 312193323Sed case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 313193323Sed } 314193323Sed } 315193323Sed 316193323Sed return Result; 317193323Sed} 318193323Sed 319193323Sed//===----------------------------------------------------------------------===// 320193323Sed// SDNode Profile Support 321193323Sed//===----------------------------------------------------------------------===// 322193323Sed 323193323Sed/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 324193323Sed/// 325193323Sedstatic void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 326193323Sed ID.AddInteger(OpC); 327193323Sed} 328193323Sed 329193323Sed/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 330193323Sed/// solely with their pointer. 331193323Sedstatic void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 332193323Sed ID.AddPointer(VTList.VTs); 333193323Sed} 334193323Sed 335193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336193323Sed/// 337193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 338193323Sed const SDValue *Ops, unsigned NumOps) { 339193323Sed for (; NumOps; --NumOps, ++Ops) { 340193323Sed ID.AddPointer(Ops->getNode()); 341193323Sed ID.AddInteger(Ops->getResNo()); 342193323Sed } 343193323Sed} 344193323Sed 345193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 346193323Sed/// 347193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 348193323Sed const SDUse *Ops, unsigned NumOps) { 349193323Sed for (; NumOps; --NumOps, ++Ops) { 350193323Sed ID.AddPointer(Ops->getNode()); 351193323Sed ID.AddInteger(Ops->getResNo()); 352193323Sed } 353193323Sed} 354193323Sed 355193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, 356193323Sed unsigned short OpC, SDVTList VTList, 357193323Sed const SDValue *OpList, unsigned N) { 358193323Sed AddNodeIDOpcode(ID, OpC); 359193323Sed AddNodeIDValueTypes(ID, VTList); 360193323Sed AddNodeIDOperands(ID, OpList, N); 361193323Sed} 362193323Sed 363193323Sed/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 364193323Sed/// the NodeID data. 365193323Sedstatic void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 366193323Sed switch (N->getOpcode()) { 367195098Sed case ISD::TargetExternalSymbol: 368195098Sed case ISD::ExternalSymbol: 369198090Srdivacky llvm_unreachable("Should only be used on nodes with operands"); 370193323Sed default: break; // Normal nodes don't need extra info. 371193323Sed case ISD::TargetConstant: 372193323Sed case ISD::Constant: 373193323Sed ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 374193323Sed break; 375193323Sed case ISD::TargetConstantFP: 376193323Sed case ISD::ConstantFP: { 377193323Sed ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 378193323Sed break; 379193323Sed } 380193323Sed case ISD::TargetGlobalAddress: 381193323Sed case ISD::GlobalAddress: 382193323Sed case ISD::TargetGlobalTLSAddress: 383193323Sed case ISD::GlobalTLSAddress: { 384193323Sed const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 385193323Sed ID.AddPointer(GA->getGlobal()); 386193323Sed ID.AddInteger(GA->getOffset()); 387195098Sed ID.AddInteger(GA->getTargetFlags()); 388239462Sdim ID.AddInteger(GA->getAddressSpace()); 389193323Sed break; 390193323Sed } 391193323Sed case ISD::BasicBlock: 392193323Sed ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 393193323Sed break; 394193323Sed case ISD::Register: 395193323Sed ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 396193323Sed break; 397234353Sdim case ISD::RegisterMask: 398234353Sdim ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 399234353Sdim break; 400193323Sed case ISD::SRCVALUE: 401193323Sed ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 402193323Sed break; 403193323Sed case ISD::FrameIndex: 404193323Sed case ISD::TargetFrameIndex: 405193323Sed ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 406193323Sed break; 407193323Sed case ISD::JumpTable: 408193323Sed case ISD::TargetJumpTable: 409193323Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 410195098Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 411193323Sed break; 412193323Sed case ISD::ConstantPool: 413193323Sed case ISD::TargetConstantPool: { 414193323Sed const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 415193323Sed ID.AddInteger(CP->getAlignment()); 416193323Sed ID.AddInteger(CP->getOffset()); 417193323Sed if (CP->isMachineConstantPoolEntry()) 418226633Sdim CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 419193323Sed else 420193323Sed ID.AddPointer(CP->getConstVal()); 421195098Sed ID.AddInteger(CP->getTargetFlags()); 422193323Sed break; 423193323Sed } 424239462Sdim case ISD::TargetIndex: { 425239462Sdim const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 426239462Sdim ID.AddInteger(TI->getIndex()); 427239462Sdim ID.AddInteger(TI->getOffset()); 428239462Sdim ID.AddInteger(TI->getTargetFlags()); 429239462Sdim break; 430239462Sdim } 431193323Sed case ISD::LOAD: { 432193323Sed const LoadSDNode *LD = cast<LoadSDNode>(N); 433193323Sed ID.AddInteger(LD->getMemoryVT().getRawBits()); 434193323Sed ID.AddInteger(LD->getRawSubclassData()); 435239462Sdim ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 436193323Sed break; 437193323Sed } 438193323Sed case ISD::STORE: { 439193323Sed const StoreSDNode *ST = cast<StoreSDNode>(N); 440193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 441193323Sed ID.AddInteger(ST->getRawSubclassData()); 442239462Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 443193323Sed break; 444193323Sed } 445193323Sed case ISD::ATOMIC_CMP_SWAP: 446193323Sed case ISD::ATOMIC_SWAP: 447193323Sed case ISD::ATOMIC_LOAD_ADD: 448193323Sed case ISD::ATOMIC_LOAD_SUB: 449193323Sed case ISD::ATOMIC_LOAD_AND: 450193323Sed case ISD::ATOMIC_LOAD_OR: 451193323Sed case ISD::ATOMIC_LOAD_XOR: 452193323Sed case ISD::ATOMIC_LOAD_NAND: 453193323Sed case ISD::ATOMIC_LOAD_MIN: 454193323Sed case ISD::ATOMIC_LOAD_MAX: 455193323Sed case ISD::ATOMIC_LOAD_UMIN: 456226633Sdim case ISD::ATOMIC_LOAD_UMAX: 457226633Sdim case ISD::ATOMIC_LOAD: 458226633Sdim case ISD::ATOMIC_STORE: { 459193323Sed const AtomicSDNode *AT = cast<AtomicSDNode>(N); 460193323Sed ID.AddInteger(AT->getMemoryVT().getRawBits()); 461193323Sed ID.AddInteger(AT->getRawSubclassData()); 462239462Sdim ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 463193323Sed break; 464193323Sed } 465239462Sdim case ISD::PREFETCH: { 466239462Sdim const MemSDNode *PF = cast<MemSDNode>(N); 467239462Sdim ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 468239462Sdim break; 469239462Sdim } 470193323Sed case ISD::VECTOR_SHUFFLE: { 471193323Sed const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 472198090Srdivacky for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 473193323Sed i != e; ++i) 474193323Sed ID.AddInteger(SVN->getMaskElt(i)); 475193323Sed break; 476193323Sed } 477198892Srdivacky case ISD::TargetBlockAddress: 478198892Srdivacky case ISD::BlockAddress: { 479243830Sdim const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 480243830Sdim ID.AddPointer(BA->getBlockAddress()); 481243830Sdim ID.AddInteger(BA->getOffset()); 482243830Sdim ID.AddInteger(BA->getTargetFlags()); 483198892Srdivacky break; 484198892Srdivacky } 485193323Sed } // end switch (N->getOpcode()) 486239462Sdim 487239462Sdim // Target specific memory nodes could also have address spaces to check. 488239462Sdim if (N->isTargetMemoryOpcode()) 489239462Sdim ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 490193323Sed} 491193323Sed 492193323Sed/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 493193323Sed/// data. 494193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 495193323Sed AddNodeIDOpcode(ID, N->getOpcode()); 496193323Sed // Add the return value info. 497193323Sed AddNodeIDValueTypes(ID, N->getVTList()); 498193323Sed // Add the operand info. 499193323Sed AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 500193323Sed 501193323Sed // Handle SDNode leafs with special info. 502193323Sed AddNodeIDCustom(ID, N); 503193323Sed} 504193323Sed 505193323Sed/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 506204642Srdivacky/// the CSE map that carries volatility, temporalness, indexing mode, and 507193323Sed/// extension/truncation information. 508193323Sed/// 509193323Sedstatic inline unsigned 510204642SrdivackyencodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 511234353Sdim bool isNonTemporal, bool isInvariant) { 512193323Sed assert((ConvType & 3) == ConvType && 513193323Sed "ConvType may not require more than 2 bits!"); 514193323Sed assert((AM & 7) == AM && 515193323Sed "AM may not require more than 3 bits!"); 516193323Sed return ConvType | 517193323Sed (AM << 2) | 518204642Srdivacky (isVolatile << 5) | 519234353Sdim (isNonTemporal << 6) | 520234353Sdim (isInvariant << 7); 521193323Sed} 522193323Sed 523193323Sed//===----------------------------------------------------------------------===// 524193323Sed// SelectionDAG Class 525193323Sed//===----------------------------------------------------------------------===// 526193323Sed 527193323Sed/// doNotCSE - Return true if CSE should not be performed for this node. 528193323Sedstatic bool doNotCSE(SDNode *N) { 529218893Sdim if (N->getValueType(0) == MVT::Glue) 530193323Sed return true; // Never CSE anything that produces a flag. 531193323Sed 532193323Sed switch (N->getOpcode()) { 533193323Sed default: break; 534193323Sed case ISD::HANDLENODE: 535193323Sed case ISD::EH_LABEL: 536193323Sed return true; // Never CSE these nodes. 537193323Sed } 538193323Sed 539193323Sed // Check that remaining values produced are not flags. 540193323Sed for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 541218893Sdim if (N->getValueType(i) == MVT::Glue) 542193323Sed return true; // Never CSE anything that produces a flag. 543193323Sed 544193323Sed return false; 545193323Sed} 546193323Sed 547193323Sed/// RemoveDeadNodes - This method deletes all unreachable nodes in the 548193323Sed/// SelectionDAG. 549193323Sedvoid SelectionDAG::RemoveDeadNodes() { 550193323Sed // Create a dummy node (which is not added to allnodes), that adds a reference 551193323Sed // to the root node, preventing it from being deleted. 552193323Sed HandleSDNode Dummy(getRoot()); 553193323Sed 554193323Sed SmallVector<SDNode*, 128> DeadNodes; 555193323Sed 556193323Sed // Add all obviously-dead nodes to the DeadNodes worklist. 557193323Sed for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 558193323Sed if (I->use_empty()) 559193323Sed DeadNodes.push_back(I); 560193323Sed 561193323Sed RemoveDeadNodes(DeadNodes); 562193323Sed 563193323Sed // If the root changed (e.g. it was a dead load, update the root). 564193323Sed setRoot(Dummy.getValue()); 565193323Sed} 566193323Sed 567193323Sed/// RemoveDeadNodes - This method deletes the unreachable nodes in the 568193323Sed/// given list, and any nodes that become unreachable as a result. 569239462Sdimvoid SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 570193323Sed 571193323Sed // Process the worklist, deleting the nodes and adding their uses to the 572193323Sed // worklist. 573193323Sed while (!DeadNodes.empty()) { 574193323Sed SDNode *N = DeadNodes.pop_back_val(); 575193323Sed 576239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 577239462Sdim DUL->NodeDeleted(N, 0); 578193323Sed 579193323Sed // Take the node out of the appropriate CSE map. 580193323Sed RemoveNodeFromCSEMaps(N); 581193323Sed 582193323Sed // Next, brutally remove the operand list. This is safe to do, as there are 583193323Sed // no cycles in the graph. 584193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 585193323Sed SDUse &Use = *I++; 586193323Sed SDNode *Operand = Use.getNode(); 587193323Sed Use.set(SDValue()); 588193323Sed 589193323Sed // Now that we removed this operand, see if there are no uses of it left. 590193323Sed if (Operand->use_empty()) 591193323Sed DeadNodes.push_back(Operand); 592193323Sed } 593193323Sed 594193323Sed DeallocateNode(N); 595193323Sed } 596193323Sed} 597193323Sed 598239462Sdimvoid SelectionDAG::RemoveDeadNode(SDNode *N){ 599193323Sed SmallVector<SDNode*, 16> DeadNodes(1, N); 600234353Sdim 601234353Sdim // Create a dummy node that adds a reference to the root node, preventing 602234353Sdim // it from being deleted. (This matters if the root is an operand of the 603234353Sdim // dead node.) 604234353Sdim HandleSDNode Dummy(getRoot()); 605234353Sdim 606239462Sdim RemoveDeadNodes(DeadNodes); 607193323Sed} 608193323Sed 609193323Sedvoid SelectionDAG::DeleteNode(SDNode *N) { 610193323Sed // First take this out of the appropriate CSE map. 611193323Sed RemoveNodeFromCSEMaps(N); 612193323Sed 613193323Sed // Finally, remove uses due to operands of this node, remove from the 614193323Sed // AllNodes list, and delete the node. 615193323Sed DeleteNodeNotInCSEMaps(N); 616193323Sed} 617193323Sed 618193323Sedvoid SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 619193323Sed assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 620193323Sed assert(N->use_empty() && "Cannot delete a node that is not dead!"); 621193323Sed 622193323Sed // Drop all of the operands and decrement used node's use counts. 623193323Sed N->DropOperands(); 624193323Sed 625193323Sed DeallocateNode(N); 626193323Sed} 627193323Sed 628274696Sdimvoid SDDbgInfo::erase(const SDNode *Node) { 629274696Sdim DbgValMapType::iterator I = DbgValMap.find(Node); 630274696Sdim if (I == DbgValMap.end()) 631274696Sdim return; 632274696Sdim for (unsigned J = 0, N = I->second.size(); J != N; ++J) 633274696Sdim I->second[J]->setIsInvalidated(); 634274696Sdim DbgValMap.erase(I); 635274696Sdim} 636274696Sdim 637193323Sedvoid SelectionDAG::DeallocateNode(SDNode *N) { 638193323Sed if (N->OperandsNeedDelete) 639193323Sed delete[] N->OperandList; 640193323Sed 641193323Sed // Set the opcode to DELETED_NODE to help catch bugs when node 642193323Sed // memory is reallocated. 643193323Sed N->NodeType = ISD::DELETED_NODE; 644193323Sed 645193323Sed NodeAllocator.Deallocate(AllNodes.remove(N)); 646200581Srdivacky 647274696Sdim // If any of the SDDbgValue nodes refer to this SDNode, invalidate 648274696Sdim // them and forget about that node. 649274696Sdim DbgInfo->erase(N); 650193323Sed} 651193323Sed 652193323Sed/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 653193323Sed/// correspond to it. This is useful when we're about to delete or repurpose 654193323Sed/// the node. We don't want future request for structurally identical nodes 655193323Sed/// to return N anymore. 656193323Sedbool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 657193323Sed bool Erased = false; 658193323Sed switch (N->getOpcode()) { 659193323Sed case ISD::HANDLENODE: return false; // noop. 660193323Sed case ISD::CONDCODE: 661193323Sed assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 662193323Sed "Cond code doesn't exist!"); 663193323Sed Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 664193323Sed CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 665193323Sed break; 666193323Sed case ISD::ExternalSymbol: 667193323Sed Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 668193323Sed break; 669195098Sed case ISD::TargetExternalSymbol: { 670195098Sed ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 671195098Sed Erased = TargetExternalSymbols.erase( 672195098Sed std::pair<std::string,unsigned char>(ESN->getSymbol(), 673195098Sed ESN->getTargetFlags())); 674193323Sed break; 675195098Sed } 676193323Sed case ISD::VALUETYPE: { 677198090Srdivacky EVT VT = cast<VTSDNode>(N)->getVT(); 678193323Sed if (VT.isExtended()) { 679193323Sed Erased = ExtendedValueTypeNodes.erase(VT); 680193323Sed } else { 681198090Srdivacky Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 682198090Srdivacky ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 683193323Sed } 684193323Sed break; 685193323Sed } 686193323Sed default: 687193323Sed // Remove it from the CSE Map. 688218893Sdim assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 689218893Sdim assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 690193323Sed Erased = CSEMap.RemoveNode(N); 691193323Sed break; 692193323Sed } 693193323Sed#ifndef NDEBUG 694193323Sed // Verify that the node was actually in one of the CSE maps, unless it has a 695193323Sed // flag result (which cannot be CSE'd) or is one of the special cases that are 696193323Sed // not subject to CSE. 697218893Sdim if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 698193323Sed !N->isMachineOpcode() && !doNotCSE(N)) { 699193323Sed N->dump(this); 700202375Srdivacky dbgs() << "\n"; 701198090Srdivacky llvm_unreachable("Node is not in map!"); 702193323Sed } 703193323Sed#endif 704193323Sed return Erased; 705193323Sed} 706193323Sed 707193323Sed/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 708193323Sed/// maps and modified in place. Add it back to the CSE maps, unless an identical 709193323Sed/// node already exists, in which case transfer all its users to the existing 710193323Sed/// node. This transfer can potentially trigger recursive merging. 711193323Sed/// 712193323Sedvoid 713239462SdimSelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 714193323Sed // For node types that aren't CSE'd, just act as if no identical node 715193323Sed // already exists. 716193323Sed if (!doNotCSE(N)) { 717193323Sed SDNode *Existing = CSEMap.GetOrInsertNode(N); 718193323Sed if (Existing != N) { 719193323Sed // If there was already an existing matching node, use ReplaceAllUsesWith 720193323Sed // to replace the dead one with the existing one. This can cause 721193323Sed // recursive merging of other unrelated nodes down the line. 722239462Sdim ReplaceAllUsesWith(N, Existing); 723193323Sed 724239462Sdim // N is now dead. Inform the listeners and delete it. 725239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 726239462Sdim DUL->NodeDeleted(N, Existing); 727193323Sed DeleteNodeNotInCSEMaps(N); 728193323Sed return; 729193323Sed } 730193323Sed } 731193323Sed 732239462Sdim // If the node doesn't already exist, we updated it. Inform listeners. 733239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 734239462Sdim DUL->NodeUpdated(N); 735193323Sed} 736193323Sed 737193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 738193323Sed/// were replaced with those specified. If this node is never memoized, 739193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 740193323Sed/// node already exists with these operands, the slot will be non-null. 741193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 742193323Sed void *&InsertPos) { 743193323Sed if (doNotCSE(N)) 744193323Sed return 0; 745193323Sed 746193323Sed SDValue Ops[] = { Op }; 747193323Sed FoldingSetNodeID ID; 748193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 749193323Sed AddNodeIDCustom(ID, N); 750200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 751200581Srdivacky return Node; 752193323Sed} 753193323Sed 754193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 755193323Sed/// were replaced with those specified. If this node is never memoized, 756193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 757193323Sed/// node already exists with these operands, the slot will be non-null. 758193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 759193323Sed SDValue Op1, SDValue Op2, 760193323Sed void *&InsertPos) { 761193323Sed if (doNotCSE(N)) 762193323Sed return 0; 763193323Sed 764193323Sed SDValue Ops[] = { Op1, Op2 }; 765193323Sed FoldingSetNodeID ID; 766193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 767193323Sed AddNodeIDCustom(ID, N); 768200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 769200581Srdivacky return Node; 770193323Sed} 771193323Sed 772193323Sed 773193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 774193323Sed/// were replaced with those specified. If this node is never memoized, 775193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 776193323Sed/// node already exists with these operands, the slot will be non-null. 777193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 778193323Sed const SDValue *Ops,unsigned NumOps, 779193323Sed void *&InsertPos) { 780193323Sed if (doNotCSE(N)) 781193323Sed return 0; 782193323Sed 783193323Sed FoldingSetNodeID ID; 784193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 785193323Sed AddNodeIDCustom(ID, N); 786200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 787200581Srdivacky return Node; 788193323Sed} 789193323Sed 790218893Sdim#ifndef NDEBUG 791218893Sdim/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 792218893Sdimstatic void VerifyNodeCommon(SDNode *N) { 793193323Sed switch (N->getOpcode()) { 794193323Sed default: 795193323Sed break; 796193323Sed case ISD::BUILD_PAIR: { 797198090Srdivacky EVT VT = N->getValueType(0); 798193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 799193323Sed assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 800193323Sed "Wrong return type!"); 801193323Sed assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 802193323Sed assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 803193323Sed "Mismatched operand types!"); 804193323Sed assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 805193323Sed "Wrong operand type!"); 806193323Sed assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 807193323Sed "Wrong return type size"); 808193323Sed break; 809193323Sed } 810193323Sed case ISD::BUILD_VECTOR: { 811193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 812193323Sed assert(N->getValueType(0).isVector() && "Wrong return type!"); 813193323Sed assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 814193323Sed "Wrong number of operands!"); 815198090Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 816226633Sdim for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 817193323Sed assert((I->getValueType() == EltVT || 818193323Sed (EltVT.isInteger() && I->getValueType().isInteger() && 819193323Sed EltVT.bitsLE(I->getValueType()))) && 820193323Sed "Wrong operand type!"); 821226633Sdim assert(I->getValueType() == N->getOperand(0).getValueType() && 822226633Sdim "Operands must all have the same type"); 823226633Sdim } 824193323Sed break; 825193323Sed } 826193323Sed } 827193323Sed} 828193323Sed 829218893Sdim/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 830218893Sdimstatic void VerifySDNode(SDNode *N) { 831218893Sdim // The SDNode allocators cannot be used to allocate nodes with fields that are 832218893Sdim // not present in an SDNode! 833218893Sdim assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 834218893Sdim assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 835218893Sdim assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 836218893Sdim assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 837218893Sdim assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 838218893Sdim assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 839218893Sdim assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 840218893Sdim assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 841218893Sdim assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 842218893Sdim assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 843218893Sdim assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 844218893Sdim assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 845218893Sdim assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 846218893Sdim assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 847218893Sdim assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 848218893Sdim assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 849218893Sdim assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 850218893Sdim assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 851218893Sdim assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 852218893Sdim 853218893Sdim VerifyNodeCommon(N); 854218893Sdim} 855218893Sdim 856218893Sdim/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 857218893Sdim/// invalid. 858218893Sdimstatic void VerifyMachineNode(SDNode *N) { 859218893Sdim // The MachineNode allocators cannot be used to allocate nodes with fields 860218893Sdim // that are not present in a MachineNode! 861218893Sdim // Currently there are no such nodes. 862218893Sdim 863218893Sdim VerifyNodeCommon(N); 864218893Sdim} 865218893Sdim#endif // NDEBUG 866218893Sdim 867198090Srdivacky/// getEVTAlignment - Compute the default alignment value for the 868193323Sed/// given type. 869193323Sed/// 870198090Srdivackyunsigned SelectionDAG::getEVTAlignment(EVT VT) const { 871226633Sdim Type *Ty = VT == MVT::iPTR ? 872198090Srdivacky PointerType::get(Type::getInt8Ty(*getContext()), 0) : 873198090Srdivacky VT.getTypeForEVT(*getContext()); 874193323Sed 875263508Sdim return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty); 876193323Sed} 877193323Sed 878193323Sed// EntryNode could meaningfully have debug info if we can find it... 879234353SdimSelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 880263508Sdim : TM(tm), TSI(*tm.getSelectionDAGInfo()), TTI(0), TLI(0), OptLevel(OL), 881263508Sdim EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 882263508Sdim Root(getEntryNode()), NewNodesMustHaveLegalTypes(false), 883263508Sdim UpdateListeners(0) { 884193323Sed AllNodes.push_back(&EntryNode); 885205218Srdivacky DbgInfo = new SDDbgInfo(); 886193323Sed} 887193323Sed 888263508Sdimvoid SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti, 889263508Sdim const TargetLowering *tli) { 890193323Sed MF = &mf; 891249423Sdim TTI = tti; 892263508Sdim TLI = tli; 893198090Srdivacky Context = &mf.getFunction()->getContext(); 894193323Sed} 895193323Sed 896193323SedSelectionDAG::~SelectionDAG() { 897239462Sdim assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 898193323Sed allnodes_clear(); 899205218Srdivacky delete DbgInfo; 900193323Sed} 901193323Sed 902193323Sedvoid SelectionDAG::allnodes_clear() { 903193323Sed assert(&*AllNodes.begin() == &EntryNode); 904193323Sed AllNodes.remove(AllNodes.begin()); 905193323Sed while (!AllNodes.empty()) 906193323Sed DeallocateNode(AllNodes.begin()); 907193323Sed} 908193323Sed 909193323Sedvoid SelectionDAG::clear() { 910193323Sed allnodes_clear(); 911193323Sed OperandAllocator.Reset(); 912193323Sed CSEMap.clear(); 913193323Sed 914193323Sed ExtendedValueTypeNodes.clear(); 915193323Sed ExternalSymbols.clear(); 916193323Sed TargetExternalSymbols.clear(); 917193323Sed std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 918193323Sed static_cast<CondCodeSDNode*>(0)); 919193323Sed std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 920193323Sed static_cast<SDNode*>(0)); 921193323Sed 922193323Sed EntryNode.UseList = 0; 923193323Sed AllNodes.push_back(&EntryNode); 924193323Sed Root = getEntryNode(); 925206083Srdivacky DbgInfo->clear(); 926193323Sed} 927193323Sed 928263508SdimSDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 929226633Sdim return VT.bitsGT(Op.getValueType()) ? 930226633Sdim getNode(ISD::ANY_EXTEND, DL, VT, Op) : 931226633Sdim getNode(ISD::TRUNCATE, DL, VT, Op); 932226633Sdim} 933226633Sdim 934263508SdimSDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 935198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 936198090Srdivacky getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 937198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 938198090Srdivacky} 939198090Srdivacky 940263508SdimSDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 941198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 942198090Srdivacky getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 943198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 944198090Srdivacky} 945198090Srdivacky 946263508SdimSDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) { 947200581Srdivacky assert(!VT.isVector() && 948200581Srdivacky "getZeroExtendInReg should use the vector element type instead of " 949200581Srdivacky "the vector type!"); 950193323Sed if (Op.getValueType() == VT) return Op; 951200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 952200581Srdivacky APInt Imm = APInt::getLowBitsSet(BitWidth, 953193323Sed VT.getSizeInBits()); 954193323Sed return getNode(ISD::AND, DL, Op.getValueType(), Op, 955193323Sed getConstant(Imm, Op.getValueType())); 956193323Sed} 957193323Sed 958193323Sed/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 959193323Sed/// 960263508SdimSDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) { 961204642Srdivacky EVT EltVT = VT.getScalarType(); 962193323Sed SDValue NegOne = 963193323Sed getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 964193323Sed return getNode(ISD::XOR, DL, VT, Val, NegOne); 965193323Sed} 966193323Sed 967198090SrdivackySDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 968204642Srdivacky EVT EltVT = VT.getScalarType(); 969193323Sed assert((EltVT.getSizeInBits() >= 64 || 970193323Sed (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 971193323Sed "getConstant with a uint64_t value that doesn't fit in the type!"); 972193323Sed return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 973193323Sed} 974193323Sed 975198090SrdivackySDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 976198090Srdivacky return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 977193323Sed} 978193323Sed 979198090SrdivackySDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 980193323Sed assert(VT.isInteger() && "Cannot create FP integer constant!"); 981193323Sed 982204642Srdivacky EVT EltVT = VT.getScalarType(); 983226633Sdim const ConstantInt *Elt = &Val; 984226633Sdim 985263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 986263508Sdim 987226633Sdim // In some cases the vector type is legal but the element type is illegal and 988226633Sdim // needs to be promoted, for example v8i8 on ARM. In this case, promote the 989226633Sdim // inserted value (the type does not need to match the vector element type). 990226633Sdim // Any extra bits introduced will be truncated away. 991263508Sdim if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 992226633Sdim TargetLowering::TypePromoteInteger) { 993263508Sdim EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 994226633Sdim APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 995226633Sdim Elt = ConstantInt::get(*getContext(), NewVal); 996226633Sdim } 997263508Sdim // In other cases the element type is illegal and needs to be expanded, for 998263508Sdim // example v2i64 on MIPS32. In this case, find the nearest legal type, split 999263508Sdim // the value into n parts and use a vector type with n-times the elements. 1000263508Sdim // Then bitcast to the type requested. 1001263508Sdim // Legalizing constants too early makes the DAGCombiner's job harder so we 1002263508Sdim // only legalize if the DAG tells us we must produce legal types. 1003263508Sdim else if (NewNodesMustHaveLegalTypes && VT.isVector() && 1004263508Sdim TLI->getTypeAction(*getContext(), EltVT) == 1005263508Sdim TargetLowering::TypeExpandInteger) { 1006263508Sdim APInt NewVal = Elt->getValue(); 1007263508Sdim EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1008263508Sdim unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1009263508Sdim unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1010263508Sdim EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1011226633Sdim 1012263508Sdim // Check the temporary vector is the correct size. If this fails then 1013263508Sdim // getTypeToTransformTo() probably returned a type whose size (in bits) 1014263508Sdim // isn't a power-of-2 factor of the requested type size. 1015263508Sdim assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1016263508Sdim 1017263508Sdim SmallVector<SDValue, 2> EltParts; 1018263508Sdim for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { 1019263508Sdim EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) 1020263508Sdim .trunc(ViaEltSizeInBits), 1021263508Sdim ViaEltVT, isT)); 1022263508Sdim } 1023263508Sdim 1024263508Sdim // EltParts is currently in little endian order. If we actually want 1025263508Sdim // big-endian order then reverse it now. 1026263508Sdim if (TLI->isBigEndian()) 1027263508Sdim std::reverse(EltParts.begin(), EltParts.end()); 1028263508Sdim 1029263508Sdim // The elements must be reversed when the element order is different 1030263508Sdim // to the endianness of the elements (because the BITCAST is itself a 1031263508Sdim // vector shuffle in this situation). However, we do not need any code to 1032263508Sdim // perform this reversal because getConstant() is producing a vector 1033263508Sdim // splat. 1034263508Sdim // This situation occurs in MIPS MSA. 1035263508Sdim 1036263508Sdim SmallVector<SDValue, 8> Ops; 1037263508Sdim for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) 1038263508Sdim Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); 1039263508Sdim 1040263508Sdim SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT, 1041263508Sdim getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT, 1042263508Sdim &Ops[0], Ops.size())); 1043263508Sdim return Result; 1044263508Sdim } 1045263508Sdim 1046226633Sdim assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1047193323Sed "APInt size does not match type size!"); 1048193323Sed unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1049193323Sed FoldingSetNodeID ID; 1050193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1051226633Sdim ID.AddPointer(Elt); 1052193323Sed void *IP = 0; 1053193323Sed SDNode *N = NULL; 1054201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1055193323Sed if (!VT.isVector()) 1056193323Sed return SDValue(N, 0); 1057201360Srdivacky 1058193323Sed if (!N) { 1059226633Sdim N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 1060193323Sed CSEMap.InsertNode(N, IP); 1061193323Sed AllNodes.push_back(N); 1062193323Sed } 1063193323Sed 1064193323Sed SDValue Result(N, 0); 1065193323Sed if (VT.isVector()) { 1066193323Sed SmallVector<SDValue, 8> Ops; 1067193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1068263508Sdim Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1069193323Sed } 1070193323Sed return Result; 1071193323Sed} 1072193323Sed 1073193323SedSDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 1074263508Sdim return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget); 1075193323Sed} 1076193323Sed 1077193323Sed 1078198090SrdivackySDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 1079198090Srdivacky return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 1080193323Sed} 1081193323Sed 1082198090SrdivackySDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 1083193323Sed assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1084193323Sed 1085204642Srdivacky EVT EltVT = VT.getScalarType(); 1086193323Sed 1087193323Sed // Do the map lookup using the actual bit pattern for the floating point 1088193323Sed // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1089193323Sed // we don't have issues with SNANs. 1090193323Sed unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1091193323Sed FoldingSetNodeID ID; 1092193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1093193323Sed ID.AddPointer(&V); 1094193323Sed void *IP = 0; 1095193323Sed SDNode *N = NULL; 1096201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1097193323Sed if (!VT.isVector()) 1098193323Sed return SDValue(N, 0); 1099201360Srdivacky 1100193323Sed if (!N) { 1101205407Srdivacky N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1102193323Sed CSEMap.InsertNode(N, IP); 1103193323Sed AllNodes.push_back(N); 1104193323Sed } 1105193323Sed 1106193323Sed SDValue Result(N, 0); 1107193323Sed if (VT.isVector()) { 1108193323Sed SmallVector<SDValue, 8> Ops; 1109193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1110263508Sdim // FIXME SDLoc info might be appropriate here 1111263508Sdim Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1112193323Sed } 1113193323Sed return Result; 1114193323Sed} 1115193323Sed 1116198090SrdivackySDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1117204642Srdivacky EVT EltVT = VT.getScalarType(); 1118193323Sed if (EltVT==MVT::f32) 1119193323Sed return getConstantFP(APFloat((float)Val), VT, isTarget); 1120208599Srdivacky else if (EltVT==MVT::f64) 1121193323Sed return getConstantFP(APFloat(Val), VT, isTarget); 1122249423Sdim else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 1123249423Sdim EltVT==MVT::f16) { 1124208599Srdivacky bool ignored; 1125208599Srdivacky APFloat apf = APFloat(Val); 1126249423Sdim apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1127208599Srdivacky &ignored); 1128208599Srdivacky return getConstantFP(apf, VT, isTarget); 1129234353Sdim } else 1130234353Sdim llvm_unreachable("Unsupported type in getConstantFP"); 1131193323Sed} 1132193323Sed 1133263508SdimSDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL, 1134198090Srdivacky EVT VT, int64_t Offset, 1135195098Sed bool isTargetGA, 1136195098Sed unsigned char TargetFlags) { 1137195098Sed assert((TargetFlags == 0 || isTargetGA) && 1138195098Sed "Cannot set target flags on target-independent globals"); 1139263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1140198090Srdivacky 1141193323Sed // Truncate (with sign-extension) the offset value to the pointer size. 1142263508Sdim unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 1143193323Sed if (BitWidth < 64) 1144243830Sdim Offset = SignExtend64(Offset, BitWidth); 1145193323Sed 1146193323Sed const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1147193323Sed if (!GVar) { 1148193323Sed // If GV is an alias then use the aliasee for determining thread-localness. 1149193323Sed if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1150193323Sed GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1151193323Sed } 1152193323Sed 1153195098Sed unsigned Opc; 1154193323Sed if (GVar && GVar->isThreadLocal()) 1155193323Sed Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1156193323Sed else 1157193323Sed Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1158193323Sed 1159193323Sed FoldingSetNodeID ID; 1160193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1161193323Sed ID.AddPointer(GV); 1162193323Sed ID.AddInteger(Offset); 1163195098Sed ID.AddInteger(TargetFlags); 1164239462Sdim ID.AddInteger(GV->getType()->getAddressSpace()); 1165193323Sed void *IP = 0; 1166201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1167193323Sed return SDValue(E, 0); 1168201360Srdivacky 1169263508Sdim SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(), 1170263508Sdim DL.getDebugLoc(), GV, VT, 1171205407Srdivacky Offset, TargetFlags); 1172193323Sed CSEMap.InsertNode(N, IP); 1173193323Sed AllNodes.push_back(N); 1174193323Sed return SDValue(N, 0); 1175193323Sed} 1176193323Sed 1177198090SrdivackySDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1178193323Sed unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1179193323Sed FoldingSetNodeID ID; 1180193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1181193323Sed ID.AddInteger(FI); 1182193323Sed void *IP = 0; 1183201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1184193323Sed return SDValue(E, 0); 1185201360Srdivacky 1186205407Srdivacky SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1187193323Sed CSEMap.InsertNode(N, IP); 1188193323Sed AllNodes.push_back(N); 1189193323Sed return SDValue(N, 0); 1190193323Sed} 1191193323Sed 1192198090SrdivackySDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1193195098Sed unsigned char TargetFlags) { 1194195098Sed assert((TargetFlags == 0 || isTarget) && 1195195098Sed "Cannot set target flags on target-independent jump tables"); 1196193323Sed unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1197193323Sed FoldingSetNodeID ID; 1198193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1199193323Sed ID.AddInteger(JTI); 1200195098Sed ID.AddInteger(TargetFlags); 1201193323Sed void *IP = 0; 1202201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1203193323Sed return SDValue(E, 0); 1204201360Srdivacky 1205205407Srdivacky SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1206205407Srdivacky TargetFlags); 1207193323Sed CSEMap.InsertNode(N, IP); 1208193323Sed AllNodes.push_back(N); 1209193323Sed return SDValue(N, 0); 1210193323Sed} 1211193323Sed 1212207618SrdivackySDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1213193323Sed unsigned Alignment, int Offset, 1214198090Srdivacky bool isTarget, 1215195098Sed unsigned char TargetFlags) { 1216195098Sed assert((TargetFlags == 0 || isTarget) && 1217195098Sed "Cannot set target flags on target-independent globals"); 1218193323Sed if (Alignment == 0) 1219263508Sdim Alignment = 1220263508Sdim TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1221193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1222193323Sed FoldingSetNodeID ID; 1223193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1224193323Sed ID.AddInteger(Alignment); 1225193323Sed ID.AddInteger(Offset); 1226193323Sed ID.AddPointer(C); 1227195098Sed ID.AddInteger(TargetFlags); 1228193323Sed void *IP = 0; 1229201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1230193323Sed return SDValue(E, 0); 1231201360Srdivacky 1232205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1233205407Srdivacky Alignment, TargetFlags); 1234193323Sed CSEMap.InsertNode(N, IP); 1235193323Sed AllNodes.push_back(N); 1236193323Sed return SDValue(N, 0); 1237193323Sed} 1238193323Sed 1239193323Sed 1240198090SrdivackySDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1241193323Sed unsigned Alignment, int Offset, 1242195098Sed bool isTarget, 1243195098Sed unsigned char TargetFlags) { 1244195098Sed assert((TargetFlags == 0 || isTarget) && 1245195098Sed "Cannot set target flags on target-independent globals"); 1246193323Sed if (Alignment == 0) 1247263508Sdim Alignment = 1248263508Sdim TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1249193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1250193323Sed FoldingSetNodeID ID; 1251193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1252193323Sed ID.AddInteger(Alignment); 1253193323Sed ID.AddInteger(Offset); 1254226633Sdim C->addSelectionDAGCSEId(ID); 1255195098Sed ID.AddInteger(TargetFlags); 1256193323Sed void *IP = 0; 1257201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1258193323Sed return SDValue(E, 0); 1259201360Srdivacky 1260205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1261205407Srdivacky Alignment, TargetFlags); 1262193323Sed CSEMap.InsertNode(N, IP); 1263193323Sed AllNodes.push_back(N); 1264193323Sed return SDValue(N, 0); 1265193323Sed} 1266193323Sed 1267239462SdimSDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1268239462Sdim unsigned char TargetFlags) { 1269239462Sdim FoldingSetNodeID ID; 1270239462Sdim AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 1271239462Sdim ID.AddInteger(Index); 1272239462Sdim ID.AddInteger(Offset); 1273239462Sdim ID.AddInteger(TargetFlags); 1274239462Sdim void *IP = 0; 1275239462Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1276239462Sdim return SDValue(E, 0); 1277239462Sdim 1278239462Sdim SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 1279239462Sdim TargetFlags); 1280239462Sdim CSEMap.InsertNode(N, IP); 1281239462Sdim AllNodes.push_back(N); 1282239462Sdim return SDValue(N, 0); 1283239462Sdim} 1284239462Sdim 1285193323SedSDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1286193323Sed FoldingSetNodeID ID; 1287193323Sed AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1288193323Sed ID.AddPointer(MBB); 1289193323Sed void *IP = 0; 1290201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1291193323Sed return SDValue(E, 0); 1292201360Srdivacky 1293205407Srdivacky SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1294193323Sed CSEMap.InsertNode(N, IP); 1295193323Sed AllNodes.push_back(N); 1296193323Sed return SDValue(N, 0); 1297193323Sed} 1298193323Sed 1299198090SrdivackySDValue SelectionDAG::getValueType(EVT VT) { 1300198090Srdivacky if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1301198090Srdivacky ValueTypeNodes.size()) 1302198090Srdivacky ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1303193323Sed 1304193323Sed SDNode *&N = VT.isExtended() ? 1305198090Srdivacky ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1306193323Sed 1307193323Sed if (N) return SDValue(N, 0); 1308205407Srdivacky N = new (NodeAllocator) VTSDNode(VT); 1309193323Sed AllNodes.push_back(N); 1310193323Sed return SDValue(N, 0); 1311193323Sed} 1312193323Sed 1313198090SrdivackySDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1314193323Sed SDNode *&N = ExternalSymbols[Sym]; 1315193323Sed if (N) return SDValue(N, 0); 1316205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1317193323Sed AllNodes.push_back(N); 1318193323Sed return SDValue(N, 0); 1319193323Sed} 1320193323Sed 1321198090SrdivackySDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1322195098Sed unsigned char TargetFlags) { 1323195098Sed SDNode *&N = 1324195098Sed TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1325195098Sed TargetFlags)]; 1326193323Sed if (N) return SDValue(N, 0); 1327205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1328193323Sed AllNodes.push_back(N); 1329193323Sed return SDValue(N, 0); 1330193323Sed} 1331193323Sed 1332193323SedSDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1333193323Sed if ((unsigned)Cond >= CondCodeNodes.size()) 1334193323Sed CondCodeNodes.resize(Cond+1); 1335193323Sed 1336193323Sed if (CondCodeNodes[Cond] == 0) { 1337205407Srdivacky CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1338193323Sed CondCodeNodes[Cond] = N; 1339193323Sed AllNodes.push_back(N); 1340193323Sed } 1341201360Srdivacky 1342193323Sed return SDValue(CondCodeNodes[Cond], 0); 1343193323Sed} 1344193323Sed 1345193323Sed// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1346193323Sed// the shuffle mask M that point at N1 to point at N2, and indices that point 1347193323Sed// N2 to point at N1. 1348193323Sedstatic void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1349193323Sed std::swap(N1, N2); 1350193323Sed int NElts = M.size(); 1351193323Sed for (int i = 0; i != NElts; ++i) { 1352193323Sed if (M[i] >= NElts) 1353193323Sed M[i] -= NElts; 1354193323Sed else if (M[i] >= 0) 1355193323Sed M[i] += NElts; 1356193323Sed } 1357193323Sed} 1358193323Sed 1359263508SdimSDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, 1360193323Sed SDValue N2, const int *Mask) { 1361263508Sdim assert(VT == N1.getValueType() && VT == N2.getValueType() && 1362263508Sdim "Invalid VECTOR_SHUFFLE"); 1363193323Sed 1364193323Sed // Canonicalize shuffle undef, undef -> undef 1365193323Sed if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1366198090Srdivacky return getUNDEF(VT); 1367193323Sed 1368198090Srdivacky // Validate that all indices in Mask are within the range of the elements 1369193323Sed // input to the shuffle. 1370193323Sed unsigned NElts = VT.getVectorNumElements(); 1371193323Sed SmallVector<int, 8> MaskVec; 1372193323Sed for (unsigned i = 0; i != NElts; ++i) { 1373193323Sed assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1374193323Sed MaskVec.push_back(Mask[i]); 1375193323Sed } 1376198090Srdivacky 1377193323Sed // Canonicalize shuffle v, v -> v, undef 1378193323Sed if (N1 == N2) { 1379193323Sed N2 = getUNDEF(VT); 1380193323Sed for (unsigned i = 0; i != NElts; ++i) 1381193323Sed if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1382193323Sed } 1383198090Srdivacky 1384193323Sed // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1385193323Sed if (N1.getOpcode() == ISD::UNDEF) 1386193323Sed commuteShuffle(N1, N2, MaskVec); 1387198090Srdivacky 1388193323Sed // Canonicalize all index into lhs, -> shuffle lhs, undef 1389193323Sed // Canonicalize all index into rhs, -> shuffle rhs, undef 1390193323Sed bool AllLHS = true, AllRHS = true; 1391193323Sed bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1392193323Sed for (unsigned i = 0; i != NElts; ++i) { 1393193323Sed if (MaskVec[i] >= (int)NElts) { 1394193323Sed if (N2Undef) 1395193323Sed MaskVec[i] = -1; 1396193323Sed else 1397193323Sed AllLHS = false; 1398193323Sed } else if (MaskVec[i] >= 0) { 1399193323Sed AllRHS = false; 1400193323Sed } 1401193323Sed } 1402193323Sed if (AllLHS && AllRHS) 1403193323Sed return getUNDEF(VT); 1404193323Sed if (AllLHS && !N2Undef) 1405193323Sed N2 = getUNDEF(VT); 1406193323Sed if (AllRHS) { 1407193323Sed N1 = getUNDEF(VT); 1408193323Sed commuteShuffle(N1, N2, MaskVec); 1409193323Sed } 1410198090Srdivacky 1411263508Sdim // If Identity shuffle return that node. 1412193323Sed bool Identity = true; 1413193323Sed for (unsigned i = 0; i != NElts; ++i) { 1414193323Sed if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1415193323Sed } 1416263508Sdim if (Identity && NElts) 1417193323Sed return N1; 1418193323Sed 1419193323Sed FoldingSetNodeID ID; 1420193323Sed SDValue Ops[2] = { N1, N2 }; 1421193323Sed AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1422193323Sed for (unsigned i = 0; i != NElts; ++i) 1423193323Sed ID.AddInteger(MaskVec[i]); 1424198090Srdivacky 1425193323Sed void* IP = 0; 1426201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1427193323Sed return SDValue(E, 0); 1428198090Srdivacky 1429193323Sed // Allocate the mask array for the node out of the BumpPtrAllocator, since 1430193323Sed // SDNode doesn't have access to it. This memory will be "leaked" when 1431193323Sed // the node is deallocated, but recovered when the NodeAllocator is released. 1432193323Sed int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1433193323Sed memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1434198090Srdivacky 1435205407Srdivacky ShuffleVectorSDNode *N = 1436263508Sdim new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), 1437263508Sdim dl.getDebugLoc(), N1, N2, 1438263508Sdim MaskAlloc); 1439193323Sed CSEMap.InsertNode(N, IP); 1440193323Sed AllNodes.push_back(N); 1441193323Sed return SDValue(N, 0); 1442193323Sed} 1443193323Sed 1444263508SdimSDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, 1445193323Sed SDValue Val, SDValue DTy, 1446193323Sed SDValue STy, SDValue Rnd, SDValue Sat, 1447193323Sed ISD::CvtCode Code) { 1448193323Sed // If the src and dest types are the same and the conversion is between 1449193323Sed // integer types of the same sign or two floats, no conversion is necessary. 1450193323Sed if (DTy == STy && 1451193323Sed (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1452193323Sed return Val; 1453193323Sed 1454193323Sed FoldingSetNodeID ID; 1455199481Srdivacky SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1456199481Srdivacky AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1457193323Sed void* IP = 0; 1458201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1459193323Sed return SDValue(E, 0); 1460201360Srdivacky 1461263508Sdim CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), 1462263508Sdim dl.getDebugLoc(), 1463263508Sdim Ops, 5, Code); 1464193323Sed CSEMap.InsertNode(N, IP); 1465193323Sed AllNodes.push_back(N); 1466193323Sed return SDValue(N, 0); 1467193323Sed} 1468193323Sed 1469198090SrdivackySDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1470193323Sed FoldingSetNodeID ID; 1471193323Sed AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1472193323Sed ID.AddInteger(RegNo); 1473193323Sed void *IP = 0; 1474201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1475193323Sed return SDValue(E, 0); 1476201360Srdivacky 1477205407Srdivacky SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1478193323Sed CSEMap.InsertNode(N, IP); 1479193323Sed AllNodes.push_back(N); 1480193323Sed return SDValue(N, 0); 1481193323Sed} 1482193323Sed 1483234353SdimSDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1484234353Sdim FoldingSetNodeID ID; 1485234353Sdim AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1486234353Sdim ID.AddPointer(RegMask); 1487234353Sdim void *IP = 0; 1488234353Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1489234353Sdim return SDValue(E, 0); 1490234353Sdim 1491234353Sdim SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1492234353Sdim CSEMap.InsertNode(N, IP); 1493234353Sdim AllNodes.push_back(N); 1494234353Sdim return SDValue(N, 0); 1495234353Sdim} 1496234353Sdim 1497263508SdimSDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { 1498193323Sed FoldingSetNodeID ID; 1499193323Sed SDValue Ops[] = { Root }; 1500205218Srdivacky AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1501205218Srdivacky ID.AddPointer(Label); 1502193323Sed void *IP = 0; 1503201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1504193323Sed return SDValue(E, 0); 1505218893Sdim 1506263508Sdim SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), 1507263508Sdim dl.getDebugLoc(), Root, Label); 1508193323Sed CSEMap.InsertNode(N, IP); 1509193323Sed AllNodes.push_back(N); 1510193323Sed return SDValue(N, 0); 1511193323Sed} 1512193323Sed 1513205218Srdivacky 1514207618SrdivackySDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1515243830Sdim int64_t Offset, 1516199989Srdivacky bool isTarget, 1517199989Srdivacky unsigned char TargetFlags) { 1518198892Srdivacky unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1519198892Srdivacky 1520198892Srdivacky FoldingSetNodeID ID; 1521199989Srdivacky AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1522198892Srdivacky ID.AddPointer(BA); 1523243830Sdim ID.AddInteger(Offset); 1524199989Srdivacky ID.AddInteger(TargetFlags); 1525198892Srdivacky void *IP = 0; 1526201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1527198892Srdivacky return SDValue(E, 0); 1528201360Srdivacky 1529243830Sdim SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 1530243830Sdim TargetFlags); 1531198892Srdivacky CSEMap.InsertNode(N, IP); 1532198892Srdivacky AllNodes.push_back(N); 1533198892Srdivacky return SDValue(N, 0); 1534198892Srdivacky} 1535198892Srdivacky 1536193323SedSDValue SelectionDAG::getSrcValue(const Value *V) { 1537204642Srdivacky assert((!V || V->getType()->isPointerTy()) && 1538193323Sed "SrcValue is not a pointer?"); 1539193323Sed 1540193323Sed FoldingSetNodeID ID; 1541193323Sed AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1542193323Sed ID.AddPointer(V); 1543193323Sed 1544193323Sed void *IP = 0; 1545201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1546193323Sed return SDValue(E, 0); 1547193323Sed 1548205407Srdivacky SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1549193323Sed CSEMap.InsertNode(N, IP); 1550193323Sed AllNodes.push_back(N); 1551193323Sed return SDValue(N, 0); 1552193323Sed} 1553193323Sed 1554207618Srdivacky/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1555207618SrdivackySDValue SelectionDAG::getMDNode(const MDNode *MD) { 1556207618Srdivacky FoldingSetNodeID ID; 1557207618Srdivacky AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1558207618Srdivacky ID.AddPointer(MD); 1559218893Sdim 1560207618Srdivacky void *IP = 0; 1561207618Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1562207618Srdivacky return SDValue(E, 0); 1563218893Sdim 1564207618Srdivacky SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1565207618Srdivacky CSEMap.InsertNode(N, IP); 1566207618Srdivacky AllNodes.push_back(N); 1567207618Srdivacky return SDValue(N, 0); 1568207618Srdivacky} 1569207618Srdivacky 1570263508Sdim/// getAddrSpaceCast - Return an AddrSpaceCastSDNode. 1571263508SdimSDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, 1572263508Sdim unsigned SrcAS, unsigned DestAS) { 1573263508Sdim SDValue Ops[] = {Ptr}; 1574263508Sdim FoldingSetNodeID ID; 1575263508Sdim AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), &Ops[0], 1); 1576263508Sdim ID.AddInteger(SrcAS); 1577263508Sdim ID.AddInteger(DestAS); 1578207618Srdivacky 1579263508Sdim void *IP = 0; 1580263508Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1581263508Sdim return SDValue(E, 0); 1582263508Sdim 1583263508Sdim SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(), 1584263508Sdim dl.getDebugLoc(), 1585263508Sdim VT, Ptr, SrcAS, DestAS); 1586263508Sdim CSEMap.InsertNode(N, IP); 1587263508Sdim AllNodes.push_back(N); 1588263508Sdim return SDValue(N, 0); 1589263508Sdim} 1590263508Sdim 1591193323Sed/// getShiftAmountOperand - Return the specified value casted to 1592193323Sed/// the target's desired shift amount type. 1593221345SdimSDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1594198090Srdivacky EVT OpTy = Op.getValueType(); 1595263508Sdim EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy); 1596193323Sed if (OpTy == ShTy || OpTy.isVector()) return Op; 1597193323Sed 1598193323Sed ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1599263508Sdim return getNode(Opcode, SDLoc(Op), ShTy, Op); 1600193323Sed} 1601193323Sed 1602193323Sed/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1603193323Sed/// specified value type. 1604198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1605193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1606198090Srdivacky unsigned ByteSize = VT.getStoreSize(); 1607226633Sdim Type *Ty = VT.getTypeForEVT(*getContext()); 1608263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1609193323Sed unsigned StackAlign = 1610263508Sdim std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 1611193323Sed 1612199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1613263508Sdim return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1614193323Sed} 1615193323Sed 1616193323Sed/// CreateStackTemporary - Create a stack temporary suitable for holding 1617193323Sed/// either of the specified value types. 1618198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1619193323Sed unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1620193323Sed VT2.getStoreSizeInBits())/8; 1621226633Sdim Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1622226633Sdim Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1623263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1624263508Sdim const DataLayout *TD = TLI->getDataLayout(); 1625193323Sed unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1626193323Sed TD->getPrefTypeAlignment(Ty2)); 1627193323Sed 1628193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1629199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1630263508Sdim return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1631193323Sed} 1632193323Sed 1633198090SrdivackySDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1634263508Sdim SDValue N2, ISD::CondCode Cond, SDLoc dl) { 1635193323Sed // These setcc operations always fold. 1636193323Sed switch (Cond) { 1637193323Sed default: break; 1638193323Sed case ISD::SETFALSE: 1639193323Sed case ISD::SETFALSE2: return getConstant(0, VT); 1640193323Sed case ISD::SETTRUE: 1641263508Sdim case ISD::SETTRUE2: { 1642263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1643263508Sdim TargetLowering::BooleanContent Cnt = TLI->getBooleanContents(VT.isVector()); 1644263508Sdim return getConstant( 1645263508Sdim Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT); 1646263508Sdim } 1647193323Sed 1648193323Sed case ISD::SETOEQ: 1649193323Sed case ISD::SETOGT: 1650193323Sed case ISD::SETOGE: 1651193323Sed case ISD::SETOLT: 1652193323Sed case ISD::SETOLE: 1653193323Sed case ISD::SETONE: 1654193323Sed case ISD::SETO: 1655193323Sed case ISD::SETUO: 1656193323Sed case ISD::SETUEQ: 1657193323Sed case ISD::SETUNE: 1658193323Sed assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1659193323Sed break; 1660193323Sed } 1661193323Sed 1662193323Sed if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1663193323Sed const APInt &C2 = N2C->getAPIntValue(); 1664193323Sed if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1665193323Sed const APInt &C1 = N1C->getAPIntValue(); 1666193323Sed 1667193323Sed switch (Cond) { 1668198090Srdivacky default: llvm_unreachable("Unknown integer setcc!"); 1669193323Sed case ISD::SETEQ: return getConstant(C1 == C2, VT); 1670193323Sed case ISD::SETNE: return getConstant(C1 != C2, VT); 1671193323Sed case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1672193323Sed case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1673193323Sed case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1674193323Sed case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1675193323Sed case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1676193323Sed case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1677193323Sed case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1678193323Sed case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1679193323Sed } 1680193323Sed } 1681193323Sed } 1682193323Sed if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1683193323Sed if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1684193323Sed APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1685193323Sed switch (Cond) { 1686193323Sed default: break; 1687193323Sed case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1688193323Sed return getUNDEF(VT); 1689193323Sed // fall through 1690193323Sed case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1691193323Sed case ISD::SETNE: if (R==APFloat::cmpUnordered) 1692193323Sed return getUNDEF(VT); 1693193323Sed // fall through 1694193323Sed case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1695193323Sed R==APFloat::cmpLessThan, VT); 1696193323Sed case ISD::SETLT: if (R==APFloat::cmpUnordered) 1697193323Sed return getUNDEF(VT); 1698193323Sed // fall through 1699193323Sed case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1700193323Sed case ISD::SETGT: if (R==APFloat::cmpUnordered) 1701193323Sed return getUNDEF(VT); 1702193323Sed // fall through 1703193323Sed case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1704193323Sed case ISD::SETLE: if (R==APFloat::cmpUnordered) 1705193323Sed return getUNDEF(VT); 1706193323Sed // fall through 1707193323Sed case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1708193323Sed R==APFloat::cmpEqual, VT); 1709193323Sed case ISD::SETGE: if (R==APFloat::cmpUnordered) 1710193323Sed return getUNDEF(VT); 1711193323Sed // fall through 1712193323Sed case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1713193323Sed R==APFloat::cmpEqual, VT); 1714193323Sed case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1715193323Sed case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1716193323Sed case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1717193323Sed R==APFloat::cmpEqual, VT); 1718193323Sed case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1719193323Sed case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1720193323Sed R==APFloat::cmpLessThan, VT); 1721193323Sed case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1722193323Sed R==APFloat::cmpUnordered, VT); 1723193323Sed case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1724193323Sed case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1725193323Sed } 1726193323Sed } else { 1727193323Sed // Ensure that the constant occurs on the RHS. 1728263508Sdim ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 1729263508Sdim MVT CompVT = N1.getValueType().getSimpleVT(); 1730263508Sdim if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT)) 1731263508Sdim return SDValue(); 1732263508Sdim 1733263508Sdim return getSetCC(dl, VT, N2, N1, SwappedCond); 1734193323Sed } 1735193323Sed } 1736193323Sed 1737193323Sed // Could not fold it. 1738193323Sed return SDValue(); 1739193323Sed} 1740193323Sed 1741193323Sed/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1742193323Sed/// use this predicate to simplify operations downstream. 1743193323Sedbool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1744198090Srdivacky // This predicate is not safe for vector operations. 1745198090Srdivacky if (Op.getValueType().isVector()) 1746198090Srdivacky return false; 1747198090Srdivacky 1748200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1749193323Sed return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1750193323Sed} 1751193323Sed 1752193323Sed/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1753193323Sed/// this predicate to simplify operations downstream. Mask is known to be zero 1754193323Sed/// for bits that V cannot have. 1755193323Sedbool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1756193323Sed unsigned Depth) const { 1757193323Sed APInt KnownZero, KnownOne; 1758234353Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 1759193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1760193323Sed return (KnownZero & Mask) == Mask; 1761193323Sed} 1762193323Sed 1763193323Sed/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1764193323Sed/// known to be either zero or one and return them in the KnownZero/KnownOne 1765193323Sed/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1766193323Sed/// processing. 1767234353Sdimvoid SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 1768234353Sdim APInt &KnownOne, unsigned Depth) const { 1769263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1770234353Sdim unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1771193323Sed 1772193323Sed KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1773234353Sdim if (Depth == 6) 1774193323Sed return; // Limit search depth. 1775193323Sed 1776193323Sed APInt KnownZero2, KnownOne2; 1777193323Sed 1778193323Sed switch (Op.getOpcode()) { 1779193323Sed case ISD::Constant: 1780193323Sed // We know all of the bits for a constant! 1781234353Sdim KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 1782234353Sdim KnownZero = ~KnownOne; 1783193323Sed return; 1784193323Sed case ISD::AND: 1785193323Sed // If either the LHS or the RHS are Zero, the result is zero. 1786234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1787234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1788193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1789193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1790193323Sed 1791193323Sed // Output known-1 bits are only known if set in both the LHS & RHS. 1792193323Sed KnownOne &= KnownOne2; 1793193323Sed // Output known-0 are known to be clear if zero in either the LHS | RHS. 1794193323Sed KnownZero |= KnownZero2; 1795193323Sed return; 1796193323Sed case ISD::OR: 1797234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1798234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1799193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1800193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1801193323Sed 1802193323Sed // Output known-0 bits are only known if clear in both the LHS & RHS. 1803193323Sed KnownZero &= KnownZero2; 1804193323Sed // Output known-1 are known to be set if set in either the LHS | RHS. 1805193323Sed KnownOne |= KnownOne2; 1806193323Sed return; 1807193323Sed case ISD::XOR: { 1808234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1809234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1810193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1811193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1812193323Sed 1813193323Sed // Output known-0 bits are known if clear or set in both the LHS & RHS. 1814193323Sed APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1815193323Sed // Output known-1 are known to be set if set in only one of the LHS, RHS. 1816193323Sed KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1817193323Sed KnownZero = KnownZeroOut; 1818193323Sed return; 1819193323Sed } 1820193323Sed case ISD::MUL: { 1821234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1822234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1823193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1824193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1825193323Sed 1826193323Sed // If low bits are zero in either operand, output low known-0 bits. 1827193323Sed // Also compute a conserative estimate for high known-0 bits. 1828193323Sed // More trickiness is possible, but this is sufficient for the 1829193323Sed // interesting case of alignment computation. 1830218893Sdim KnownOne.clearAllBits(); 1831193323Sed unsigned TrailZ = KnownZero.countTrailingOnes() + 1832193323Sed KnownZero2.countTrailingOnes(); 1833193323Sed unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1834193323Sed KnownZero2.countLeadingOnes(), 1835193323Sed BitWidth) - BitWidth; 1836193323Sed 1837193323Sed TrailZ = std::min(TrailZ, BitWidth); 1838193323Sed LeadZ = std::min(LeadZ, BitWidth); 1839193323Sed KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1840193323Sed APInt::getHighBitsSet(BitWidth, LeadZ); 1841193323Sed return; 1842193323Sed } 1843193323Sed case ISD::UDIV: { 1844193323Sed // For the purposes of computing leading zeros we can conservatively 1845193323Sed // treat a udiv as a logical right shift by the power of 2 known to 1846193323Sed // be less than the denominator. 1847234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1848193323Sed unsigned LeadZ = KnownZero2.countLeadingOnes(); 1849193323Sed 1850218893Sdim KnownOne2.clearAllBits(); 1851218893Sdim KnownZero2.clearAllBits(); 1852234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1853193323Sed unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1854193323Sed if (RHSUnknownLeadingOnes != BitWidth) 1855193323Sed LeadZ = std::min(BitWidth, 1856193323Sed LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1857193323Sed 1858234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 1859193323Sed return; 1860193323Sed } 1861193323Sed case ISD::SELECT: 1862234353Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 1863234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1864193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1865193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1866193323Sed 1867193323Sed // Only known if known in both the LHS and RHS. 1868193323Sed KnownOne &= KnownOne2; 1869193323Sed KnownZero &= KnownZero2; 1870193323Sed return; 1871193323Sed case ISD::SELECT_CC: 1872234353Sdim ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 1873234353Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 1874193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1875193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1876193323Sed 1877193323Sed // Only known if known in both the LHS and RHS. 1878193323Sed KnownOne &= KnownOne2; 1879193323Sed KnownZero &= KnownZero2; 1880193323Sed return; 1881193323Sed case ISD::SADDO: 1882193323Sed case ISD::UADDO: 1883193323Sed case ISD::SSUBO: 1884193323Sed case ISD::USUBO: 1885193323Sed case ISD::SMULO: 1886193323Sed case ISD::UMULO: 1887193323Sed if (Op.getResNo() != 1) 1888193323Sed return; 1889193323Sed // The boolean result conforms to getBooleanContents. Fall through. 1890193323Sed case ISD::SETCC: 1891193323Sed // If we know the result of a setcc has the top bits zero, use this info. 1892263508Sdim if (TLI->getBooleanContents(Op.getValueType().isVector()) == 1893226633Sdim TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1894193323Sed KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1895193323Sed return; 1896193323Sed case ISD::SHL: 1897193323Sed // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1898193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1899193323Sed unsigned ShAmt = SA->getZExtValue(); 1900193323Sed 1901193323Sed // If the shift count is an invalid immediate, don't do anything. 1902193323Sed if (ShAmt >= BitWidth) 1903193323Sed return; 1904193323Sed 1905234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1906193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1907193323Sed KnownZero <<= ShAmt; 1908193323Sed KnownOne <<= ShAmt; 1909193323Sed // low bits known zero. 1910193323Sed KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1911193323Sed } 1912193323Sed return; 1913193323Sed case ISD::SRL: 1914193323Sed // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1915193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1916193323Sed unsigned ShAmt = SA->getZExtValue(); 1917193323Sed 1918193323Sed // If the shift count is an invalid immediate, don't do anything. 1919193323Sed if (ShAmt >= BitWidth) 1920193323Sed return; 1921193323Sed 1922234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1923193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1924193323Sed KnownZero = KnownZero.lshr(ShAmt); 1925193323Sed KnownOne = KnownOne.lshr(ShAmt); 1926193323Sed 1927234353Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1928193323Sed KnownZero |= HighBits; // High bits known zero. 1929193323Sed } 1930193323Sed return; 1931193323Sed case ISD::SRA: 1932193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1933193323Sed unsigned ShAmt = SA->getZExtValue(); 1934193323Sed 1935193323Sed // If the shift count is an invalid immediate, don't do anything. 1936193323Sed if (ShAmt >= BitWidth) 1937193323Sed return; 1938193323Sed 1939193323Sed // If any of the demanded bits are produced by the sign extension, we also 1940193323Sed // demand the input sign bit. 1941234353Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1942193323Sed 1943234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1944193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1945193323Sed KnownZero = KnownZero.lshr(ShAmt); 1946193323Sed KnownOne = KnownOne.lshr(ShAmt); 1947193323Sed 1948193323Sed // Handle the sign bits. 1949193323Sed APInt SignBit = APInt::getSignBit(BitWidth); 1950193323Sed SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1951193323Sed 1952193323Sed if (KnownZero.intersects(SignBit)) { 1953193323Sed KnownZero |= HighBits; // New bits are known zero. 1954193323Sed } else if (KnownOne.intersects(SignBit)) { 1955193323Sed KnownOne |= HighBits; // New bits are known one. 1956193323Sed } 1957193323Sed } 1958193323Sed return; 1959193323Sed case ISD::SIGN_EXTEND_INREG: { 1960198090Srdivacky EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1961202375Srdivacky unsigned EBits = EVT.getScalarType().getSizeInBits(); 1962193323Sed 1963193323Sed // Sign extension. Compute the demanded bits in the result that are not 1964193323Sed // present in the input. 1965234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 1966193323Sed 1967193323Sed APInt InSignBit = APInt::getSignBit(EBits); 1968234353Sdim APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 1969193323Sed 1970193323Sed // If the sign extended bits are demanded, we know that the sign 1971193323Sed // bit is demanded. 1972218893Sdim InSignBit = InSignBit.zext(BitWidth); 1973193323Sed if (NewBits.getBoolValue()) 1974193323Sed InputDemandedBits |= InSignBit; 1975193323Sed 1976234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1977234353Sdim KnownOne &= InputDemandedBits; 1978234353Sdim KnownZero &= InputDemandedBits; 1979193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1980193323Sed 1981193323Sed // If the sign bit of the input is known set or clear, then we know the 1982193323Sed // top bits of the result. 1983193323Sed if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1984193323Sed KnownZero |= NewBits; 1985193323Sed KnownOne &= ~NewBits; 1986193323Sed } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1987193323Sed KnownOne |= NewBits; 1988193323Sed KnownZero &= ~NewBits; 1989193323Sed } else { // Input sign bit unknown 1990193323Sed KnownZero &= ~NewBits; 1991193323Sed KnownOne &= ~NewBits; 1992193323Sed } 1993193323Sed return; 1994193323Sed } 1995193323Sed case ISD::CTTZ: 1996234353Sdim case ISD::CTTZ_ZERO_UNDEF: 1997193323Sed case ISD::CTLZ: 1998234353Sdim case ISD::CTLZ_ZERO_UNDEF: 1999193323Sed case ISD::CTPOP: { 2000193323Sed unsigned LowBits = Log2_32(BitWidth)+1; 2001193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 2002218893Sdim KnownOne.clearAllBits(); 2003193323Sed return; 2004193323Sed } 2005193323Sed case ISD::LOAD: { 2006234353Sdim LoadSDNode *LD = cast<LoadSDNode>(Op); 2007249423Sdim // If this is a ZEXTLoad and we are looking at the loaded value. 2008249423Sdim if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 2009198090Srdivacky EVT VT = LD->getMemoryVT(); 2010202375Srdivacky unsigned MemBits = VT.getScalarType().getSizeInBits(); 2011234353Sdim KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 2012234353Sdim } else if (const MDNode *Ranges = LD->getRanges()) { 2013234353Sdim computeMaskedBitsLoad(*Ranges, KnownZero); 2014193323Sed } 2015193323Sed return; 2016193323Sed } 2017193323Sed case ISD::ZERO_EXTEND: { 2018198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2019200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2020234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2021218893Sdim KnownZero = KnownZero.trunc(InBits); 2022218893Sdim KnownOne = KnownOne.trunc(InBits); 2023234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2024218893Sdim KnownZero = KnownZero.zext(BitWidth); 2025218893Sdim KnownOne = KnownOne.zext(BitWidth); 2026193323Sed KnownZero |= NewBits; 2027193323Sed return; 2028193323Sed } 2029193323Sed case ISD::SIGN_EXTEND: { 2030198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2031200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2032234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2033193323Sed 2034218893Sdim KnownZero = KnownZero.trunc(InBits); 2035218893Sdim KnownOne = KnownOne.trunc(InBits); 2036234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2037193323Sed 2038193323Sed // Note if the sign bit is known to be zero or one. 2039193323Sed bool SignBitKnownZero = KnownZero.isNegative(); 2040193323Sed bool SignBitKnownOne = KnownOne.isNegative(); 2041193323Sed assert(!(SignBitKnownZero && SignBitKnownOne) && 2042193323Sed "Sign bit can't be known to be both zero and one!"); 2043193323Sed 2044218893Sdim KnownZero = KnownZero.zext(BitWidth); 2045218893Sdim KnownOne = KnownOne.zext(BitWidth); 2046193323Sed 2047193323Sed // If the sign bit is known zero or one, the top bits match. 2048193323Sed if (SignBitKnownZero) 2049193323Sed KnownZero |= NewBits; 2050193323Sed else if (SignBitKnownOne) 2051193323Sed KnownOne |= NewBits; 2052193323Sed return; 2053193323Sed } 2054193323Sed case ISD::ANY_EXTEND: { 2055198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2056200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2057218893Sdim KnownZero = KnownZero.trunc(InBits); 2058218893Sdim KnownOne = KnownOne.trunc(InBits); 2059234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2060218893Sdim KnownZero = KnownZero.zext(BitWidth); 2061218893Sdim KnownOne = KnownOne.zext(BitWidth); 2062193323Sed return; 2063193323Sed } 2064193323Sed case ISD::TRUNCATE: { 2065198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2066200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2067218893Sdim KnownZero = KnownZero.zext(InBits); 2068218893Sdim KnownOne = KnownOne.zext(InBits); 2069234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2070193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 2071218893Sdim KnownZero = KnownZero.trunc(BitWidth); 2072218893Sdim KnownOne = KnownOne.trunc(BitWidth); 2073193323Sed break; 2074193323Sed } 2075193323Sed case ISD::AssertZext: { 2076198090Srdivacky EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2077193323Sed APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 2078234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2079234353Sdim KnownZero |= (~InMask); 2080239462Sdim KnownOne &= (~KnownZero); 2081193323Sed return; 2082193323Sed } 2083193323Sed case ISD::FGETSIGN: 2084193323Sed // All bits are zero except the low bit. 2085193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 2086193323Sed return; 2087193323Sed 2088193323Sed case ISD::SUB: { 2089193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 2090193323Sed // We know that the top bits of C-X are clear if X contains less bits 2091193323Sed // than C (i.e. no wrap-around can happen). For example, 20-X is 2092193323Sed // positive if we can prove that X is >= 0 and < 16. 2093193323Sed if (CLHS->getAPIntValue().isNonNegative()) { 2094193323Sed unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2095193323Sed // NLZ can't be BitWidth with no sign bit 2096193323Sed APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2097234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2098193323Sed 2099193323Sed // If all of the MaskV bits are known to be zero, then we know the 2100193323Sed // output top bits are zero, because we now know that the output is 2101193323Sed // from [0-C]. 2102193323Sed if ((KnownZero2 & MaskV) == MaskV) { 2103193323Sed unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2104193323Sed // Top bits known zero. 2105234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 2106193323Sed } 2107193323Sed } 2108193323Sed } 2109193323Sed } 2110193323Sed // fall through 2111218893Sdim case ISD::ADD: 2112218893Sdim case ISD::ADDE: { 2113193323Sed // Output known-0 bits are known if clear or set in both the low clear bits 2114193323Sed // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2115193323Sed // low 3 bits clear. 2116234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 2117193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2118193323Sed unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2119193323Sed 2120234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2121193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2122193323Sed KnownZeroOut = std::min(KnownZeroOut, 2123193323Sed KnownZero2.countTrailingOnes()); 2124193323Sed 2125218893Sdim if (Op.getOpcode() == ISD::ADD) { 2126218893Sdim KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2127218893Sdim return; 2128218893Sdim } 2129218893Sdim 2130218893Sdim // With ADDE, a carry bit may be added in, so we can only use this 2131218893Sdim // information if we know (at least) that the low two bits are clear. We 2132218893Sdim // then return to the caller that the low bit is unknown but that other bits 2133218893Sdim // are known zero. 2134218893Sdim if (KnownZeroOut >= 2) // ADDE 2135218893Sdim KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2136193323Sed return; 2137193323Sed } 2138193323Sed case ISD::SREM: 2139193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2140203954Srdivacky const APInt &RA = Rem->getAPIntValue().abs(); 2141203954Srdivacky if (RA.isPowerOf2()) { 2142203954Srdivacky APInt LowBits = RA - 1; 2143234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 2144193323Sed 2145203954Srdivacky // The low bits of the first operand are unchanged by the srem. 2146203954Srdivacky KnownZero = KnownZero2 & LowBits; 2147203954Srdivacky KnownOne = KnownOne2 & LowBits; 2148203954Srdivacky 2149203954Srdivacky // If the first operand is non-negative or has all low bits zero, then 2150203954Srdivacky // the upper bits are all zero. 2151193323Sed if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2152203954Srdivacky KnownZero |= ~LowBits; 2153193323Sed 2154203954Srdivacky // If the first operand is negative and not all low bits are zero, then 2155203954Srdivacky // the upper bits are all one. 2156203954Srdivacky if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2157203954Srdivacky KnownOne |= ~LowBits; 2158193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2159193323Sed } 2160193323Sed } 2161193323Sed return; 2162193323Sed case ISD::UREM: { 2163193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2164193323Sed const APInt &RA = Rem->getAPIntValue(); 2165193323Sed if (RA.isPowerOf2()) { 2166193323Sed APInt LowBits = (RA - 1); 2167234353Sdim KnownZero |= ~LowBits; 2168234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 2169193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2170193323Sed break; 2171193323Sed } 2172193323Sed } 2173193323Sed 2174193323Sed // Since the result is less than or equal to either operand, any leading 2175193323Sed // zero bits in either operand must also exist in the result. 2176234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2177234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2178193323Sed 2179193323Sed uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2180193323Sed KnownZero2.countLeadingOnes()); 2181218893Sdim KnownOne.clearAllBits(); 2182234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 2183193323Sed return; 2184193323Sed } 2185218893Sdim case ISD::FrameIndex: 2186218893Sdim case ISD::TargetFrameIndex: 2187218893Sdim if (unsigned Align = InferPtrAlignment(Op)) { 2188218893Sdim // The low bits are known zero if the pointer is aligned. 2189218893Sdim KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2190218893Sdim return; 2191218893Sdim } 2192218893Sdim break; 2193219077Sdim 2194193323Sed default: 2195223017Sdim if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2196223017Sdim break; 2197223017Sdim // Fallthrough 2198193323Sed case ISD::INTRINSIC_WO_CHAIN: 2199193323Sed case ISD::INTRINSIC_W_CHAIN: 2200193323Sed case ISD::INTRINSIC_VOID: 2201223017Sdim // Allow the target to implement this method for its nodes. 2202263508Sdim TLI->computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 2203193323Sed return; 2204193323Sed } 2205193323Sed} 2206193323Sed 2207193323Sed/// ComputeNumSignBits - Return the number of times the sign bit of the 2208193323Sed/// register is replicated into the other bits. We know that at least 1 bit 2209193323Sed/// is always equal to the sign bit (itself), but other cases can give us 2210193323Sed/// information. For example, immediately after an "SRA X, 2", we know that 2211193323Sed/// the top 3 bits are all equal to each other, so we return 3. 2212193323Sedunsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2213263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2214198090Srdivacky EVT VT = Op.getValueType(); 2215193323Sed assert(VT.isInteger() && "Invalid VT!"); 2216200581Srdivacky unsigned VTBits = VT.getScalarType().getSizeInBits(); 2217193323Sed unsigned Tmp, Tmp2; 2218193323Sed unsigned FirstAnswer = 1; 2219193323Sed 2220193323Sed if (Depth == 6) 2221193323Sed return 1; // Limit search depth. 2222193323Sed 2223193323Sed switch (Op.getOpcode()) { 2224193323Sed default: break; 2225193323Sed case ISD::AssertSext: 2226193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2227193323Sed return VTBits-Tmp+1; 2228193323Sed case ISD::AssertZext: 2229193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2230193323Sed return VTBits-Tmp; 2231193323Sed 2232193323Sed case ISD::Constant: { 2233193323Sed const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2234219077Sdim return Val.getNumSignBits(); 2235193323Sed } 2236193323Sed 2237193323Sed case ISD::SIGN_EXTEND: 2238263508Sdim Tmp = 2239263508Sdim VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2240193323Sed return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2241193323Sed 2242193323Sed case ISD::SIGN_EXTEND_INREG: 2243193323Sed // Max of the input and what this extends. 2244202375Srdivacky Tmp = 2245202375Srdivacky cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2246193323Sed Tmp = VTBits-Tmp+1; 2247193323Sed 2248193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2249193323Sed return std::max(Tmp, Tmp2); 2250193323Sed 2251193323Sed case ISD::SRA: 2252193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2253193323Sed // SRA X, C -> adds C sign bits. 2254193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2255193323Sed Tmp += C->getZExtValue(); 2256193323Sed if (Tmp > VTBits) Tmp = VTBits; 2257193323Sed } 2258193323Sed return Tmp; 2259193323Sed case ISD::SHL: 2260193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2261193323Sed // shl destroys sign bits. 2262193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2263193323Sed if (C->getZExtValue() >= VTBits || // Bad shift. 2264193323Sed C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2265193323Sed return Tmp - C->getZExtValue(); 2266193323Sed } 2267193323Sed break; 2268193323Sed case ISD::AND: 2269193323Sed case ISD::OR: 2270193323Sed case ISD::XOR: // NOT is handled here. 2271193323Sed // Logical binary ops preserve the number of sign bits at the worst. 2272193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2273193323Sed if (Tmp != 1) { 2274193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2275193323Sed FirstAnswer = std::min(Tmp, Tmp2); 2276193323Sed // We computed what we know about the sign bits as our first 2277193323Sed // answer. Now proceed to the generic code that uses 2278193323Sed // ComputeMaskedBits, and pick whichever answer is better. 2279193323Sed } 2280193323Sed break; 2281193323Sed 2282193323Sed case ISD::SELECT: 2283193323Sed Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2284193323Sed if (Tmp == 1) return 1; // Early out. 2285193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2286193323Sed return std::min(Tmp, Tmp2); 2287193323Sed 2288193323Sed case ISD::SADDO: 2289193323Sed case ISD::UADDO: 2290193323Sed case ISD::SSUBO: 2291193323Sed case ISD::USUBO: 2292193323Sed case ISD::SMULO: 2293193323Sed case ISD::UMULO: 2294193323Sed if (Op.getResNo() != 1) 2295193323Sed break; 2296193323Sed // The boolean result conforms to getBooleanContents. Fall through. 2297193323Sed case ISD::SETCC: 2298193323Sed // If setcc returns 0/-1, all bits are sign bits. 2299263508Sdim if (TLI->getBooleanContents(Op.getValueType().isVector()) == 2300193323Sed TargetLowering::ZeroOrNegativeOneBooleanContent) 2301193323Sed return VTBits; 2302193323Sed break; 2303193323Sed case ISD::ROTL: 2304193323Sed case ISD::ROTR: 2305193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2306193323Sed unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2307193323Sed 2308193323Sed // Handle rotate right by N like a rotate left by 32-N. 2309193323Sed if (Op.getOpcode() == ISD::ROTR) 2310193323Sed RotAmt = (VTBits-RotAmt) & (VTBits-1); 2311193323Sed 2312193323Sed // If we aren't rotating out all of the known-in sign bits, return the 2313193323Sed // number that are left. This handles rotl(sext(x), 1) for example. 2314193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2315193323Sed if (Tmp > RotAmt+1) return Tmp-RotAmt; 2316193323Sed } 2317193323Sed break; 2318193323Sed case ISD::ADD: 2319193323Sed // Add can have at most one carry bit. Thus we know that the output 2320193323Sed // is, at worst, one more bit than the inputs. 2321193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2322193323Sed if (Tmp == 1) return 1; // Early out. 2323193323Sed 2324193323Sed // Special case decrementing a value (ADD X, -1): 2325193323Sed if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2326193323Sed if (CRHS->isAllOnesValue()) { 2327193323Sed APInt KnownZero, KnownOne; 2328234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2329193323Sed 2330193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2331193323Sed // sign bits set. 2332234353Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2333193323Sed return VTBits; 2334193323Sed 2335193323Sed // If we are subtracting one from a positive number, there is no carry 2336193323Sed // out of the result. 2337193323Sed if (KnownZero.isNegative()) 2338193323Sed return Tmp; 2339193323Sed } 2340193323Sed 2341193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2342193323Sed if (Tmp2 == 1) return 1; 2343234353Sdim return std::min(Tmp, Tmp2)-1; 2344193323Sed 2345193323Sed case ISD::SUB: 2346193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2347193323Sed if (Tmp2 == 1) return 1; 2348193323Sed 2349193323Sed // Handle NEG. 2350193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2351193323Sed if (CLHS->isNullValue()) { 2352193323Sed APInt KnownZero, KnownOne; 2353234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 2354193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2355193323Sed // sign bits set. 2356234353Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2357193323Sed return VTBits; 2358193323Sed 2359193323Sed // If the input is known to be positive (the sign bit is known clear), 2360193323Sed // the output of the NEG has the same number of sign bits as the input. 2361193323Sed if (KnownZero.isNegative()) 2362193323Sed return Tmp2; 2363193323Sed 2364193323Sed // Otherwise, we treat this like a SUB. 2365193323Sed } 2366193323Sed 2367193323Sed // Sub can have at most one carry bit. Thus we know that the output 2368193323Sed // is, at worst, one more bit than the inputs. 2369193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2370193323Sed if (Tmp == 1) return 1; // Early out. 2371234353Sdim return std::min(Tmp, Tmp2)-1; 2372193323Sed case ISD::TRUNCATE: 2373193323Sed // FIXME: it's tricky to do anything useful for this, but it is an important 2374193323Sed // case for targets like X86. 2375193323Sed break; 2376193323Sed } 2377193323Sed 2378249423Sdim // If we are looking at the loaded value of the SDNode. 2379249423Sdim if (Op.getResNo() == 0) { 2380249423Sdim // Handle LOADX separately here. EXTLOAD case will fallthrough. 2381249423Sdim if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 2382249423Sdim unsigned ExtType = LD->getExtensionType(); 2383249423Sdim switch (ExtType) { 2384249423Sdim default: break; 2385249423Sdim case ISD::SEXTLOAD: // '17' bits known 2386249423Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2387249423Sdim return VTBits-Tmp+1; 2388249423Sdim case ISD::ZEXTLOAD: // '16' bits known 2389249423Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2390249423Sdim return VTBits-Tmp; 2391249423Sdim } 2392193323Sed } 2393193323Sed } 2394193323Sed 2395193323Sed // Allow the target to implement this method for its nodes. 2396193323Sed if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2397193323Sed Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2398193323Sed Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2399193323Sed Op.getOpcode() == ISD::INTRINSIC_VOID) { 2400263508Sdim unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, Depth); 2401193323Sed if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2402193323Sed } 2403193323Sed 2404193323Sed // Finally, if we can prove that the top bits of the result are 0's or 1's, 2405193323Sed // use this information. 2406193323Sed APInt KnownZero, KnownOne; 2407234353Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 2408193323Sed 2409234353Sdim APInt Mask; 2410193323Sed if (KnownZero.isNegative()) { // sign bit is 0 2411193323Sed Mask = KnownZero; 2412193323Sed } else if (KnownOne.isNegative()) { // sign bit is 1; 2413193323Sed Mask = KnownOne; 2414193323Sed } else { 2415193323Sed // Nothing known. 2416193323Sed return FirstAnswer; 2417193323Sed } 2418193323Sed 2419193323Sed // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2420193323Sed // the number of identical bits in the top of the input value. 2421193323Sed Mask = ~Mask; 2422193323Sed Mask <<= Mask.getBitWidth()-VTBits; 2423193323Sed // Return # leading zeros. We use 'min' here in case Val was zero before 2424193323Sed // shifting. We don't want to return '64' as for an i32 "0". 2425193323Sed return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2426193323Sed} 2427193323Sed 2428218893Sdim/// isBaseWithConstantOffset - Return true if the specified operand is an 2429218893Sdim/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2430218893Sdim/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2431218893Sdim/// semantics as an ADD. This handles the equivalence: 2432218893Sdim/// X|Cst == X+Cst iff X&Cst = 0. 2433218893Sdimbool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2434218893Sdim if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2435218893Sdim !isa<ConstantSDNode>(Op.getOperand(1))) 2436218893Sdim return false; 2437219077Sdim 2438219077Sdim if (Op.getOpcode() == ISD::OR && 2439218893Sdim !MaskedValueIsZero(Op.getOperand(0), 2440218893Sdim cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2441218893Sdim return false; 2442219077Sdim 2443218893Sdim return true; 2444218893Sdim} 2445218893Sdim 2446218893Sdim 2447198090Srdivackybool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2448198090Srdivacky // If we're told that NaNs won't happen, assume they won't. 2449234353Sdim if (getTarget().Options.NoNaNsFPMath) 2450198090Srdivacky return true; 2451193323Sed 2452198090Srdivacky // If the value is a constant, we can obviously see if it is a NaN or not. 2453198090Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2454198090Srdivacky return !C->getValueAPF().isNaN(); 2455198090Srdivacky 2456198090Srdivacky // TODO: Recognize more cases here. 2457198090Srdivacky 2458198090Srdivacky return false; 2459198090Srdivacky} 2460198090Srdivacky 2461204642Srdivackybool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2462204642Srdivacky // If the value is a constant, we can obviously see if it is a zero or not. 2463204642Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2464204642Srdivacky return !C->isZero(); 2465204642Srdivacky 2466204642Srdivacky // TODO: Recognize more cases here. 2467223017Sdim switch (Op.getOpcode()) { 2468223017Sdim default: break; 2469223017Sdim case ISD::OR: 2470223017Sdim if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2471223017Sdim return !C->isNullValue(); 2472223017Sdim break; 2473223017Sdim } 2474204642Srdivacky 2475204642Srdivacky return false; 2476204642Srdivacky} 2477204642Srdivacky 2478204642Srdivackybool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2479204642Srdivacky // Check the obvious case. 2480204642Srdivacky if (A == B) return true; 2481204642Srdivacky 2482204642Srdivacky // For for negative and positive zero. 2483204642Srdivacky if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2484204642Srdivacky if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2485204642Srdivacky if (CA->isZero() && CB->isZero()) return true; 2486204642Srdivacky 2487204642Srdivacky // Otherwise they may not be equal. 2488204642Srdivacky return false; 2489204642Srdivacky} 2490204642Srdivacky 2491193323Sed/// getNode - Gets or creates the specified node. 2492193323Sed/// 2493263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) { 2494193323Sed FoldingSetNodeID ID; 2495193323Sed AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2496193323Sed void *IP = 0; 2497201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2498193323Sed return SDValue(E, 0); 2499201360Srdivacky 2500263508Sdim SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), 2501263508Sdim DL.getDebugLoc(), getVTList(VT)); 2502193323Sed CSEMap.InsertNode(N, IP); 2503193323Sed 2504193323Sed AllNodes.push_back(N); 2505193323Sed#ifndef NDEBUG 2506218893Sdim VerifySDNode(N); 2507193323Sed#endif 2508193323Sed return SDValue(N, 0); 2509193323Sed} 2510193323Sed 2511263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 2512198090Srdivacky EVT VT, SDValue Operand) { 2513193323Sed // Constant fold unary operations with an integer constant operand. 2514193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2515193323Sed const APInt &Val = C->getAPIntValue(); 2516193323Sed switch (Opcode) { 2517193323Sed default: break; 2518193323Sed case ISD::SIGN_EXTEND: 2519218893Sdim return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2520193323Sed case ISD::ANY_EXTEND: 2521193323Sed case ISD::ZERO_EXTEND: 2522193323Sed case ISD::TRUNCATE: 2523218893Sdim return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2524193323Sed case ISD::UINT_TO_FP: 2525193323Sed case ISD::SINT_TO_FP: { 2526249423Sdim APFloat apf(EVTToAPFloatSemantics(VT), 2527249423Sdim APInt::getNullValue(VT.getSizeInBits())); 2528193323Sed (void)apf.convertFromAPInt(Val, 2529193323Sed Opcode==ISD::SINT_TO_FP, 2530193323Sed APFloat::rmNearestTiesToEven); 2531193323Sed return getConstantFP(apf, VT); 2532193323Sed } 2533218893Sdim case ISD::BITCAST: 2534193323Sed if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2535249423Sdim return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 2536193323Sed else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2537249423Sdim return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 2538193323Sed break; 2539193323Sed case ISD::BSWAP: 2540193323Sed return getConstant(Val.byteSwap(), VT); 2541193323Sed case ISD::CTPOP: 2542193323Sed return getConstant(Val.countPopulation(), VT); 2543193323Sed case ISD::CTLZ: 2544234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2545193323Sed return getConstant(Val.countLeadingZeros(), VT); 2546193323Sed case ISD::CTTZ: 2547234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2548193323Sed return getConstant(Val.countTrailingZeros(), VT); 2549193323Sed } 2550193323Sed } 2551193323Sed 2552193323Sed // Constant fold unary operations with a floating point constant operand. 2553193323Sed if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2554193323Sed APFloat V = C->getValueAPF(); // make copy 2555243830Sdim switch (Opcode) { 2556243830Sdim case ISD::FNEG: 2557243830Sdim V.changeSign(); 2558243830Sdim return getConstantFP(V, VT); 2559243830Sdim case ISD::FABS: 2560243830Sdim V.clearSign(); 2561243830Sdim return getConstantFP(V, VT); 2562243830Sdim case ISD::FCEIL: { 2563243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 2564243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2565193323Sed return getConstantFP(V, VT); 2566243830Sdim break; 2567243830Sdim } 2568243830Sdim case ISD::FTRUNC: { 2569243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 2570243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2571193323Sed return getConstantFP(V, VT); 2572243830Sdim break; 2573243830Sdim } 2574243830Sdim case ISD::FFLOOR: { 2575243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 2576243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2577193323Sed return getConstantFP(V, VT); 2578243830Sdim break; 2579243830Sdim } 2580243830Sdim case ISD::FP_EXTEND: { 2581243830Sdim bool ignored; 2582243830Sdim // This can return overflow, underflow, or inexact; we don't care. 2583243830Sdim // FIXME need to be more flexible about rounding mode. 2584249423Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 2585243830Sdim APFloat::rmNearestTiesToEven, &ignored); 2586243830Sdim return getConstantFP(V, VT); 2587243830Sdim } 2588243830Sdim case ISD::FP_TO_SINT: 2589243830Sdim case ISD::FP_TO_UINT: { 2590243830Sdim integerPart x[2]; 2591243830Sdim bool ignored; 2592243830Sdim assert(integerPartWidth >= 64); 2593243830Sdim // FIXME need to be more flexible about rounding mode. 2594243830Sdim APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2595243830Sdim Opcode==ISD::FP_TO_SINT, 2596243830Sdim APFloat::rmTowardZero, &ignored); 2597243830Sdim if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2598193323Sed break; 2599243830Sdim APInt api(VT.getSizeInBits(), x); 2600243830Sdim return getConstant(api, VT); 2601193323Sed } 2602243830Sdim case ISD::BITCAST: 2603243830Sdim if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2604243830Sdim return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2605243830Sdim else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2606243830Sdim return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2607243830Sdim break; 2608243830Sdim } 2609193323Sed } 2610193323Sed 2611193323Sed unsigned OpOpcode = Operand.getNode()->getOpcode(); 2612193323Sed switch (Opcode) { 2613193323Sed case ISD::TokenFactor: 2614193323Sed case ISD::MERGE_VALUES: 2615193323Sed case ISD::CONCAT_VECTORS: 2616193323Sed return Operand; // Factor, merge or concat of one node? No need. 2617198090Srdivacky case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2618193323Sed case ISD::FP_EXTEND: 2619193323Sed assert(VT.isFloatingPoint() && 2620193323Sed Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2621193323Sed if (Operand.getValueType() == VT) return Operand; // noop conversion. 2622200581Srdivacky assert((!VT.isVector() || 2623200581Srdivacky VT.getVectorNumElements() == 2624200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2625200581Srdivacky "Vector element count mismatch!"); 2626193323Sed if (Operand.getOpcode() == ISD::UNDEF) 2627193323Sed return getUNDEF(VT); 2628193323Sed break; 2629193323Sed case ISD::SIGN_EXTEND: 2630193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2631193323Sed "Invalid SIGN_EXTEND!"); 2632193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2633200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2634200581Srdivacky "Invalid sext node, dst < src!"); 2635200581Srdivacky assert((!VT.isVector() || 2636200581Srdivacky VT.getVectorNumElements() == 2637200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2638200581Srdivacky "Vector element count mismatch!"); 2639193323Sed if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2640193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2641221345Sdim else if (OpOpcode == ISD::UNDEF) 2642221345Sdim // sext(undef) = 0, because the top bits will all be the same. 2643221345Sdim return getConstant(0, VT); 2644193323Sed break; 2645193323Sed case ISD::ZERO_EXTEND: 2646193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2647193323Sed "Invalid ZERO_EXTEND!"); 2648193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2649200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2650200581Srdivacky "Invalid zext node, dst < src!"); 2651200581Srdivacky assert((!VT.isVector() || 2652200581Srdivacky VT.getVectorNumElements() == 2653200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2654200581Srdivacky "Vector element count mismatch!"); 2655193323Sed if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2656193323Sed return getNode(ISD::ZERO_EXTEND, DL, VT, 2657193323Sed Operand.getNode()->getOperand(0)); 2658221345Sdim else if (OpOpcode == ISD::UNDEF) 2659221345Sdim // zext(undef) = 0, because the top bits will be zero. 2660221345Sdim return getConstant(0, VT); 2661193323Sed break; 2662193323Sed case ISD::ANY_EXTEND: 2663193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2664193323Sed "Invalid ANY_EXTEND!"); 2665193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2666200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2667200581Srdivacky "Invalid anyext node, dst < src!"); 2668200581Srdivacky assert((!VT.isVector() || 2669200581Srdivacky VT.getVectorNumElements() == 2670200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2671200581Srdivacky "Vector element count mismatch!"); 2672210299Sed 2673210299Sed if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2674210299Sed OpOpcode == ISD::ANY_EXTEND) 2675193323Sed // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2676193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2677221345Sdim else if (OpOpcode == ISD::UNDEF) 2678221345Sdim return getUNDEF(VT); 2679210299Sed 2680210299Sed // (ext (trunx x)) -> x 2681210299Sed if (OpOpcode == ISD::TRUNCATE) { 2682210299Sed SDValue OpOp = Operand.getNode()->getOperand(0); 2683210299Sed if (OpOp.getValueType() == VT) 2684210299Sed return OpOp; 2685210299Sed } 2686193323Sed break; 2687193323Sed case ISD::TRUNCATE: 2688193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2689193323Sed "Invalid TRUNCATE!"); 2690193323Sed if (Operand.getValueType() == VT) return Operand; // noop truncate 2691200581Srdivacky assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2692200581Srdivacky "Invalid truncate node, src < dst!"); 2693200581Srdivacky assert((!VT.isVector() || 2694200581Srdivacky VT.getVectorNumElements() == 2695200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2696200581Srdivacky "Vector element count mismatch!"); 2697193323Sed if (OpOpcode == ISD::TRUNCATE) 2698193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2699234353Sdim if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2700234353Sdim OpOpcode == ISD::ANY_EXTEND) { 2701193323Sed // If the source is smaller than the dest, we still need an extend. 2702200581Srdivacky if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2703200581Srdivacky .bitsLT(VT.getScalarType())) 2704193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2705234353Sdim if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2706193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2707234353Sdim return Operand.getNode()->getOperand(0); 2708193323Sed } 2709234353Sdim if (OpOpcode == ISD::UNDEF) 2710234353Sdim return getUNDEF(VT); 2711193323Sed break; 2712218893Sdim case ISD::BITCAST: 2713193323Sed // Basic sanity checking. 2714193323Sed assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2715218893Sdim && "Cannot BITCAST between types of different sizes!"); 2716193323Sed if (VT == Operand.getValueType()) return Operand; // noop conversion. 2717218893Sdim if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2718218893Sdim return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2719193323Sed if (OpOpcode == ISD::UNDEF) 2720193323Sed return getUNDEF(VT); 2721193323Sed break; 2722193323Sed case ISD::SCALAR_TO_VECTOR: 2723193323Sed assert(VT.isVector() && !Operand.getValueType().isVector() && 2724193323Sed (VT.getVectorElementType() == Operand.getValueType() || 2725193323Sed (VT.getVectorElementType().isInteger() && 2726193323Sed Operand.getValueType().isInteger() && 2727193323Sed VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2728193323Sed "Illegal SCALAR_TO_VECTOR node!"); 2729193323Sed if (OpOpcode == ISD::UNDEF) 2730193323Sed return getUNDEF(VT); 2731193323Sed // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2732193323Sed if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2733193323Sed isa<ConstantSDNode>(Operand.getOperand(1)) && 2734193323Sed Operand.getConstantOperandVal(1) == 0 && 2735193323Sed Operand.getOperand(0).getValueType() == VT) 2736193323Sed return Operand.getOperand(0); 2737193323Sed break; 2738193323Sed case ISD::FNEG: 2739193323Sed // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2740234353Sdim if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2741193323Sed return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2742193323Sed Operand.getNode()->getOperand(0)); 2743193323Sed if (OpOpcode == ISD::FNEG) // --X -> X 2744193323Sed return Operand.getNode()->getOperand(0); 2745193323Sed break; 2746193323Sed case ISD::FABS: 2747193323Sed if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2748193323Sed return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2749193323Sed break; 2750193323Sed } 2751193323Sed 2752193323Sed SDNode *N; 2753193323Sed SDVTList VTs = getVTList(VT); 2754218893Sdim if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2755193323Sed FoldingSetNodeID ID; 2756193323Sed SDValue Ops[1] = { Operand }; 2757193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2758193323Sed void *IP = 0; 2759201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2760193323Sed return SDValue(E, 0); 2761201360Srdivacky 2762263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2763263508Sdim DL.getDebugLoc(), VTs, Operand); 2764193323Sed CSEMap.InsertNode(N, IP); 2765193323Sed } else { 2766263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2767263508Sdim DL.getDebugLoc(), VTs, Operand); 2768193323Sed } 2769193323Sed 2770193323Sed AllNodes.push_back(N); 2771193323Sed#ifndef NDEBUG 2772218893Sdim VerifySDNode(N); 2773193323Sed#endif 2774193323Sed return SDValue(N, 0); 2775193323Sed} 2776193323Sed 2777249423SdimSDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 2778249423Sdim SDNode *Cst1, SDNode *Cst2) { 2779249423Sdim SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 2780249423Sdim SmallVector<SDValue, 4> Outputs; 2781249423Sdim EVT SVT = VT.getScalarType(); 2782193323Sed 2783249423Sdim ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 2784249423Sdim ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 2785249423Sdim if (Scalar1 && Scalar2) { 2786249423Sdim // Scalar instruction. 2787249423Sdim Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 2788249423Sdim } else { 2789249423Sdim // For vectors extract each constant element into Inputs so we can constant 2790249423Sdim // fold them individually. 2791249423Sdim BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 2792249423Sdim BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 2793249423Sdim if (!BV1 || !BV2) 2794249423Sdim return SDValue(); 2795249423Sdim 2796249423Sdim assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 2797249423Sdim 2798249423Sdim for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 2799249423Sdim ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 2800249423Sdim ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 2801249423Sdim if (!V1 || !V2) // Not a constant, bail. 2802249423Sdim return SDValue(); 2803249423Sdim 2804249423Sdim // Avoid BUILD_VECTOR nodes that perform implicit truncation. 2805249423Sdim // FIXME: This is valid and could be handled by truncating the APInts. 2806249423Sdim if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 2807249423Sdim return SDValue(); 2808249423Sdim 2809249423Sdim Inputs.push_back(std::make_pair(V1, V2)); 2810249423Sdim } 2811193323Sed } 2812193323Sed 2813249423Sdim // We have a number of constant values, constant fold them element by element. 2814249423Sdim for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 2815249423Sdim const APInt &C1 = Inputs[I].first->getAPIntValue(); 2816249423Sdim const APInt &C2 = Inputs[I].second->getAPIntValue(); 2817249423Sdim 2818249423Sdim switch (Opcode) { 2819249423Sdim case ISD::ADD: 2820249423Sdim Outputs.push_back(getConstant(C1 + C2, SVT)); 2821249423Sdim break; 2822249423Sdim case ISD::SUB: 2823249423Sdim Outputs.push_back(getConstant(C1 - C2, SVT)); 2824249423Sdim break; 2825249423Sdim case ISD::MUL: 2826249423Sdim Outputs.push_back(getConstant(C1 * C2, SVT)); 2827249423Sdim break; 2828249423Sdim case ISD::UDIV: 2829249423Sdim if (!C2.getBoolValue()) 2830249423Sdim return SDValue(); 2831249423Sdim Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 2832249423Sdim break; 2833249423Sdim case ISD::UREM: 2834249423Sdim if (!C2.getBoolValue()) 2835249423Sdim return SDValue(); 2836249423Sdim Outputs.push_back(getConstant(C1.urem(C2), SVT)); 2837249423Sdim break; 2838249423Sdim case ISD::SDIV: 2839249423Sdim if (!C2.getBoolValue()) 2840249423Sdim return SDValue(); 2841249423Sdim Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 2842249423Sdim break; 2843249423Sdim case ISD::SREM: 2844249423Sdim if (!C2.getBoolValue()) 2845249423Sdim return SDValue(); 2846249423Sdim Outputs.push_back(getConstant(C1.srem(C2), SVT)); 2847249423Sdim break; 2848249423Sdim case ISD::AND: 2849249423Sdim Outputs.push_back(getConstant(C1 & C2, SVT)); 2850249423Sdim break; 2851249423Sdim case ISD::OR: 2852249423Sdim Outputs.push_back(getConstant(C1 | C2, SVT)); 2853249423Sdim break; 2854249423Sdim case ISD::XOR: 2855249423Sdim Outputs.push_back(getConstant(C1 ^ C2, SVT)); 2856249423Sdim break; 2857249423Sdim case ISD::SHL: 2858249423Sdim Outputs.push_back(getConstant(C1 << C2, SVT)); 2859249423Sdim break; 2860249423Sdim case ISD::SRL: 2861249423Sdim Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 2862249423Sdim break; 2863249423Sdim case ISD::SRA: 2864249423Sdim Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 2865249423Sdim break; 2866249423Sdim case ISD::ROTL: 2867249423Sdim Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 2868249423Sdim break; 2869249423Sdim case ISD::ROTR: 2870249423Sdim Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 2871249423Sdim break; 2872249423Sdim default: 2873249423Sdim return SDValue(); 2874249423Sdim } 2875249423Sdim } 2876249423Sdim 2877249423Sdim // Handle the scalar case first. 2878251662Sdim if (Scalar1 && Scalar2) 2879249423Sdim return Outputs.back(); 2880249423Sdim 2881249423Sdim // Otherwise build a big vector out of the scalar elements we generated. 2882263508Sdim return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(), 2883249423Sdim Outputs.size()); 2884193323Sed} 2885193323Sed 2886263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, 2887249423Sdim SDValue N2) { 2888193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2889193323Sed ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2890193323Sed switch (Opcode) { 2891193323Sed default: break; 2892193323Sed case ISD::TokenFactor: 2893193323Sed assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2894193323Sed N2.getValueType() == MVT::Other && "Invalid token factor!"); 2895193323Sed // Fold trivial token factors. 2896193323Sed if (N1.getOpcode() == ISD::EntryToken) return N2; 2897193323Sed if (N2.getOpcode() == ISD::EntryToken) return N1; 2898193323Sed if (N1 == N2) return N1; 2899193323Sed break; 2900193323Sed case ISD::CONCAT_VECTORS: 2901239462Sdim // Concat of UNDEFs is UNDEF. 2902239462Sdim if (N1.getOpcode() == ISD::UNDEF && 2903239462Sdim N2.getOpcode() == ISD::UNDEF) 2904239462Sdim return getUNDEF(VT); 2905239462Sdim 2906193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2907193323Sed // one big BUILD_VECTOR. 2908193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 2909193323Sed N2.getOpcode() == ISD::BUILD_VECTOR) { 2910212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2911212904Sdim N1.getNode()->op_end()); 2912210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2913193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2914193323Sed } 2915193323Sed break; 2916193323Sed case ISD::AND: 2917208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2918208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2919193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2920193323Sed // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2921193323Sed // worth handling here. 2922193323Sed if (N2C && N2C->isNullValue()) 2923193323Sed return N2; 2924193323Sed if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2925193323Sed return N1; 2926193323Sed break; 2927193323Sed case ISD::OR: 2928193323Sed case ISD::XOR: 2929193323Sed case ISD::ADD: 2930193323Sed case ISD::SUB: 2931208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2932208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2933193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2934193323Sed // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2935193323Sed // it's worth handling here. 2936193323Sed if (N2C && N2C->isNullValue()) 2937193323Sed return N1; 2938193323Sed break; 2939193323Sed case ISD::UDIV: 2940193323Sed case ISD::UREM: 2941193323Sed case ISD::MULHU: 2942193323Sed case ISD::MULHS: 2943193323Sed case ISD::MUL: 2944193323Sed case ISD::SDIV: 2945193323Sed case ISD::SREM: 2946193323Sed assert(VT.isInteger() && "This operator does not apply to FP types!"); 2947208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2948208599Srdivacky N1.getValueType() == VT && "Binary operator types must match!"); 2949208599Srdivacky break; 2950193323Sed case ISD::FADD: 2951193323Sed case ISD::FSUB: 2952193323Sed case ISD::FMUL: 2953193323Sed case ISD::FDIV: 2954193323Sed case ISD::FREM: 2955234353Sdim if (getTarget().Options.UnsafeFPMath) { 2956193323Sed if (Opcode == ISD::FADD) { 2957193323Sed // 0+x --> x 2958193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2959193323Sed if (CFP->getValueAPF().isZero()) 2960193323Sed return N2; 2961193323Sed // x+0 --> x 2962193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2963193323Sed if (CFP->getValueAPF().isZero()) 2964193323Sed return N1; 2965193323Sed } else if (Opcode == ISD::FSUB) { 2966193323Sed // x-0 --> x 2967193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2968193323Sed if (CFP->getValueAPF().isZero()) 2969193323Sed return N1; 2970243830Sdim } else if (Opcode == ISD::FMUL) { 2971243830Sdim ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 2972243830Sdim SDValue V = N2; 2973243830Sdim 2974243830Sdim // If the first operand isn't the constant, try the second 2975243830Sdim if (!CFP) { 2976243830Sdim CFP = dyn_cast<ConstantFPSDNode>(N2); 2977243830Sdim V = N1; 2978243830Sdim } 2979243830Sdim 2980243830Sdim if (CFP) { 2981243830Sdim // 0*x --> 0 2982243830Sdim if (CFP->isZero()) 2983243830Sdim return SDValue(CFP,0); 2984243830Sdim // 1*x --> x 2985243830Sdim if (CFP->isExactlyValue(1.0)) 2986243830Sdim return V; 2987243830Sdim } 2988193323Sed } 2989193323Sed } 2990208599Srdivacky assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2991193323Sed assert(N1.getValueType() == N2.getValueType() && 2992193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2993193323Sed break; 2994193323Sed case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2995193323Sed assert(N1.getValueType() == VT && 2996193323Sed N1.getValueType().isFloatingPoint() && 2997193323Sed N2.getValueType().isFloatingPoint() && 2998193323Sed "Invalid FCOPYSIGN!"); 2999193323Sed break; 3000193323Sed case ISD::SHL: 3001193323Sed case ISD::SRA: 3002193323Sed case ISD::SRL: 3003193323Sed case ISD::ROTL: 3004193323Sed case ISD::ROTR: 3005193323Sed assert(VT == N1.getValueType() && 3006193323Sed "Shift operators return type must be the same as their first arg"); 3007193323Sed assert(VT.isInteger() && N2.getValueType().isInteger() && 3008193323Sed "Shifts only work on integers"); 3009249423Sdim assert((!VT.isVector() || VT == N2.getValueType()) && 3010249423Sdim "Vector shift amounts must be in the same as their first arg"); 3011218893Sdim // Verify that the shift amount VT is bit enough to hold valid shift 3012218893Sdim // amounts. This catches things like trying to shift an i1024 value by an 3013218893Sdim // i8, which is easy to fall into in generic code that uses 3014218893Sdim // TLI.getShiftAmount(). 3015218893Sdim assert(N2.getValueType().getSizeInBits() >= 3016219077Sdim Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 3017218893Sdim "Invalid use of small shift amount with oversized value!"); 3018193323Sed 3019193323Sed // Always fold shifts of i1 values so the code generator doesn't need to 3020193323Sed // handle them. Since we know the size of the shift has to be less than the 3021193323Sed // size of the value, the shift/rotate count is guaranteed to be zero. 3022193323Sed if (VT == MVT::i1) 3023193323Sed return N1; 3024202375Srdivacky if (N2C && N2C->isNullValue()) 3025202375Srdivacky return N1; 3026193323Sed break; 3027193323Sed case ISD::FP_ROUND_INREG: { 3028198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3029193323Sed assert(VT == N1.getValueType() && "Not an inreg round!"); 3030193323Sed assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 3031193323Sed "Cannot FP_ROUND_INREG integer types"); 3032202375Srdivacky assert(EVT.isVector() == VT.isVector() && 3033202375Srdivacky "FP_ROUND_INREG type should be vector iff the operand " 3034202375Srdivacky "type is vector!"); 3035202375Srdivacky assert((!EVT.isVector() || 3036202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3037202375Srdivacky "Vector element counts must match in FP_ROUND_INREG"); 3038193323Sed assert(EVT.bitsLE(VT) && "Not rounding down!"); 3039226633Sdim (void)EVT; 3040193323Sed if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 3041193323Sed break; 3042193323Sed } 3043193323Sed case ISD::FP_ROUND: 3044193323Sed assert(VT.isFloatingPoint() && 3045193323Sed N1.getValueType().isFloatingPoint() && 3046193323Sed VT.bitsLE(N1.getValueType()) && 3047193323Sed isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 3048193323Sed if (N1.getValueType() == VT) return N1; // noop conversion. 3049193323Sed break; 3050193323Sed case ISD::AssertSext: 3051193323Sed case ISD::AssertZext: { 3052198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3053193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 3054193323Sed assert(VT.isInteger() && EVT.isInteger() && 3055193323Sed "Cannot *_EXTEND_INREG FP types"); 3056200581Srdivacky assert(!EVT.isVector() && 3057200581Srdivacky "AssertSExt/AssertZExt type should be the vector element type " 3058200581Srdivacky "rather than the vector type!"); 3059193323Sed assert(EVT.bitsLE(VT) && "Not extending!"); 3060193323Sed if (VT == EVT) return N1; // noop assertion. 3061193323Sed break; 3062193323Sed } 3063193323Sed case ISD::SIGN_EXTEND_INREG: { 3064198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3065193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 3066193323Sed assert(VT.isInteger() && EVT.isInteger() && 3067193323Sed "Cannot *_EXTEND_INREG FP types"); 3068202375Srdivacky assert(EVT.isVector() == VT.isVector() && 3069202375Srdivacky "SIGN_EXTEND_INREG type should be vector iff the operand " 3070202375Srdivacky "type is vector!"); 3071202375Srdivacky assert((!EVT.isVector() || 3072202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3073202375Srdivacky "Vector element counts must match in SIGN_EXTEND_INREG"); 3074202375Srdivacky assert(EVT.bitsLE(VT) && "Not extending!"); 3075193323Sed if (EVT == VT) return N1; // Not actually extending 3076193323Sed 3077193323Sed if (N1C) { 3078193323Sed APInt Val = N1C->getAPIntValue(); 3079202375Srdivacky unsigned FromBits = EVT.getScalarType().getSizeInBits(); 3080193323Sed Val <<= Val.getBitWidth()-FromBits; 3081193323Sed Val = Val.ashr(Val.getBitWidth()-FromBits); 3082193323Sed return getConstant(Val, VT); 3083193323Sed } 3084193323Sed break; 3085193323Sed } 3086193323Sed case ISD::EXTRACT_VECTOR_ELT: 3087193323Sed // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 3088193323Sed if (N1.getOpcode() == ISD::UNDEF) 3089193323Sed return getUNDEF(VT); 3090193323Sed 3091193323Sed // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 3092193323Sed // expanding copies of large vectors from registers. 3093193323Sed if (N2C && 3094193323Sed N1.getOpcode() == ISD::CONCAT_VECTORS && 3095193323Sed N1.getNumOperands() > 0) { 3096193323Sed unsigned Factor = 3097193323Sed N1.getOperand(0).getValueType().getVectorNumElements(); 3098193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 3099193323Sed N1.getOperand(N2C->getZExtValue() / Factor), 3100193323Sed getConstant(N2C->getZExtValue() % Factor, 3101193323Sed N2.getValueType())); 3102193323Sed } 3103193323Sed 3104193323Sed // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 3105193323Sed // expanding large vector constants. 3106193323Sed if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 3107193323Sed SDValue Elt = N1.getOperand(N2C->getZExtValue()); 3108243830Sdim 3109243830Sdim if (VT != Elt.getValueType()) 3110193323Sed // If the vector element type is not legal, the BUILD_VECTOR operands 3111243830Sdim // are promoted and implicitly truncated, and the result implicitly 3112243830Sdim // extended. Make that explicit here. 3113243830Sdim Elt = getAnyExtOrTrunc(Elt, DL, VT); 3114243830Sdim 3115193323Sed return Elt; 3116193323Sed } 3117193323Sed 3118193323Sed // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 3119193323Sed // operations are lowered to scalars. 3120193323Sed if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 3121203954Srdivacky // If the indices are the same, return the inserted element else 3122203954Srdivacky // if the indices are known different, extract the element from 3123193323Sed // the original vector. 3124207618Srdivacky SDValue N1Op2 = N1.getOperand(2); 3125207618Srdivacky ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 3126207618Srdivacky 3127207618Srdivacky if (N1Op2C && N2C) { 3128207618Srdivacky if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 3129207618Srdivacky if (VT == N1.getOperand(1).getValueType()) 3130207618Srdivacky return N1.getOperand(1); 3131207618Srdivacky else 3132207618Srdivacky return getSExtOrTrunc(N1.getOperand(1), DL, VT); 3133207618Srdivacky } 3134207618Srdivacky 3135193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 3136207618Srdivacky } 3137193323Sed } 3138193323Sed break; 3139193323Sed case ISD::EXTRACT_ELEMENT: 3140193323Sed assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 3141193323Sed assert(!N1.getValueType().isVector() && !VT.isVector() && 3142193323Sed (N1.getValueType().isInteger() == VT.isInteger()) && 3143226633Sdim N1.getValueType() != VT && 3144193323Sed "Wrong types for EXTRACT_ELEMENT!"); 3145193323Sed 3146193323Sed // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 3147193323Sed // 64-bit integers into 32-bit parts. Instead of building the extract of 3148193323Sed // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 3149193323Sed if (N1.getOpcode() == ISD::BUILD_PAIR) 3150193323Sed return N1.getOperand(N2C->getZExtValue()); 3151193323Sed 3152193323Sed // EXTRACT_ELEMENT of a constant int is also very common. 3153193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 3154193323Sed unsigned ElementSize = VT.getSizeInBits(); 3155193323Sed unsigned Shift = ElementSize * N2C->getZExtValue(); 3156193323Sed APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 3157193323Sed return getConstant(ShiftedVal.trunc(ElementSize), VT); 3158193323Sed } 3159193323Sed break; 3160218893Sdim case ISD::EXTRACT_SUBVECTOR: { 3161218893Sdim SDValue Index = N2; 3162218893Sdim if (VT.isSimple() && N1.getValueType().isSimple()) { 3163218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3164218893Sdim "Extract subvector VTs must be a vectors!"); 3165263508Sdim assert(VT.getVectorElementType() == 3166263508Sdim N1.getValueType().getVectorElementType() && 3167218893Sdim "Extract subvector VTs must have the same element type!"); 3168263508Sdim assert(VT.getSimpleVT() <= N1.getSimpleValueType() && 3169218893Sdim "Extract subvector must be from larger vector to smaller vector!"); 3170218893Sdim 3171218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3172218893Sdim assert((VT.getVectorNumElements() + 3173218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3174218893Sdim <= N1.getValueType().getVectorNumElements()) 3175218893Sdim && "Extract subvector overflow!"); 3176218893Sdim } 3177218893Sdim 3178218893Sdim // Trivial extraction. 3179263508Sdim if (VT.getSimpleVT() == N1.getSimpleValueType()) 3180218893Sdim return N1; 3181218893Sdim } 3182193323Sed break; 3183193323Sed } 3184218893Sdim } 3185193323Sed 3186249423Sdim // Perform trivial constant folding. 3187249423Sdim SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 3188249423Sdim if (SV.getNode()) return SV; 3189249423Sdim 3190249423Sdim // Canonicalize constant to RHS if commutative. 3191249423Sdim if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 3192249423Sdim std::swap(N1C, N2C); 3193249423Sdim std::swap(N1, N2); 3194193323Sed } 3195193323Sed 3196193323Sed // Constant fold FP operations. 3197193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 3198193323Sed ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 3199193323Sed if (N1CFP) { 3200193323Sed if (!N2CFP && isCommutativeBinOp(Opcode)) { 3201249423Sdim // Canonicalize constant to RHS if commutative. 3202193323Sed std::swap(N1CFP, N2CFP); 3203193323Sed std::swap(N1, N2); 3204243830Sdim } else if (N2CFP) { 3205193323Sed APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3206193323Sed APFloat::opStatus s; 3207193323Sed switch (Opcode) { 3208193323Sed case ISD::FADD: 3209193323Sed s = V1.add(V2, APFloat::rmNearestTiesToEven); 3210193323Sed if (s != APFloat::opInvalidOp) 3211193323Sed return getConstantFP(V1, VT); 3212193323Sed break; 3213193323Sed case ISD::FSUB: 3214193323Sed s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3215193323Sed if (s!=APFloat::opInvalidOp) 3216193323Sed return getConstantFP(V1, VT); 3217193323Sed break; 3218193323Sed case ISD::FMUL: 3219193323Sed s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3220193323Sed if (s!=APFloat::opInvalidOp) 3221193323Sed return getConstantFP(V1, VT); 3222193323Sed break; 3223193323Sed case ISD::FDIV: 3224193323Sed s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3225193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3226193323Sed return getConstantFP(V1, VT); 3227193323Sed break; 3228193323Sed case ISD::FREM : 3229193323Sed s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3230193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3231193323Sed return getConstantFP(V1, VT); 3232193323Sed break; 3233193323Sed case ISD::FCOPYSIGN: 3234193323Sed V1.copySign(V2); 3235193323Sed return getConstantFP(V1, VT); 3236193323Sed default: break; 3237193323Sed } 3238193323Sed } 3239234353Sdim 3240234353Sdim if (Opcode == ISD::FP_ROUND) { 3241234353Sdim APFloat V = N1CFP->getValueAPF(); // make copy 3242234353Sdim bool ignored; 3243234353Sdim // This can return overflow, underflow, or inexact; we don't care. 3244234353Sdim // FIXME need to be more flexible about rounding mode. 3245249423Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 3246234353Sdim APFloat::rmNearestTiesToEven, &ignored); 3247234353Sdim return getConstantFP(V, VT); 3248234353Sdim } 3249193323Sed } 3250193323Sed 3251193323Sed // Canonicalize an UNDEF to the RHS, even over a constant. 3252193323Sed if (N1.getOpcode() == ISD::UNDEF) { 3253193323Sed if (isCommutativeBinOp(Opcode)) { 3254193323Sed std::swap(N1, N2); 3255193323Sed } else { 3256193323Sed switch (Opcode) { 3257193323Sed case ISD::FP_ROUND_INREG: 3258193323Sed case ISD::SIGN_EXTEND_INREG: 3259193323Sed case ISD::SUB: 3260193323Sed case ISD::FSUB: 3261193323Sed case ISD::FDIV: 3262193323Sed case ISD::FREM: 3263193323Sed case ISD::SRA: 3264193323Sed return N1; // fold op(undef, arg2) -> undef 3265193323Sed case ISD::UDIV: 3266193323Sed case ISD::SDIV: 3267193323Sed case ISD::UREM: 3268193323Sed case ISD::SREM: 3269193323Sed case ISD::SRL: 3270193323Sed case ISD::SHL: 3271193323Sed if (!VT.isVector()) 3272193323Sed return getConstant(0, VT); // fold op(undef, arg2) -> 0 3273193323Sed // For vectors, we can't easily build an all zero vector, just return 3274193323Sed // the LHS. 3275193323Sed return N2; 3276193323Sed } 3277193323Sed } 3278193323Sed } 3279193323Sed 3280193323Sed // Fold a bunch of operators when the RHS is undef. 3281193323Sed if (N2.getOpcode() == ISD::UNDEF) { 3282193323Sed switch (Opcode) { 3283193323Sed case ISD::XOR: 3284193323Sed if (N1.getOpcode() == ISD::UNDEF) 3285193323Sed // Handle undef ^ undef -> 0 special case. This is a common 3286193323Sed // idiom (misuse). 3287193323Sed return getConstant(0, VT); 3288193323Sed // fallthrough 3289193323Sed case ISD::ADD: 3290193323Sed case ISD::ADDC: 3291193323Sed case ISD::ADDE: 3292193323Sed case ISD::SUB: 3293193574Sed case ISD::UDIV: 3294193574Sed case ISD::SDIV: 3295193574Sed case ISD::UREM: 3296193574Sed case ISD::SREM: 3297193574Sed return N2; // fold op(arg1, undef) -> undef 3298193323Sed case ISD::FADD: 3299193323Sed case ISD::FSUB: 3300193323Sed case ISD::FMUL: 3301193323Sed case ISD::FDIV: 3302193323Sed case ISD::FREM: 3303234353Sdim if (getTarget().Options.UnsafeFPMath) 3304193574Sed return N2; 3305193574Sed break; 3306193323Sed case ISD::MUL: 3307193323Sed case ISD::AND: 3308193323Sed case ISD::SRL: 3309193323Sed case ISD::SHL: 3310193323Sed if (!VT.isVector()) 3311193323Sed return getConstant(0, VT); // fold op(arg1, undef) -> 0 3312193323Sed // For vectors, we can't easily build an all zero vector, just return 3313193323Sed // the LHS. 3314193323Sed return N1; 3315193323Sed case ISD::OR: 3316193323Sed if (!VT.isVector()) 3317193323Sed return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3318193323Sed // For vectors, we can't easily build an all one vector, just return 3319193323Sed // the LHS. 3320193323Sed return N1; 3321193323Sed case ISD::SRA: 3322193323Sed return N1; 3323193323Sed } 3324193323Sed } 3325193323Sed 3326193323Sed // Memoize this node if possible. 3327193323Sed SDNode *N; 3328193323Sed SDVTList VTs = getVTList(VT); 3329218893Sdim if (VT != MVT::Glue) { 3330193323Sed SDValue Ops[] = { N1, N2 }; 3331193323Sed FoldingSetNodeID ID; 3332193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3333193323Sed void *IP = 0; 3334201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3335193323Sed return SDValue(E, 0); 3336201360Srdivacky 3337263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3338263508Sdim DL.getDebugLoc(), VTs, N1, N2); 3339193323Sed CSEMap.InsertNode(N, IP); 3340193323Sed } else { 3341263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3342263508Sdim DL.getDebugLoc(), VTs, N1, N2); 3343193323Sed } 3344193323Sed 3345193323Sed AllNodes.push_back(N); 3346193323Sed#ifndef NDEBUG 3347218893Sdim VerifySDNode(N); 3348193323Sed#endif 3349193323Sed return SDValue(N, 0); 3350193323Sed} 3351193323Sed 3352263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3353193323Sed SDValue N1, SDValue N2, SDValue N3) { 3354193323Sed // Perform various simplifications. 3355193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3356193323Sed switch (Opcode) { 3357263508Sdim case ISD::FMA: { 3358263508Sdim ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 3359263508Sdim ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 3360263508Sdim ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 3361263508Sdim if (N1CFP && N2CFP && N3CFP) { 3362263508Sdim APFloat V1 = N1CFP->getValueAPF(); 3363263508Sdim const APFloat &V2 = N2CFP->getValueAPF(); 3364263508Sdim const APFloat &V3 = N3CFP->getValueAPF(); 3365263508Sdim APFloat::opStatus s = 3366263508Sdim V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 3367263508Sdim if (s != APFloat::opInvalidOp) 3368263508Sdim return getConstantFP(V1, VT); 3369263508Sdim } 3370263508Sdim break; 3371263508Sdim } 3372193323Sed case ISD::CONCAT_VECTORS: 3373193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3374193323Sed // one big BUILD_VECTOR. 3375193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 3376193323Sed N2.getOpcode() == ISD::BUILD_VECTOR && 3377193323Sed N3.getOpcode() == ISD::BUILD_VECTOR) { 3378212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3379212904Sdim N1.getNode()->op_end()); 3380210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3381210299Sed Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3382193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3383193323Sed } 3384193323Sed break; 3385193323Sed case ISD::SETCC: { 3386193323Sed // Use FoldSetCC to simplify SETCC's. 3387193323Sed SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3388193323Sed if (Simp.getNode()) return Simp; 3389193323Sed break; 3390193323Sed } 3391193323Sed case ISD::SELECT: 3392193323Sed if (N1C) { 3393193323Sed if (N1C->getZExtValue()) 3394234353Sdim return N2; // select true, X, Y -> X 3395234353Sdim return N3; // select false, X, Y -> Y 3396193323Sed } 3397193323Sed 3398193323Sed if (N2 == N3) return N2; // select C, X, X -> X 3399193323Sed break; 3400193323Sed case ISD::VECTOR_SHUFFLE: 3401198090Srdivacky llvm_unreachable("should use getVectorShuffle constructor!"); 3402218893Sdim case ISD::INSERT_SUBVECTOR: { 3403218893Sdim SDValue Index = N3; 3404218893Sdim if (VT.isSimple() && N1.getValueType().isSimple() 3405218893Sdim && N2.getValueType().isSimple()) { 3406218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3407218893Sdim N2.getValueType().isVector() && 3408218893Sdim "Insert subvector VTs must be a vectors"); 3409218893Sdim assert(VT == N1.getValueType() && 3410218893Sdim "Dest and insert subvector source types must match!"); 3411263508Sdim assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && 3412218893Sdim "Insert subvector must be from smaller vector to larger vector!"); 3413218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3414218893Sdim assert((N2.getValueType().getVectorNumElements() + 3415218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3416218893Sdim <= VT.getVectorNumElements()) 3417218893Sdim && "Insert subvector overflow!"); 3418218893Sdim } 3419218893Sdim 3420218893Sdim // Trivial insertion. 3421263508Sdim if (VT.getSimpleVT() == N2.getSimpleValueType()) 3422218893Sdim return N2; 3423218893Sdim } 3424218893Sdim break; 3425218893Sdim } 3426218893Sdim case ISD::BITCAST: 3427193323Sed // Fold bit_convert nodes from a type to themselves. 3428193323Sed if (N1.getValueType() == VT) 3429193323Sed return N1; 3430193323Sed break; 3431193323Sed } 3432193323Sed 3433193323Sed // Memoize node if it doesn't produce a flag. 3434193323Sed SDNode *N; 3435193323Sed SDVTList VTs = getVTList(VT); 3436218893Sdim if (VT != MVT::Glue) { 3437193323Sed SDValue Ops[] = { N1, N2, N3 }; 3438193323Sed FoldingSetNodeID ID; 3439193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3440193323Sed void *IP = 0; 3441201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3442193323Sed return SDValue(E, 0); 3443201360Srdivacky 3444263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3445263508Sdim DL.getDebugLoc(), VTs, N1, N2, N3); 3446193323Sed CSEMap.InsertNode(N, IP); 3447193323Sed } else { 3448263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3449263508Sdim DL.getDebugLoc(), VTs, N1, N2, N3); 3450193323Sed } 3451200581Srdivacky 3452193323Sed AllNodes.push_back(N); 3453193323Sed#ifndef NDEBUG 3454218893Sdim VerifySDNode(N); 3455193323Sed#endif 3456193323Sed return SDValue(N, 0); 3457193323Sed} 3458193323Sed 3459263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3460193323Sed SDValue N1, SDValue N2, SDValue N3, 3461193323Sed SDValue N4) { 3462193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 3463193323Sed return getNode(Opcode, DL, VT, Ops, 4); 3464193323Sed} 3465193323Sed 3466263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3467193323Sed SDValue N1, SDValue N2, SDValue N3, 3468193323Sed SDValue N4, SDValue N5) { 3469193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3470193323Sed return getNode(Opcode, DL, VT, Ops, 5); 3471193323Sed} 3472193323Sed 3473198090Srdivacky/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3474198090Srdivacky/// the incoming stack arguments to be loaded from the stack. 3475198090SrdivackySDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3476198090Srdivacky SmallVector<SDValue, 8> ArgChains; 3477198090Srdivacky 3478198090Srdivacky // Include the original chain at the beginning of the list. When this is 3479198090Srdivacky // used by target LowerCall hooks, this helps legalize find the 3480198090Srdivacky // CALLSEQ_BEGIN node. 3481198090Srdivacky ArgChains.push_back(Chain); 3482198090Srdivacky 3483198090Srdivacky // Add a chain value for each stack argument. 3484198090Srdivacky for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3485198090Srdivacky UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3486198090Srdivacky if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3487198090Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3488198090Srdivacky if (FI->getIndex() < 0) 3489198090Srdivacky ArgChains.push_back(SDValue(L, 1)); 3490198090Srdivacky 3491198090Srdivacky // Build a tokenfactor for all the chains. 3492263508Sdim return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, 3493198090Srdivacky &ArgChains[0], ArgChains.size()); 3494198090Srdivacky} 3495198090Srdivacky 3496193323Sed/// getMemsetValue - Vectorized representation of the memset value 3497193323Sed/// operand. 3498198090Srdivackystatic SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3499263508Sdim SDLoc dl) { 3500206124Srdivacky assert(Value.getOpcode() != ISD::UNDEF); 3501206124Srdivacky 3502204642Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits(); 3503193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3504249423Sdim assert(C->getAPIntValue().getBitWidth() == 8); 3505249423Sdim APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 3506193323Sed if (VT.isInteger()) 3507193323Sed return DAG.getConstant(Val, VT); 3508249423Sdim return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 3509193323Sed } 3510193323Sed 3511193323Sed Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3512218893Sdim if (NumBits > 8) { 3513218893Sdim // Use a multiplication with 0x010101... to extend the input to the 3514218893Sdim // required length. 3515249423Sdim APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 3516218893Sdim Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3517193323Sed } 3518193323Sed 3519193323Sed return Value; 3520193323Sed} 3521193323Sed 3522193323Sed/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3523193323Sed/// used when a memcpy is turned into a memset when the source is a constant 3524193323Sed/// string ptr. 3525263508Sdimstatic SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, 3526234353Sdim const TargetLowering &TLI, StringRef Str) { 3527193323Sed // Handle vector with all elements zero. 3528193323Sed if (Str.empty()) { 3529193323Sed if (VT.isInteger()) 3530193323Sed return DAG.getConstant(0, VT); 3531218893Sdim else if (VT == MVT::f32 || VT == MVT::f64) 3532206083Srdivacky return DAG.getConstantFP(0.0, VT); 3533206083Srdivacky else if (VT.isVector()) { 3534206083Srdivacky unsigned NumElts = VT.getVectorNumElements(); 3535206083Srdivacky MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3536218893Sdim return DAG.getNode(ISD::BITCAST, dl, VT, 3537206083Srdivacky DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3538206083Srdivacky EltVT, NumElts))); 3539206083Srdivacky } else 3540206083Srdivacky llvm_unreachable("Expected type!"); 3541193323Sed } 3542193323Sed 3543193323Sed assert(!VT.isVector() && "Can't handle vector type here!"); 3544249423Sdim unsigned NumVTBits = VT.getSizeInBits(); 3545249423Sdim unsigned NumVTBytes = NumVTBits / 8; 3546234353Sdim unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 3547234353Sdim 3548249423Sdim APInt Val(NumVTBits, 0); 3549234353Sdim if (TLI.isLittleEndian()) { 3550234353Sdim for (unsigned i = 0; i != NumBytes; ++i) 3551234353Sdim Val |= (uint64_t)(unsigned char)Str[i] << i*8; 3552234353Sdim } else { 3553234353Sdim for (unsigned i = 0; i != NumBytes; ++i) 3554234353Sdim Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 3555193323Sed } 3556234353Sdim 3557249423Sdim // If the "cost" of materializing the integer immediate is 1 or free, then 3558249423Sdim // it is cost effective to turn the load into the immediate. 3559249423Sdim const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 3560249423Sdim if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 3561249423Sdim return DAG.getConstant(Val, VT); 3562249423Sdim return SDValue(0, 0); 3563193323Sed} 3564193323Sed 3565193323Sed/// getMemBasePlusOffset - Returns base and offset node for the 3566193323Sed/// 3567263508Sdimstatic SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, 3568193323Sed SelectionDAG &DAG) { 3569198090Srdivacky EVT VT = Base.getValueType(); 3570263508Sdim return DAG.getNode(ISD::ADD, dl, 3571193323Sed VT, Base, DAG.getConstant(Offset, VT)); 3572193323Sed} 3573193323Sed 3574193323Sed/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3575193323Sed/// 3576234353Sdimstatic bool isMemSrcFromString(SDValue Src, StringRef &Str) { 3577193323Sed unsigned SrcDelta = 0; 3578193323Sed GlobalAddressSDNode *G = NULL; 3579193323Sed if (Src.getOpcode() == ISD::GlobalAddress) 3580193323Sed G = cast<GlobalAddressSDNode>(Src); 3581193323Sed else if (Src.getOpcode() == ISD::ADD && 3582193323Sed Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3583193323Sed Src.getOperand(1).getOpcode() == ISD::Constant) { 3584193323Sed G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3585193323Sed SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3586193323Sed } 3587193323Sed if (!G) 3588193323Sed return false; 3589193323Sed 3590234353Sdim return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 3591193323Sed} 3592193323Sed 3593206083Srdivacky/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3594206083Srdivacky/// to replace the memset / memcpy. Return true if the number of memory ops 3595206083Srdivacky/// is below the threshold. It returns the types of the sequence of 3596206083Srdivacky/// memory ops to perform memset / memcpy by reference. 3597206083Srdivackystatic bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3598206083Srdivacky unsigned Limit, uint64_t Size, 3599206083Srdivacky unsigned DstAlign, unsigned SrcAlign, 3600249423Sdim bool IsMemset, 3601249423Sdim bool ZeroMemset, 3602207618Srdivacky bool MemcpyStrSrc, 3603249423Sdim bool AllowOverlap, 3604206083Srdivacky SelectionDAG &DAG, 3605206083Srdivacky const TargetLowering &TLI) { 3606206083Srdivacky assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3607206083Srdivacky "Expecting memcpy / memset source to meet alignment requirement!"); 3608224145Sdim // If 'SrcAlign' is zero, that means the memory operation does not need to 3609224145Sdim // load the value, i.e. memset or memcpy from constant string. Otherwise, 3610224145Sdim // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3611224145Sdim // is the specified alignment of the memory operation. If it is zero, that 3612224145Sdim // means it's possible to change the alignment of the destination. 3613224145Sdim // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3614224145Sdim // not need to be loaded. 3615206124Srdivacky EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3616249423Sdim IsMemset, ZeroMemset, MemcpyStrSrc, 3617207618Srdivacky DAG.getMachineFunction()); 3618193323Sed 3619204961Srdivacky if (VT == MVT::Other) { 3620243830Sdim if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 3621206083Srdivacky TLI.allowsUnalignedMemoryAccesses(VT)) { 3622206274Srdivacky VT = TLI.getPointerTy(); 3623193323Sed } else { 3624206083Srdivacky switch (DstAlign & 7) { 3625193323Sed case 0: VT = MVT::i64; break; 3626193323Sed case 4: VT = MVT::i32; break; 3627193323Sed case 2: VT = MVT::i16; break; 3628193323Sed default: VT = MVT::i8; break; 3629193323Sed } 3630193323Sed } 3631193323Sed 3632193323Sed MVT LVT = MVT::i64; 3633193323Sed while (!TLI.isTypeLegal(LVT)) 3634198090Srdivacky LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3635193323Sed assert(LVT.isInteger()); 3636193323Sed 3637193323Sed if (VT.bitsGT(LVT)) 3638193323Sed VT = LVT; 3639193323Sed } 3640193323Sed 3641193323Sed unsigned NumMemOps = 0; 3642193323Sed while (Size != 0) { 3643193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3644193323Sed while (VTSize > Size) { 3645193323Sed // For now, only use non-vector load / store's for the left-over pieces. 3646249423Sdim EVT NewVT = VT; 3647249423Sdim unsigned NewVTSize; 3648249423Sdim 3649249423Sdim bool Found = false; 3650206083Srdivacky if (VT.isVector() || VT.isFloatingPoint()) { 3651249423Sdim NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 3652249423Sdim if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 3653249423Sdim TLI.isSafeMemOpType(NewVT.getSimpleVT())) 3654249423Sdim Found = true; 3655249423Sdim else if (NewVT == MVT::i64 && 3656249423Sdim TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 3657249423Sdim TLI.isSafeMemOpType(MVT::f64)) { 3658249423Sdim // i64 is usually not legal on 32-bit targets, but f64 may be. 3659249423Sdim NewVT = MVT::f64; 3660249423Sdim Found = true; 3661249423Sdim } 3662193323Sed } 3663249423Sdim 3664249423Sdim if (!Found) { 3665249423Sdim do { 3666249423Sdim NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 3667249423Sdim if (NewVT == MVT::i8) 3668249423Sdim break; 3669249423Sdim } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 3670249423Sdim } 3671249423Sdim NewVTSize = NewVT.getSizeInBits() / 8; 3672249423Sdim 3673249423Sdim // If the new VT cannot cover all of the remaining bits, then consider 3674249423Sdim // issuing a (or a pair of) unaligned and overlapping load / store. 3675249423Sdim // FIXME: Only does this for 64-bit or more since we don't have proper 3676249423Sdim // cost model for unaligned load / store. 3677249423Sdim bool Fast; 3678249423Sdim if (NumMemOps && AllowOverlap && 3679249423Sdim VTSize >= 8 && NewVTSize < Size && 3680249423Sdim TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 3681249423Sdim VTSize = Size; 3682249423Sdim else { 3683249423Sdim VT = NewVT; 3684249423Sdim VTSize = NewVTSize; 3685249423Sdim } 3686193323Sed } 3687193323Sed 3688193323Sed if (++NumMemOps > Limit) 3689193323Sed return false; 3690249423Sdim 3691193323Sed MemOps.push_back(VT); 3692193323Sed Size -= VTSize; 3693193323Sed } 3694193323Sed 3695193323Sed return true; 3696193323Sed} 3697193323Sed 3698263508Sdimstatic SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3699206083Srdivacky SDValue Chain, SDValue Dst, 3700206083Srdivacky SDValue Src, uint64_t Size, 3701206274Srdivacky unsigned Align, bool isVol, 3702206274Srdivacky bool AlwaysInline, 3703218893Sdim MachinePointerInfo DstPtrInfo, 3704218893Sdim MachinePointerInfo SrcPtrInfo) { 3705206124Srdivacky // Turn a memcpy of undef to nop. 3706206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3707206124Srdivacky return Chain; 3708193323Sed 3709193323Sed // Expand memcpy to a series of load and store ops if the size operand falls 3710193323Sed // below a certain threshold. 3711218893Sdim // TODO: In the AlwaysInline case, if the size is big then generate a loop 3712218893Sdim // rather than maybe a humongous number of loads and stores. 3713206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3714198090Srdivacky std::vector<EVT> MemOps; 3715206083Srdivacky bool DstAlignCanChange = false; 3716218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3717218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3718243830Sdim bool OptSize = 3719249423Sdim MF.getFunction()->getAttributes(). 3720249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3721206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3722206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3723206083Srdivacky DstAlignCanChange = true; 3724206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3725206083Srdivacky if (Align > SrcAlign) 3726206083Srdivacky SrcAlign = Align; 3727234353Sdim StringRef Str; 3728206083Srdivacky bool CopyFromStr = isMemSrcFromString(Src, Str); 3729206083Srdivacky bool isZeroStr = CopyFromStr && Str.empty(); 3730218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3731218893Sdim 3732206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3733206083Srdivacky (DstAlignCanChange ? 0 : Align), 3734207618Srdivacky (isZeroStr ? 0 : SrcAlign), 3735249423Sdim false, false, CopyFromStr, true, DAG, TLI)) 3736193323Sed return SDValue(); 3737193323Sed 3738206083Srdivacky if (DstAlignCanChange) { 3739226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3740243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3741249423Sdim 3742249423Sdim // Don't promote to an alignment that would require dynamic stack 3743263508Sdim // realignment. 3744249423Sdim const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 3745249423Sdim if (!TRI->needsStackRealignment(MF)) 3746249423Sdim while (NewAlign > Align && 3747249423Sdim TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 3748249423Sdim NewAlign /= 2; 3749249423Sdim 3750206083Srdivacky if (NewAlign > Align) { 3751206083Srdivacky // Give the stack frame object a larger alignment if needed. 3752206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3753206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3754206083Srdivacky Align = NewAlign; 3755206083Srdivacky } 3756206083Srdivacky } 3757193323Sed 3758193323Sed SmallVector<SDValue, 8> OutChains; 3759193323Sed unsigned NumMemOps = MemOps.size(); 3760193323Sed uint64_t SrcOff = 0, DstOff = 0; 3761198090Srdivacky for (unsigned i = 0; i != NumMemOps; ++i) { 3762198090Srdivacky EVT VT = MemOps[i]; 3763193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3764193323Sed SDValue Value, Store; 3765193323Sed 3766249423Sdim if (VTSize > Size) { 3767249423Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3768249423Sdim // pair. Adjust the offset accordingly. 3769249423Sdim assert(i == NumMemOps-1 && i != 0); 3770249423Sdim SrcOff -= VTSize - Size; 3771249423Sdim DstOff -= VTSize - Size; 3772249423Sdim } 3773249423Sdim 3774206083Srdivacky if (CopyFromStr && 3775206083Srdivacky (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3776193323Sed // It's unlikely a store of a vector immediate can be done in a single 3777193323Sed // instruction. It would require a load from a constantpool first. 3778206083Srdivacky // We only handle zero vectors here. 3779193323Sed // FIXME: Handle other cases where store of vector immediate is done in 3780193323Sed // a single instruction. 3781234353Sdim Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 3782249423Sdim if (Value.getNode()) 3783249423Sdim Store = DAG.getStore(Chain, dl, Value, 3784263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3785249423Sdim DstPtrInfo.getWithOffset(DstOff), isVol, 3786249423Sdim false, Align); 3787249423Sdim } 3788249423Sdim 3789249423Sdim if (!Store.getNode()) { 3790194710Sed // The type might not be legal for the target. This should only happen 3791194710Sed // if the type is smaller than a legal type, as on PPC, so the right 3792195098Sed // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3793195098Sed // to Load/Store if NVT==VT. 3794194710Sed // FIXME does the case above also need this? 3795198090Srdivacky EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3796195098Sed assert(NVT.bitsGE(VT)); 3797218893Sdim Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3798263508Sdim getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3799218893Sdim SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3800206083Srdivacky MinAlign(SrcAlign, SrcOff)); 3801195098Sed Store = DAG.getTruncStore(Chain, dl, Value, 3802263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3803218893Sdim DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3804218893Sdim false, Align); 3805193323Sed } 3806193323Sed OutChains.push_back(Store); 3807193323Sed SrcOff += VTSize; 3808193323Sed DstOff += VTSize; 3809249423Sdim Size -= VTSize; 3810193323Sed } 3811193323Sed 3812193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3813193323Sed &OutChains[0], OutChains.size()); 3814193323Sed} 3815193323Sed 3816263508Sdimstatic SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3817206083Srdivacky SDValue Chain, SDValue Dst, 3818206083Srdivacky SDValue Src, uint64_t Size, 3819206274Srdivacky unsigned Align, bool isVol, 3820206274Srdivacky bool AlwaysInline, 3821218893Sdim MachinePointerInfo DstPtrInfo, 3822218893Sdim MachinePointerInfo SrcPtrInfo) { 3823206124Srdivacky // Turn a memmove of undef to nop. 3824206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3825206124Srdivacky return Chain; 3826193323Sed 3827193323Sed // Expand memmove to a series of load and store ops if the size operand falls 3828193323Sed // below a certain threshold. 3829206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3830198090Srdivacky std::vector<EVT> MemOps; 3831206083Srdivacky bool DstAlignCanChange = false; 3832218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3833218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3834249423Sdim bool OptSize = MF.getFunction()->getAttributes(). 3835249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3836206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3837206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3838206083Srdivacky DstAlignCanChange = true; 3839206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3840206083Srdivacky if (Align > SrcAlign) 3841206083Srdivacky SrcAlign = Align; 3842218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3843206083Srdivacky 3844206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3845249423Sdim (DstAlignCanChange ? 0 : Align), SrcAlign, 3846249423Sdim false, false, false, false, DAG, TLI)) 3847193323Sed return SDValue(); 3848193323Sed 3849206083Srdivacky if (DstAlignCanChange) { 3850226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3851243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3852206083Srdivacky if (NewAlign > Align) { 3853206083Srdivacky // Give the stack frame object a larger alignment if needed. 3854206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3855206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3856206083Srdivacky Align = NewAlign; 3857206083Srdivacky } 3858206083Srdivacky } 3859206083Srdivacky 3860193323Sed uint64_t SrcOff = 0, DstOff = 0; 3861193323Sed SmallVector<SDValue, 8> LoadValues; 3862193323Sed SmallVector<SDValue, 8> LoadChains; 3863193323Sed SmallVector<SDValue, 8> OutChains; 3864193323Sed unsigned NumMemOps = MemOps.size(); 3865193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3866198090Srdivacky EVT VT = MemOps[i]; 3867193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3868263508Sdim SDValue Value; 3869193323Sed 3870193323Sed Value = DAG.getLoad(VT, dl, Chain, 3871263508Sdim getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3872218893Sdim SrcPtrInfo.getWithOffset(SrcOff), isVol, 3873234353Sdim false, false, SrcAlign); 3874193323Sed LoadValues.push_back(Value); 3875193323Sed LoadChains.push_back(Value.getValue(1)); 3876193323Sed SrcOff += VTSize; 3877193323Sed } 3878193323Sed Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3879193323Sed &LoadChains[0], LoadChains.size()); 3880193323Sed OutChains.clear(); 3881193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3882198090Srdivacky EVT VT = MemOps[i]; 3883193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3884263508Sdim SDValue Store; 3885193323Sed 3886193323Sed Store = DAG.getStore(Chain, dl, LoadValues[i], 3887263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3888218893Sdim DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3889193323Sed OutChains.push_back(Store); 3890193323Sed DstOff += VTSize; 3891193323Sed } 3892193323Sed 3893193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3894193323Sed &OutChains[0], OutChains.size()); 3895193323Sed} 3896193323Sed 3897263508Sdim/// \brief Lower the call to 'memset' intrinsic function into a series of store 3898263508Sdim/// operations. 3899263508Sdim/// 3900263508Sdim/// \param DAG Selection DAG where lowered code is placed. 3901263508Sdim/// \param dl Link to corresponding IR location. 3902263508Sdim/// \param Chain Control flow dependency. 3903263508Sdim/// \param Dst Pointer to destination memory location. 3904263508Sdim/// \param Src Value of byte to write into the memory. 3905263508Sdim/// \param Size Number of bytes to write. 3906263508Sdim/// \param Align Alignment of the destination in bytes. 3907263508Sdim/// \param isVol True if destination is volatile. 3908263508Sdim/// \param DstPtrInfo IR information on the memory pointer. 3909263508Sdim/// \returns New head in the control flow, if lowering was successful, empty 3910263508Sdim/// SDValue otherwise. 3911263508Sdim/// 3912263508Sdim/// The function tries to replace 'llvm.memset' intrinsic with several store 3913263508Sdim/// operations and value calculation code. This is usually profitable for small 3914263508Sdim/// memory size. 3915263508Sdimstatic SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, 3916206083Srdivacky SDValue Chain, SDValue Dst, 3917206083Srdivacky SDValue Src, uint64_t Size, 3918206274Srdivacky unsigned Align, bool isVol, 3919218893Sdim MachinePointerInfo DstPtrInfo) { 3920206124Srdivacky // Turn a memset of undef to nop. 3921206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3922206124Srdivacky return Chain; 3923193323Sed 3924193323Sed // Expand memset to a series of load/store ops if the size operand 3925193323Sed // falls below a certain threshold. 3926206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3927198090Srdivacky std::vector<EVT> MemOps; 3928206083Srdivacky bool DstAlignCanChange = false; 3929218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3930218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3931249423Sdim bool OptSize = MF.getFunction()->getAttributes(). 3932249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3933206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3934206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3935206083Srdivacky DstAlignCanChange = true; 3936234353Sdim bool IsZeroVal = 3937206124Srdivacky isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3938218893Sdim if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3939206083Srdivacky Size, (DstAlignCanChange ? 0 : Align), 0, 3940249423Sdim true, IsZeroVal, false, true, DAG, TLI)) 3941193323Sed return SDValue(); 3942193323Sed 3943206083Srdivacky if (DstAlignCanChange) { 3944226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3945243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3946206083Srdivacky if (NewAlign > Align) { 3947206083Srdivacky // Give the stack frame object a larger alignment if needed. 3948206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3949206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3950206083Srdivacky Align = NewAlign; 3951206083Srdivacky } 3952206083Srdivacky } 3953206083Srdivacky 3954193323Sed SmallVector<SDValue, 8> OutChains; 3955193323Sed uint64_t DstOff = 0; 3956193323Sed unsigned NumMemOps = MemOps.size(); 3957218893Sdim 3958218893Sdim // Find the largest store and generate the bit pattern for it. 3959218893Sdim EVT LargestVT = MemOps[0]; 3960218893Sdim for (unsigned i = 1; i < NumMemOps; i++) 3961218893Sdim if (MemOps[i].bitsGT(LargestVT)) 3962218893Sdim LargestVT = MemOps[i]; 3963218893Sdim SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3964218893Sdim 3965193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3966198090Srdivacky EVT VT = MemOps[i]; 3967249423Sdim unsigned VTSize = VT.getSizeInBits() / 8; 3968249423Sdim if (VTSize > Size) { 3969249423Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3970249423Sdim // pair. Adjust the offset accordingly. 3971249423Sdim assert(i == NumMemOps-1 && i != 0); 3972249423Sdim DstOff -= VTSize - Size; 3973249423Sdim } 3974218893Sdim 3975218893Sdim // If this store is smaller than the largest store see whether we can get 3976218893Sdim // the smaller value for free with a truncate. 3977218893Sdim SDValue Value = MemSetValue; 3978218893Sdim if (VT.bitsLT(LargestVT)) { 3979218893Sdim if (!LargestVT.isVector() && !VT.isVector() && 3980218893Sdim TLI.isTruncateFree(LargestVT, VT)) 3981218893Sdim Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3982218893Sdim else 3983218893Sdim Value = getMemsetValue(Src, VT, DAG, dl); 3984218893Sdim } 3985218893Sdim assert(Value.getValueType() == VT && "Value with wrong type."); 3986193323Sed SDValue Store = DAG.getStore(Chain, dl, Value, 3987263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3988218893Sdim DstPtrInfo.getWithOffset(DstOff), 3989218893Sdim isVol, false, Align); 3990193323Sed OutChains.push_back(Store); 3991218893Sdim DstOff += VT.getSizeInBits() / 8; 3992249423Sdim Size -= VTSize; 3993193323Sed } 3994193323Sed 3995193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3996193323Sed &OutChains[0], OutChains.size()); 3997193323Sed} 3998193323Sed 3999263508SdimSDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, 4000193323Sed SDValue Src, SDValue Size, 4001206274Srdivacky unsigned Align, bool isVol, bool AlwaysInline, 4002218893Sdim MachinePointerInfo DstPtrInfo, 4003218893Sdim MachinePointerInfo SrcPtrInfo) { 4004249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4005193323Sed 4006193323Sed // Check to see if we should lower the memcpy to loads and stores first. 4007193323Sed // For cases within the target-specified limits, this is the best choice. 4008193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4009193323Sed if (ConstantSize) { 4010193323Sed // Memcpy with size zero? Just return the original chain. 4011193323Sed if (ConstantSize->isNullValue()) 4012193323Sed return Chain; 4013193323Sed 4014206083Srdivacky SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4015206083Srdivacky ConstantSize->getZExtValue(),Align, 4016218893Sdim isVol, false, DstPtrInfo, SrcPtrInfo); 4017193323Sed if (Result.getNode()) 4018193323Sed return Result; 4019193323Sed } 4020193323Sed 4021193323Sed // Then check to see if we should lower the memcpy with target-specific 4022193323Sed // code. If the target chooses to do this, this is the next best. 4023193323Sed SDValue Result = 4024208599Srdivacky TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 4025206274Srdivacky isVol, AlwaysInline, 4026218893Sdim DstPtrInfo, SrcPtrInfo); 4027193323Sed if (Result.getNode()) 4028193323Sed return Result; 4029193323Sed 4030193323Sed // If we really need inline code and the target declined to provide it, 4031193323Sed // use a (potentially long) sequence of loads and stores. 4032193323Sed if (AlwaysInline) { 4033193323Sed assert(ConstantSize && "AlwaysInline requires a constant size!"); 4034193323Sed return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4035206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 4036218893Sdim true, DstPtrInfo, SrcPtrInfo); 4037193323Sed } 4038193323Sed 4039206274Srdivacky // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 4040206274Srdivacky // memcpy is not guaranteed to be safe. libc memcpys aren't required to 4041206274Srdivacky // respect volatile, so they may do things like read or write memory 4042206274Srdivacky // beyond the given memory regions. But fixing this isn't easy, and most 4043206274Srdivacky // people don't care. 4044206274Srdivacky 4045263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4046263508Sdim 4047193323Sed // Emit a library call. 4048193323Sed TargetLowering::ArgListTy Args; 4049193323Sed TargetLowering::ArgListEntry Entry; 4050263508Sdim Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4051193323Sed Entry.Node = Dst; Args.push_back(Entry); 4052193323Sed Entry.Node = Src; Args.push_back(Entry); 4053193323Sed Entry.Node = Size; Args.push_back(Entry); 4054263508Sdim // FIXME: pass in SDLoc 4055239462Sdim TargetLowering:: 4056239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4057198090Srdivacky false, false, false, false, 0, 4058263508Sdim TLI->getLibcallCallingConv(RTLIB::MEMCPY), 4059234353Sdim /*isTailCall=*/false, 4060234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4061263508Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 4062263508Sdim TLI->getPointerTy()), 4063204642Srdivacky Args, *this, dl); 4064263508Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4065239462Sdim 4066193323Sed return CallResult.second; 4067193323Sed} 4068193323Sed 4069263508SdimSDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, 4070193323Sed SDValue Src, SDValue Size, 4071206274Srdivacky unsigned Align, bool isVol, 4072218893Sdim MachinePointerInfo DstPtrInfo, 4073218893Sdim MachinePointerInfo SrcPtrInfo) { 4074249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4075193323Sed 4076193323Sed // Check to see if we should lower the memmove to loads and stores first. 4077193323Sed // For cases within the target-specified limits, this is the best choice. 4078193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4079193323Sed if (ConstantSize) { 4080193323Sed // Memmove with size zero? Just return the original chain. 4081193323Sed if (ConstantSize->isNullValue()) 4082193323Sed return Chain; 4083193323Sed 4084193323Sed SDValue Result = 4085193323Sed getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 4086206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 4087218893Sdim false, DstPtrInfo, SrcPtrInfo); 4088193323Sed if (Result.getNode()) 4089193323Sed return Result; 4090193323Sed } 4091193323Sed 4092193323Sed // Then check to see if we should lower the memmove with target-specific 4093193323Sed // code. If the target chooses to do this, this is the next best. 4094193323Sed SDValue Result = 4095208599Srdivacky TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4096218893Sdim DstPtrInfo, SrcPtrInfo); 4097193323Sed if (Result.getNode()) 4098193323Sed return Result; 4099193323Sed 4100207618Srdivacky // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 4101207618Srdivacky // not be safe. See memcpy above for more details. 4102207618Srdivacky 4103263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4104263508Sdim 4105193323Sed // Emit a library call. 4106193323Sed TargetLowering::ArgListTy Args; 4107193323Sed TargetLowering::ArgListEntry Entry; 4108263508Sdim Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4109193323Sed Entry.Node = Dst; Args.push_back(Entry); 4110193323Sed Entry.Node = Src; Args.push_back(Entry); 4111193323Sed Entry.Node = Size; Args.push_back(Entry); 4112263508Sdim // FIXME: pass in SDLoc 4113239462Sdim TargetLowering:: 4114239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4115198090Srdivacky false, false, false, false, 0, 4116263508Sdim TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 4117234353Sdim /*isTailCall=*/false, 4118234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4119263508Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 4120263508Sdim TLI->getPointerTy()), 4121204642Srdivacky Args, *this, dl); 4122263508Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4123239462Sdim 4124193323Sed return CallResult.second; 4125193323Sed} 4126193323Sed 4127263508SdimSDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, 4128193323Sed SDValue Src, SDValue Size, 4129206274Srdivacky unsigned Align, bool isVol, 4130218893Sdim MachinePointerInfo DstPtrInfo) { 4131249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4132193323Sed 4133193323Sed // Check to see if we should lower the memset to stores first. 4134193323Sed // For cases within the target-specified limits, this is the best choice. 4135193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4136193323Sed if (ConstantSize) { 4137193323Sed // Memset with size zero? Just return the original chain. 4138193323Sed if (ConstantSize->isNullValue()) 4139193323Sed return Chain; 4140193323Sed 4141206274Srdivacky SDValue Result = 4142206274Srdivacky getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 4143218893Sdim Align, isVol, DstPtrInfo); 4144206274Srdivacky 4145193323Sed if (Result.getNode()) 4146193323Sed return Result; 4147193323Sed } 4148193323Sed 4149193323Sed // Then check to see if we should lower the memset with target-specific 4150193323Sed // code. If the target chooses to do this, this is the next best. 4151193323Sed SDValue Result = 4152208599Srdivacky TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4153218893Sdim DstPtrInfo); 4154193323Sed if (Result.getNode()) 4155193323Sed return Result; 4156193323Sed 4157218893Sdim // Emit a library call. 4158263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4159263508Sdim Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext()); 4160193323Sed TargetLowering::ArgListTy Args; 4161193323Sed TargetLowering::ArgListEntry Entry; 4162193323Sed Entry.Node = Dst; Entry.Ty = IntPtrTy; 4163193323Sed Args.push_back(Entry); 4164193323Sed // Extend or truncate the argument to be an i32 value for the call. 4165193323Sed if (Src.getValueType().bitsGT(MVT::i32)) 4166193323Sed Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 4167193323Sed else 4168193323Sed Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 4169198090Srdivacky Entry.Node = Src; 4170198090Srdivacky Entry.Ty = Type::getInt32Ty(*getContext()); 4171198090Srdivacky Entry.isSExt = true; 4172193323Sed Args.push_back(Entry); 4173198090Srdivacky Entry.Node = Size; 4174198090Srdivacky Entry.Ty = IntPtrTy; 4175198090Srdivacky Entry.isSExt = false; 4176193323Sed Args.push_back(Entry); 4177263508Sdim // FIXME: pass in SDLoc 4178239462Sdim TargetLowering:: 4179239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4180198090Srdivacky false, false, false, false, 0, 4181263508Sdim TLI->getLibcallCallingConv(RTLIB::MEMSET), 4182234353Sdim /*isTailCall=*/false, 4183234353Sdim /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 4184263508Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 4185263508Sdim TLI->getPointerTy()), 4186204642Srdivacky Args, *this, dl); 4187263508Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4188239462Sdim 4189193323Sed return CallResult.second; 4190193323Sed} 4191193323Sed 4192263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4193263508Sdim SDVTList VTList, SDValue* Ops, unsigned NumOps, 4194263508Sdim MachineMemOperand *MMO, 4195263508Sdim AtomicOrdering Ordering, 4196263508Sdim SynchronizationScope SynchScope) { 4197263508Sdim FoldingSetNodeID ID; 4198263508Sdim ID.AddInteger(MemVT.getRawBits()); 4199263508Sdim AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4200263508Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4201263508Sdim void* IP = 0; 4202263508Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4203263508Sdim cast<AtomicSDNode>(E)->refineAlignment(MMO); 4204263508Sdim return SDValue(E, 0); 4205263508Sdim } 4206263508Sdim 4207263508Sdim // Allocate the operands array for the node out of the BumpPtrAllocator, since 4208263508Sdim // SDNode doesn't have access to it. This memory will be "leaked" when 4209263508Sdim // the node is deallocated, but recovered when the allocator is released. 4210263508Sdim // If the number of operands is less than 5 we use AtomicSDNode's internal 4211263508Sdim // storage. 4212263508Sdim SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) : 0; 4213263508Sdim 4214263508Sdim SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), 4215263508Sdim dl.getDebugLoc(), VTList, MemVT, 4216263508Sdim Ops, DynOps, NumOps, MMO, 4217263508Sdim Ordering, SynchScope); 4218263508Sdim CSEMap.InsertNode(N, IP); 4219263508Sdim AllNodes.push_back(N); 4220263508Sdim return SDValue(N, 0); 4221263508Sdim} 4222263508Sdim 4223263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4224218893Sdim SDValue Chain, SDValue Ptr, SDValue Cmp, 4225218893Sdim SDValue Swp, MachinePointerInfo PtrInfo, 4226226633Sdim unsigned Alignment, 4227226633Sdim AtomicOrdering Ordering, 4228243830Sdim SynchronizationScope SynchScope) { 4229198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4230198090Srdivacky Alignment = getEVTAlignment(MemVT); 4231198090Srdivacky 4232198090Srdivacky MachineFunction &MF = getMachineFunction(); 4233198090Srdivacky 4234243830Sdim // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 4235198090Srdivacky // For now, atomics are considered to be volatile always. 4236226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4237226633Sdim // orderings in the memoperand. 4238243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4239243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4240243830Sdim Flags |= MachineMemOperand::MOLoad; 4241243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4242243830Sdim Flags |= MachineMemOperand::MOStore; 4243198090Srdivacky 4244198090Srdivacky MachineMemOperand *MMO = 4245218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 4246198090Srdivacky 4247226633Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 4248226633Sdim Ordering, SynchScope); 4249198090Srdivacky} 4250198090Srdivacky 4251263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4252198090Srdivacky SDValue Chain, 4253198090Srdivacky SDValue Ptr, SDValue Cmp, 4254226633Sdim SDValue Swp, MachineMemOperand *MMO, 4255226633Sdim AtomicOrdering Ordering, 4256226633Sdim SynchronizationScope SynchScope) { 4257193323Sed assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 4258193323Sed assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 4259193323Sed 4260198090Srdivacky EVT VT = Cmp.getValueType(); 4261193323Sed 4262193323Sed SDVTList VTs = getVTList(VT, MVT::Other); 4263193323Sed SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 4264263508Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, Ordering, SynchScope); 4265193323Sed} 4266193323Sed 4267263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4268193323Sed SDValue Chain, 4269193323Sed SDValue Ptr, SDValue Val, 4270193323Sed const Value* PtrVal, 4271226633Sdim unsigned Alignment, 4272226633Sdim AtomicOrdering Ordering, 4273226633Sdim SynchronizationScope SynchScope) { 4274198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4275198090Srdivacky Alignment = getEVTAlignment(MemVT); 4276198090Srdivacky 4277198090Srdivacky MachineFunction &MF = getMachineFunction(); 4278243830Sdim // An atomic store does not load. An atomic load does not store. 4279226633Sdim // (An atomicrmw obviously both loads and stores.) 4280243830Sdim // For now, atomics are considered to be volatile always, and they are 4281243830Sdim // chained as such. 4282226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4283226633Sdim // orderings in the memoperand. 4284243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4285243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4286243830Sdim Flags |= MachineMemOperand::MOLoad; 4287243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4288243830Sdim Flags |= MachineMemOperand::MOStore; 4289198090Srdivacky 4290198090Srdivacky MachineMemOperand *MMO = 4291218893Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4292198090Srdivacky MemVT.getStoreSize(), Alignment); 4293198090Srdivacky 4294226633Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 4295226633Sdim Ordering, SynchScope); 4296198090Srdivacky} 4297198090Srdivacky 4298263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4299198090Srdivacky SDValue Chain, 4300198090Srdivacky SDValue Ptr, SDValue Val, 4301226633Sdim MachineMemOperand *MMO, 4302226633Sdim AtomicOrdering Ordering, 4303226633Sdim SynchronizationScope SynchScope) { 4304193323Sed assert((Opcode == ISD::ATOMIC_LOAD_ADD || 4305193323Sed Opcode == ISD::ATOMIC_LOAD_SUB || 4306193323Sed Opcode == ISD::ATOMIC_LOAD_AND || 4307193323Sed Opcode == ISD::ATOMIC_LOAD_OR || 4308193323Sed Opcode == ISD::ATOMIC_LOAD_XOR || 4309193323Sed Opcode == ISD::ATOMIC_LOAD_NAND || 4310193323Sed Opcode == ISD::ATOMIC_LOAD_MIN || 4311193323Sed Opcode == ISD::ATOMIC_LOAD_MAX || 4312193323Sed Opcode == ISD::ATOMIC_LOAD_UMIN || 4313193323Sed Opcode == ISD::ATOMIC_LOAD_UMAX || 4314226633Sdim Opcode == ISD::ATOMIC_SWAP || 4315226633Sdim Opcode == ISD::ATOMIC_STORE) && 4316193323Sed "Invalid Atomic Op"); 4317193323Sed 4318198090Srdivacky EVT VT = Val.getValueType(); 4319193323Sed 4320226633Sdim SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 4321226633Sdim getVTList(VT, MVT::Other); 4322193323Sed SDValue Ops[] = {Chain, Ptr, Val}; 4323263508Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope); 4324193323Sed} 4325193323Sed 4326263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4327226633Sdim EVT VT, SDValue Chain, 4328226633Sdim SDValue Ptr, 4329226633Sdim const Value* PtrVal, 4330226633Sdim unsigned Alignment, 4331226633Sdim AtomicOrdering Ordering, 4332226633Sdim SynchronizationScope SynchScope) { 4333226633Sdim if (Alignment == 0) // Ensure that codegen never sees alignment 0 4334226633Sdim Alignment = getEVTAlignment(MemVT); 4335226633Sdim 4336226633Sdim MachineFunction &MF = getMachineFunction(); 4337243830Sdim // An atomic store does not load. An atomic load does not store. 4338243830Sdim // (An atomicrmw obviously both loads and stores.) 4339243830Sdim // For now, atomics are considered to be volatile always, and they are 4340243830Sdim // chained as such. 4341226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4342226633Sdim // orderings in the memoperand. 4343243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4344243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4345243830Sdim Flags |= MachineMemOperand::MOLoad; 4346243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4347243830Sdim Flags |= MachineMemOperand::MOStore; 4348226633Sdim 4349226633Sdim MachineMemOperand *MMO = 4350226633Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4351226633Sdim MemVT.getStoreSize(), Alignment); 4352226633Sdim 4353226633Sdim return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4354226633Sdim Ordering, SynchScope); 4355226633Sdim} 4356226633Sdim 4357263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4358226633Sdim EVT VT, SDValue Chain, 4359226633Sdim SDValue Ptr, 4360226633Sdim MachineMemOperand *MMO, 4361226633Sdim AtomicOrdering Ordering, 4362226633Sdim SynchronizationScope SynchScope) { 4363226633Sdim assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4364226633Sdim 4365226633Sdim SDVTList VTs = getVTList(VT, MVT::Other); 4366226633Sdim SDValue Ops[] = {Chain, Ptr}; 4367263508Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 2, MMO, Ordering, SynchScope); 4368226633Sdim} 4369226633Sdim 4370193323Sed/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4371193323SedSDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4372263508Sdim SDLoc dl) { 4373193323Sed if (NumOps == 1) 4374193323Sed return Ops[0]; 4375193323Sed 4376198090Srdivacky SmallVector<EVT, 4> VTs; 4377193323Sed VTs.reserve(NumOps); 4378193323Sed for (unsigned i = 0; i < NumOps; ++i) 4379193323Sed VTs.push_back(Ops[i].getValueType()); 4380193323Sed return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4381193323Sed Ops, NumOps); 4382193323Sed} 4383193323Sed 4384193323SedSDValue 4385263508SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, 4386198090Srdivacky const EVT *VTs, unsigned NumVTs, 4387193323Sed const SDValue *Ops, unsigned NumOps, 4388218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4389193323Sed unsigned Align, bool Vol, 4390193323Sed bool ReadMem, bool WriteMem) { 4391193323Sed return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4392218893Sdim MemVT, PtrInfo, Align, Vol, 4393193323Sed ReadMem, WriteMem); 4394193323Sed} 4395193323Sed 4396193323SedSDValue 4397263508SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4398193323Sed const SDValue *Ops, unsigned NumOps, 4399218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4400193323Sed unsigned Align, bool Vol, 4401193323Sed bool ReadMem, bool WriteMem) { 4402198090Srdivacky if (Align == 0) // Ensure that codegen never sees alignment 0 4403198090Srdivacky Align = getEVTAlignment(MemVT); 4404198090Srdivacky 4405198090Srdivacky MachineFunction &MF = getMachineFunction(); 4406198090Srdivacky unsigned Flags = 0; 4407198090Srdivacky if (WriteMem) 4408198090Srdivacky Flags |= MachineMemOperand::MOStore; 4409198090Srdivacky if (ReadMem) 4410198090Srdivacky Flags |= MachineMemOperand::MOLoad; 4411198090Srdivacky if (Vol) 4412198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4413198090Srdivacky MachineMemOperand *MMO = 4414218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4415198090Srdivacky 4416198090Srdivacky return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4417198090Srdivacky} 4418198090Srdivacky 4419198090SrdivackySDValue 4420263508SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4421198090Srdivacky const SDValue *Ops, unsigned NumOps, 4422198090Srdivacky EVT MemVT, MachineMemOperand *MMO) { 4423198090Srdivacky assert((Opcode == ISD::INTRINSIC_VOID || 4424198090Srdivacky Opcode == ISD::INTRINSIC_W_CHAIN || 4425218893Sdim Opcode == ISD::PREFETCH || 4426243830Sdim Opcode == ISD::LIFETIME_START || 4427243830Sdim Opcode == ISD::LIFETIME_END || 4428198090Srdivacky (Opcode <= INT_MAX && 4429198090Srdivacky (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4430198090Srdivacky "Opcode is not a memory-accessing opcode!"); 4431198090Srdivacky 4432193323Sed // Memoize the node unless it returns a flag. 4433193323Sed MemIntrinsicSDNode *N; 4434218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4435193323Sed FoldingSetNodeID ID; 4436193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4437239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4438193323Sed void *IP = 0; 4439198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4440198090Srdivacky cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4441193323Sed return SDValue(E, 0); 4442198090Srdivacky } 4443193323Sed 4444263508Sdim N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4445263508Sdim dl.getDebugLoc(), VTList, Ops, 4446263508Sdim NumOps, MemVT, MMO); 4447193323Sed CSEMap.InsertNode(N, IP); 4448193323Sed } else { 4449263508Sdim N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4450263508Sdim dl.getDebugLoc(), VTList, Ops, 4451263508Sdim NumOps, MemVT, MMO); 4452193323Sed } 4453193323Sed AllNodes.push_back(N); 4454193323Sed return SDValue(N, 0); 4455193323Sed} 4456193323Sed 4457218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4458218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4459218893Sdim/// code generator has many cases where it doesn't bother passing in a 4460218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4461218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4462218893Sdim // If this is FI+Offset, we can model it. 4463218893Sdim if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4464218893Sdim return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4465218893Sdim 4466218893Sdim // If this is (FI+Offset1)+Offset2, we can model it. 4467218893Sdim if (Ptr.getOpcode() != ISD::ADD || 4468218893Sdim !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4469218893Sdim !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4470218893Sdim return MachinePointerInfo(); 4471218893Sdim 4472218893Sdim int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4473218893Sdim return MachinePointerInfo::getFixedStack(FI, Offset+ 4474218893Sdim cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4475218893Sdim} 4476218893Sdim 4477218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4478218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4479218893Sdim/// code generator has many cases where it doesn't bother passing in a 4480218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4481218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4482218893Sdim // If the 'Offset' value isn't a constant, we can't handle this. 4483218893Sdim if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4484218893Sdim return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4485218893Sdim if (OffsetOp.getOpcode() == ISD::UNDEF) 4486218893Sdim return InferPointerInfo(Ptr); 4487218893Sdim return MachinePointerInfo(); 4488218893Sdim} 4489218893Sdim 4490218893Sdim 4491193323SedSDValue 4492210299SedSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4493263508Sdim EVT VT, SDLoc dl, SDValue Chain, 4494193323Sed SDValue Ptr, SDValue Offset, 4495218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4496234353Sdim bool isVolatile, bool isNonTemporal, bool isInvariant, 4497234353Sdim unsigned Alignment, const MDNode *TBAAInfo, 4498234353Sdim const MDNode *Ranges) { 4499243830Sdim assert(Chain.getValueType() == MVT::Other && 4500224145Sdim "Invalid chain type"); 4501193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4502198090Srdivacky Alignment = getEVTAlignment(VT); 4503193323Sed 4504198090Srdivacky unsigned Flags = MachineMemOperand::MOLoad; 4505198090Srdivacky if (isVolatile) 4506198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4507203954Srdivacky if (isNonTemporal) 4508203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4509234353Sdim if (isInvariant) 4510234353Sdim Flags |= MachineMemOperand::MOInvariant; 4511218893Sdim 4512218893Sdim // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4513218893Sdim // clients. 4514218893Sdim if (PtrInfo.V == 0) 4515218893Sdim PtrInfo = InferPointerInfo(Ptr, Offset); 4516218893Sdim 4517218893Sdim MachineFunction &MF = getMachineFunction(); 4518198090Srdivacky MachineMemOperand *MMO = 4519218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4520234353Sdim TBAAInfo, Ranges); 4521210299Sed return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4522198090Srdivacky} 4523198090Srdivacky 4524198090SrdivackySDValue 4525218893SdimSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4526263508Sdim EVT VT, SDLoc dl, SDValue Chain, 4527198090Srdivacky SDValue Ptr, SDValue Offset, EVT MemVT, 4528198090Srdivacky MachineMemOperand *MMO) { 4529198090Srdivacky if (VT == MemVT) { 4530193323Sed ExtType = ISD::NON_EXTLOAD; 4531193323Sed } else if (ExtType == ISD::NON_EXTLOAD) { 4532198090Srdivacky assert(VT == MemVT && "Non-extending load from different memory type!"); 4533193323Sed } else { 4534193323Sed // Extending load. 4535200581Srdivacky assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4536200581Srdivacky "Should only be an extending load, not truncating!"); 4537198090Srdivacky assert(VT.isInteger() == MemVT.isInteger() && 4538193323Sed "Cannot convert from FP to Int or Int -> FP!"); 4539200581Srdivacky assert(VT.isVector() == MemVT.isVector() && 4540200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4541200581Srdivacky assert((!VT.isVector() || 4542200581Srdivacky VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4543200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4544193323Sed } 4545193323Sed 4546193323Sed bool Indexed = AM != ISD::UNINDEXED; 4547193323Sed assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4548193323Sed "Unindexed load with an offset!"); 4549193323Sed 4550193323Sed SDVTList VTs = Indexed ? 4551193323Sed getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4552193323Sed SDValue Ops[] = { Chain, Ptr, Offset }; 4553193323Sed FoldingSetNodeID ID; 4554193323Sed AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4555198090Srdivacky ID.AddInteger(MemVT.getRawBits()); 4556204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4557243830Sdim MMO->isNonTemporal(), 4558234353Sdim MMO->isInvariant())); 4559239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4560193323Sed void *IP = 0; 4561198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4562198090Srdivacky cast<LoadSDNode>(E)->refineAlignment(MMO); 4563193323Sed return SDValue(E, 0); 4564198090Srdivacky } 4565263508Sdim SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), 4566263508Sdim dl.getDebugLoc(), VTs, AM, ExtType, 4567205407Srdivacky MemVT, MMO); 4568193323Sed CSEMap.InsertNode(N, IP); 4569193323Sed AllNodes.push_back(N); 4570193323Sed return SDValue(N, 0); 4571193323Sed} 4572193323Sed 4573263508SdimSDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4574193323Sed SDValue Chain, SDValue Ptr, 4575218893Sdim MachinePointerInfo PtrInfo, 4576203954Srdivacky bool isVolatile, bool isNonTemporal, 4577243830Sdim bool isInvariant, unsigned Alignment, 4578234353Sdim const MDNode *TBAAInfo, 4579234353Sdim const MDNode *Ranges) { 4580193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4581210299Sed return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4582234353Sdim PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4583234353Sdim TBAAInfo, Ranges); 4584193323Sed} 4585193323Sed 4586263508SdimSDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4587263508Sdim SDValue Chain, SDValue Ptr, 4588263508Sdim MachineMemOperand *MMO) { 4589263508Sdim SDValue Undef = getUNDEF(Ptr.getValueType()); 4590263508Sdim return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4591263508Sdim VT, MMO); 4592263508Sdim} 4593263508Sdim 4594263508SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4595193323Sed SDValue Chain, SDValue Ptr, 4596218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4597203954Srdivacky bool isVolatile, bool isNonTemporal, 4598218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4599193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4600210299Sed return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4601234353Sdim PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4602218893Sdim TBAAInfo); 4603193323Sed} 4604193323Sed 4605218893Sdim 4606263508SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4607263508Sdim SDValue Chain, SDValue Ptr, EVT MemVT, 4608263508Sdim MachineMemOperand *MMO) { 4609263508Sdim SDValue Undef = getUNDEF(Ptr.getValueType()); 4610263508Sdim return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4611263508Sdim MemVT, MMO); 4612263508Sdim} 4613263508Sdim 4614193323SedSDValue 4615263508SdimSelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, 4616193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4617193323Sed LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4618193323Sed assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4619193323Sed "Load is already a indexed load!"); 4620210299Sed return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4621218893Sdim LD->getChain(), Base, Offset, LD->getPointerInfo(), 4622243830Sdim LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4623234353Sdim false, LD->getAlignment()); 4624193323Sed} 4625193323Sed 4626263508SdimSDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4627218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4628203954Srdivacky bool isVolatile, bool isNonTemporal, 4629218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4630243830Sdim assert(Chain.getValueType() == MVT::Other && 4631224145Sdim "Invalid chain type"); 4632193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4633198090Srdivacky Alignment = getEVTAlignment(Val.getValueType()); 4634193323Sed 4635198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4636198090Srdivacky if (isVolatile) 4637198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4638203954Srdivacky if (isNonTemporal) 4639203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4640218893Sdim 4641218893Sdim if (PtrInfo.V == 0) 4642218893Sdim PtrInfo = InferPointerInfo(Ptr); 4643218893Sdim 4644218893Sdim MachineFunction &MF = getMachineFunction(); 4645198090Srdivacky MachineMemOperand *MMO = 4646218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, 4647218893Sdim Val.getValueType().getStoreSize(), Alignment, 4648218893Sdim TBAAInfo); 4649198090Srdivacky 4650198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4651198090Srdivacky} 4652198090Srdivacky 4653263508SdimSDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4654198090Srdivacky SDValue Ptr, MachineMemOperand *MMO) { 4655243830Sdim assert(Chain.getValueType() == MVT::Other && 4656224145Sdim "Invalid chain type"); 4657198090Srdivacky EVT VT = Val.getValueType(); 4658193323Sed SDVTList VTs = getVTList(MVT::Other); 4659193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4660193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4661193323Sed FoldingSetNodeID ID; 4662193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4663193323Sed ID.AddInteger(VT.getRawBits()); 4664204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4665234353Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4666239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4667193323Sed void *IP = 0; 4668198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4669198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4670193323Sed return SDValue(E, 0); 4671198090Srdivacky } 4672263508Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4673263508Sdim dl.getDebugLoc(), VTs, 4674263508Sdim ISD::UNINDEXED, false, VT, MMO); 4675193323Sed CSEMap.InsertNode(N, IP); 4676193323Sed AllNodes.push_back(N); 4677193323Sed return SDValue(N, 0); 4678193323Sed} 4679193323Sed 4680263508SdimSDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4681218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4682218893Sdim EVT SVT,bool isVolatile, bool isNonTemporal, 4683218893Sdim unsigned Alignment, 4684218893Sdim const MDNode *TBAAInfo) { 4685243830Sdim assert(Chain.getValueType() == MVT::Other && 4686224145Sdim "Invalid chain type"); 4687198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4688198090Srdivacky Alignment = getEVTAlignment(SVT); 4689193323Sed 4690198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4691198090Srdivacky if (isVolatile) 4692198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4693203954Srdivacky if (isNonTemporal) 4694203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4695218893Sdim 4696218893Sdim if (PtrInfo.V == 0) 4697218893Sdim PtrInfo = InferPointerInfo(Ptr); 4698218893Sdim 4699218893Sdim MachineFunction &MF = getMachineFunction(); 4700198090Srdivacky MachineMemOperand *MMO = 4701218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4702218893Sdim TBAAInfo); 4703198090Srdivacky 4704198090Srdivacky return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4705198090Srdivacky} 4706198090Srdivacky 4707263508SdimSDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4708198090Srdivacky SDValue Ptr, EVT SVT, 4709198090Srdivacky MachineMemOperand *MMO) { 4710198090Srdivacky EVT VT = Val.getValueType(); 4711198090Srdivacky 4712243830Sdim assert(Chain.getValueType() == MVT::Other && 4713224145Sdim "Invalid chain type"); 4714193323Sed if (VT == SVT) 4715198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4716193323Sed 4717200581Srdivacky assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4718200581Srdivacky "Should only be a truncating store, not extending!"); 4719193323Sed assert(VT.isInteger() == SVT.isInteger() && 4720193323Sed "Can't do FP-INT conversion!"); 4721200581Srdivacky assert(VT.isVector() == SVT.isVector() && 4722200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4723200581Srdivacky assert((!VT.isVector() || 4724200581Srdivacky VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4725200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4726193323Sed 4727193323Sed SDVTList VTs = getVTList(MVT::Other); 4728193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4729193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4730193323Sed FoldingSetNodeID ID; 4731193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4732193323Sed ID.AddInteger(SVT.getRawBits()); 4733204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4734234353Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4735239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4736193323Sed void *IP = 0; 4737198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4738198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4739193323Sed return SDValue(E, 0); 4740198090Srdivacky } 4741263508Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4742263508Sdim dl.getDebugLoc(), VTs, 4743263508Sdim ISD::UNINDEXED, true, SVT, MMO); 4744193323Sed CSEMap.InsertNode(N, IP); 4745193323Sed AllNodes.push_back(N); 4746193323Sed return SDValue(N, 0); 4747193323Sed} 4748193323Sed 4749193323SedSDValue 4750263508SdimSelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, 4751193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4752193323Sed StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4753193323Sed assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4754193323Sed "Store is already a indexed store!"); 4755193323Sed SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4756193323Sed SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4757193323Sed FoldingSetNodeID ID; 4758193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4759193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 4760193323Sed ID.AddInteger(ST->getRawSubclassData()); 4761239462Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 4762193323Sed void *IP = 0; 4763201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4764193323Sed return SDValue(E, 0); 4765201360Srdivacky 4766263508Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4767263508Sdim dl.getDebugLoc(), VTs, AM, 4768205407Srdivacky ST->isTruncatingStore(), 4769205407Srdivacky ST->getMemoryVT(), 4770205407Srdivacky ST->getMemOperand()); 4771193323Sed CSEMap.InsertNode(N, IP); 4772193323Sed AllNodes.push_back(N); 4773193323Sed return SDValue(N, 0); 4774193323Sed} 4775193323Sed 4776263508SdimSDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl, 4777193323Sed SDValue Chain, SDValue Ptr, 4778210299Sed SDValue SV, 4779210299Sed unsigned Align) { 4780210299Sed SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4781210299Sed return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4782193323Sed} 4783193323Sed 4784263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4785193323Sed const SDUse *Ops, unsigned NumOps) { 4786193323Sed switch (NumOps) { 4787193323Sed case 0: return getNode(Opcode, DL, VT); 4788193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4789193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4790193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4791193323Sed default: break; 4792193323Sed } 4793193323Sed 4794193323Sed // Copy from an SDUse array into an SDValue array for use with 4795193323Sed // the regular getNode logic. 4796193323Sed SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4797193323Sed return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4798193323Sed} 4799193323Sed 4800263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4801193323Sed const SDValue *Ops, unsigned NumOps) { 4802193323Sed switch (NumOps) { 4803193323Sed case 0: return getNode(Opcode, DL, VT); 4804193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4805193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4806193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4807193323Sed default: break; 4808193323Sed } 4809193323Sed 4810193323Sed switch (Opcode) { 4811193323Sed default: break; 4812193323Sed case ISD::SELECT_CC: { 4813193323Sed assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4814193323Sed assert(Ops[0].getValueType() == Ops[1].getValueType() && 4815193323Sed "LHS and RHS of condition must have same type!"); 4816193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4817193323Sed "True and False arms of SelectCC must have same type!"); 4818193323Sed assert(Ops[2].getValueType() == VT && 4819193323Sed "select_cc node must be of same type as true and false value!"); 4820193323Sed break; 4821193323Sed } 4822193323Sed case ISD::BR_CC: { 4823193323Sed assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4824193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4825193323Sed "LHS/RHS of comparison should match types!"); 4826193323Sed break; 4827193323Sed } 4828193323Sed } 4829193323Sed 4830193323Sed // Memoize nodes. 4831193323Sed SDNode *N; 4832193323Sed SDVTList VTs = getVTList(VT); 4833193323Sed 4834218893Sdim if (VT != MVT::Glue) { 4835193323Sed FoldingSetNodeID ID; 4836193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4837193323Sed void *IP = 0; 4838193323Sed 4839201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4840193323Sed return SDValue(E, 0); 4841193323Sed 4842263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4843263508Sdim VTs, Ops, NumOps); 4844193323Sed CSEMap.InsertNode(N, IP); 4845193323Sed } else { 4846263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4847263508Sdim VTs, Ops, NumOps); 4848193323Sed } 4849193323Sed 4850193323Sed AllNodes.push_back(N); 4851193323Sed#ifndef NDEBUG 4852218893Sdim VerifySDNode(N); 4853193323Sed#endif 4854193323Sed return SDValue(N, 0); 4855193323Sed} 4856193323Sed 4857263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4858249423Sdim ArrayRef<EVT> ResultTys, 4859193323Sed const SDValue *Ops, unsigned NumOps) { 4860193323Sed return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4861193323Sed Ops, NumOps); 4862193323Sed} 4863193323Sed 4864263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4865198090Srdivacky const EVT *VTs, unsigned NumVTs, 4866193323Sed const SDValue *Ops, unsigned NumOps) { 4867193323Sed if (NumVTs == 1) 4868193323Sed return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4869193323Sed return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4870193323Sed} 4871193323Sed 4872263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4873193323Sed const SDValue *Ops, unsigned NumOps) { 4874193323Sed if (VTList.NumVTs == 1) 4875193323Sed return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4876193323Sed 4877198090Srdivacky#if 0 4878193323Sed switch (Opcode) { 4879193323Sed // FIXME: figure out how to safely handle things like 4880193323Sed // int foo(int x) { return 1 << (x & 255); } 4881193323Sed // int bar() { return foo(256); } 4882193323Sed case ISD::SRA_PARTS: 4883193323Sed case ISD::SRL_PARTS: 4884193323Sed case ISD::SHL_PARTS: 4885193323Sed if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4886193323Sed cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4887193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4888193323Sed else if (N3.getOpcode() == ISD::AND) 4889193323Sed if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4890193323Sed // If the and is only masking out bits that cannot effect the shift, 4891193323Sed // eliminate the and. 4892202375Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4893193323Sed if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4894193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4895193323Sed } 4896193323Sed break; 4897198090Srdivacky } 4898193323Sed#endif 4899193323Sed 4900193323Sed // Memoize the node unless it returns a flag. 4901193323Sed SDNode *N; 4902218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4903193323Sed FoldingSetNodeID ID; 4904193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4905193323Sed void *IP = 0; 4906201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4907193323Sed return SDValue(E, 0); 4908201360Srdivacky 4909193323Sed if (NumOps == 1) { 4910263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4911263508Sdim DL.getDebugLoc(), VTList, Ops[0]); 4912193323Sed } else if (NumOps == 2) { 4913263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4914263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4915263508Sdim Ops[1]); 4916193323Sed } else if (NumOps == 3) { 4917263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4918263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4919263508Sdim Ops[1], Ops[2]); 4920193323Sed } else { 4921263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4922263508Sdim VTList, Ops, NumOps); 4923193323Sed } 4924193323Sed CSEMap.InsertNode(N, IP); 4925193323Sed } else { 4926193323Sed if (NumOps == 1) { 4927263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4928263508Sdim DL.getDebugLoc(), VTList, Ops[0]); 4929193323Sed } else if (NumOps == 2) { 4930263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4931263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4932263508Sdim Ops[1]); 4933193323Sed } else if (NumOps == 3) { 4934263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4935263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4936263508Sdim Ops[1], Ops[2]); 4937193323Sed } else { 4938263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4939263508Sdim VTList, Ops, NumOps); 4940193323Sed } 4941193323Sed } 4942193323Sed AllNodes.push_back(N); 4943193323Sed#ifndef NDEBUG 4944218893Sdim VerifySDNode(N); 4945193323Sed#endif 4946193323Sed return SDValue(N, 0); 4947193323Sed} 4948193323Sed 4949263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) { 4950193323Sed return getNode(Opcode, DL, VTList, 0, 0); 4951193323Sed} 4952193323Sed 4953263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4954193323Sed SDValue N1) { 4955193323Sed SDValue Ops[] = { N1 }; 4956193323Sed return getNode(Opcode, DL, VTList, Ops, 1); 4957193323Sed} 4958193323Sed 4959263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4960193323Sed SDValue N1, SDValue N2) { 4961193323Sed SDValue Ops[] = { N1, N2 }; 4962193323Sed return getNode(Opcode, DL, VTList, Ops, 2); 4963193323Sed} 4964193323Sed 4965263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4966193323Sed SDValue N1, SDValue N2, SDValue N3) { 4967193323Sed SDValue Ops[] = { N1, N2, N3 }; 4968193323Sed return getNode(Opcode, DL, VTList, Ops, 3); 4969193323Sed} 4970193323Sed 4971263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4972193323Sed SDValue N1, SDValue N2, SDValue N3, 4973193323Sed SDValue N4) { 4974193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 4975193323Sed return getNode(Opcode, DL, VTList, Ops, 4); 4976193323Sed} 4977193323Sed 4978263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4979193323Sed SDValue N1, SDValue N2, SDValue N3, 4980193323Sed SDValue N4, SDValue N5) { 4981193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4982193323Sed return getNode(Opcode, DL, VTList, Ops, 5); 4983193323Sed} 4984193323Sed 4985198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT) { 4986193323Sed return makeVTList(SDNode::getValueTypeList(VT), 1); 4987193323Sed} 4988193323Sed 4989198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4990263508Sdim FoldingSetNodeID ID; 4991263508Sdim ID.AddInteger(2U); 4992263508Sdim ID.AddInteger(VT1.getRawBits()); 4993263508Sdim ID.AddInteger(VT2.getRawBits()); 4994193323Sed 4995263508Sdim void *IP = 0; 4996263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4997263508Sdim if (Result == NULL) { 4998263508Sdim EVT *Array = Allocator.Allocate<EVT>(2); 4999263508Sdim Array[0] = VT1; 5000263508Sdim Array[1] = VT2; 5001263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 5002263508Sdim VTListMap.InsertNode(Result, IP); 5003263508Sdim } 5004263508Sdim return Result->getSDVTList(); 5005193323Sed} 5006193323Sed 5007198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 5008263508Sdim FoldingSetNodeID ID; 5009263508Sdim ID.AddInteger(3U); 5010263508Sdim ID.AddInteger(VT1.getRawBits()); 5011263508Sdim ID.AddInteger(VT2.getRawBits()); 5012263508Sdim ID.AddInteger(VT3.getRawBits()); 5013193323Sed 5014263508Sdim void *IP = 0; 5015263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5016263508Sdim if (Result == NULL) { 5017263508Sdim EVT *Array = Allocator.Allocate<EVT>(3); 5018263508Sdim Array[0] = VT1; 5019263508Sdim Array[1] = VT2; 5020263508Sdim Array[2] = VT3; 5021263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 5022263508Sdim VTListMap.InsertNode(Result, IP); 5023263508Sdim } 5024263508Sdim return Result->getSDVTList(); 5025193323Sed} 5026193323Sed 5027198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 5028263508Sdim FoldingSetNodeID ID; 5029263508Sdim ID.AddInteger(4U); 5030263508Sdim ID.AddInteger(VT1.getRawBits()); 5031263508Sdim ID.AddInteger(VT2.getRawBits()); 5032263508Sdim ID.AddInteger(VT3.getRawBits()); 5033263508Sdim ID.AddInteger(VT4.getRawBits()); 5034193323Sed 5035263508Sdim void *IP = 0; 5036263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5037263508Sdim if (Result == NULL) { 5038263508Sdim EVT *Array = Allocator.Allocate<EVT>(4); 5039263508Sdim Array[0] = VT1; 5040263508Sdim Array[1] = VT2; 5041263508Sdim Array[2] = VT3; 5042263508Sdim Array[3] = VT4; 5043263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 5044263508Sdim VTListMap.InsertNode(Result, IP); 5045263508Sdim } 5046263508Sdim return Result->getSDVTList(); 5047193323Sed} 5048193323Sed 5049198090SrdivackySDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 5050263508Sdim FoldingSetNodeID ID; 5051263508Sdim ID.AddInteger(NumVTs); 5052263508Sdim for (unsigned index = 0; index < NumVTs; index++) { 5053263508Sdim ID.AddInteger(VTs[index].getRawBits()); 5054193323Sed } 5055193323Sed 5056263508Sdim void *IP = 0; 5057263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5058263508Sdim if (Result == NULL) { 5059263508Sdim EVT *Array = Allocator.Allocate<EVT>(NumVTs); 5060263508Sdim std::copy(VTs, VTs + NumVTs, Array); 5061263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 5062263508Sdim VTListMap.InsertNode(Result, IP); 5063193323Sed } 5064263508Sdim return Result->getSDVTList(); 5065193323Sed} 5066193323Sed 5067193323Sed 5068193323Sed/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 5069193323Sed/// specified operands. If the resultant node already exists in the DAG, 5070193323Sed/// this does not modify the specified node, instead it returns the node that 5071193323Sed/// already exists. If the resultant node does not exist in the DAG, the 5072193323Sed/// input node is returned. As a degenerate case, if you specify the same 5073193323Sed/// input operands as the node already has, the input node is returned. 5074210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 5075193323Sed assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 5076193323Sed 5077193323Sed // Check to see if there is no change. 5078210299Sed if (Op == N->getOperand(0)) return N; 5079193323Sed 5080193323Sed // See if the modified node already exists. 5081193323Sed void *InsertPos = 0; 5082193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 5083210299Sed return Existing; 5084193323Sed 5085193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5086193323Sed if (InsertPos) 5087193323Sed if (!RemoveNodeFromCSEMaps(N)) 5088193323Sed InsertPos = 0; 5089193323Sed 5090193323Sed // Now we update the operands. 5091193323Sed N->OperandList[0].set(Op); 5092193323Sed 5093193323Sed // If this gets put into a CSE map, add it. 5094193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5095210299Sed return N; 5096193323Sed} 5097193323Sed 5098210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 5099193323Sed assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 5100193323Sed 5101193323Sed // Check to see if there is no change. 5102193323Sed if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 5103210299Sed return N; // No operands changed, just return the input node. 5104193323Sed 5105193323Sed // See if the modified node already exists. 5106193323Sed void *InsertPos = 0; 5107193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 5108210299Sed return Existing; 5109193323Sed 5110193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5111193323Sed if (InsertPos) 5112193323Sed if (!RemoveNodeFromCSEMaps(N)) 5113193323Sed InsertPos = 0; 5114193323Sed 5115193323Sed // Now we update the operands. 5116193323Sed if (N->OperandList[0] != Op1) 5117193323Sed N->OperandList[0].set(Op1); 5118193323Sed if (N->OperandList[1] != Op2) 5119193323Sed N->OperandList[1].set(Op2); 5120193323Sed 5121193323Sed // If this gets put into a CSE map, add it. 5122193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5123210299Sed return N; 5124193323Sed} 5125193323Sed 5126210299SedSDNode *SelectionDAG:: 5127210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 5128193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5129193323Sed return UpdateNodeOperands(N, Ops, 3); 5130193323Sed} 5131193323Sed 5132210299SedSDNode *SelectionDAG:: 5133210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5134193323Sed SDValue Op3, SDValue Op4) { 5135193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 5136193323Sed return UpdateNodeOperands(N, Ops, 4); 5137193323Sed} 5138193323Sed 5139210299SedSDNode *SelectionDAG:: 5140210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5141193323Sed SDValue Op3, SDValue Op4, SDValue Op5) { 5142193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 5143193323Sed return UpdateNodeOperands(N, Ops, 5); 5144193323Sed} 5145193323Sed 5146210299SedSDNode *SelectionDAG:: 5147210299SedUpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 5148193323Sed assert(N->getNumOperands() == NumOps && 5149193323Sed "Update with wrong number of operands"); 5150193323Sed 5151193323Sed // Check to see if there is no change. 5152193323Sed bool AnyChange = false; 5153193323Sed for (unsigned i = 0; i != NumOps; ++i) { 5154193323Sed if (Ops[i] != N->getOperand(i)) { 5155193323Sed AnyChange = true; 5156193323Sed break; 5157193323Sed } 5158193323Sed } 5159193323Sed 5160193323Sed // No operands changed, just return the input node. 5161210299Sed if (!AnyChange) return N; 5162193323Sed 5163193323Sed // See if the modified node already exists. 5164193323Sed void *InsertPos = 0; 5165193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 5166210299Sed return Existing; 5167193323Sed 5168193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5169193323Sed if (InsertPos) 5170193323Sed if (!RemoveNodeFromCSEMaps(N)) 5171193323Sed InsertPos = 0; 5172193323Sed 5173193323Sed // Now we update the operands. 5174193323Sed for (unsigned i = 0; i != NumOps; ++i) 5175193323Sed if (N->OperandList[i] != Ops[i]) 5176193323Sed N->OperandList[i].set(Ops[i]); 5177193323Sed 5178193323Sed // If this gets put into a CSE map, add it. 5179193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5180210299Sed return N; 5181193323Sed} 5182193323Sed 5183193323Sed/// DropOperands - Release the operands and set this node to have 5184193323Sed/// zero operands. 5185193323Sedvoid SDNode::DropOperands() { 5186193323Sed // Unlike the code in MorphNodeTo that does this, we don't need to 5187193323Sed // watch for dead nodes here. 5188193323Sed for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 5189193323Sed SDUse &Use = *I++; 5190193323Sed Use.set(SDValue()); 5191193323Sed } 5192193323Sed} 5193193323Sed 5194193323Sed/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 5195193323Sed/// machine opcode. 5196193323Sed/// 5197193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5198198090Srdivacky EVT VT) { 5199193323Sed SDVTList VTs = getVTList(VT); 5200193323Sed return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 5201193323Sed} 5202193323Sed 5203193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5204198090Srdivacky EVT VT, SDValue Op1) { 5205193323Sed SDVTList VTs = getVTList(VT); 5206193323Sed SDValue Ops[] = { Op1 }; 5207193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5208193323Sed} 5209193323Sed 5210193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5211198090Srdivacky EVT VT, SDValue Op1, 5212193323Sed SDValue Op2) { 5213193323Sed SDVTList VTs = getVTList(VT); 5214193323Sed SDValue Ops[] = { Op1, Op2 }; 5215193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5216193323Sed} 5217193323Sed 5218193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5219198090Srdivacky EVT VT, SDValue Op1, 5220193323Sed SDValue Op2, SDValue Op3) { 5221193323Sed SDVTList VTs = getVTList(VT); 5222193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5223193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5224193323Sed} 5225193323Sed 5226193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5227198090Srdivacky EVT VT, const SDValue *Ops, 5228193323Sed unsigned NumOps) { 5229193323Sed SDVTList VTs = getVTList(VT); 5230193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5231193323Sed} 5232193323Sed 5233193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5234198090Srdivacky EVT VT1, EVT VT2, const SDValue *Ops, 5235193323Sed unsigned NumOps) { 5236193323Sed SDVTList VTs = getVTList(VT1, VT2); 5237193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5238193323Sed} 5239193323Sed 5240193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5241198090Srdivacky EVT VT1, EVT VT2) { 5242193323Sed SDVTList VTs = getVTList(VT1, VT2); 5243193323Sed return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 5244193323Sed} 5245193323Sed 5246193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5247198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5248193323Sed const SDValue *Ops, unsigned NumOps) { 5249193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5250193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5251193323Sed} 5252193323Sed 5253193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5254198090Srdivacky EVT VT1, EVT VT2, EVT VT3, EVT VT4, 5255193323Sed const SDValue *Ops, unsigned NumOps) { 5256193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5257193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5258193323Sed} 5259193323Sed 5260193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5261198090Srdivacky EVT VT1, EVT VT2, 5262193323Sed SDValue Op1) { 5263193323Sed SDVTList VTs = getVTList(VT1, VT2); 5264193323Sed SDValue Ops[] = { Op1 }; 5265193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5266193323Sed} 5267193323Sed 5268193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5269198090Srdivacky EVT VT1, EVT VT2, 5270193323Sed SDValue Op1, SDValue Op2) { 5271193323Sed SDVTList VTs = getVTList(VT1, VT2); 5272193323Sed SDValue Ops[] = { Op1, Op2 }; 5273193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5274193323Sed} 5275193323Sed 5276193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5277198090Srdivacky EVT VT1, EVT VT2, 5278193323Sed SDValue Op1, SDValue Op2, 5279193323Sed SDValue Op3) { 5280193323Sed SDVTList VTs = getVTList(VT1, VT2); 5281193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5282193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5283193323Sed} 5284193323Sed 5285193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5286198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5287193323Sed SDValue Op1, SDValue Op2, 5288193323Sed SDValue Op3) { 5289193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5290193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5291193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5292193323Sed} 5293193323Sed 5294193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5295193323Sed SDVTList VTs, const SDValue *Ops, 5296193323Sed unsigned NumOps) { 5297204642Srdivacky N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 5298204642Srdivacky // Reset the NodeID to -1. 5299204642Srdivacky N->setNodeId(-1); 5300204642Srdivacky return N; 5301193323Sed} 5302193323Sed 5303263508Sdim/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away 5304234353Sdim/// the line number information on the merged node since it is not possible to 5305234353Sdim/// preserve the information that operation is associated with multiple lines. 5306234353Sdim/// This will make the debugger working better at -O0, were there is a higher 5307234353Sdim/// probability having other instructions associated with that line. 5308234353Sdim/// 5309263508Sdim/// For IROrder, we keep the smaller of the two 5310263508SdimSDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { 5311234353Sdim DebugLoc NLoc = N->getDebugLoc(); 5312263508Sdim if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && 5313263508Sdim (OLoc.getDebugLoc() != NLoc)) { 5314234353Sdim N->setDebugLoc(DebugLoc()); 5315234353Sdim } 5316263508Sdim unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 5317263508Sdim N->setIROrder(Order); 5318234353Sdim return N; 5319234353Sdim} 5320234353Sdim 5321204642Srdivacky/// MorphNodeTo - This *mutates* the specified node to have the specified 5322193323Sed/// return type, opcode, and operands. 5323193323Sed/// 5324193323Sed/// Note that MorphNodeTo returns the resultant node. If there is already a 5325193323Sed/// node of the specified opcode and operands, it returns that node instead of 5326263508Sdim/// the current one. Note that the SDLoc need not be the same. 5327193323Sed/// 5328193323Sed/// Using MorphNodeTo is faster than creating a new node and swapping it in 5329193323Sed/// with ReplaceAllUsesWith both because it often avoids allocating a new 5330193323Sed/// node, and because it doesn't require CSE recalculation for any of 5331193323Sed/// the node's users. 5332193323Sed/// 5333193323SedSDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 5334193323Sed SDVTList VTs, const SDValue *Ops, 5335193323Sed unsigned NumOps) { 5336193323Sed // If an identical node already exists, use it. 5337193323Sed void *IP = 0; 5338218893Sdim if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 5339193323Sed FoldingSetNodeID ID; 5340193323Sed AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 5341201360Srdivacky if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 5342263508Sdim return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N)); 5343193323Sed } 5344193323Sed 5345193323Sed if (!RemoveNodeFromCSEMaps(N)) 5346193323Sed IP = 0; 5347193323Sed 5348193323Sed // Start the morphing. 5349193323Sed N->NodeType = Opc; 5350193323Sed N->ValueList = VTs.VTs; 5351193323Sed N->NumValues = VTs.NumVTs; 5352193323Sed 5353193323Sed // Clear the operands list, updating used nodes to remove this from their 5354193323Sed // use list. Keep track of any operands that become dead as a result. 5355193323Sed SmallPtrSet<SDNode*, 16> DeadNodeSet; 5356193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 5357193323Sed SDUse &Use = *I++; 5358193323Sed SDNode *Used = Use.getNode(); 5359193323Sed Use.set(SDValue()); 5360193323Sed if (Used->use_empty()) 5361193323Sed DeadNodeSet.insert(Used); 5362193323Sed } 5363193323Sed 5364198090Srdivacky if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 5365198090Srdivacky // Initialize the memory references information. 5366198090Srdivacky MN->setMemRefs(0, 0); 5367198090Srdivacky // If NumOps is larger than the # of operands we can have in a 5368198090Srdivacky // MachineSDNode, reallocate the operand list. 5369198090Srdivacky if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5370198090Srdivacky if (MN->OperandsNeedDelete) 5371198090Srdivacky delete[] MN->OperandList; 5372198090Srdivacky if (NumOps > array_lengthof(MN->LocalOperands)) 5373198090Srdivacky // We're creating a final node that will live unmorphed for the 5374198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5375198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5376198090Srdivacky MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5377205407Srdivacky Ops, NumOps); 5378198090Srdivacky else 5379198090Srdivacky MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5380198090Srdivacky MN->OperandsNeedDelete = false; 5381198090Srdivacky } else 5382198090Srdivacky MN->InitOperands(MN->OperandList, Ops, NumOps); 5383198090Srdivacky } else { 5384198090Srdivacky // If NumOps is larger than the # of operands we currently have, reallocate 5385198090Srdivacky // the operand list. 5386198090Srdivacky if (NumOps > N->NumOperands) { 5387198090Srdivacky if (N->OperandsNeedDelete) 5388198090Srdivacky delete[] N->OperandList; 5389198090Srdivacky N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5390193323Sed N->OperandsNeedDelete = true; 5391198090Srdivacky } else 5392198396Srdivacky N->InitOperands(N->OperandList, Ops, NumOps); 5393193323Sed } 5394193323Sed 5395193323Sed // Delete any nodes that are still dead after adding the uses for the 5396193323Sed // new operands. 5397204642Srdivacky if (!DeadNodeSet.empty()) { 5398204642Srdivacky SmallVector<SDNode *, 16> DeadNodes; 5399204642Srdivacky for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5400204642Srdivacky E = DeadNodeSet.end(); I != E; ++I) 5401204642Srdivacky if ((*I)->use_empty()) 5402204642Srdivacky DeadNodes.push_back(*I); 5403204642Srdivacky RemoveDeadNodes(DeadNodes); 5404204642Srdivacky } 5405193323Sed 5406193323Sed if (IP) 5407193323Sed CSEMap.InsertNode(N, IP); // Memoize the new node. 5408193323Sed return N; 5409193323Sed} 5410193323Sed 5411193323Sed 5412198090Srdivacky/// getMachineNode - These are used for target selectors to create a new node 5413198090Srdivacky/// with specified return type(s), MachineInstr opcode, and operands. 5414193323Sed/// 5415198090Srdivacky/// Note that getMachineNode returns the resultant node. If there is already a 5416193323Sed/// node of the specified opcode and operands, it returns that node instead of 5417193323Sed/// the current one. 5418198090SrdivackyMachineSDNode * 5419263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) { 5420198090Srdivacky SDVTList VTs = getVTList(VT); 5421251662Sdim return getMachineNode(Opcode, dl, VTs, None); 5422193323Sed} 5423193323Sed 5424198090SrdivackyMachineSDNode * 5425263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) { 5426198090Srdivacky SDVTList VTs = getVTList(VT); 5427198090Srdivacky SDValue Ops[] = { Op1 }; 5428251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5429193323Sed} 5430193323Sed 5431198090SrdivackyMachineSDNode * 5432263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5433198090Srdivacky SDValue Op1, SDValue Op2) { 5434198090Srdivacky SDVTList VTs = getVTList(VT); 5435198090Srdivacky SDValue Ops[] = { Op1, Op2 }; 5436251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5437193323Sed} 5438193323Sed 5439198090SrdivackyMachineSDNode * 5440263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5441198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5442198090Srdivacky SDVTList VTs = getVTList(VT); 5443198090Srdivacky SDValue Ops[] = { Op1, Op2, Op3 }; 5444251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5445193323Sed} 5446193323Sed 5447198090SrdivackyMachineSDNode * 5448263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5449251662Sdim ArrayRef<SDValue> Ops) { 5450198090Srdivacky SDVTList VTs = getVTList(VT); 5451251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5452193323Sed} 5453193323Sed 5454198090SrdivackyMachineSDNode * 5455263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) { 5456193323Sed SDVTList VTs = getVTList(VT1, VT2); 5457251662Sdim return getMachineNode(Opcode, dl, VTs, None); 5458193323Sed} 5459193323Sed 5460198090SrdivackyMachineSDNode * 5461263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5462198090Srdivacky EVT VT1, EVT VT2, SDValue Op1) { 5463193323Sed SDVTList VTs = getVTList(VT1, VT2); 5464198090Srdivacky SDValue Ops[] = { Op1 }; 5465251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5466193323Sed} 5467193323Sed 5468198090SrdivackyMachineSDNode * 5469263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5470198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5471193323Sed SDVTList VTs = getVTList(VT1, VT2); 5472193323Sed SDValue Ops[] = { Op1, Op2 }; 5473251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5474193323Sed} 5475193323Sed 5476198090SrdivackyMachineSDNode * 5477263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5478198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, 5479198090Srdivacky SDValue Op2, SDValue Op3) { 5480193323Sed SDVTList VTs = getVTList(VT1, VT2); 5481193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5482251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5483193323Sed} 5484193323Sed 5485198090SrdivackyMachineSDNode * 5486263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5487198090Srdivacky EVT VT1, EVT VT2, 5488251662Sdim ArrayRef<SDValue> Ops) { 5489193323Sed SDVTList VTs = getVTList(VT1, VT2); 5490251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5491193323Sed} 5492193323Sed 5493198090SrdivackyMachineSDNode * 5494263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5495198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5496198090Srdivacky SDValue Op1, SDValue Op2) { 5497193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5498193323Sed SDValue Ops[] = { Op1, Op2 }; 5499251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5500193323Sed} 5501193323Sed 5502198090SrdivackyMachineSDNode * 5503263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5504198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5505198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5506193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5507193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5508251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5509193323Sed} 5510193323Sed 5511198090SrdivackyMachineSDNode * 5512263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5513198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5514251662Sdim ArrayRef<SDValue> Ops) { 5515193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5516251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5517193323Sed} 5518193323Sed 5519198090SrdivackyMachineSDNode * 5520263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, 5521198090Srdivacky EVT VT2, EVT VT3, EVT VT4, 5522251662Sdim ArrayRef<SDValue> Ops) { 5523193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5524251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5525193323Sed} 5526193323Sed 5527198090SrdivackyMachineSDNode * 5528263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5529249423Sdim ArrayRef<EVT> ResultTys, 5530251662Sdim ArrayRef<SDValue> Ops) { 5531198090Srdivacky SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5532251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5533193323Sed} 5534193323Sed 5535198090SrdivackyMachineSDNode * 5536263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, 5537251662Sdim ArrayRef<SDValue> OpsArray) { 5538218893Sdim bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5539198090Srdivacky MachineSDNode *N; 5540218893Sdim void *IP = 0; 5541251662Sdim const SDValue *Ops = OpsArray.data(); 5542251662Sdim unsigned NumOps = OpsArray.size(); 5543198090Srdivacky 5544198090Srdivacky if (DoCSE) { 5545198090Srdivacky FoldingSetNodeID ID; 5546198090Srdivacky AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5547198090Srdivacky IP = 0; 5548234353Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5549263508Sdim return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); 5550234353Sdim } 5551198090Srdivacky } 5552198090Srdivacky 5553198090Srdivacky // Allocate a new MachineSDNode. 5554263508Sdim N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), 5555263508Sdim DL.getDebugLoc(), VTs); 5556198090Srdivacky 5557198090Srdivacky // Initialize the operands list. 5558198090Srdivacky if (NumOps > array_lengthof(N->LocalOperands)) 5559198090Srdivacky // We're creating a final node that will live unmorphed for the 5560198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5561198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5562198090Srdivacky N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5563198090Srdivacky Ops, NumOps); 5564198090Srdivacky else 5565198090Srdivacky N->InitOperands(N->LocalOperands, Ops, NumOps); 5566198090Srdivacky N->OperandsNeedDelete = false; 5567198090Srdivacky 5568198090Srdivacky if (DoCSE) 5569198090Srdivacky CSEMap.InsertNode(N, IP); 5570198090Srdivacky 5571198090Srdivacky AllNodes.push_back(N); 5572198090Srdivacky#ifndef NDEBUG 5573218893Sdim VerifyMachineNode(N); 5574198090Srdivacky#endif 5575198090Srdivacky return N; 5576198090Srdivacky} 5577198090Srdivacky 5578198090Srdivacky/// getTargetExtractSubreg - A convenience function for creating 5579203954Srdivacky/// TargetOpcode::EXTRACT_SUBREG nodes. 5580198090SrdivackySDValue 5581263508SdimSelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, 5582198090Srdivacky SDValue Operand) { 5583198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5584203954Srdivacky SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5585198090Srdivacky VT, Operand, SRIdxVal); 5586198090Srdivacky return SDValue(Subreg, 0); 5587198090Srdivacky} 5588198090Srdivacky 5589198090Srdivacky/// getTargetInsertSubreg - A convenience function for creating 5590203954Srdivacky/// TargetOpcode::INSERT_SUBREG nodes. 5591198090SrdivackySDValue 5592263508SdimSelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, 5593198090Srdivacky SDValue Operand, SDValue Subreg) { 5594198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5595203954Srdivacky SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5596198090Srdivacky VT, Operand, Subreg, SRIdxVal); 5597198090Srdivacky return SDValue(Result, 0); 5598198090Srdivacky} 5599198090Srdivacky 5600193323Sed/// getNodeIfExists - Get the specified node if it's already available, or 5601193323Sed/// else return NULL. 5602193323SedSDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5603193323Sed const SDValue *Ops, unsigned NumOps) { 5604218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5605193323Sed FoldingSetNodeID ID; 5606193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5607193323Sed void *IP = 0; 5608201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5609193323Sed return E; 5610193323Sed } 5611193323Sed return NULL; 5612193323Sed} 5613193323Sed 5614206083Srdivacky/// getDbgValue - Creates a SDDbgValue node. 5615206083Srdivacky/// 5616206083SrdivackySDDbgValue * 5617206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5618206083Srdivacky DebugLoc DL, unsigned O) { 5619206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5620206083Srdivacky} 5621206083Srdivacky 5622206083SrdivackySDDbgValue * 5623207618SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5624206083Srdivacky DebugLoc DL, unsigned O) { 5625206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5626206083Srdivacky} 5627206083Srdivacky 5628206083SrdivackySDDbgValue * 5629206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5630206083Srdivacky DebugLoc DL, unsigned O) { 5631206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5632206083Srdivacky} 5633206083Srdivacky 5634204792Srdivackynamespace { 5635204792Srdivacky 5636204792Srdivacky/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5637204792Srdivacky/// pointed to by a use iterator is deleted, increment the use iterator 5638204792Srdivacky/// so that it doesn't dangle. 5639204792Srdivacky/// 5640204792Srdivackyclass RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5641204792Srdivacky SDNode::use_iterator &UI; 5642204792Srdivacky SDNode::use_iterator &UE; 5643204792Srdivacky 5644204792Srdivacky virtual void NodeDeleted(SDNode *N, SDNode *E) { 5645204792Srdivacky // Increment the iterator as needed. 5646204792Srdivacky while (UI != UE && N == *UI) 5647204792Srdivacky ++UI; 5648204792Srdivacky } 5649204792Srdivacky 5650204792Srdivackypublic: 5651239462Sdim RAUWUpdateListener(SelectionDAG &d, 5652204792Srdivacky SDNode::use_iterator &ui, 5653204792Srdivacky SDNode::use_iterator &ue) 5654239462Sdim : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 5655204792Srdivacky}; 5656204792Srdivacky 5657204792Srdivacky} 5658204792Srdivacky 5659193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5660193323Sed/// This can cause recursive merging of nodes in the DAG. 5661193323Sed/// 5662193323Sed/// This version assumes From has a single result value. 5663193323Sed/// 5664239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 5665193323Sed SDNode *From = FromN.getNode(); 5666193323Sed assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5667193323Sed "Cannot replace with this method!"); 5668193323Sed assert(From != To.getNode() && "Cannot replace uses of with self"); 5669193323Sed 5670193323Sed // Iterate over all the existing uses of From. New uses will be added 5671193323Sed // to the beginning of the use list, which we avoid visiting. 5672193323Sed // This specifically avoids visiting uses of From that arise while the 5673193323Sed // replacement is happening, because any such uses would be the result 5674193323Sed // of CSE: If an existing node looks like From after one of its operands 5675193323Sed // is replaced by To, we don't want to replace of all its users with To 5676193323Sed // too. See PR3018 for more info. 5677193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5678239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5679193323Sed while (UI != UE) { 5680193323Sed SDNode *User = *UI; 5681193323Sed 5682193323Sed // This node is about to morph, remove its old self from the CSE maps. 5683193323Sed RemoveNodeFromCSEMaps(User); 5684193323Sed 5685193323Sed // A user can appear in a use list multiple times, and when this 5686193323Sed // happens the uses are usually next to each other in the list. 5687193323Sed // To help reduce the number of CSE recomputations, process all 5688193323Sed // the uses of this user that we can find this way. 5689193323Sed do { 5690193323Sed SDUse &Use = UI.getUse(); 5691193323Sed ++UI; 5692193323Sed Use.set(To); 5693193323Sed } while (UI != UE && *UI == User); 5694193323Sed 5695193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5696193323Sed // already exists there, recursively merge the results together. 5697239462Sdim AddModifiedNodeToCSEMaps(User); 5698193323Sed } 5699234353Sdim 5700234353Sdim // If we just RAUW'd the root, take note. 5701234353Sdim if (FromN == getRoot()) 5702234353Sdim setRoot(To); 5703193323Sed} 5704193323Sed 5705193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5706193323Sed/// This can cause recursive merging of nodes in the DAG. 5707193323Sed/// 5708193323Sed/// This version assumes that for each value of From, there is a 5709193323Sed/// corresponding value in To in the same position with the same type. 5710193323Sed/// 5711239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 5712193323Sed#ifndef NDEBUG 5713193323Sed for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5714193323Sed assert((!From->hasAnyUseOfValue(i) || 5715193323Sed From->getValueType(i) == To->getValueType(i)) && 5716193323Sed "Cannot use this version of ReplaceAllUsesWith!"); 5717193323Sed#endif 5718193323Sed 5719193323Sed // Handle the trivial case. 5720193323Sed if (From == To) 5721193323Sed return; 5722193323Sed 5723193323Sed // Iterate over just the existing users of From. See the comments in 5724193323Sed // the ReplaceAllUsesWith above. 5725193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5726239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5727193323Sed while (UI != UE) { 5728193323Sed SDNode *User = *UI; 5729193323Sed 5730193323Sed // This node is about to morph, remove its old self from the CSE maps. 5731193323Sed RemoveNodeFromCSEMaps(User); 5732193323Sed 5733193323Sed // A user can appear in a use list multiple times, and when this 5734193323Sed // happens the uses are usually next to each other in the list. 5735193323Sed // To help reduce the number of CSE recomputations, process all 5736193323Sed // the uses of this user that we can find this way. 5737193323Sed do { 5738193323Sed SDUse &Use = UI.getUse(); 5739193323Sed ++UI; 5740193323Sed Use.setNode(To); 5741193323Sed } while (UI != UE && *UI == User); 5742193323Sed 5743193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5744193323Sed // already exists there, recursively merge the results together. 5745239462Sdim AddModifiedNodeToCSEMaps(User); 5746193323Sed } 5747234353Sdim 5748234353Sdim // If we just RAUW'd the root, take note. 5749234353Sdim if (From == getRoot().getNode()) 5750234353Sdim setRoot(SDValue(To, getRoot().getResNo())); 5751193323Sed} 5752193323Sed 5753193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5754193323Sed/// This can cause recursive merging of nodes in the DAG. 5755193323Sed/// 5756193323Sed/// This version can replace From with any result values. To must match the 5757193323Sed/// number and types of values returned by From. 5758239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 5759193323Sed if (From->getNumValues() == 1) // Handle the simple case efficiently. 5760239462Sdim return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 5761193323Sed 5762193323Sed // Iterate over just the existing users of From. See the comments in 5763193323Sed // the ReplaceAllUsesWith above. 5764193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5765239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5766193323Sed while (UI != UE) { 5767193323Sed SDNode *User = *UI; 5768193323Sed 5769193323Sed // This node is about to morph, remove its old self from the CSE maps. 5770193323Sed RemoveNodeFromCSEMaps(User); 5771193323Sed 5772193323Sed // A user can appear in a use list multiple times, and when this 5773193323Sed // happens the uses are usually next to each other in the list. 5774193323Sed // To help reduce the number of CSE recomputations, process all 5775193323Sed // the uses of this user that we can find this way. 5776193323Sed do { 5777193323Sed SDUse &Use = UI.getUse(); 5778193323Sed const SDValue &ToOp = To[Use.getResNo()]; 5779193323Sed ++UI; 5780193323Sed Use.set(ToOp); 5781193323Sed } while (UI != UE && *UI == User); 5782193323Sed 5783193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5784193323Sed // already exists there, recursively merge the results together. 5785239462Sdim AddModifiedNodeToCSEMaps(User); 5786193323Sed } 5787234353Sdim 5788234353Sdim // If we just RAUW'd the root, take note. 5789234353Sdim if (From == getRoot().getNode()) 5790234353Sdim setRoot(SDValue(To[getRoot().getResNo()])); 5791193323Sed} 5792193323Sed 5793193323Sed/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5794193323Sed/// uses of other values produced by From.getNode() alone. The Deleted 5795193323Sed/// vector is handled the same way as for ReplaceAllUsesWith. 5796239462Sdimvoid SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 5797193323Sed // Handle the really simple, really trivial case efficiently. 5798193323Sed if (From == To) return; 5799193323Sed 5800193323Sed // Handle the simple, trivial, case efficiently. 5801193323Sed if (From.getNode()->getNumValues() == 1) { 5802239462Sdim ReplaceAllUsesWith(From, To); 5803193323Sed return; 5804193323Sed } 5805193323Sed 5806193323Sed // Iterate over just the existing users of From. See the comments in 5807193323Sed // the ReplaceAllUsesWith above. 5808193323Sed SDNode::use_iterator UI = From.getNode()->use_begin(), 5809193323Sed UE = From.getNode()->use_end(); 5810239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5811193323Sed while (UI != UE) { 5812193323Sed SDNode *User = *UI; 5813193323Sed bool UserRemovedFromCSEMaps = false; 5814193323Sed 5815193323Sed // A user can appear in a use list multiple times, and when this 5816193323Sed // happens the uses are usually next to each other in the list. 5817193323Sed // To help reduce the number of CSE recomputations, process all 5818193323Sed // the uses of this user that we can find this way. 5819193323Sed do { 5820193323Sed SDUse &Use = UI.getUse(); 5821193323Sed 5822193323Sed // Skip uses of different values from the same node. 5823193323Sed if (Use.getResNo() != From.getResNo()) { 5824193323Sed ++UI; 5825193323Sed continue; 5826193323Sed } 5827193323Sed 5828193323Sed // If this node hasn't been modified yet, it's still in the CSE maps, 5829193323Sed // so remove its old self from the CSE maps. 5830193323Sed if (!UserRemovedFromCSEMaps) { 5831193323Sed RemoveNodeFromCSEMaps(User); 5832193323Sed UserRemovedFromCSEMaps = true; 5833193323Sed } 5834193323Sed 5835193323Sed ++UI; 5836193323Sed Use.set(To); 5837193323Sed } while (UI != UE && *UI == User); 5838193323Sed 5839193323Sed // We are iterating over all uses of the From node, so if a use 5840193323Sed // doesn't use the specific value, no changes are made. 5841193323Sed if (!UserRemovedFromCSEMaps) 5842193323Sed continue; 5843193323Sed 5844193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5845193323Sed // already exists there, recursively merge the results together. 5846239462Sdim AddModifiedNodeToCSEMaps(User); 5847193323Sed } 5848234353Sdim 5849234353Sdim // If we just RAUW'd the root, take note. 5850234353Sdim if (From == getRoot()) 5851234353Sdim setRoot(To); 5852193323Sed} 5853193323Sed 5854193323Sednamespace { 5855193323Sed /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5856193323Sed /// to record information about a use. 5857193323Sed struct UseMemo { 5858193323Sed SDNode *User; 5859193323Sed unsigned Index; 5860193323Sed SDUse *Use; 5861193323Sed }; 5862193323Sed 5863193323Sed /// operator< - Sort Memos by User. 5864193323Sed bool operator<(const UseMemo &L, const UseMemo &R) { 5865193323Sed return (intptr_t)L.User < (intptr_t)R.User; 5866193323Sed } 5867193323Sed} 5868193323Sed 5869193323Sed/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5870193323Sed/// uses of other values produced by From.getNode() alone. The same value 5871193323Sed/// may appear in both the From and To list. The Deleted vector is 5872193323Sed/// handled the same way as for ReplaceAllUsesWith. 5873193323Sedvoid SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5874193323Sed const SDValue *To, 5875239462Sdim unsigned Num){ 5876193323Sed // Handle the simple, trivial case efficiently. 5877193323Sed if (Num == 1) 5878239462Sdim return ReplaceAllUsesOfValueWith(*From, *To); 5879193323Sed 5880193323Sed // Read up all the uses and make records of them. This helps 5881193323Sed // processing new uses that are introduced during the 5882193323Sed // replacement process. 5883193323Sed SmallVector<UseMemo, 4> Uses; 5884193323Sed for (unsigned i = 0; i != Num; ++i) { 5885193323Sed unsigned FromResNo = From[i].getResNo(); 5886193323Sed SDNode *FromNode = From[i].getNode(); 5887193323Sed for (SDNode::use_iterator UI = FromNode->use_begin(), 5888193323Sed E = FromNode->use_end(); UI != E; ++UI) { 5889193323Sed SDUse &Use = UI.getUse(); 5890193323Sed if (Use.getResNo() == FromResNo) { 5891193323Sed UseMemo Memo = { *UI, i, &Use }; 5892193323Sed Uses.push_back(Memo); 5893193323Sed } 5894193323Sed } 5895193323Sed } 5896193323Sed 5897193323Sed // Sort the uses, so that all the uses from a given User are together. 5898193323Sed std::sort(Uses.begin(), Uses.end()); 5899193323Sed 5900193323Sed for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5901193323Sed UseIndex != UseIndexEnd; ) { 5902193323Sed // We know that this user uses some value of From. If it is the right 5903193323Sed // value, update it. 5904193323Sed SDNode *User = Uses[UseIndex].User; 5905193323Sed 5906193323Sed // This node is about to morph, remove its old self from the CSE maps. 5907193323Sed RemoveNodeFromCSEMaps(User); 5908193323Sed 5909193323Sed // The Uses array is sorted, so all the uses for a given User 5910193323Sed // are next to each other in the list. 5911193323Sed // To help reduce the number of CSE recomputations, process all 5912193323Sed // the uses of this user that we can find this way. 5913193323Sed do { 5914193323Sed unsigned i = Uses[UseIndex].Index; 5915193323Sed SDUse &Use = *Uses[UseIndex].Use; 5916193323Sed ++UseIndex; 5917193323Sed 5918193323Sed Use.set(To[i]); 5919193323Sed } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5920193323Sed 5921193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5922193323Sed // already exists there, recursively merge the results together. 5923239462Sdim AddModifiedNodeToCSEMaps(User); 5924193323Sed } 5925193323Sed} 5926193323Sed 5927193323Sed/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5928193323Sed/// based on their topological order. It returns the maximum id and a vector 5929193323Sed/// of the SDNodes* in assigned order by reference. 5930193323Sedunsigned SelectionDAG::AssignTopologicalOrder() { 5931193323Sed 5932193323Sed unsigned DAGSize = 0; 5933193323Sed 5934193323Sed // SortedPos tracks the progress of the algorithm. Nodes before it are 5935193323Sed // sorted, nodes after it are unsorted. When the algorithm completes 5936193323Sed // it is at the end of the list. 5937193323Sed allnodes_iterator SortedPos = allnodes_begin(); 5938193323Sed 5939193323Sed // Visit all the nodes. Move nodes with no operands to the front of 5940193323Sed // the list immediately. Annotate nodes that do have operands with their 5941193323Sed // operand count. Before we do this, the Node Id fields of the nodes 5942193323Sed // may contain arbitrary values. After, the Node Id fields for nodes 5943193323Sed // before SortedPos will contain the topological sort index, and the 5944193323Sed // Node Id fields for nodes At SortedPos and after will contain the 5945193323Sed // count of outstanding operands. 5946193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5947193323Sed SDNode *N = I++; 5948202878Srdivacky checkForCycles(N); 5949193323Sed unsigned Degree = N->getNumOperands(); 5950193323Sed if (Degree == 0) { 5951193323Sed // A node with no uses, add it to the result array immediately. 5952193323Sed N->setNodeId(DAGSize++); 5953193323Sed allnodes_iterator Q = N; 5954193323Sed if (Q != SortedPos) 5955193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5956202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5957193323Sed ++SortedPos; 5958193323Sed } else { 5959193323Sed // Temporarily use the Node Id as scratch space for the degree count. 5960193323Sed N->setNodeId(Degree); 5961193323Sed } 5962193323Sed } 5963193323Sed 5964239462Sdim // Visit all the nodes. As we iterate, move nodes into sorted order, 5965193323Sed // such that by the time the end is reached all nodes will be sorted. 5966193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5967193323Sed SDNode *N = I; 5968202878Srdivacky checkForCycles(N); 5969202878Srdivacky // N is in sorted position, so all its uses have one less operand 5970202878Srdivacky // that needs to be sorted. 5971193323Sed for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5972193323Sed UI != UE; ++UI) { 5973193323Sed SDNode *P = *UI; 5974193323Sed unsigned Degree = P->getNodeId(); 5975202878Srdivacky assert(Degree != 0 && "Invalid node degree"); 5976193323Sed --Degree; 5977193323Sed if (Degree == 0) { 5978193323Sed // All of P's operands are sorted, so P may sorted now. 5979193323Sed P->setNodeId(DAGSize++); 5980193323Sed if (P != SortedPos) 5981193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5982202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5983193323Sed ++SortedPos; 5984193323Sed } else { 5985193323Sed // Update P's outstanding operand count. 5986193323Sed P->setNodeId(Degree); 5987193323Sed } 5988193323Sed } 5989202878Srdivacky if (I == SortedPos) { 5990203954Srdivacky#ifndef NDEBUG 5991203954Srdivacky SDNode *S = ++I; 5992203954Srdivacky dbgs() << "Overran sorted position:\n"; 5993202878Srdivacky S->dumprFull(); 5994203954Srdivacky#endif 5995203954Srdivacky llvm_unreachable(0); 5996202878Srdivacky } 5997193323Sed } 5998193323Sed 5999193323Sed assert(SortedPos == AllNodes.end() && 6000193323Sed "Topological sort incomplete!"); 6001193323Sed assert(AllNodes.front().getOpcode() == ISD::EntryToken && 6002193323Sed "First node in topological sort is not the entry token!"); 6003193323Sed assert(AllNodes.front().getNodeId() == 0 && 6004193323Sed "First node in topological sort has non-zero id!"); 6005193323Sed assert(AllNodes.front().getNumOperands() == 0 && 6006193323Sed "First node in topological sort has operands!"); 6007193323Sed assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 6008193323Sed "Last node in topologic sort has unexpected id!"); 6009193323Sed assert(AllNodes.back().use_empty() && 6010193323Sed "Last node in topologic sort has users!"); 6011193323Sed assert(DAGSize == allnodes_size() && "Node count mismatch!"); 6012193323Sed return DAGSize; 6013193323Sed} 6014193323Sed 6015206083Srdivacky/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 6016206083Srdivacky/// value is produced by SD. 6017207618Srdivackyvoid SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 6018207618Srdivacky DbgInfo->add(DB, SD, isParameter); 6019206083Srdivacky if (SD) 6020206083Srdivacky SD->setHasDebugValue(true); 6021205218Srdivacky} 6022201360Srdivacky 6023218893Sdim/// TransferDbgValues - Transfer SDDbgValues. 6024218893Sdimvoid SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 6025218893Sdim if (From == To || !From.getNode()->getHasDebugValue()) 6026218893Sdim return; 6027218893Sdim SDNode *FromNode = From.getNode(); 6028218893Sdim SDNode *ToNode = To.getNode(); 6029224145Sdim ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 6030218893Sdim SmallVector<SDDbgValue *, 2> ClonedDVs; 6031224145Sdim for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 6032218893Sdim I != E; ++I) { 6033218893Sdim SDDbgValue *Dbg = *I; 6034218893Sdim if (Dbg->getKind() == SDDbgValue::SDNODE) { 6035218893Sdim SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 6036218893Sdim Dbg->getOffset(), Dbg->getDebugLoc(), 6037218893Sdim Dbg->getOrder()); 6038218893Sdim ClonedDVs.push_back(Clone); 6039218893Sdim } 6040218893Sdim } 6041263508Sdim for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(), 6042218893Sdim E = ClonedDVs.end(); I != E; ++I) 6043218893Sdim AddDbgValue(*I, ToNode, false); 6044218893Sdim} 6045218893Sdim 6046193323Sed//===----------------------------------------------------------------------===// 6047193323Sed// SDNode Class 6048193323Sed//===----------------------------------------------------------------------===// 6049193323Sed 6050193323SedHandleSDNode::~HandleSDNode() { 6051193323Sed DropOperands(); 6052193323Sed} 6053193323Sed 6054263508SdimGlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 6055263508Sdim DebugLoc DL, const GlobalValue *GA, 6056198090Srdivacky EVT VT, int64_t o, unsigned char TF) 6057263508Sdim : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 6058207618Srdivacky TheGlobal = GA; 6059193323Sed} 6060193323Sed 6061263508SdimAddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, 6062263508Sdim SDValue X, unsigned SrcAS, 6063263508Sdim unsigned DestAS) 6064263508Sdim : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), 6065263508Sdim SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 6066263508Sdim 6067263508SdimMemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6068263508Sdim EVT memvt, MachineMemOperand *mmo) 6069263508Sdim : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 6070204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6071234353Sdim MMO->isNonTemporal(), MMO->isInvariant()); 6072198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6073204642Srdivacky assert(isNonTemporal() == MMO->isNonTemporal() && 6074204642Srdivacky "Non-temporal encoding error!"); 6075198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6076193323Sed} 6077193323Sed 6078263508SdimMemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6079218893Sdim const SDValue *Ops, unsigned NumOps, EVT memvt, 6080198090Srdivacky MachineMemOperand *mmo) 6081263508Sdim : SDNode(Opc, Order, dl, VTs, Ops, NumOps), 6082198090Srdivacky MemoryVT(memvt), MMO(mmo) { 6083204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6084234353Sdim MMO->isNonTemporal(), MMO->isInvariant()); 6085198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6086198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6087193323Sed} 6088193323Sed 6089193323Sed/// Profile - Gather unique data for the node. 6090193323Sed/// 6091193323Sedvoid SDNode::Profile(FoldingSetNodeID &ID) const { 6092193323Sed AddNodeIDNode(ID, this); 6093193323Sed} 6094193323Sed 6095198090Srdivackynamespace { 6096198090Srdivacky struct EVTArray { 6097198090Srdivacky std::vector<EVT> VTs; 6098218893Sdim 6099198090Srdivacky EVTArray() { 6100198090Srdivacky VTs.reserve(MVT::LAST_VALUETYPE); 6101198090Srdivacky for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 6102198090Srdivacky VTs.push_back(MVT((MVT::SimpleValueType)i)); 6103198090Srdivacky } 6104198090Srdivacky }; 6105198090Srdivacky} 6106198090Srdivacky 6107198090Srdivackystatic ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 6108198090Srdivackystatic ManagedStatic<EVTArray> SimpleVTArray; 6109195098Sedstatic ManagedStatic<sys::SmartMutex<true> > VTMutex; 6110195098Sed 6111193323Sed/// getValueTypeList - Return a pointer to the specified value type. 6112193323Sed/// 6113198090Srdivackyconst EVT *SDNode::getValueTypeList(EVT VT) { 6114193323Sed if (VT.isExtended()) { 6115198090Srdivacky sys::SmartScopedLock<true> Lock(*VTMutex); 6116195098Sed return &(*EVTs->insert(VT).first); 6117193323Sed } else { 6118218893Sdim assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 6119208599Srdivacky "Value type out of range!"); 6120198090Srdivacky return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 6121193323Sed } 6122193323Sed} 6123193323Sed 6124193323Sed/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 6125193323Sed/// indicated value. This method ignores uses of other values defined by this 6126193323Sed/// operation. 6127193323Sedbool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 6128193323Sed assert(Value < getNumValues() && "Bad value!"); 6129193323Sed 6130193323Sed // TODO: Only iterate over uses of a given value of the node 6131193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 6132193323Sed if (UI.getUse().getResNo() == Value) { 6133193323Sed if (NUses == 0) 6134193323Sed return false; 6135193323Sed --NUses; 6136193323Sed } 6137193323Sed } 6138193323Sed 6139193323Sed // Found exactly the right number of uses? 6140193323Sed return NUses == 0; 6141193323Sed} 6142193323Sed 6143193323Sed 6144193323Sed/// hasAnyUseOfValue - Return true if there are any use of the indicated 6145193323Sed/// value. This method ignores uses of other values defined by this operation. 6146193323Sedbool SDNode::hasAnyUseOfValue(unsigned Value) const { 6147193323Sed assert(Value < getNumValues() && "Bad value!"); 6148193323Sed 6149193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 6150193323Sed if (UI.getUse().getResNo() == Value) 6151193323Sed return true; 6152193323Sed 6153193323Sed return false; 6154193323Sed} 6155193323Sed 6156193323Sed 6157193323Sed/// isOnlyUserOf - Return true if this node is the only use of N. 6158193323Sed/// 6159193323Sedbool SDNode::isOnlyUserOf(SDNode *N) const { 6160193323Sed bool Seen = false; 6161193323Sed for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 6162193323Sed SDNode *User = *I; 6163193323Sed if (User == this) 6164193323Sed Seen = true; 6165193323Sed else 6166193323Sed return false; 6167193323Sed } 6168193323Sed 6169193323Sed return Seen; 6170193323Sed} 6171193323Sed 6172193323Sed/// isOperand - Return true if this node is an operand of N. 6173193323Sed/// 6174193323Sedbool SDValue::isOperandOf(SDNode *N) const { 6175193323Sed for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6176193323Sed if (*this == N->getOperand(i)) 6177193323Sed return true; 6178193323Sed return false; 6179193323Sed} 6180193323Sed 6181193323Sedbool SDNode::isOperandOf(SDNode *N) const { 6182193323Sed for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 6183193323Sed if (this == N->OperandList[i].getNode()) 6184193323Sed return true; 6185193323Sed return false; 6186193323Sed} 6187193323Sed 6188193323Sed/// reachesChainWithoutSideEffects - Return true if this operand (which must 6189193323Sed/// be a chain) reaches the specified operand without crossing any 6190218893Sdim/// side-effecting instructions on any chain path. In practice, this looks 6191218893Sdim/// through token factors and non-volatile loads. In order to remain efficient, 6192218893Sdim/// this only looks a couple of nodes in, it does not do an exhaustive search. 6193193323Sedbool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 6194193323Sed unsigned Depth) const { 6195193323Sed if (*this == Dest) return true; 6196193323Sed 6197193323Sed // Don't search too deeply, we just want to be able to see through 6198193323Sed // TokenFactor's etc. 6199193323Sed if (Depth == 0) return false; 6200193323Sed 6201193323Sed // If this is a token factor, all inputs to the TF happen in parallel. If any 6202218893Sdim // of the operands of the TF does not reach dest, then we cannot do the xform. 6203193323Sed if (getOpcode() == ISD::TokenFactor) { 6204193323Sed for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6205218893Sdim if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 6206218893Sdim return false; 6207218893Sdim return true; 6208193323Sed } 6209193323Sed 6210193323Sed // Loads don't have side effects, look through them. 6211193323Sed if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 6212193323Sed if (!Ld->isVolatile()) 6213193323Sed return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 6214193323Sed } 6215193323Sed return false; 6216193323Sed} 6217193323Sed 6218224145Sdim/// hasPredecessor - Return true if N is a predecessor of this node. 6219224145Sdim/// N is either an operand of this node, or can be reached by recursively 6220224145Sdim/// traversing up the operands. 6221224145Sdim/// NOTE: This is an expensive method. Use it carefully. 6222224145Sdimbool SDNode::hasPredecessor(const SDNode *N) const { 6223224145Sdim SmallPtrSet<const SDNode *, 32> Visited; 6224224145Sdim SmallVector<const SDNode *, 16> Worklist; 6225224145Sdim return hasPredecessorHelper(N, Visited, Worklist); 6226224145Sdim} 6227198892Srdivacky 6228263508Sdimbool 6229263508SdimSDNode::hasPredecessorHelper(const SDNode *N, 6230263508Sdim SmallPtrSet<const SDNode *, 32> &Visited, 6231263508Sdim SmallVectorImpl<const SDNode *> &Worklist) const { 6232224145Sdim if (Visited.empty()) { 6233224145Sdim Worklist.push_back(this); 6234224145Sdim } else { 6235224145Sdim // Take a look in the visited set. If we've already encountered this node 6236224145Sdim // we needn't search further. 6237224145Sdim if (Visited.count(N)) 6238224145Sdim return true; 6239224145Sdim } 6240224145Sdim 6241224145Sdim // Haven't visited N yet. Continue the search. 6242224145Sdim while (!Worklist.empty()) { 6243224145Sdim const SDNode *M = Worklist.pop_back_val(); 6244224145Sdim for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 6245224145Sdim SDNode *Op = M->getOperand(i).getNode(); 6246198892Srdivacky if (Visited.insert(Op)) 6247198892Srdivacky Worklist.push_back(Op); 6248224145Sdim if (Op == N) 6249224145Sdim return true; 6250198892Srdivacky } 6251224145Sdim } 6252198892Srdivacky 6253198892Srdivacky return false; 6254193323Sed} 6255193323Sed 6256193323Seduint64_t SDNode::getConstantOperandVal(unsigned Num) const { 6257193323Sed assert(Num < NumOperands && "Invalid child # of SDNode!"); 6258193323Sed return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 6259193323Sed} 6260193323Sed 6261199989SrdivackySDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6262199989Srdivacky assert(N->getNumValues() == 1 && 6263199989Srdivacky "Can't unroll a vector with multiple results!"); 6264199989Srdivacky 6265199989Srdivacky EVT VT = N->getValueType(0); 6266199989Srdivacky unsigned NE = VT.getVectorNumElements(); 6267199989Srdivacky EVT EltVT = VT.getVectorElementType(); 6268263508Sdim SDLoc dl(N); 6269199989Srdivacky 6270199989Srdivacky SmallVector<SDValue, 8> Scalars; 6271199989Srdivacky SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6272199989Srdivacky 6273199989Srdivacky // If ResNE is 0, fully unroll the vector op. 6274199989Srdivacky if (ResNE == 0) 6275199989Srdivacky ResNE = NE; 6276199989Srdivacky else if (NE > ResNE) 6277199989Srdivacky NE = ResNE; 6278199989Srdivacky 6279199989Srdivacky unsigned i; 6280199989Srdivacky for (i= 0; i != NE; ++i) { 6281207618Srdivacky for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6282199989Srdivacky SDValue Operand = N->getOperand(j); 6283199989Srdivacky EVT OperandVT = Operand.getValueType(); 6284199989Srdivacky if (OperandVT.isVector()) { 6285199989Srdivacky // A vector operand; extract a single element. 6286263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6287199989Srdivacky EVT OperandEltVT = OperandVT.getVectorElementType(); 6288199989Srdivacky Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6289199989Srdivacky OperandEltVT, 6290199989Srdivacky Operand, 6291263508Sdim getConstant(i, TLI->getVectorIdxTy())); 6292199989Srdivacky } else { 6293199989Srdivacky // A scalar operand; just use it as is. 6294199989Srdivacky Operands[j] = Operand; 6295199989Srdivacky } 6296199989Srdivacky } 6297199989Srdivacky 6298199989Srdivacky switch (N->getOpcode()) { 6299199989Srdivacky default: 6300199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6301199989Srdivacky &Operands[0], Operands.size())); 6302199989Srdivacky break; 6303226633Sdim case ISD::VSELECT: 6304226633Sdim Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6305226633Sdim &Operands[0], Operands.size())); 6306226633Sdim break; 6307199989Srdivacky case ISD::SHL: 6308199989Srdivacky case ISD::SRA: 6309199989Srdivacky case ISD::SRL: 6310199989Srdivacky case ISD::ROTL: 6311199989Srdivacky case ISD::ROTR: 6312199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6313263508Sdim getShiftAmountOperand(Operands[0].getValueType(), 6314263508Sdim Operands[1]))); 6315199989Srdivacky break; 6316202375Srdivacky case ISD::SIGN_EXTEND_INREG: 6317202375Srdivacky case ISD::FP_ROUND_INREG: { 6318202375Srdivacky EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6319202375Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6320202375Srdivacky Operands[0], 6321202375Srdivacky getValueType(ExtVT))); 6322199989Srdivacky } 6323202375Srdivacky } 6324199989Srdivacky } 6325199989Srdivacky 6326199989Srdivacky for (; i < ResNE; ++i) 6327199989Srdivacky Scalars.push_back(getUNDEF(EltVT)); 6328199989Srdivacky 6329199989Srdivacky return getNode(ISD::BUILD_VECTOR, dl, 6330199989Srdivacky EVT::getVectorVT(*getContext(), EltVT, ResNE), 6331199989Srdivacky &Scalars[0], Scalars.size()); 6332199989Srdivacky} 6333199989Srdivacky 6334200581Srdivacky 6335218893Sdim/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6336218893Sdim/// location that is 'Dist' units away from the location that the 'Base' load 6337200581Srdivacky/// is loading from. 6338218893Sdimbool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6339200581Srdivacky unsigned Bytes, int Dist) const { 6340200581Srdivacky if (LD->getChain() != Base->getChain()) 6341200581Srdivacky return false; 6342200581Srdivacky EVT VT = LD->getValueType(0); 6343200581Srdivacky if (VT.getSizeInBits() / 8 != Bytes) 6344200581Srdivacky return false; 6345200581Srdivacky 6346200581Srdivacky SDValue Loc = LD->getOperand(1); 6347200581Srdivacky SDValue BaseLoc = Base->getOperand(1); 6348200581Srdivacky if (Loc.getOpcode() == ISD::FrameIndex) { 6349200581Srdivacky if (BaseLoc.getOpcode() != ISD::FrameIndex) 6350200581Srdivacky return false; 6351200581Srdivacky const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6352200581Srdivacky int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6353200581Srdivacky int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6354200581Srdivacky int FS = MFI->getObjectSize(FI); 6355200581Srdivacky int BFS = MFI->getObjectSize(BFI); 6356200581Srdivacky if (FS != BFS || FS != (int)Bytes) return false; 6357200581Srdivacky return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6358200581Srdivacky } 6359200581Srdivacky 6360218893Sdim // Handle X+C 6361218893Sdim if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6362218893Sdim cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6363218893Sdim return true; 6364218893Sdim 6365207618Srdivacky const GlobalValue *GV1 = NULL; 6366207618Srdivacky const GlobalValue *GV2 = NULL; 6367200581Srdivacky int64_t Offset1 = 0; 6368200581Srdivacky int64_t Offset2 = 0; 6369263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6370263508Sdim bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6371263508Sdim bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6372200581Srdivacky if (isGA1 && isGA2 && GV1 == GV2) 6373200581Srdivacky return Offset1 == (Offset2 + Dist*Bytes); 6374200581Srdivacky return false; 6375200581Srdivacky} 6376200581Srdivacky 6377200581Srdivacky 6378200581Srdivacky/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6379200581Srdivacky/// it cannot be inferred. 6380200581Srdivackyunsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6381200581Srdivacky // If this is a GlobalAddress + cst, return the alignment. 6382207618Srdivacky const GlobalValue *GV; 6383200581Srdivacky int64_t GVOffset = 0; 6384263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6385263508Sdim if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6386263508Sdim unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 6387234353Sdim APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6388234353Sdim llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 6389263508Sdim TLI->getDataLayout()); 6390234353Sdim unsigned AlignBits = KnownZero.countTrailingOnes(); 6391234353Sdim unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6392234353Sdim if (Align) 6393234353Sdim return MinAlign(Align, GVOffset); 6394206083Srdivacky } 6395200581Srdivacky 6396200581Srdivacky // If this is a direct reference to a stack slot, use information about the 6397200581Srdivacky // stack slot's alignment. 6398200581Srdivacky int FrameIdx = 1 << 31; 6399200581Srdivacky int64_t FrameOffset = 0; 6400200581Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6401200581Srdivacky FrameIdx = FI->getIndex(); 6402218893Sdim } else if (isBaseWithConstantOffset(Ptr) && 6403200581Srdivacky isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6404218893Sdim // Handle FI+Cst 6405200581Srdivacky FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6406200581Srdivacky FrameOffset = Ptr.getConstantOperandVal(1); 6407200581Srdivacky } 6408200581Srdivacky 6409200581Srdivacky if (FrameIdx != (1 << 31)) { 6410200581Srdivacky const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6411200581Srdivacky unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6412200581Srdivacky FrameOffset); 6413200581Srdivacky return FIInfoAlign; 6414200581Srdivacky } 6415200581Srdivacky 6416200581Srdivacky return 0; 6417200581Srdivacky} 6418200581Srdivacky 6419263508Sdim/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 6420263508Sdim/// which is split (or expanded) into two not necessarily identical pieces. 6421263508Sdimstd::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 6422263508Sdim // Currently all types are split in half. 6423263508Sdim EVT LoVT, HiVT; 6424263508Sdim if (!VT.isVector()) { 6425263508Sdim LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 6426263508Sdim } else { 6427263508Sdim unsigned NumElements = VT.getVectorNumElements(); 6428263508Sdim assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 6429263508Sdim LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), 6430263508Sdim NumElements/2); 6431263508Sdim } 6432263508Sdim return std::make_pair(LoVT, HiVT); 6433263508Sdim} 6434263508Sdim 6435263508Sdim/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 6436263508Sdim/// low/high part. 6437263508Sdimstd::pair<SDValue, SDValue> 6438263508SdimSelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 6439263508Sdim const EVT &HiVT) { 6440263508Sdim assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <= 6441263508Sdim N.getValueType().getVectorNumElements() && 6442263508Sdim "More vector elements requested than available!"); 6443263508Sdim SDValue Lo, Hi; 6444263508Sdim Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, 6445263508Sdim getConstant(0, TLI->getVectorIdxTy())); 6446263508Sdim Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 6447263508Sdim getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy())); 6448263508Sdim return std::make_pair(Lo, Hi); 6449263508Sdim} 6450263508Sdim 6451193323Sed// getAddressSpace - Return the address space this GlobalAddress belongs to. 6452193323Sedunsigned GlobalAddressSDNode::getAddressSpace() const { 6453193323Sed return getGlobal()->getType()->getAddressSpace(); 6454193323Sed} 6455193323Sed 6456193323Sed 6457226633SdimType *ConstantPoolSDNode::getType() const { 6458193323Sed if (isMachineConstantPoolEntry()) 6459193323Sed return Val.MachineCPVal->getType(); 6460193323Sed return Val.ConstVal->getType(); 6461193323Sed} 6462193323Sed 6463193323Sedbool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6464193323Sed APInt &SplatUndef, 6465193323Sed unsigned &SplatBitSize, 6466193323Sed bool &HasAnyUndefs, 6467199481Srdivacky unsigned MinSplatBits, 6468199481Srdivacky bool isBigEndian) { 6469198090Srdivacky EVT VT = getValueType(0); 6470193323Sed assert(VT.isVector() && "Expected a vector type"); 6471193323Sed unsigned sz = VT.getSizeInBits(); 6472193323Sed if (MinSplatBits > sz) 6473193323Sed return false; 6474193323Sed 6475193323Sed SplatValue = APInt(sz, 0); 6476193323Sed SplatUndef = APInt(sz, 0); 6477193323Sed 6478193323Sed // Get the bits. Bits with undefined values (when the corresponding element 6479193323Sed // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6480193323Sed // in SplatValue. If any of the values are not constant, give up and return 6481193323Sed // false. 6482193323Sed unsigned int nOps = getNumOperands(); 6483193323Sed assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6484193323Sed unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6485199481Srdivacky 6486199481Srdivacky for (unsigned j = 0; j < nOps; ++j) { 6487199481Srdivacky unsigned i = isBigEndian ? nOps-1-j : j; 6488193323Sed SDValue OpVal = getOperand(i); 6489199481Srdivacky unsigned BitPos = j * EltBitSize; 6490193323Sed 6491193323Sed if (OpVal.getOpcode() == ISD::UNDEF) 6492199481Srdivacky SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6493193323Sed else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6494218893Sdim SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6495207618Srdivacky zextOrTrunc(sz) << BitPos; 6496193323Sed else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6497193323Sed SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6498193323Sed else 6499193323Sed return false; 6500193323Sed } 6501193323Sed 6502193323Sed // The build_vector is all constants or undefs. Find the smallest element 6503193323Sed // size that splats the vector. 6504193323Sed 6505193323Sed HasAnyUndefs = (SplatUndef != 0); 6506193323Sed while (sz > 8) { 6507193323Sed 6508193323Sed unsigned HalfSize = sz / 2; 6509218893Sdim APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6510218893Sdim APInt LowValue = SplatValue.trunc(HalfSize); 6511218893Sdim APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6512218893Sdim APInt LowUndef = SplatUndef.trunc(HalfSize); 6513193323Sed 6514193323Sed // If the two halves do not match (ignoring undef bits), stop here. 6515193323Sed if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6516193323Sed MinSplatBits > HalfSize) 6517193323Sed break; 6518193323Sed 6519193323Sed SplatValue = HighValue | LowValue; 6520193323Sed SplatUndef = HighUndef & LowUndef; 6521198090Srdivacky 6522193323Sed sz = HalfSize; 6523193323Sed } 6524193323Sed 6525193323Sed SplatBitSize = sz; 6526193323Sed return true; 6527193323Sed} 6528193323Sed 6529198090Srdivackybool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6530193323Sed // Find the first non-undef value in the shuffle mask. 6531193323Sed unsigned i, e; 6532193323Sed for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6533193323Sed /* search */; 6534193323Sed 6535193323Sed assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6536198090Srdivacky 6537193323Sed // Make sure all remaining elements are either undef or the same as the first 6538193323Sed // non-undef value. 6539193323Sed for (int Idx = Mask[i]; i != e; ++i) 6540193323Sed if (Mask[i] >= 0 && Mask[i] != Idx) 6541193323Sed return false; 6542193323Sed return true; 6543193323Sed} 6544202878Srdivacky 6545204642Srdivacky#ifdef XDEBUG 6546202878Srdivackystatic void checkForCyclesHelper(const SDNode *N, 6547204642Srdivacky SmallPtrSet<const SDNode*, 32> &Visited, 6548204642Srdivacky SmallPtrSet<const SDNode*, 32> &Checked) { 6549204642Srdivacky // If this node has already been checked, don't check it again. 6550204642Srdivacky if (Checked.count(N)) 6551204642Srdivacky return; 6552218893Sdim 6553204642Srdivacky // If a node has already been visited on this depth-first walk, reject it as 6554204642Srdivacky // a cycle. 6555204642Srdivacky if (!Visited.insert(N)) { 6556202878Srdivacky dbgs() << "Offending node:\n"; 6557202878Srdivacky N->dumprFull(); 6558204642Srdivacky errs() << "Detected cycle in SelectionDAG\n"; 6559204642Srdivacky abort(); 6560202878Srdivacky } 6561218893Sdim 6562204642Srdivacky for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6563204642Srdivacky checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6564218893Sdim 6565204642Srdivacky Checked.insert(N); 6566204642Srdivacky Visited.erase(N); 6567202878Srdivacky} 6568204642Srdivacky#endif 6569202878Srdivacky 6570202878Srdivackyvoid llvm::checkForCycles(const llvm::SDNode *N) { 6571202878Srdivacky#ifdef XDEBUG 6572263508Sdim assert(N && "Checking nonexistent SDNode"); 6573204642Srdivacky SmallPtrSet<const SDNode*, 32> visited; 6574204642Srdivacky SmallPtrSet<const SDNode*, 32> checked; 6575204642Srdivacky checkForCyclesHelper(N, visited, checked); 6576202878Srdivacky#endif 6577202878Srdivacky} 6578202878Srdivacky 6579202878Srdivackyvoid llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6580202878Srdivacky checkForCycles(DAG->getRoot().getNode()); 6581202878Srdivacky} 6582