LegalizeDAG.cpp revision 249423
1193323Sed//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file implements the SelectionDAG::Legalize method. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13193323Sed 14249423Sdim#include "llvm/CodeGen/SelectionDAG.h" 15249423Sdim#include "llvm/ADT/SmallPtrSet.h" 16249423Sdim#include "llvm/ADT/SmallVector.h" 17249423Sdim#include "llvm/ADT/Triple.h" 18218893Sdim#include "llvm/CodeGen/Analysis.h" 19193323Sed#include "llvm/CodeGen/MachineFunction.h" 20193323Sed#include "llvm/CodeGen/MachineJumpTableInfo.h" 21249423Sdim#include "llvm/DebugInfo.h" 22249423Sdim#include "llvm/IR/CallingConv.h" 23249423Sdim#include "llvm/IR/Constants.h" 24249423Sdim#include "llvm/IR/DataLayout.h" 25249423Sdim#include "llvm/IR/DerivedTypes.h" 26249423Sdim#include "llvm/IR/Function.h" 27249423Sdim#include "llvm/IR/LLVMContext.h" 28202375Srdivacky#include "llvm/Support/Debug.h" 29210299Sed#include "llvm/Support/ErrorHandling.h" 30193323Sed#include "llvm/Support/MathExtras.h" 31198090Srdivacky#include "llvm/Support/raw_ostream.h" 32249423Sdim#include "llvm/Target/TargetFrameLowering.h" 33249423Sdim#include "llvm/Target/TargetLowering.h" 34249423Sdim#include "llvm/Target/TargetMachine.h" 35193323Sedusing namespace llvm; 36193323Sed 37193323Sed//===----------------------------------------------------------------------===// 38193323Sed/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 39193323Sed/// hacks on it until the target machine can handle it. This involves 40193323Sed/// eliminating value sizes the machine cannot handle (promoting small sizes to 41193323Sed/// large sizes or splitting up large values into small values) as well as 42193323Sed/// eliminating operations the machine cannot handle. 43193323Sed/// 44193323Sed/// This code also does a small amount of optimization and recognition of idioms 45193323Sed/// as part of its processing. For example, if a target does not support a 46193323Sed/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 47193323Sed/// will attempt merge setcc and brc instructions into brcc's. 48193323Sed/// 49193323Sednamespace { 50234353Sdimclass SelectionDAGLegalize : public SelectionDAG::DAGUpdateListener { 51207618Srdivacky const TargetMachine &TM; 52207618Srdivacky const TargetLowering &TLI; 53193323Sed SelectionDAG &DAG; 54193323Sed 55234353Sdim /// LegalizePosition - The iterator for walking through the node list. 56234353Sdim SelectionDAG::allnodes_iterator LegalizePosition; 57193323Sed 58234353Sdim /// LegalizedNodes - The set of nodes which have already been legalized. 59234353Sdim SmallPtrSet<SDNode *, 16> LegalizedNodes; 60193323Sed 61234353Sdim // Libcall insertion helpers. 62226633Sdim 63193323Sedpublic: 64223017Sdim explicit SelectionDAGLegalize(SelectionDAG &DAG); 65193323Sed 66193323Sed void LegalizeDAG(); 67193323Sed 68193323Sedprivate: 69234353Sdim /// LegalizeOp - Legalizes the given operation. 70234353Sdim void LegalizeOp(SDNode *Node); 71193323Sed 72193574Sed SDValue OptimizeFloatStore(StoreSDNode *ST); 73193574Sed 74239462Sdim void LegalizeLoadOps(SDNode *Node); 75239462Sdim void LegalizeStoreOps(SDNode *Node); 76239462Sdim 77193323Sed /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 78193323Sed /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 79193323Sed /// is necessary to spill the vector being inserted into to memory, perform 80193323Sed /// the insert there, and then read the result back. 81193323Sed SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 82193323Sed SDValue Idx, DebugLoc dl); 83193323Sed SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 84193323Sed SDValue Idx, DebugLoc dl); 85193323Sed 86193323Sed /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 87193323Sed /// performs the same shuffe in terms of order or result bytes, but on a type 88193323Sed /// whose vector element type is narrower than the original shuffle type. 89193323Sed /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 90198090Srdivacky SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, 91210299Sed SDValue N1, SDValue N2, 92234353Sdim ArrayRef<int> Mask) const; 93193323Sed 94198090Srdivacky void LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 95193323Sed DebugLoc dl); 96193323Sed 97193323Sed SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 98221345Sdim SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, 99221345Sdim unsigned NumOps, bool isSigned, DebugLoc dl); 100221345Sdim 101210299Sed std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, 102210299Sed SDNode *Node, bool isSigned); 103193323Sed SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 104193323Sed RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 105249423Sdim RTLIB::Libcall Call_F128, 106193323Sed RTLIB::Libcall Call_PPCF128); 107199481Srdivacky SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 108199481Srdivacky RTLIB::Libcall Call_I8, 109199481Srdivacky RTLIB::Libcall Call_I16, 110199481Srdivacky RTLIB::Libcall Call_I32, 111199481Srdivacky RTLIB::Libcall Call_I64, 112193323Sed RTLIB::Libcall Call_I128); 113221345Sdim void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 114249423Sdim void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 115193323Sed 116198090Srdivacky SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, DebugLoc dl); 117193323Sed SDValue ExpandBUILD_VECTOR(SDNode *Node); 118193323Sed SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 119193323Sed void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 120193323Sed SmallVectorImpl<SDValue> &Results); 121193323Sed SDValue ExpandFCOPYSIGN(SDNode *Node); 122198090Srdivacky SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, 123193323Sed DebugLoc dl); 124198090Srdivacky SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, 125193323Sed DebugLoc dl); 126198090Srdivacky SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, 127193323Sed DebugLoc dl); 128193323Sed 129193323Sed SDValue ExpandBSWAP(SDValue Op, DebugLoc dl); 130193323Sed SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl); 131193323Sed 132193323Sed SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 133218893Sdim SDValue ExpandInsertToVectorThroughStack(SDValue Op); 134193574Sed SDValue ExpandVectorBuildThroughStack(SDNode* Node); 135193323Sed 136234353Sdim SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 137234353Sdim 138210299Sed std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); 139210299Sed 140234353Sdim void ExpandNode(SDNode *Node); 141234353Sdim void PromoteNode(SDNode *Node); 142234353Sdim 143234353Sdim void ForgetNode(SDNode *N) { 144234353Sdim LegalizedNodes.erase(N); 145234353Sdim if (LegalizePosition == SelectionDAG::allnodes_iterator(N)) 146234353Sdim ++LegalizePosition; 147234353Sdim } 148234353Sdim 149234353Sdimpublic: 150234353Sdim // DAGUpdateListener implementation. 151234353Sdim virtual void NodeDeleted(SDNode *N, SDNode *E) { 152234353Sdim ForgetNode(N); 153234353Sdim } 154234353Sdim virtual void NodeUpdated(SDNode *N) {} 155234353Sdim 156234353Sdim // Node replacement helpers 157234353Sdim void ReplacedNode(SDNode *N) { 158234353Sdim if (N->use_empty()) { 159239462Sdim DAG.RemoveDeadNode(N); 160234353Sdim } else { 161234353Sdim ForgetNode(N); 162234353Sdim } 163234353Sdim } 164234353Sdim void ReplaceNode(SDNode *Old, SDNode *New) { 165239462Sdim DAG.ReplaceAllUsesWith(Old, New); 166234353Sdim ReplacedNode(Old); 167234353Sdim } 168234353Sdim void ReplaceNode(SDValue Old, SDValue New) { 169239462Sdim DAG.ReplaceAllUsesWith(Old, New); 170234353Sdim ReplacedNode(Old.getNode()); 171234353Sdim } 172234353Sdim void ReplaceNode(SDNode *Old, const SDValue *New) { 173239462Sdim DAG.ReplaceAllUsesWith(Old, New); 174234353Sdim ReplacedNode(Old); 175234353Sdim } 176193323Sed}; 177193323Sed} 178193323Sed 179193323Sed/// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 180193323Sed/// performs the same shuffe in terms of order or result bytes, but on a type 181193323Sed/// whose vector element type is narrower than the original shuffle type. 182193323Sed/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 183210299SedSDValue 184210299SedSelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, 185193323Sed SDValue N1, SDValue N2, 186234353Sdim ArrayRef<int> Mask) const { 187193323Sed unsigned NumMaskElts = VT.getVectorNumElements(); 188193323Sed unsigned NumDestElts = NVT.getVectorNumElements(); 189193323Sed unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 190193323Sed 191193323Sed assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 192193323Sed 193193323Sed if (NumEltsGrowth == 1) 194193323Sed return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); 195210299Sed 196193323Sed SmallVector<int, 8> NewMask; 197193323Sed for (unsigned i = 0; i != NumMaskElts; ++i) { 198193323Sed int Idx = Mask[i]; 199193323Sed for (unsigned j = 0; j != NumEltsGrowth; ++j) { 200210299Sed if (Idx < 0) 201193323Sed NewMask.push_back(-1); 202193323Sed else 203193323Sed NewMask.push_back(Idx * NumEltsGrowth + j); 204193323Sed } 205193323Sed } 206193323Sed assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 207193323Sed assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 208193323Sed return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); 209193323Sed} 210193323Sed 211223017SdimSelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) 212239462Sdim : SelectionDAG::DAGUpdateListener(dag), 213239462Sdim TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), 214224145Sdim DAG(dag) { 215193323Sed} 216193323Sed 217193323Sedvoid SelectionDAGLegalize::LegalizeDAG() { 218193323Sed DAG.AssignTopologicalOrder(); 219193323Sed 220234353Sdim // Visit all the nodes. We start in topological order, so that we see 221234353Sdim // nodes with their original operands intact. Legalization can produce 222234353Sdim // new nodes which may themselves need to be legalized. Iterate until all 223234353Sdim // nodes have been legalized. 224234353Sdim for (;;) { 225234353Sdim bool AnyLegalized = false; 226234353Sdim for (LegalizePosition = DAG.allnodes_end(); 227234353Sdim LegalizePosition != DAG.allnodes_begin(); ) { 228234353Sdim --LegalizePosition; 229193323Sed 230234353Sdim SDNode *N = LegalizePosition; 231234353Sdim if (LegalizedNodes.insert(N)) { 232234353Sdim AnyLegalized = true; 233234353Sdim LegalizeOp(N); 234234353Sdim } 235193323Sed } 236234353Sdim if (!AnyLegalized) 237218893Sdim break; 238193323Sed 239193323Sed } 240193323Sed 241234353Sdim // Remove dead nodes now. 242234353Sdim DAG.RemoveDeadNodes(); 243193323Sed} 244193323Sed 245193323Sed/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 246193323Sed/// a load from the constant pool. 247234353SdimSDValue 248234353SdimSelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 249193323Sed bool Extend = false; 250193323Sed DebugLoc dl = CFP->getDebugLoc(); 251193323Sed 252193323Sed // If a FP immediate is precise when represented as a float and if the 253193323Sed // target can do an extending load from float to double, we put it into 254193323Sed // the constant pool as a float, even if it's is statically typed as a 255193323Sed // double. This shrinks FP constants and canonicalizes them for targets where 256193323Sed // an FP extending load is the same cost as a normal load (such as on the x87 257193323Sed // fp stack or PPC FP unit). 258198090Srdivacky EVT VT = CFP->getValueType(0); 259193323Sed ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 260193323Sed if (!UseCP) { 261193323Sed assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 262193323Sed return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 263193323Sed (VT == MVT::f64) ? MVT::i64 : MVT::i32); 264193323Sed } 265193323Sed 266198090Srdivacky EVT OrigVT = VT; 267198090Srdivacky EVT SVT = VT; 268193323Sed while (SVT != MVT::f32) { 269198090Srdivacky SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 270210299Sed if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && 271193323Sed // Only do this if the target has a native EXTLOAD instruction from 272193323Sed // smaller type. 273193323Sed TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 274193323Sed TLI.ShouldShrinkFPConstant(OrigVT)) { 275226633Sdim Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 276193323Sed LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 277193323Sed VT = SVT; 278193323Sed Extend = true; 279193323Sed } 280193323Sed } 281193323Sed 282193323Sed SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 283193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 284234353Sdim if (Extend) { 285234353Sdim SDValue Result = 286234353Sdim DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, 287234353Sdim DAG.getEntryNode(), 288234353Sdim CPIdx, MachinePointerInfo::getConstantPool(), 289234353Sdim VT, false, false, Alignment); 290234353Sdim return Result; 291234353Sdim } 292234353Sdim SDValue Result = 293234353Sdim DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 294234353Sdim MachinePointerInfo::getConstantPool(), false, false, false, 295234353Sdim Alignment); 296234353Sdim return Result; 297193323Sed} 298193323Sed 299193323Sed/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 300234353Sdimstatic void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 301234353Sdim const TargetLowering &TLI, 302234353Sdim SelectionDAGLegalize *DAGLegalize) { 303234353Sdim assert(ST->getAddressingMode() == ISD::UNINDEXED && 304234353Sdim "unaligned indexed stores not implemented!"); 305193323Sed SDValue Chain = ST->getChain(); 306193323Sed SDValue Ptr = ST->getBasePtr(); 307193323Sed SDValue Val = ST->getValue(); 308198090Srdivacky EVT VT = Val.getValueType(); 309193323Sed int Alignment = ST->getAlignment(); 310193323Sed DebugLoc dl = ST->getDebugLoc(); 311193323Sed if (ST->getMemoryVT().isFloatingPoint() || 312193323Sed ST->getMemoryVT().isVector()) { 313198090Srdivacky EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 314193323Sed if (TLI.isTypeLegal(intVT)) { 315193323Sed // Expand to a bitconvert of the value to the integer type of the 316193323Sed // same size, then a (misaligned) int store. 317193323Sed // FIXME: Does not handle truncating floating point stores! 318218893Sdim SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); 319234353Sdim Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), 320234353Sdim ST->isVolatile(), ST->isNonTemporal(), Alignment); 321234353Sdim DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 322234353Sdim return; 323223017Sdim } 324223017Sdim // Do a (aligned) store to a stack slot, then copy from the stack slot 325223017Sdim // to the final destination using (unaligned) integer loads and stores. 326223017Sdim EVT StoredVT = ST->getMemoryVT(); 327249423Sdim MVT RegVT = 328223017Sdim TLI.getRegisterType(*DAG.getContext(), 329223017Sdim EVT::getIntegerVT(*DAG.getContext(), 330223017Sdim StoredVT.getSizeInBits())); 331223017Sdim unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 332223017Sdim unsigned RegBytes = RegVT.getSizeInBits() / 8; 333223017Sdim unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 334193323Sed 335223017Sdim // Make sure the stack slot is also aligned for the register type. 336223017Sdim SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 337193323Sed 338223017Sdim // Perform the original store, only redirected to the stack slot. 339223017Sdim SDValue Store = DAG.getTruncStore(Chain, dl, 340223017Sdim Val, StackPtr, MachinePointerInfo(), 341223017Sdim StoredVT, false, false, 0); 342223017Sdim SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 343223017Sdim SmallVector<SDValue, 8> Stores; 344223017Sdim unsigned Offset = 0; 345193323Sed 346223017Sdim // Do all but one copies using the full register width. 347223017Sdim for (unsigned i = 1; i < NumRegs; i++) { 348223017Sdim // Load one integer register's worth from the stack slot. 349223017Sdim SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, 350223017Sdim MachinePointerInfo(), 351234353Sdim false, false, false, 0); 352223017Sdim // Store it to the final location. Remember the store. 353223017Sdim Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 354223017Sdim ST->getPointerInfo().getWithOffset(Offset), 355223017Sdim ST->isVolatile(), ST->isNonTemporal(), 356223017Sdim MinAlign(ST->getAlignment(), Offset))); 357223017Sdim // Increment the pointers. 358223017Sdim Offset += RegBytes; 359223017Sdim StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 360223017Sdim Increment); 361223017Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 362223017Sdim } 363193323Sed 364223017Sdim // The last store may be partial. Do a truncating store. On big-endian 365223017Sdim // machines this requires an extending load from the stack slot to ensure 366223017Sdim // that the bits are in the right place. 367223017Sdim EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 368223017Sdim 8 * (StoredBytes - Offset)); 369193323Sed 370223017Sdim // Load from the stack slot. 371223017Sdim SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 372223017Sdim MachinePointerInfo(), 373223017Sdim MemVT, false, false, 0); 374193323Sed 375223017Sdim Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 376223017Sdim ST->getPointerInfo() 377223017Sdim .getWithOffset(Offset), 378223017Sdim MemVT, ST->isVolatile(), 379223017Sdim ST->isNonTemporal(), 380223017Sdim MinAlign(ST->getAlignment(), Offset))); 381223017Sdim // The order of the stores doesn't matter - say it with a TokenFactor. 382234353Sdim SDValue Result = 383234353Sdim DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 384234353Sdim Stores.size()); 385234353Sdim DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 386234353Sdim return; 387193323Sed } 388193323Sed assert(ST->getMemoryVT().isInteger() && 389193323Sed !ST->getMemoryVT().isVector() && 390193323Sed "Unaligned store of unknown type."); 391193323Sed // Get the half-size VT 392201360Srdivacky EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); 393193323Sed int NumBits = NewStoredVT.getSizeInBits(); 394193323Sed int IncrementSize = NumBits / 8; 395193323Sed 396193323Sed // Divide the stored value in two parts. 397219077Sdim SDValue ShiftAmount = DAG.getConstant(NumBits, 398219077Sdim TLI.getShiftAmountTy(Val.getValueType())); 399193323Sed SDValue Lo = Val; 400193323Sed SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 401193323Sed 402193323Sed // Store the two parts 403193323Sed SDValue Store1, Store2; 404193323Sed Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 405218893Sdim ST->getPointerInfo(), NewStoredVT, 406203954Srdivacky ST->isVolatile(), ST->isNonTemporal(), Alignment); 407193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 408193323Sed DAG.getConstant(IncrementSize, TLI.getPointerTy())); 409193323Sed Alignment = MinAlign(Alignment, IncrementSize); 410193323Sed Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 411218893Sdim ST->getPointerInfo().getWithOffset(IncrementSize), 412203954Srdivacky NewStoredVT, ST->isVolatile(), ST->isNonTemporal(), 413203954Srdivacky Alignment); 414193323Sed 415234353Sdim SDValue Result = 416234353Sdim DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 417234353Sdim DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 418193323Sed} 419193323Sed 420193323Sed/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 421234353Sdimstatic void 422234353SdimExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 423234353Sdim const TargetLowering &TLI, 424234353Sdim SDValue &ValResult, SDValue &ChainResult) { 425234353Sdim assert(LD->getAddressingMode() == ISD::UNINDEXED && 426234353Sdim "unaligned indexed loads not implemented!"); 427193323Sed SDValue Chain = LD->getChain(); 428193323Sed SDValue Ptr = LD->getBasePtr(); 429198090Srdivacky EVT VT = LD->getValueType(0); 430198090Srdivacky EVT LoadedVT = LD->getMemoryVT(); 431193323Sed DebugLoc dl = LD->getDebugLoc(); 432193323Sed if (VT.isFloatingPoint() || VT.isVector()) { 433198090Srdivacky EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); 434239462Sdim if (TLI.isTypeLegal(intVT) && TLI.isTypeLegal(LoadedVT)) { 435193323Sed // Expand to a (misaligned) integer load of the same size, 436193323Sed // then bitconvert to floating point or vector. 437218893Sdim SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, LD->getPointerInfo(), 438218893Sdim LD->isVolatile(), 439234353Sdim LD->isNonTemporal(), 440234353Sdim LD->isInvariant(), LD->getAlignment()); 441218893Sdim SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); 442239462Sdim if (LoadedVT != VT) 443239462Sdim Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : 444239462Sdim ISD::ANY_EXTEND, dl, VT, Result); 445193323Sed 446234353Sdim ValResult = Result; 447234353Sdim ChainResult = Chain; 448234353Sdim return; 449218893Sdim } 450193323Sed 451218893Sdim // Copy the value to a (aligned) stack slot using (unaligned) integer 452218893Sdim // loads and stores, then do a (aligned) load from the stack slot. 453249423Sdim MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); 454218893Sdim unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 455218893Sdim unsigned RegBytes = RegVT.getSizeInBits() / 8; 456218893Sdim unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 457193323Sed 458218893Sdim // Make sure the stack slot is also aligned for the register type. 459218893Sdim SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 460193323Sed 461218893Sdim SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 462218893Sdim SmallVector<SDValue, 8> Stores; 463218893Sdim SDValue StackPtr = StackBase; 464218893Sdim unsigned Offset = 0; 465193323Sed 466218893Sdim // Do all but one copies using the full register width. 467218893Sdim for (unsigned i = 1; i < NumRegs; i++) { 468218893Sdim // Load one integer register's worth from the original location. 469218893Sdim SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, 470218893Sdim LD->getPointerInfo().getWithOffset(Offset), 471218893Sdim LD->isVolatile(), LD->isNonTemporal(), 472234353Sdim LD->isInvariant(), 473218893Sdim MinAlign(LD->getAlignment(), Offset)); 474193323Sed // Follow the load with a store to the stack slot. Remember the store. 475218893Sdim Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 476218893Sdim MachinePointerInfo(), false, false, 0)); 477218893Sdim // Increment the pointers. 478218893Sdim Offset += RegBytes; 479218893Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 480218893Sdim StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 481218893Sdim Increment); 482218893Sdim } 483193323Sed 484218893Sdim // The last copy may be partial. Do an extending load. 485218893Sdim EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 486218893Sdim 8 * (LoadedBytes - Offset)); 487218893Sdim SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 488218893Sdim LD->getPointerInfo().getWithOffset(Offset), 489218893Sdim MemVT, LD->isVolatile(), 490218893Sdim LD->isNonTemporal(), 491218893Sdim MinAlign(LD->getAlignment(), Offset)); 492218893Sdim // Follow the load with a store to the stack slot. Remember the store. 493218893Sdim // On big-endian machines this requires a truncating store to ensure 494218893Sdim // that the bits end up in the right place. 495218893Sdim Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 496218893Sdim MachinePointerInfo(), MemVT, 497218893Sdim false, false, 0)); 498193323Sed 499218893Sdim // The order of the stores doesn't matter - say it with a TokenFactor. 500218893Sdim SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 501218893Sdim Stores.size()); 502193323Sed 503218893Sdim // Finally, perform the original load only redirected to the stack slot. 504218893Sdim Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 505218893Sdim MachinePointerInfo(), LoadedVT, false, false, 0); 506218893Sdim 507218893Sdim // Callers expect a MERGE_VALUES node. 508234353Sdim ValResult = Load; 509234353Sdim ChainResult = TF; 510234353Sdim return; 511193323Sed } 512193323Sed assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 513193323Sed "Unaligned load of unsupported type."); 514193323Sed 515193323Sed // Compute the new VT that is half the size of the old one. This is an 516193323Sed // integer MVT. 517193323Sed unsigned NumBits = LoadedVT.getSizeInBits(); 518198090Srdivacky EVT NewLoadedVT; 519198090Srdivacky NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2); 520193323Sed NumBits >>= 1; 521193323Sed 522193323Sed unsigned Alignment = LD->getAlignment(); 523193323Sed unsigned IncrementSize = NumBits / 8; 524193323Sed ISD::LoadExtType HiExtType = LD->getExtensionType(); 525193323Sed 526193323Sed // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 527193323Sed if (HiExtType == ISD::NON_EXTLOAD) 528193323Sed HiExtType = ISD::ZEXTLOAD; 529193323Sed 530193323Sed // Load the value in two parts 531193323Sed SDValue Lo, Hi; 532193323Sed if (TLI.isLittleEndian()) { 533218893Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), 534218893Sdim NewLoadedVT, LD->isVolatile(), 535203954Srdivacky LD->isNonTemporal(), Alignment); 536193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 537193323Sed DAG.getConstant(IncrementSize, TLI.getPointerTy())); 538218893Sdim Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, 539218893Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 540218893Sdim NewLoadedVT, LD->isVolatile(), 541210299Sed LD->isNonTemporal(), MinAlign(Alignment,IncrementSize)); 542193323Sed } else { 543218893Sdim Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), 544218893Sdim NewLoadedVT, LD->isVolatile(), 545203954Srdivacky LD->isNonTemporal(), Alignment); 546193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 547193323Sed DAG.getConstant(IncrementSize, TLI.getPointerTy())); 548218893Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, 549218893Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 550218893Sdim NewLoadedVT, LD->isVolatile(), 551210299Sed LD->isNonTemporal(), MinAlign(Alignment,IncrementSize)); 552193323Sed } 553193323Sed 554193323Sed // aggregate the two parts 555219077Sdim SDValue ShiftAmount = DAG.getConstant(NumBits, 556219077Sdim TLI.getShiftAmountTy(Hi.getValueType())); 557193323Sed SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 558193323Sed Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 559193323Sed 560193323Sed SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 561193323Sed Hi.getValue(1)); 562193323Sed 563234353Sdim ValResult = Result; 564234353Sdim ChainResult = TF; 565193323Sed} 566193323Sed 567193323Sed/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 568193323Sed/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 569193323Sed/// is necessary to spill the vector being inserted into to memory, perform 570193323Sed/// the insert there, and then read the result back. 571193323SedSDValue SelectionDAGLegalize:: 572193323SedPerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 573193323Sed DebugLoc dl) { 574193323Sed SDValue Tmp1 = Vec; 575193323Sed SDValue Tmp2 = Val; 576193323Sed SDValue Tmp3 = Idx; 577193323Sed 578193323Sed // If the target doesn't support this, we have to spill the input vector 579193323Sed // to a temporary stack slot, update the element, then reload it. This is 580193323Sed // badness. We could also load the value into a vector register (either 581193323Sed // with a "move to register" or "extload into register" instruction, then 582193323Sed // permute it into place, if the idx is a constant and if the idx is 583193323Sed // supported by the target. 584198090Srdivacky EVT VT = Tmp1.getValueType(); 585198090Srdivacky EVT EltVT = VT.getVectorElementType(); 586198090Srdivacky EVT IdxVT = Tmp3.getValueType(); 587198090Srdivacky EVT PtrVT = TLI.getPointerTy(); 588193323Sed SDValue StackPtr = DAG.CreateStackTemporary(VT); 589193323Sed 590193323Sed int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 591193323Sed 592193323Sed // Store the vector. 593193323Sed SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 594218893Sdim MachinePointerInfo::getFixedStack(SPFI), 595203954Srdivacky false, false, 0); 596193323Sed 597193323Sed // Truncate or zero extend offset to target pointer type. 598193323Sed unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 599193323Sed Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 600193323Sed // Add the offset to the index. 601193323Sed unsigned EltSize = EltVT.getSizeInBits()/8; 602193323Sed Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 603193323Sed SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 604193323Sed // Store the scalar value. 605218893Sdim Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT, 606203954Srdivacky false, false, 0); 607193323Sed // Load the updated vector. 608193323Sed return DAG.getLoad(VT, dl, Ch, StackPtr, 609234353Sdim MachinePointerInfo::getFixedStack(SPFI), false, false, 610234353Sdim false, 0); 611193323Sed} 612193323Sed 613193323Sed 614193323SedSDValue SelectionDAGLegalize:: 615193323SedExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, DebugLoc dl) { 616193323Sed if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 617193323Sed // SCALAR_TO_VECTOR requires that the type of the value being inserted 618193323Sed // match the element type of the vector being created, except for 619193323Sed // integers in which case the inserted value can be over width. 620198090Srdivacky EVT EltVT = Vec.getValueType().getVectorElementType(); 621193323Sed if (Val.getValueType() == EltVT || 622193323Sed (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 623193323Sed SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 624193323Sed Vec.getValueType(), Val); 625193323Sed 626193323Sed unsigned NumElts = Vec.getValueType().getVectorNumElements(); 627193323Sed // We generate a shuffle of InVec and ScVec, so the shuffle mask 628193323Sed // should be 0,1,2,3,4,5... with the appropriate element replaced with 629193323Sed // elt 0 of the RHS. 630193323Sed SmallVector<int, 8> ShufOps; 631193323Sed for (unsigned i = 0; i != NumElts; ++i) 632193323Sed ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 633193323Sed 634193323Sed return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, 635193323Sed &ShufOps[0]); 636193323Sed } 637193323Sed } 638193323Sed return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 639193323Sed} 640193323Sed 641193574SedSDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 642193574Sed // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 643193574Sed // FIXME: We shouldn't do this for TargetConstantFP's. 644193574Sed // FIXME: move this to the DAG Combiner! Note that we can't regress due 645193574Sed // to phase ordering between legalized code and the dag combiner. This 646193574Sed // probably means that we need to integrate dag combiner and legalizer 647193574Sed // together. 648193574Sed // We generally can't do this one for long doubles. 649239462Sdim SDValue Chain = ST->getChain(); 650239462Sdim SDValue Ptr = ST->getBasePtr(); 651193574Sed unsigned Alignment = ST->getAlignment(); 652193574Sed bool isVolatile = ST->isVolatile(); 653203954Srdivacky bool isNonTemporal = ST->isNonTemporal(); 654193574Sed DebugLoc dl = ST->getDebugLoc(); 655193574Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 656193574Sed if (CFP->getValueType(0) == MVT::f32 && 657224145Sdim TLI.isTypeLegal(MVT::i32)) { 658239462Sdim SDValue Con = DAG.getConstant(CFP->getValueAPF(). 659193574Sed bitcastToAPInt().zextOrTrunc(32), 660193574Sed MVT::i32); 661239462Sdim return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 662218893Sdim isVolatile, isNonTemporal, Alignment); 663218893Sdim } 664218893Sdim 665218893Sdim if (CFP->getValueType(0) == MVT::f64) { 666193574Sed // If this target supports 64-bit registers, do a single 64-bit store. 667224145Sdim if (TLI.isTypeLegal(MVT::i64)) { 668239462Sdim SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 669193574Sed zextOrTrunc(64), MVT::i64); 670239462Sdim return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 671218893Sdim isVolatile, isNonTemporal, Alignment); 672218893Sdim } 673218893Sdim 674224145Sdim if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 675193574Sed // Otherwise, if the target supports 32-bit registers, use 2 32-bit 676193574Sed // stores. If the target supports neither 32- nor 64-bits, this 677193574Sed // xform is certainly not worth it. 678193574Sed const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 679218893Sdim SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32); 680193574Sed SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 681193574Sed if (TLI.isBigEndian()) std::swap(Lo, Hi); 682193574Sed 683239462Sdim Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile, 684218893Sdim isNonTemporal, Alignment); 685239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 686193574Sed DAG.getIntPtrConstant(4)); 687239462Sdim Hi = DAG.getStore(Chain, dl, Hi, Ptr, 688218893Sdim ST->getPointerInfo().getWithOffset(4), 689203954Srdivacky isVolatile, isNonTemporal, MinAlign(Alignment, 4U)); 690193574Sed 691193574Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 692193574Sed } 693193574Sed } 694193574Sed } 695221345Sdim return SDValue(0, 0); 696193574Sed} 697193574Sed 698239462Sdimvoid SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { 699239462Sdim StoreSDNode *ST = cast<StoreSDNode>(Node); 700239462Sdim SDValue Chain = ST->getChain(); 701239462Sdim SDValue Ptr = ST->getBasePtr(); 702239462Sdim DebugLoc dl = Node->getDebugLoc(); 703239462Sdim 704239462Sdim unsigned Alignment = ST->getAlignment(); 705239462Sdim bool isVolatile = ST->isVolatile(); 706239462Sdim bool isNonTemporal = ST->isNonTemporal(); 707239462Sdim 708239462Sdim if (!ST->isTruncatingStore()) { 709239462Sdim if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 710239462Sdim ReplaceNode(ST, OptStore); 711239462Sdim return; 712239462Sdim } 713239462Sdim 714239462Sdim { 715239462Sdim SDValue Value = ST->getValue(); 716249423Sdim MVT VT = Value.getSimpleValueType(); 717239462Sdim switch (TLI.getOperationAction(ISD::STORE, VT)) { 718239462Sdim default: llvm_unreachable("This action is not supported yet!"); 719239462Sdim case TargetLowering::Legal: 720239462Sdim // If this is an unaligned store and the target doesn't support it, 721239462Sdim // expand it. 722239462Sdim if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 723239462Sdim Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 724243830Sdim unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 725239462Sdim if (ST->getAlignment() < ABIAlignment) 726239462Sdim ExpandUnalignedStore(cast<StoreSDNode>(Node), 727239462Sdim DAG, TLI, this); 728239462Sdim } 729239462Sdim break; 730239462Sdim case TargetLowering::Custom: { 731239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 732239462Sdim if (Res.getNode()) 733239462Sdim ReplaceNode(SDValue(Node, 0), Res); 734239462Sdim return; 735239462Sdim } 736239462Sdim case TargetLowering::Promote: { 737249423Sdim MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); 738249423Sdim assert(NVT.getSizeInBits() == VT.getSizeInBits() && 739249423Sdim "Can only promote stores to same size type"); 740249423Sdim Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); 741239462Sdim SDValue Result = 742239462Sdim DAG.getStore(Chain, dl, Value, Ptr, 743239462Sdim ST->getPointerInfo(), isVolatile, 744239462Sdim isNonTemporal, Alignment); 745239462Sdim ReplaceNode(SDValue(Node, 0), Result); 746239462Sdim break; 747239462Sdim } 748239462Sdim } 749239462Sdim return; 750239462Sdim } 751239462Sdim } else { 752239462Sdim SDValue Value = ST->getValue(); 753239462Sdim 754239462Sdim EVT StVT = ST->getMemoryVT(); 755239462Sdim unsigned StWidth = StVT.getSizeInBits(); 756239462Sdim 757239462Sdim if (StWidth != StVT.getStoreSizeInBits()) { 758239462Sdim // Promote to a byte-sized store with upper bits zero if not 759239462Sdim // storing an integral number of bytes. For example, promote 760239462Sdim // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 761239462Sdim EVT NVT = EVT::getIntegerVT(*DAG.getContext(), 762239462Sdim StVT.getStoreSizeInBits()); 763239462Sdim Value = DAG.getZeroExtendInReg(Value, dl, StVT); 764239462Sdim SDValue Result = 765239462Sdim DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 766239462Sdim NVT, isVolatile, isNonTemporal, Alignment); 767239462Sdim ReplaceNode(SDValue(Node, 0), Result); 768239462Sdim } else if (StWidth & (StWidth - 1)) { 769239462Sdim // If not storing a power-of-2 number of bits, expand as two stores. 770239462Sdim assert(!StVT.isVector() && "Unsupported truncstore!"); 771239462Sdim unsigned RoundWidth = 1 << Log2_32(StWidth); 772239462Sdim assert(RoundWidth < StWidth); 773239462Sdim unsigned ExtraWidth = StWidth - RoundWidth; 774239462Sdim assert(ExtraWidth < RoundWidth); 775239462Sdim assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 776239462Sdim "Store size not an integral number of bytes!"); 777239462Sdim EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 778239462Sdim EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 779239462Sdim SDValue Lo, Hi; 780239462Sdim unsigned IncrementSize; 781239462Sdim 782239462Sdim if (TLI.isLittleEndian()) { 783239462Sdim // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 784239462Sdim // Store the bottom RoundWidth bits. 785239462Sdim Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 786239462Sdim RoundVT, 787239462Sdim isVolatile, isNonTemporal, Alignment); 788239462Sdim 789239462Sdim // Store the remaining ExtraWidth bits. 790239462Sdim IncrementSize = RoundWidth / 8; 791239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 792239462Sdim DAG.getIntPtrConstant(IncrementSize)); 793239462Sdim Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 794239462Sdim DAG.getConstant(RoundWidth, 795239462Sdim TLI.getShiftAmountTy(Value.getValueType()))); 796239462Sdim Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, 797239462Sdim ST->getPointerInfo().getWithOffset(IncrementSize), 798239462Sdim ExtraVT, isVolatile, isNonTemporal, 799239462Sdim MinAlign(Alignment, IncrementSize)); 800239462Sdim } else { 801239462Sdim // Big endian - avoid unaligned stores. 802239462Sdim // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 803239462Sdim // Store the top RoundWidth bits. 804239462Sdim Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 805239462Sdim DAG.getConstant(ExtraWidth, 806239462Sdim TLI.getShiftAmountTy(Value.getValueType()))); 807239462Sdim Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), 808239462Sdim RoundVT, isVolatile, isNonTemporal, Alignment); 809239462Sdim 810239462Sdim // Store the remaining ExtraWidth bits. 811239462Sdim IncrementSize = RoundWidth / 8; 812239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 813239462Sdim DAG.getIntPtrConstant(IncrementSize)); 814239462Sdim Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, 815239462Sdim ST->getPointerInfo().getWithOffset(IncrementSize), 816239462Sdim ExtraVT, isVolatile, isNonTemporal, 817239462Sdim MinAlign(Alignment, IncrementSize)); 818239462Sdim } 819239462Sdim 820239462Sdim // The order of the stores doesn't matter. 821239462Sdim SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 822239462Sdim ReplaceNode(SDValue(Node, 0), Result); 823239462Sdim } else { 824249423Sdim switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(), 825249423Sdim StVT.getSimpleVT())) { 826239462Sdim default: llvm_unreachable("This action is not supported yet!"); 827239462Sdim case TargetLowering::Legal: 828239462Sdim // If this is an unaligned store and the target doesn't support it, 829239462Sdim // expand it. 830239462Sdim if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 831239462Sdim Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 832243830Sdim unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 833239462Sdim if (ST->getAlignment() < ABIAlignment) 834239462Sdim ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); 835239462Sdim } 836239462Sdim break; 837239462Sdim case TargetLowering::Custom: { 838239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 839239462Sdim if (Res.getNode()) 840239462Sdim ReplaceNode(SDValue(Node, 0), Res); 841239462Sdim return; 842239462Sdim } 843239462Sdim case TargetLowering::Expand: 844239462Sdim assert(!StVT.isVector() && 845239462Sdim "Vector Stores are handled in LegalizeVectorOps"); 846239462Sdim 847239462Sdim // TRUNCSTORE:i16 i32 -> STORE i16 848239462Sdim assert(TLI.isTypeLegal(StVT) && 849239462Sdim "Do not know how to expand this store!"); 850239462Sdim Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); 851239462Sdim SDValue Result = 852239462Sdim DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 853239462Sdim isVolatile, isNonTemporal, Alignment); 854239462Sdim ReplaceNode(SDValue(Node, 0), Result); 855239462Sdim break; 856239462Sdim } 857239462Sdim } 858239462Sdim } 859239462Sdim} 860239462Sdim 861239462Sdimvoid SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { 862239462Sdim LoadSDNode *LD = cast<LoadSDNode>(Node); 863239462Sdim SDValue Chain = LD->getChain(); // The chain. 864239462Sdim SDValue Ptr = LD->getBasePtr(); // The base pointer. 865239462Sdim SDValue Value; // The value returned by the load op. 866239462Sdim DebugLoc dl = Node->getDebugLoc(); 867239462Sdim 868239462Sdim ISD::LoadExtType ExtType = LD->getExtensionType(); 869239462Sdim if (ExtType == ISD::NON_EXTLOAD) { 870249423Sdim MVT VT = Node->getSimpleValueType(0); 871239462Sdim SDValue RVal = SDValue(Node, 0); 872239462Sdim SDValue RChain = SDValue(Node, 1); 873239462Sdim 874239462Sdim switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 875239462Sdim default: llvm_unreachable("This action is not supported yet!"); 876239462Sdim case TargetLowering::Legal: 877243830Sdim // If this is an unaligned load and the target doesn't support it, 878243830Sdim // expand it. 879243830Sdim if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 880243830Sdim Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 881243830Sdim unsigned ABIAlignment = 882243830Sdim TLI.getDataLayout()->getABITypeAlignment(Ty); 883243830Sdim if (LD->getAlignment() < ABIAlignment){ 884243830Sdim ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain); 885243830Sdim } 886243830Sdim } 887243830Sdim break; 888239462Sdim case TargetLowering::Custom: { 889243830Sdim SDValue Res = TLI.LowerOperation(RVal, DAG); 890243830Sdim if (Res.getNode()) { 891243830Sdim RVal = Res; 892243830Sdim RChain = Res.getValue(1); 893243830Sdim } 894243830Sdim break; 895239462Sdim } 896239462Sdim case TargetLowering::Promote: { 897249423Sdim MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 898249423Sdim assert(NVT.getSizeInBits() == VT.getSizeInBits() && 899249423Sdim "Can only promote loads to same size type"); 900239462Sdim 901239462Sdim SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), 902239462Sdim LD->isVolatile(), LD->isNonTemporal(), 903239462Sdim LD->isInvariant(), LD->getAlignment()); 904239462Sdim RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); 905239462Sdim RChain = Res.getValue(1); 906239462Sdim break; 907239462Sdim } 908239462Sdim } 909239462Sdim if (RChain.getNode() != Node) { 910239462Sdim assert(RVal.getNode() != Node && "Load must be completely replaced"); 911239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); 912239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); 913239462Sdim ReplacedNode(Node); 914239462Sdim } 915239462Sdim return; 916239462Sdim } 917239462Sdim 918239462Sdim EVT SrcVT = LD->getMemoryVT(); 919239462Sdim unsigned SrcWidth = SrcVT.getSizeInBits(); 920239462Sdim unsigned Alignment = LD->getAlignment(); 921239462Sdim bool isVolatile = LD->isVolatile(); 922239462Sdim bool isNonTemporal = LD->isNonTemporal(); 923239462Sdim 924239462Sdim if (SrcWidth != SrcVT.getStoreSizeInBits() && 925239462Sdim // Some targets pretend to have an i1 loading operation, and actually 926239462Sdim // load an i8. This trick is correct for ZEXTLOAD because the top 7 927239462Sdim // bits are guaranteed to be zero; it helps the optimizers understand 928239462Sdim // that these bits are zero. It is also useful for EXTLOAD, since it 929239462Sdim // tells the optimizers that those bits are undefined. It would be 930239462Sdim // nice to have an effective generic way of getting these benefits... 931239462Sdim // Until such a way is found, don't insist on promoting i1 here. 932239462Sdim (SrcVT != MVT::i1 || 933239462Sdim TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 934239462Sdim // Promote to a byte-sized load if not loading an integral number of 935239462Sdim // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 936239462Sdim unsigned NewWidth = SrcVT.getStoreSizeInBits(); 937239462Sdim EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 938239462Sdim SDValue Ch; 939239462Sdim 940239462Sdim // The extra bits are guaranteed to be zero, since we stored them that 941239462Sdim // way. A zext load from NVT thus automatically gives zext from SrcVT. 942239462Sdim 943239462Sdim ISD::LoadExtType NewExtType = 944239462Sdim ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 945239462Sdim 946239462Sdim SDValue Result = 947239462Sdim DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 948239462Sdim Chain, Ptr, LD->getPointerInfo(), 949239462Sdim NVT, isVolatile, isNonTemporal, Alignment); 950239462Sdim 951239462Sdim Ch = Result.getValue(1); // The chain. 952239462Sdim 953239462Sdim if (ExtType == ISD::SEXTLOAD) 954239462Sdim // Having the top bits zero doesn't help when sign extending. 955239462Sdim Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 956239462Sdim Result.getValueType(), 957239462Sdim Result, DAG.getValueType(SrcVT)); 958239462Sdim else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 959239462Sdim // All the top bits are guaranteed to be zero - inform the optimizers. 960239462Sdim Result = DAG.getNode(ISD::AssertZext, dl, 961239462Sdim Result.getValueType(), Result, 962239462Sdim DAG.getValueType(SrcVT)); 963239462Sdim 964239462Sdim Value = Result; 965239462Sdim Chain = Ch; 966239462Sdim } else if (SrcWidth & (SrcWidth - 1)) { 967239462Sdim // If not loading a power-of-2 number of bits, expand as two loads. 968239462Sdim assert(!SrcVT.isVector() && "Unsupported extload!"); 969239462Sdim unsigned RoundWidth = 1 << Log2_32(SrcWidth); 970239462Sdim assert(RoundWidth < SrcWidth); 971239462Sdim unsigned ExtraWidth = SrcWidth - RoundWidth; 972239462Sdim assert(ExtraWidth < RoundWidth); 973239462Sdim assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 974239462Sdim "Load size not an integral number of bytes!"); 975239462Sdim EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 976239462Sdim EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 977239462Sdim SDValue Lo, Hi, Ch; 978239462Sdim unsigned IncrementSize; 979239462Sdim 980239462Sdim if (TLI.isLittleEndian()) { 981239462Sdim // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 982239462Sdim // Load the bottom RoundWidth bits. 983239462Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), 984239462Sdim Chain, Ptr, 985239462Sdim LD->getPointerInfo(), RoundVT, isVolatile, 986239462Sdim isNonTemporal, Alignment); 987239462Sdim 988239462Sdim // Load the remaining ExtraWidth bits. 989239462Sdim IncrementSize = RoundWidth / 8; 990239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 991239462Sdim DAG.getIntPtrConstant(IncrementSize)); 992239462Sdim Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 993239462Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 994239462Sdim ExtraVT, isVolatile, isNonTemporal, 995239462Sdim MinAlign(Alignment, IncrementSize)); 996239462Sdim 997239462Sdim // Build a factor node to remember that this load is independent of 998239462Sdim // the other one. 999239462Sdim Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1000239462Sdim Hi.getValue(1)); 1001239462Sdim 1002239462Sdim // Move the top bits to the right place. 1003239462Sdim Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1004239462Sdim DAG.getConstant(RoundWidth, 1005239462Sdim TLI.getShiftAmountTy(Hi.getValueType()))); 1006239462Sdim 1007239462Sdim // Join the hi and lo parts. 1008239462Sdim Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1009239462Sdim } else { 1010239462Sdim // Big endian - avoid unaligned loads. 1011239462Sdim // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 1012239462Sdim // Load the top RoundWidth bits. 1013239462Sdim Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 1014239462Sdim LD->getPointerInfo(), RoundVT, isVolatile, 1015239462Sdim isNonTemporal, Alignment); 1016239462Sdim 1017239462Sdim // Load the remaining ExtraWidth bits. 1018239462Sdim IncrementSize = RoundWidth / 8; 1019239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1020239462Sdim DAG.getIntPtrConstant(IncrementSize)); 1021239462Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, 1022239462Sdim dl, Node->getValueType(0), Chain, Ptr, 1023239462Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 1024239462Sdim ExtraVT, isVolatile, isNonTemporal, 1025239462Sdim MinAlign(Alignment, IncrementSize)); 1026239462Sdim 1027239462Sdim // Build a factor node to remember that this load is independent of 1028239462Sdim // the other one. 1029239462Sdim Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1030239462Sdim Hi.getValue(1)); 1031239462Sdim 1032239462Sdim // Move the top bits to the right place. 1033239462Sdim Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1034239462Sdim DAG.getConstant(ExtraWidth, 1035239462Sdim TLI.getShiftAmountTy(Hi.getValueType()))); 1036239462Sdim 1037239462Sdim // Join the hi and lo parts. 1038239462Sdim Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1039239462Sdim } 1040239462Sdim 1041239462Sdim Chain = Ch; 1042239462Sdim } else { 1043239462Sdim bool isCustom = false; 1044249423Sdim switch (TLI.getLoadExtAction(ExtType, SrcVT.getSimpleVT())) { 1045239462Sdim default: llvm_unreachable("This action is not supported yet!"); 1046239462Sdim case TargetLowering::Custom: 1047239462Sdim isCustom = true; 1048239462Sdim // FALLTHROUGH 1049239462Sdim case TargetLowering::Legal: { 1050239462Sdim Value = SDValue(Node, 0); 1051239462Sdim Chain = SDValue(Node, 1); 1052239462Sdim 1053239462Sdim if (isCustom) { 1054239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1055239462Sdim if (Res.getNode()) { 1056239462Sdim Value = Res; 1057239462Sdim Chain = Res.getValue(1); 1058239462Sdim } 1059239462Sdim } else { 1060239462Sdim // If this is an unaligned load and the target doesn't support it, 1061239462Sdim // expand it. 1062239462Sdim if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 1063239462Sdim Type *Ty = 1064239462Sdim LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1065239462Sdim unsigned ABIAlignment = 1066243830Sdim TLI.getDataLayout()->getABITypeAlignment(Ty); 1067239462Sdim if (LD->getAlignment() < ABIAlignment){ 1068239462Sdim ExpandUnalignedLoad(cast<LoadSDNode>(Node), 1069239462Sdim DAG, TLI, Value, Chain); 1070239462Sdim } 1071239462Sdim } 1072239462Sdim } 1073239462Sdim break; 1074239462Sdim } 1075239462Sdim case TargetLowering::Expand: 1076239462Sdim if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && TLI.isTypeLegal(SrcVT)) { 1077239462Sdim SDValue Load = DAG.getLoad(SrcVT, dl, Chain, Ptr, 1078239462Sdim LD->getPointerInfo(), 1079239462Sdim LD->isVolatile(), LD->isNonTemporal(), 1080239462Sdim LD->isInvariant(), LD->getAlignment()); 1081239462Sdim unsigned ExtendOp; 1082239462Sdim switch (ExtType) { 1083239462Sdim case ISD::EXTLOAD: 1084239462Sdim ExtendOp = (SrcVT.isFloatingPoint() ? 1085239462Sdim ISD::FP_EXTEND : ISD::ANY_EXTEND); 1086239462Sdim break; 1087239462Sdim case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break; 1088239462Sdim case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break; 1089239462Sdim default: llvm_unreachable("Unexpected extend load type!"); 1090239462Sdim } 1091239462Sdim Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 1092239462Sdim Chain = Load.getValue(1); 1093239462Sdim break; 1094239462Sdim } 1095239462Sdim 1096239462Sdim assert(!SrcVT.isVector() && 1097239462Sdim "Vector Loads are handled in LegalizeVectorOps"); 1098239462Sdim 1099239462Sdim // FIXME: This does not work for vectors on most targets. Sign- and 1100239462Sdim // zero-extend operations are currently folded into extending loads, 1101239462Sdim // whether they are legal or not, and then we end up here without any 1102239462Sdim // support for legalizing them. 1103239462Sdim assert(ExtType != ISD::EXTLOAD && 1104239462Sdim "EXTLOAD should always be supported!"); 1105239462Sdim // Turn the unsupported load into an EXTLOAD followed by an explicit 1106239462Sdim // zero/sign extend inreg. 1107239462Sdim SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0), 1108239462Sdim Chain, Ptr, LD->getPointerInfo(), SrcVT, 1109239462Sdim LD->isVolatile(), LD->isNonTemporal(), 1110239462Sdim LD->getAlignment()); 1111239462Sdim SDValue ValRes; 1112239462Sdim if (ExtType == ISD::SEXTLOAD) 1113239462Sdim ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 1114239462Sdim Result.getValueType(), 1115239462Sdim Result, DAG.getValueType(SrcVT)); 1116239462Sdim else 1117239462Sdim ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); 1118239462Sdim Value = ValRes; 1119239462Sdim Chain = Result.getValue(1); 1120239462Sdim break; 1121239462Sdim } 1122239462Sdim } 1123239462Sdim 1124239462Sdim // Since loads produce two values, make sure to remember that we legalized 1125239462Sdim // both of them. 1126239462Sdim if (Chain.getNode() != Node) { 1127239462Sdim assert(Value.getNode() != Node && "Load must be completely replaced"); 1128239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); 1129239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 1130239462Sdim ReplacedNode(Node); 1131239462Sdim } 1132239462Sdim} 1133239462Sdim 1134224145Sdim/// LegalizeOp - Return a legal replacement for the given operation, with 1135224145Sdim/// all legal operands. 1136234353Sdimvoid SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 1137234353Sdim if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 1138234353Sdim return; 1139193323Sed 1140193323Sed for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1141224145Sdim assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 1142224145Sdim TargetLowering::TypeLegal && 1143193323Sed "Unexpected illegal type!"); 1144193323Sed 1145193323Sed for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1146224145Sdim assert((TLI.getTypeAction(*DAG.getContext(), 1147224145Sdim Node->getOperand(i).getValueType()) == 1148224145Sdim TargetLowering::TypeLegal || 1149193323Sed Node->getOperand(i).getOpcode() == ISD::TargetConstant) && 1150193323Sed "Unexpected illegal type!"); 1151193323Sed 1152193323Sed // Figure out the correct action; the way to query this varies by opcode 1153218893Sdim TargetLowering::LegalizeAction Action = TargetLowering::Legal; 1154193323Sed bool SimpleFinishLegalizing = true; 1155193323Sed switch (Node->getOpcode()) { 1156193323Sed case ISD::INTRINSIC_W_CHAIN: 1157193323Sed case ISD::INTRINSIC_WO_CHAIN: 1158193323Sed case ISD::INTRINSIC_VOID: 1159193323Sed case ISD::STACKSAVE: 1160193323Sed Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1161193323Sed break; 1162234353Sdim case ISD::VAARG: 1163234353Sdim Action = TLI.getOperationAction(Node->getOpcode(), 1164234353Sdim Node->getValueType(0)); 1165234353Sdim if (Action != TargetLowering::Promote) 1166234353Sdim Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1167234353Sdim break; 1168193323Sed case ISD::SINT_TO_FP: 1169193323Sed case ISD::UINT_TO_FP: 1170193323Sed case ISD::EXTRACT_VECTOR_ELT: 1171193323Sed Action = TLI.getOperationAction(Node->getOpcode(), 1172193323Sed Node->getOperand(0).getValueType()); 1173193323Sed break; 1174193323Sed case ISD::FP_ROUND_INREG: 1175193323Sed case ISD::SIGN_EXTEND_INREG: { 1176198090Srdivacky EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1177193323Sed Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 1178193323Sed break; 1179193323Sed } 1180226633Sdim case ISD::ATOMIC_STORE: { 1181226633Sdim Action = TLI.getOperationAction(Node->getOpcode(), 1182226633Sdim Node->getOperand(2).getValueType()); 1183226633Sdim break; 1184226633Sdim } 1185193323Sed case ISD::SELECT_CC: 1186193323Sed case ISD::SETCC: 1187193323Sed case ISD::BR_CC: { 1188193323Sed unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 1189193323Sed Node->getOpcode() == ISD::SETCC ? 2 : 1; 1190193323Sed unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; 1191249423Sdim MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); 1192193323Sed ISD::CondCode CCCode = 1193193323Sed cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 1194193323Sed Action = TLI.getCondCodeAction(CCCode, OpVT); 1195193323Sed if (Action == TargetLowering::Legal) { 1196193323Sed if (Node->getOpcode() == ISD::SELECT_CC) 1197193323Sed Action = TLI.getOperationAction(Node->getOpcode(), 1198193323Sed Node->getValueType(0)); 1199193323Sed else 1200193323Sed Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 1201193323Sed } 1202193323Sed break; 1203193323Sed } 1204193323Sed case ISD::LOAD: 1205193323Sed case ISD::STORE: 1206193323Sed // FIXME: Model these properly. LOAD and STORE are complicated, and 1207193323Sed // STORE expects the unlegalized operand in some cases. 1208193323Sed SimpleFinishLegalizing = false; 1209193323Sed break; 1210193323Sed case ISD::CALLSEQ_START: 1211193323Sed case ISD::CALLSEQ_END: 1212193323Sed // FIXME: This shouldn't be necessary. These nodes have special properties 1213193323Sed // dealing with the recursive nature of legalization. Removing this 1214193323Sed // special case should be done as part of making LegalizeDAG non-recursive. 1215193323Sed SimpleFinishLegalizing = false; 1216193323Sed break; 1217193323Sed case ISD::EXTRACT_ELEMENT: 1218193323Sed case ISD::FLT_ROUNDS_: 1219193323Sed case ISD::SADDO: 1220193323Sed case ISD::SSUBO: 1221193323Sed case ISD::UADDO: 1222193323Sed case ISD::USUBO: 1223193323Sed case ISD::SMULO: 1224193323Sed case ISD::UMULO: 1225193323Sed case ISD::FPOWI: 1226193323Sed case ISD::MERGE_VALUES: 1227193323Sed case ISD::EH_RETURN: 1228193323Sed case ISD::FRAME_TO_ARGS_OFFSET: 1229210299Sed case ISD::EH_SJLJ_SETJMP: 1230210299Sed case ISD::EH_SJLJ_LONGJMP: 1231193323Sed // These operations lie about being legal: when they claim to be legal, 1232193323Sed // they should actually be expanded. 1233193323Sed Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1234193323Sed if (Action == TargetLowering::Legal) 1235193323Sed Action = TargetLowering::Expand; 1236193323Sed break; 1237226633Sdim case ISD::INIT_TRAMPOLINE: 1238226633Sdim case ISD::ADJUST_TRAMPOLINE: 1239193323Sed case ISD::FRAMEADDR: 1240193323Sed case ISD::RETURNADDR: 1241193323Sed // These operations lie about being legal: when they claim to be legal, 1242193323Sed // they should actually be custom-lowered. 1243193323Sed Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1244193323Sed if (Action == TargetLowering::Legal) 1245193323Sed Action = TargetLowering::Custom; 1246193323Sed break; 1247243830Sdim case ISD::DEBUGTRAP: 1248243830Sdim Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1249243830Sdim if (Action == TargetLowering::Expand) { 1250243830Sdim // replace ISD::DEBUGTRAP with ISD::TRAP 1251243830Sdim SDValue NewVal; 1252243830Sdim NewVal = DAG.getNode(ISD::TRAP, Node->getDebugLoc(), Node->getVTList(), 1253243830Sdim Node->getOperand(0)); 1254243830Sdim ReplaceNode(Node, NewVal.getNode()); 1255243830Sdim LegalizeOp(NewVal.getNode()); 1256243830Sdim return; 1257243830Sdim } 1258243830Sdim break; 1259243830Sdim 1260193323Sed default: 1261193323Sed if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1262193323Sed Action = TargetLowering::Legal; 1263193323Sed } else { 1264193323Sed Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1265193323Sed } 1266193323Sed break; 1267193323Sed } 1268193323Sed 1269193323Sed if (SimpleFinishLegalizing) { 1270239462Sdim SDNode *NewNode = Node; 1271193323Sed switch (Node->getOpcode()) { 1272193323Sed default: break; 1273193323Sed case ISD::SHL: 1274193323Sed case ISD::SRL: 1275193323Sed case ISD::SRA: 1276193323Sed case ISD::ROTL: 1277193323Sed case ISD::ROTR: 1278193323Sed // Legalizing shifts/rotates requires adjusting the shift amount 1279193323Sed // to the appropriate width. 1280239462Sdim if (!Node->getOperand(1).getValueType().isVector()) { 1281239462Sdim SDValue SAO = 1282239462Sdim DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1283239462Sdim Node->getOperand(1)); 1284234353Sdim HandleSDNode Handle(SAO); 1285234353Sdim LegalizeOp(SAO.getNode()); 1286239462Sdim NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1287239462Sdim Handle.getValue()); 1288234353Sdim } 1289193323Sed break; 1290198090Srdivacky case ISD::SRL_PARTS: 1291198090Srdivacky case ISD::SRA_PARTS: 1292198090Srdivacky case ISD::SHL_PARTS: 1293198090Srdivacky // Legalizing shifts/rotates requires adjusting the shift amount 1294198090Srdivacky // to the appropriate width. 1295239462Sdim if (!Node->getOperand(2).getValueType().isVector()) { 1296239462Sdim SDValue SAO = 1297239462Sdim DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1298239462Sdim Node->getOperand(2)); 1299234353Sdim HandleSDNode Handle(SAO); 1300234353Sdim LegalizeOp(SAO.getNode()); 1301239462Sdim NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1302239462Sdim Node->getOperand(1), 1303239462Sdim Handle.getValue()); 1304234353Sdim } 1305198090Srdivacky break; 1306193323Sed } 1307193323Sed 1308234353Sdim if (NewNode != Node) { 1309239462Sdim DAG.ReplaceAllUsesWith(Node, NewNode); 1310234353Sdim for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1311234353Sdim DAG.TransferDbgValues(SDValue(Node, i), SDValue(NewNode, i)); 1312234353Sdim ReplacedNode(Node); 1313234353Sdim Node = NewNode; 1314234353Sdim } 1315193323Sed switch (Action) { 1316193323Sed case TargetLowering::Legal: 1317234353Sdim return; 1318239462Sdim case TargetLowering::Custom: { 1319193323Sed // FIXME: The handling for custom lowering with multiple results is 1320193323Sed // a complete mess. 1321239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1322239462Sdim if (Res.getNode()) { 1323234353Sdim SmallVector<SDValue, 8> ResultVals; 1324193323Sed for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 1325193323Sed if (e == 1) 1326239462Sdim ResultVals.push_back(Res); 1327193323Sed else 1328239462Sdim ResultVals.push_back(Res.getValue(i)); 1329193323Sed } 1330239462Sdim if (Res.getNode() != Node || Res.getResNo() != 0) { 1331239462Sdim DAG.ReplaceAllUsesWith(Node, ResultVals.data()); 1332234353Sdim for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1333234353Sdim DAG.TransferDbgValues(SDValue(Node, i), ResultVals[i]); 1334234353Sdim ReplacedNode(Node); 1335234353Sdim } 1336234353Sdim return; 1337193323Sed } 1338239462Sdim } 1339193323Sed // FALL THROUGH 1340193323Sed case TargetLowering::Expand: 1341234353Sdim ExpandNode(Node); 1342234353Sdim return; 1343193323Sed case TargetLowering::Promote: 1344234353Sdim PromoteNode(Node); 1345234353Sdim return; 1346193323Sed } 1347193323Sed } 1348193323Sed 1349193323Sed switch (Node->getOpcode()) { 1350193323Sed default: 1351193323Sed#ifndef NDEBUG 1352202375Srdivacky dbgs() << "NODE: "; 1353202375Srdivacky Node->dump( &DAG); 1354202375Srdivacky dbgs() << "\n"; 1355193323Sed#endif 1356234353Sdim llvm_unreachable("Do not know how to legalize this operator!"); 1357193323Sed 1358234353Sdim case ISD::CALLSEQ_START: 1359234353Sdim case ISD::CALLSEQ_END: 1360226633Sdim break; 1361193323Sed case ISD::LOAD: { 1362239462Sdim return LegalizeLoadOps(Node); 1363193323Sed } 1364193323Sed case ISD::STORE: { 1365239462Sdim return LegalizeStoreOps(Node); 1366193323Sed } 1367193323Sed } 1368193323Sed} 1369193323Sed 1370193323SedSDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1371193323Sed SDValue Vec = Op.getOperand(0); 1372193323Sed SDValue Idx = Op.getOperand(1); 1373193323Sed DebugLoc dl = Op.getDebugLoc(); 1374193323Sed // Store the value to a temporary stack slot, then LOAD the returned part. 1375193323Sed SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1376218893Sdim SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1377218893Sdim MachinePointerInfo(), false, false, 0); 1378193323Sed 1379193323Sed // Add the offset to the index. 1380193323Sed unsigned EltSize = 1381193323Sed Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1382193323Sed Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1383193323Sed DAG.getConstant(EltSize, Idx.getValueType())); 1384193323Sed 1385193323Sed if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 1386193323Sed Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 1387193323Sed else 1388193323Sed Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 1389193323Sed 1390193323Sed StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 1391193323Sed 1392198090Srdivacky if (Op.getValueType().isVector()) 1393218893Sdim return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), 1394234353Sdim false, false, false, 0); 1395218893Sdim return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1396218893Sdim MachinePointerInfo(), 1397218893Sdim Vec.getValueType().getVectorElementType(), 1398218893Sdim false, false, 0); 1399218893Sdim} 1400218893Sdim 1401218893SdimSDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1402218893Sdim assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1403218893Sdim 1404218893Sdim SDValue Vec = Op.getOperand(0); 1405218893Sdim SDValue Part = Op.getOperand(1); 1406218893Sdim SDValue Idx = Op.getOperand(2); 1407218893Sdim DebugLoc dl = Op.getDebugLoc(); 1408218893Sdim 1409218893Sdim // Store the value to a temporary stack slot, then LOAD the returned part. 1410218893Sdim 1411218893Sdim SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1412218893Sdim int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1413218893Sdim MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1414218893Sdim 1415218893Sdim // First store the whole vector. 1416218893Sdim SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1417218893Sdim false, false, 0); 1418218893Sdim 1419218893Sdim // Then store the inserted part. 1420218893Sdim 1421218893Sdim // Add the offset to the index. 1422218893Sdim unsigned EltSize = 1423218893Sdim Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1424218893Sdim 1425218893Sdim Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1426218893Sdim DAG.getConstant(EltSize, Idx.getValueType())); 1427218893Sdim 1428218893Sdim if (Idx.getValueType().bitsGT(TLI.getPointerTy())) 1429218893Sdim Idx = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Idx); 1430198090Srdivacky else 1431218893Sdim Idx = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Idx); 1432218893Sdim 1433218893Sdim SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1434218893Sdim StackPtr); 1435218893Sdim 1436218893Sdim // Store the subvector. 1437218893Sdim Ch = DAG.getStore(DAG.getEntryNode(), dl, Part, SubStackPtr, 1438218893Sdim MachinePointerInfo(), false, false, 0); 1439218893Sdim 1440218893Sdim // Finally, load the updated vector. 1441218893Sdim return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo, 1442234353Sdim false, false, false, 0); 1443193323Sed} 1444193323Sed 1445193574SedSDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1446193574Sed // We can't handle this case efficiently. Allocate a sufficiently 1447193574Sed // aligned object on the stack, store each element into it, then load 1448193574Sed // the result as a vector. 1449193574Sed // Create the stack frame object. 1450198090Srdivacky EVT VT = Node->getValueType(0); 1451199989Srdivacky EVT EltVT = VT.getVectorElementType(); 1452193574Sed DebugLoc dl = Node->getDebugLoc(); 1453193574Sed SDValue FIPtr = DAG.CreateStackTemporary(VT); 1454193574Sed int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1455218893Sdim MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1456193574Sed 1457193574Sed // Emit a store of each element to the stack slot. 1458193574Sed SmallVector<SDValue, 8> Stores; 1459199989Srdivacky unsigned TypeByteSize = EltVT.getSizeInBits() / 8; 1460193574Sed // Store (in the right endianness) the elements to memory. 1461193574Sed for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1462193574Sed // Ignore undef elements. 1463193574Sed if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1464193574Sed 1465193574Sed unsigned Offset = TypeByteSize*i; 1466193574Sed 1467193574Sed SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 1468193574Sed Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 1469193574Sed 1470204642Srdivacky // If the destination vector element type is narrower than the source 1471204642Srdivacky // element type, only store the bits necessary. 1472204642Srdivacky if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { 1473199989Srdivacky Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1474218893Sdim Node->getOperand(i), Idx, 1475218893Sdim PtrInfo.getWithOffset(Offset), 1476203954Srdivacky EltVT, false, false, 0)); 1477203954Srdivacky } else 1478210299Sed Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, 1479218893Sdim Node->getOperand(i), Idx, 1480218893Sdim PtrInfo.getWithOffset(Offset), 1481203954Srdivacky false, false, 0)); 1482193574Sed } 1483193574Sed 1484193574Sed SDValue StoreChain; 1485193574Sed if (!Stores.empty()) // Not all undef elements? 1486193574Sed StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1487193574Sed &Stores[0], Stores.size()); 1488193574Sed else 1489193574Sed StoreChain = DAG.getEntryNode(); 1490193574Sed 1491193574Sed // Result is a load from the stack slot. 1492234353Sdim return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, 1493234353Sdim false, false, false, 0); 1494193574Sed} 1495193574Sed 1496193323SedSDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) { 1497193323Sed DebugLoc dl = Node->getDebugLoc(); 1498193323Sed SDValue Tmp1 = Node->getOperand(0); 1499193323Sed SDValue Tmp2 = Node->getOperand(1); 1500205218Srdivacky 1501205218Srdivacky // Get the sign bit of the RHS. First obtain a value that has the same 1502205218Srdivacky // sign as the sign bit, i.e. negative if and only if the sign bit is 1. 1503193323Sed SDValue SignBit; 1504205218Srdivacky EVT FloatVT = Tmp2.getValueType(); 1505205218Srdivacky EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits()); 1506224145Sdim if (TLI.isTypeLegal(IVT)) { 1507205218Srdivacky // Convert to an integer with the same sign bit. 1508218893Sdim SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); 1509193323Sed } else { 1510205218Srdivacky // Store the float to memory, then load the sign part out as an integer. 1511205218Srdivacky MVT LoadTy = TLI.getPointerTy(); 1512205218Srdivacky // First create a temporary that is aligned for both the load and store. 1513205218Srdivacky SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1514205218Srdivacky // Then store the float to it. 1515193323Sed SDValue Ch = 1516218893Sdim DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(), 1517203954Srdivacky false, false, 0); 1518205218Srdivacky if (TLI.isBigEndian()) { 1519205218Srdivacky assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1520205218Srdivacky // Load out a legal integer with the same sign bit as the float. 1521218893Sdim SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(), 1522234353Sdim false, false, false, 0); 1523205218Srdivacky } else { // Little endian 1524205218Srdivacky SDValue LoadPtr = StackPtr; 1525205218Srdivacky // The float may be wider than the integer we are going to load. Advance 1526205218Srdivacky // the pointer so that the loaded integer will contain the sign bit. 1527205218Srdivacky unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits(); 1528205218Srdivacky unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8; 1529205218Srdivacky LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), 1530205218Srdivacky LoadPtr, DAG.getIntPtrConstant(ByteOffset)); 1531205218Srdivacky // Load a legal integer containing the sign bit. 1532218893Sdim SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(), 1533234353Sdim false, false, false, 0); 1534205218Srdivacky // Move the sign bit to the top bit of the loaded integer. 1535205218Srdivacky unsigned BitShift = LoadTy.getSizeInBits() - 1536205218Srdivacky (FloatVT.getSizeInBits() - 8 * ByteOffset); 1537205218Srdivacky assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?"); 1538205218Srdivacky if (BitShift) 1539205218Srdivacky SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, 1540219077Sdim DAG.getConstant(BitShift, 1541219077Sdim TLI.getShiftAmountTy(SignBit.getValueType()))); 1542205218Srdivacky } 1543193323Sed } 1544205218Srdivacky // Now get the sign bit proper, by seeing whether the value is negative. 1545205218Srdivacky SignBit = DAG.getSetCC(dl, TLI.getSetCCResultType(SignBit.getValueType()), 1546205218Srdivacky SignBit, DAG.getConstant(0, SignBit.getValueType()), 1547205218Srdivacky ISD::SETLT); 1548193323Sed // Get the absolute value of the result. 1549193323Sed SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 1550193323Sed // Select between the nabs and abs value based on the sign bit of 1551193323Sed // the input. 1552193323Sed return DAG.getNode(ISD::SELECT, dl, AbsVal.getValueType(), SignBit, 1553193323Sed DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), 1554193323Sed AbsVal); 1555193323Sed} 1556193323Sed 1557193323Sedvoid SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1558193323Sed SmallVectorImpl<SDValue> &Results) { 1559193323Sed unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1560193323Sed assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1561193323Sed " not tell us which reg is the stack pointer!"); 1562193323Sed DebugLoc dl = Node->getDebugLoc(); 1563198090Srdivacky EVT VT = Node->getValueType(0); 1564193323Sed SDValue Tmp1 = SDValue(Node, 0); 1565193323Sed SDValue Tmp2 = SDValue(Node, 1); 1566193323Sed SDValue Tmp3 = Node->getOperand(2); 1567193323Sed SDValue Chain = Tmp1.getOperand(0); 1568193323Sed 1569193323Sed // Chain the dynamic stack allocation so that it doesn't modify the stack 1570193323Sed // pointer when other instructions are using the stack. 1571193323Sed Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); 1572193323Sed 1573193323Sed SDValue Size = Tmp2.getOperand(1); 1574193323Sed SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1575193323Sed Chain = SP.getValue(1); 1576193323Sed unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1577218893Sdim unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 1578193323Sed if (Align > StackAlign) 1579193323Sed SP = DAG.getNode(ISD::AND, dl, VT, SP, 1580193323Sed DAG.getConstant(-(uint64_t)Align, VT)); 1581193323Sed Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1582193323Sed Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1583193323Sed 1584193323Sed Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1585193323Sed DAG.getIntPtrConstant(0, true), SDValue()); 1586193323Sed 1587193323Sed Results.push_back(Tmp1); 1588193323Sed Results.push_back(Tmp2); 1589193323Sed} 1590193323Sed 1591193323Sed/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 1592198396Srdivacky/// condition code CC on the current target. This routine expands SETCC with 1593193323Sed/// illegal condition code into AND / OR of multiple SETCC values. 1594198090Srdivackyvoid SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, 1595193323Sed SDValue &LHS, SDValue &RHS, 1596193323Sed SDValue &CC, 1597193323Sed DebugLoc dl) { 1598249423Sdim MVT OpVT = LHS.getSimpleValueType(); 1599193323Sed ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1600193323Sed switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1601234353Sdim default: llvm_unreachable("Unknown condition code action!"); 1602193323Sed case TargetLowering::Legal: 1603193323Sed // Nothing to do. 1604193323Sed break; 1605193323Sed case TargetLowering::Expand: { 1606193323Sed ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1607243830Sdim ISD::CondCode InvCC = ISD::SETCC_INVALID; 1608193323Sed unsigned Opc = 0; 1609193323Sed switch (CCCode) { 1610234353Sdim default: llvm_unreachable("Don't know how to expand this condition!"); 1611243830Sdim case ISD::SETO: 1612243830Sdim assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) 1613243830Sdim == TargetLowering::Legal 1614243830Sdim && "If SETO is expanded, SETOEQ must be legal!"); 1615243830Sdim CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; 1616243830Sdim case ISD::SETUO: 1617243830Sdim assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) 1618243830Sdim == TargetLowering::Legal 1619243830Sdim && "If SETUO is expanded, SETUNE must be legal!"); 1620243830Sdim CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; 1621243830Sdim case ISD::SETOEQ: 1622243830Sdim case ISD::SETOGT: 1623243830Sdim case ISD::SETOGE: 1624243830Sdim case ISD::SETOLT: 1625243830Sdim case ISD::SETOLE: 1626243830Sdim case ISD::SETONE: 1627243830Sdim case ISD::SETUEQ: 1628243830Sdim case ISD::SETUNE: 1629243830Sdim case ISD::SETUGT: 1630243830Sdim case ISD::SETUGE: 1631243830Sdim case ISD::SETULT: 1632243830Sdim case ISD::SETULE: 1633243830Sdim // If we are floating point, assign and break, otherwise fall through. 1634243830Sdim if (!OpVT.isInteger()) { 1635243830Sdim // We can use the 4th bit to tell if we are the unordered 1636243830Sdim // or ordered version of the opcode. 1637243830Sdim CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1638243830Sdim Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; 1639243830Sdim CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); 1640243830Sdim break; 1641243830Sdim } 1642243830Sdim // Fallthrough if we are unsigned integer. 1643243830Sdim case ISD::SETLE: 1644243830Sdim case ISD::SETGT: 1645243830Sdim case ISD::SETGE: 1646243830Sdim case ISD::SETLT: 1647243830Sdim case ISD::SETNE: 1648243830Sdim case ISD::SETEQ: 1649243830Sdim InvCC = ISD::getSetCCSwappedOperands(CCCode); 1650243830Sdim if (TLI.getCondCodeAction(InvCC, OpVT) == TargetLowering::Expand) { 1651243830Sdim // We only support using the inverted operation and not a 1652243830Sdim // different manner of supporting expanding these cases. 1653243830Sdim llvm_unreachable("Don't know how to expand this condition!"); 1654243830Sdim } 1655243830Sdim LHS = DAG.getSetCC(dl, VT, RHS, LHS, InvCC); 1656243830Sdim RHS = SDValue(); 1657243830Sdim CC = SDValue(); 1658243830Sdim return; 1659193323Sed } 1660243830Sdim 1661243830Sdim SDValue SetCC1, SetCC2; 1662243830Sdim if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { 1663243830Sdim // If we aren't the ordered or unorder operation, 1664243830Sdim // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). 1665243830Sdim SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 1666243830Sdim SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 1667243830Sdim } else { 1668243830Sdim // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) 1669243830Sdim SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); 1670243830Sdim SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); 1671243830Sdim } 1672193323Sed LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1673193323Sed RHS = SDValue(); 1674193323Sed CC = SDValue(); 1675193323Sed break; 1676193323Sed } 1677193323Sed } 1678193323Sed} 1679193323Sed 1680193323Sed/// EmitStackConvert - Emit a store/load combination to the stack. This stores 1681193323Sed/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1682193323Sed/// a load from the stack slot to DestVT, extending it if needed. 1683193323Sed/// The resultant code need not be legal. 1684193323SedSDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 1685198090Srdivacky EVT SlotVT, 1686198090Srdivacky EVT DestVT, 1687193323Sed DebugLoc dl) { 1688193323Sed // Create the stack frame object. 1689193323Sed unsigned SrcAlign = 1690243830Sdim TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType(). 1691198090Srdivacky getTypeForEVT(*DAG.getContext())); 1692193323Sed SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 1693193323Sed 1694193323Sed FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1695193323Sed int SPFI = StackPtrFI->getIndex(); 1696218893Sdim MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); 1697193323Sed 1698193323Sed unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 1699193323Sed unsigned SlotSize = SlotVT.getSizeInBits(); 1700193323Sed unsigned DestSize = DestVT.getSizeInBits(); 1701226633Sdim Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1702243830Sdim unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType); 1703193323Sed 1704193323Sed // Emit a store to the stack slot. Use a truncstore if the input value is 1705193323Sed // later than DestVT. 1706193323Sed SDValue Store; 1707193323Sed 1708193323Sed if (SrcSize > SlotSize) 1709193323Sed Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1710218893Sdim PtrInfo, SlotVT, false, false, SrcAlign); 1711193323Sed else { 1712193323Sed assert(SrcSize == SlotSize && "Invalid store"); 1713193323Sed Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1714218893Sdim PtrInfo, false, false, SrcAlign); 1715193323Sed } 1716193323Sed 1717193323Sed // Result is a load from the stack slot. 1718193323Sed if (SlotSize == DestSize) 1719218893Sdim return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, 1720234353Sdim false, false, false, DestAlign); 1721193323Sed 1722193323Sed assert(SlotSize < DestSize && "Unknown extension!"); 1723218893Sdim return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, 1724218893Sdim PtrInfo, SlotVT, false, false, DestAlign); 1725193323Sed} 1726193323Sed 1727193323SedSDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1728193323Sed DebugLoc dl = Node->getDebugLoc(); 1729193323Sed // Create a vector sized/aligned stack slot, store the value to element #0, 1730193323Sed // then load the whole vector back out. 1731193323Sed SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1732193323Sed 1733193323Sed FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1734193323Sed int SPFI = StackPtrFI->getIndex(); 1735193323Sed 1736193323Sed SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), 1737193323Sed StackPtr, 1738218893Sdim MachinePointerInfo::getFixedStack(SPFI), 1739203954Srdivacky Node->getValueType(0).getVectorElementType(), 1740203954Srdivacky false, false, 0); 1741193323Sed return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 1742218893Sdim MachinePointerInfo::getFixedStack(SPFI), 1743234353Sdim false, false, false, 0); 1744193323Sed} 1745193323Sed 1746193323Sed 1747193323Sed/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 1748193323Sed/// support the operation, but do support the resultant vector type. 1749193323SedSDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1750193323Sed unsigned NumElems = Node->getNumOperands(); 1751193630Sed SDValue Value1, Value2; 1752193323Sed DebugLoc dl = Node->getDebugLoc(); 1753198090Srdivacky EVT VT = Node->getValueType(0); 1754198090Srdivacky EVT OpVT = Node->getOperand(0).getValueType(); 1755198090Srdivacky EVT EltVT = VT.getVectorElementType(); 1756193323Sed 1757193323Sed // If the only non-undef value is the low element, turn this into a 1758193323Sed // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 1759193323Sed bool isOnlyLowElement = true; 1760193630Sed bool MoreThanTwoValues = false; 1761193323Sed bool isConstant = true; 1762193630Sed for (unsigned i = 0; i < NumElems; ++i) { 1763193323Sed SDValue V = Node->getOperand(i); 1764193630Sed if (V.getOpcode() == ISD::UNDEF) 1765193630Sed continue; 1766193630Sed if (i > 0) 1767193323Sed isOnlyLowElement = false; 1768193630Sed if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 1769193630Sed isConstant = false; 1770193323Sed 1771193630Sed if (!Value1.getNode()) { 1772193630Sed Value1 = V; 1773193630Sed } else if (!Value2.getNode()) { 1774193630Sed if (V != Value1) 1775193630Sed Value2 = V; 1776193630Sed } else if (V != Value1 && V != Value2) { 1777193630Sed MoreThanTwoValues = true; 1778193630Sed } 1779193323Sed } 1780193323Sed 1781193630Sed if (!Value1.getNode()) 1782193630Sed return DAG.getUNDEF(VT); 1783193630Sed 1784193630Sed if (isOnlyLowElement) 1785193323Sed return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 1786193323Sed 1787193323Sed // If all elements are constants, create a load from the constant pool. 1788193323Sed if (isConstant) { 1789234353Sdim SmallVector<Constant*, 16> CV; 1790193323Sed for (unsigned i = 0, e = NumElems; i != e; ++i) { 1791193323Sed if (ConstantFPSDNode *V = 1792193323Sed dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 1793193323Sed CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 1794193323Sed } else if (ConstantSDNode *V = 1795193323Sed dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 1796199481Srdivacky if (OpVT==EltVT) 1797199481Srdivacky CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 1798199481Srdivacky else { 1799199481Srdivacky // If OpVT and EltVT don't match, EltVT is not legal and the 1800199481Srdivacky // element values have been promoted/truncated earlier. Undo this; 1801199481Srdivacky // we don't want a v16i8 to become a v16i32 for example. 1802199481Srdivacky const ConstantInt *CI = V->getConstantIntValue(); 1803199481Srdivacky CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 1804199481Srdivacky CI->getZExtValue())); 1805199481Srdivacky } 1806193323Sed } else { 1807193323Sed assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 1808226633Sdim Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 1809193323Sed CV.push_back(UndefValue::get(OpNTy)); 1810193323Sed } 1811193323Sed } 1812193323Sed Constant *CP = ConstantVector::get(CV); 1813193323Sed SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 1814193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 1815193323Sed return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 1816218893Sdim MachinePointerInfo::getConstantPool(), 1817234353Sdim false, false, false, Alignment); 1818193323Sed } 1819193323Sed 1820193630Sed if (!MoreThanTwoValues) { 1821193630Sed SmallVector<int, 8> ShuffleVec(NumElems, -1); 1822193630Sed for (unsigned i = 0; i < NumElems; ++i) { 1823193630Sed SDValue V = Node->getOperand(i); 1824193630Sed if (V.getOpcode() == ISD::UNDEF) 1825193630Sed continue; 1826193630Sed ShuffleVec[i] = V == Value1 ? 0 : NumElems; 1827193630Sed } 1828193630Sed if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 1829193323Sed // Get the splatted value into the low element of a vector register. 1830193630Sed SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 1831193630Sed SDValue Vec2; 1832193630Sed if (Value2.getNode()) 1833193630Sed Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 1834193630Sed else 1835193630Sed Vec2 = DAG.getUNDEF(VT); 1836193323Sed 1837193323Sed // Return shuffle(LowValVec, undef, <0,0,0,0>) 1838193630Sed return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); 1839193323Sed } 1840193323Sed } 1841193323Sed 1842193574Sed // Otherwise, we can't handle this case efficiently. 1843193574Sed return ExpandVectorBuildThroughStack(Node); 1844193323Sed} 1845193323Sed 1846193323Sed// ExpandLibCall - Expand a node into a call to a libcall. If the result value 1847193323Sed// does not fit into a register, return the lo part and set the hi part to the 1848193323Sed// by-reg argument. If it does fit into a single register, return the result 1849193323Sed// and leave the Hi part unset. 1850193323SedSDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 1851193323Sed bool isSigned) { 1852193323Sed TargetLowering::ArgListTy Args; 1853193323Sed TargetLowering::ArgListEntry Entry; 1854193323Sed for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1855198090Srdivacky EVT ArgVT = Node->getOperand(i).getValueType(); 1856226633Sdim Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1857193323Sed Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 1858193323Sed Entry.isSExt = isSigned; 1859193323Sed Entry.isZExt = !isSigned; 1860193323Sed Args.push_back(Entry); 1861193323Sed } 1862193323Sed SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1863193323Sed TLI.getPointerTy()); 1864193323Sed 1865226633Sdim Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1866218893Sdim 1867234353Sdim // By default, the input chain to this libcall is the entry node of the 1868234353Sdim // function. If the libcall is going to be emitted as a tail call then 1869234353Sdim // TLI.isUsedByReturnOnly will change it to the right chain if the return 1870234353Sdim // node which is being folded has a non-entry input chain. 1871234353Sdim SDValue InChain = DAG.getEntryNode(); 1872234353Sdim 1873218893Sdim // isTailCall may be true since the callee does not reference caller stack 1874218893Sdim // frame. Check if it's in the right position. 1875234353Sdim SDValue TCChain = InChain; 1876249423Sdim bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain); 1877234353Sdim if (isTailCall) 1878234353Sdim InChain = TCChain; 1879234353Sdim 1880239462Sdim TargetLowering:: 1881239462Sdim CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 1882218893Sdim 0, TLI.getLibcallCallingConv(LC), isTailCall, 1883234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1884204642Srdivacky Callee, Args, DAG, Node->getDebugLoc()); 1885239462Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 1886193323Sed 1887239462Sdim 1888218893Sdim if (!CallInfo.second.getNode()) 1889218893Sdim // It's a tailcall, return the chain (which is the DAG root). 1890218893Sdim return DAG.getRoot(); 1891218893Sdim 1892221345Sdim return CallInfo.first; 1893221345Sdim} 1894221345Sdim 1895223017Sdim/// ExpandLibCall - Generate a libcall taking the given operands as arguments 1896221345Sdim/// and returning a result of type RetVT. 1897221345SdimSDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, 1898221345Sdim const SDValue *Ops, unsigned NumOps, 1899221345Sdim bool isSigned, DebugLoc dl) { 1900221345Sdim TargetLowering::ArgListTy Args; 1901221345Sdim Args.reserve(NumOps); 1902223017Sdim 1903221345Sdim TargetLowering::ArgListEntry Entry; 1904221345Sdim for (unsigned i = 0; i != NumOps; ++i) { 1905221345Sdim Entry.Node = Ops[i]; 1906221345Sdim Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); 1907221345Sdim Entry.isSExt = isSigned; 1908221345Sdim Entry.isZExt = !isSigned; 1909221345Sdim Args.push_back(Entry); 1910221345Sdim } 1911221345Sdim SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1912221345Sdim TLI.getPointerTy()); 1913223017Sdim 1914226633Sdim Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 1915239462Sdim TargetLowering:: 1916239462Sdim CallLoweringInfo CLI(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, 1917239462Sdim false, 0, TLI.getLibcallCallingConv(LC), 1918239462Sdim /*isTailCall=*/false, 1919234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1920221345Sdim Callee, Args, DAG, dl); 1921239462Sdim std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); 1922223017Sdim 1923193323Sed return CallInfo.first; 1924193323Sed} 1925193323Sed 1926210299Sed// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to 1927210299Sed// ExpandLibCall except that the first operand is the in-chain. 1928210299Sedstd::pair<SDValue, SDValue> 1929210299SedSelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, 1930210299Sed SDNode *Node, 1931210299Sed bool isSigned) { 1932210299Sed SDValue InChain = Node->getOperand(0); 1933210299Sed 1934210299Sed TargetLowering::ArgListTy Args; 1935210299Sed TargetLowering::ArgListEntry Entry; 1936210299Sed for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { 1937210299Sed EVT ArgVT = Node->getOperand(i).getValueType(); 1938226633Sdim Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1939210299Sed Entry.Node = Node->getOperand(i); 1940210299Sed Entry.Ty = ArgTy; 1941210299Sed Entry.isSExt = isSigned; 1942210299Sed Entry.isZExt = !isSigned; 1943210299Sed Args.push_back(Entry); 1944210299Sed } 1945210299Sed SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1946210299Sed TLI.getPointerTy()); 1947210299Sed 1948226633Sdim Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1949239462Sdim TargetLowering:: 1950239462Sdim CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 1951218893Sdim 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 1952234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1953210299Sed Callee, Args, DAG, Node->getDebugLoc()); 1954239462Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 1955210299Sed 1956210299Sed return CallInfo; 1957210299Sed} 1958210299Sed 1959193323SedSDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 1960193323Sed RTLIB::Libcall Call_F32, 1961193323Sed RTLIB::Libcall Call_F64, 1962193323Sed RTLIB::Libcall Call_F80, 1963249423Sdim RTLIB::Libcall Call_F128, 1964193323Sed RTLIB::Libcall Call_PPCF128) { 1965193323Sed RTLIB::Libcall LC; 1966198090Srdivacky switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1967234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 1968193323Sed case MVT::f32: LC = Call_F32; break; 1969193323Sed case MVT::f64: LC = Call_F64; break; 1970193323Sed case MVT::f80: LC = Call_F80; break; 1971249423Sdim case MVT::f128: LC = Call_F128; break; 1972193323Sed case MVT::ppcf128: LC = Call_PPCF128; break; 1973193323Sed } 1974193323Sed return ExpandLibCall(LC, Node, false); 1975193323Sed} 1976193323Sed 1977193323SedSDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 1978199481Srdivacky RTLIB::Libcall Call_I8, 1979193323Sed RTLIB::Libcall Call_I16, 1980193323Sed RTLIB::Libcall Call_I32, 1981193323Sed RTLIB::Libcall Call_I64, 1982193323Sed RTLIB::Libcall Call_I128) { 1983193323Sed RTLIB::Libcall LC; 1984198090Srdivacky switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 1985234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 1986199481Srdivacky case MVT::i8: LC = Call_I8; break; 1987199481Srdivacky case MVT::i16: LC = Call_I16; break; 1988199481Srdivacky case MVT::i32: LC = Call_I32; break; 1989199481Srdivacky case MVT::i64: LC = Call_I64; break; 1990193323Sed case MVT::i128: LC = Call_I128; break; 1991193323Sed } 1992193323Sed return ExpandLibCall(LC, Node, isSigned); 1993193323Sed} 1994193323Sed 1995221345Sdim/// isDivRemLibcallAvailable - Return true if divmod libcall is available. 1996221345Sdimstatic bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, 1997221345Sdim const TargetLowering &TLI) { 1998221345Sdim RTLIB::Libcall LC; 1999221345Sdim switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2000234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2001221345Sdim case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2002221345Sdim case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2003221345Sdim case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2004221345Sdim case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2005221345Sdim case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2006221345Sdim } 2007221345Sdim 2008221345Sdim return TLI.getLibcallName(LC) != 0; 2009221345Sdim} 2010221345Sdim 2011239462Sdim/// useDivRem - Only issue divrem libcall if both quotient and remainder are 2012221345Sdim/// needed. 2013239462Sdimstatic bool useDivRem(SDNode *Node, bool isSigned, bool isDIV) { 2014239462Sdim // The other use might have been replaced with a divrem already. 2015239462Sdim unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 2016221345Sdim unsigned OtherOpcode = 0; 2017221345Sdim if (isSigned) 2018221345Sdim OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; 2019221345Sdim else 2020221345Sdim OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; 2021221345Sdim 2022221345Sdim SDValue Op0 = Node->getOperand(0); 2023221345Sdim SDValue Op1 = Node->getOperand(1); 2024221345Sdim for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2025221345Sdim UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2026221345Sdim SDNode *User = *UI; 2027221345Sdim if (User == Node) 2028221345Sdim continue; 2029239462Sdim if ((User->getOpcode() == OtherOpcode || User->getOpcode() == DivRemOpc) && 2030221345Sdim User->getOperand(0) == Op0 && 2031221345Sdim User->getOperand(1) == Op1) 2032221345Sdim return true; 2033221345Sdim } 2034221345Sdim return false; 2035221345Sdim} 2036221345Sdim 2037221345Sdim/// ExpandDivRemLibCall - Issue libcalls to __{u}divmod to compute div / rem 2038221345Sdim/// pairs. 2039221345Sdimvoid 2040221345SdimSelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 2041221345Sdim SmallVectorImpl<SDValue> &Results) { 2042221345Sdim unsigned Opcode = Node->getOpcode(); 2043221345Sdim bool isSigned = Opcode == ISD::SDIVREM; 2044221345Sdim 2045221345Sdim RTLIB::Libcall LC; 2046221345Sdim switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2047234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2048221345Sdim case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2049221345Sdim case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2050221345Sdim case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2051221345Sdim case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2052221345Sdim case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2053221345Sdim } 2054221345Sdim 2055221345Sdim // The input chain to this libcall is the entry node of the function. 2056221345Sdim // Legalizing the call will automatically add the previous call to the 2057221345Sdim // dependence. 2058221345Sdim SDValue InChain = DAG.getEntryNode(); 2059221345Sdim 2060221345Sdim EVT RetVT = Node->getValueType(0); 2061226633Sdim Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2062221345Sdim 2063221345Sdim TargetLowering::ArgListTy Args; 2064221345Sdim TargetLowering::ArgListEntry Entry; 2065221345Sdim for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 2066221345Sdim EVT ArgVT = Node->getOperand(i).getValueType(); 2067226633Sdim Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2068221345Sdim Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 2069221345Sdim Entry.isSExt = isSigned; 2070221345Sdim Entry.isZExt = !isSigned; 2071221345Sdim Args.push_back(Entry); 2072221345Sdim } 2073221345Sdim 2074221345Sdim // Also pass the return address of the remainder. 2075221345Sdim SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 2076221345Sdim Entry.Node = FIPtr; 2077221345Sdim Entry.Ty = RetTy->getPointerTo(); 2078221345Sdim Entry.isSExt = isSigned; 2079221345Sdim Entry.isZExt = !isSigned; 2080221345Sdim Args.push_back(Entry); 2081221345Sdim 2082221345Sdim SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2083221345Sdim TLI.getPointerTy()); 2084221345Sdim 2085221345Sdim DebugLoc dl = Node->getDebugLoc(); 2086239462Sdim TargetLowering:: 2087239462Sdim CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 2088221345Sdim 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 2089234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2090234353Sdim Callee, Args, DAG, dl); 2091239462Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2092221345Sdim 2093221345Sdim // Remainder is loaded back from the stack frame. 2094234353Sdim SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, 2095234353Sdim MachinePointerInfo(), false, false, false, 0); 2096221345Sdim Results.push_back(CallInfo.first); 2097221345Sdim Results.push_back(Rem); 2098221345Sdim} 2099221345Sdim 2100249423Sdim/// isSinCosLibcallAvailable - Return true if sincos libcall is available. 2101249423Sdimstatic bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { 2102249423Sdim RTLIB::Libcall LC; 2103249423Sdim switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2104249423Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2105249423Sdim case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2106249423Sdim case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2107249423Sdim case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2108249423Sdim case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2109249423Sdim case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2110249423Sdim } 2111249423Sdim return TLI.getLibcallName(LC) != 0; 2112249423Sdim} 2113249423Sdim 2114249423Sdim/// canCombineSinCosLibcall - Return true if sincos libcall is available and 2115249423Sdim/// can be used to combine sin and cos. 2116249423Sdimstatic bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, 2117249423Sdim const TargetMachine &TM) { 2118249423Sdim if (!isSinCosLibcallAvailable(Node, TLI)) 2119249423Sdim return false; 2120249423Sdim // GNU sin/cos functions set errno while sincos does not. Therefore 2121249423Sdim // combining sin and cos is only safe if unsafe-fpmath is enabled. 2122249423Sdim bool isGNU = Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU; 2123249423Sdim if (isGNU && !TM.Options.UnsafeFPMath) 2124249423Sdim return false; 2125249423Sdim return true; 2126249423Sdim} 2127249423Sdim 2128249423Sdim/// useSinCos - Only issue sincos libcall if both sin and cos are 2129249423Sdim/// needed. 2130249423Sdimstatic bool useSinCos(SDNode *Node) { 2131249423Sdim unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN 2132249423Sdim ? ISD::FCOS : ISD::FSIN; 2133249423Sdim 2134249423Sdim SDValue Op0 = Node->getOperand(0); 2135249423Sdim for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2136249423Sdim UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2137249423Sdim SDNode *User = *UI; 2138249423Sdim if (User == Node) 2139249423Sdim continue; 2140249423Sdim // The other user might have been turned into sincos already. 2141249423Sdim if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) 2142249423Sdim return true; 2143249423Sdim } 2144249423Sdim return false; 2145249423Sdim} 2146249423Sdim 2147249423Sdim/// ExpandSinCosLibCall - Issue libcalls to sincos to compute sin / cos 2148249423Sdim/// pairs. 2149249423Sdimvoid 2150249423SdimSelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, 2151249423Sdim SmallVectorImpl<SDValue> &Results) { 2152249423Sdim RTLIB::Libcall LC; 2153249423Sdim switch (Node->getValueType(0).getSimpleVT().SimpleTy) { 2154249423Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2155249423Sdim case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2156249423Sdim case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2157249423Sdim case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2158249423Sdim case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2159249423Sdim case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2160249423Sdim } 2161249423Sdim 2162249423Sdim // The input chain to this libcall is the entry node of the function. 2163249423Sdim // Legalizing the call will automatically add the previous call to the 2164249423Sdim // dependence. 2165249423Sdim SDValue InChain = DAG.getEntryNode(); 2166249423Sdim 2167249423Sdim EVT RetVT = Node->getValueType(0); 2168249423Sdim Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2169249423Sdim 2170249423Sdim TargetLowering::ArgListTy Args; 2171249423Sdim TargetLowering::ArgListEntry Entry; 2172249423Sdim 2173249423Sdim // Pass the argument. 2174249423Sdim Entry.Node = Node->getOperand(0); 2175249423Sdim Entry.Ty = RetTy; 2176249423Sdim Entry.isSExt = false; 2177249423Sdim Entry.isZExt = false; 2178249423Sdim Args.push_back(Entry); 2179249423Sdim 2180249423Sdim // Pass the return address of sin. 2181249423Sdim SDValue SinPtr = DAG.CreateStackTemporary(RetVT); 2182249423Sdim Entry.Node = SinPtr; 2183249423Sdim Entry.Ty = RetTy->getPointerTo(); 2184249423Sdim Entry.isSExt = false; 2185249423Sdim Entry.isZExt = false; 2186249423Sdim Args.push_back(Entry); 2187249423Sdim 2188249423Sdim // Also pass the return address of the cos. 2189249423Sdim SDValue CosPtr = DAG.CreateStackTemporary(RetVT); 2190249423Sdim Entry.Node = CosPtr; 2191249423Sdim Entry.Ty = RetTy->getPointerTo(); 2192249423Sdim Entry.isSExt = false; 2193249423Sdim Entry.isZExt = false; 2194249423Sdim Args.push_back(Entry); 2195249423Sdim 2196249423Sdim SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2197249423Sdim TLI.getPointerTy()); 2198249423Sdim 2199249423Sdim DebugLoc dl = Node->getDebugLoc(); 2200249423Sdim TargetLowering:: 2201249423Sdim CallLoweringInfo CLI(InChain, Type::getVoidTy(*DAG.getContext()), 2202249423Sdim false, false, false, false, 2203249423Sdim 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 2204249423Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2205249423Sdim Callee, Args, DAG, dl); 2206249423Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2207249423Sdim 2208249423Sdim Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, 2209249423Sdim MachinePointerInfo(), false, false, false, 0)); 2210249423Sdim Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, 2211249423Sdim MachinePointerInfo(), false, false, false, 0)); 2212249423Sdim} 2213249423Sdim 2214193323Sed/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 2215193323Sed/// INT_TO_FP operation of the specified operand when the target requests that 2216193323Sed/// we expand it. At this point, we know that the result and operand types are 2217193323Sed/// legal for the target. 2218193323SedSDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 2219193323Sed SDValue Op0, 2220198090Srdivacky EVT DestVT, 2221193323Sed DebugLoc dl) { 2222243830Sdim if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { 2223193323Sed // simple 32-bit [signed|unsigned] integer to float/double expansion 2224193323Sed 2225193323Sed // Get the stack frame index of a 8 byte buffer. 2226193323Sed SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2227193323Sed 2228193323Sed // word offset constant for Hi/Lo address computation 2229193323Sed SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); 2230193323Sed // set up Hi and Lo (into buffer) address based on endian 2231193323Sed SDValue Hi = StackSlot; 2232193323Sed SDValue Lo = DAG.getNode(ISD::ADD, dl, 2233193323Sed TLI.getPointerTy(), StackSlot, WordOff); 2234193323Sed if (TLI.isLittleEndian()) 2235193323Sed std::swap(Hi, Lo); 2236193323Sed 2237193323Sed // if signed map to unsigned space 2238193323Sed SDValue Op0Mapped; 2239193323Sed if (isSigned) { 2240193323Sed // constant used to invert sign bit (signed to unsigned mapping) 2241193323Sed SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 2242193323Sed Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 2243193323Sed } else { 2244193323Sed Op0Mapped = Op0; 2245193323Sed } 2246193323Sed // store the lo of the constructed double - based on integer input 2247193323Sed SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 2248218893Sdim Op0Mapped, Lo, MachinePointerInfo(), 2249203954Srdivacky false, false, 0); 2250193323Sed // initial hi portion of constructed double 2251193323Sed SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 2252193323Sed // store the hi of the constructed double - biased exponent 2253218893Sdim SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi, 2254218893Sdim MachinePointerInfo(), 2255218893Sdim false, false, 0); 2256193323Sed // load the constructed double 2257218893Sdim SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, 2258234353Sdim MachinePointerInfo(), false, false, false, 0); 2259193323Sed // FP constant to bias correct the final result 2260193323Sed SDValue Bias = DAG.getConstantFP(isSigned ? 2261193323Sed BitsToDouble(0x4330000080000000ULL) : 2262193323Sed BitsToDouble(0x4330000000000000ULL), 2263193323Sed MVT::f64); 2264193323Sed // subtract the bias 2265193323Sed SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2266193323Sed // final result 2267193323Sed SDValue Result; 2268193323Sed // handle final rounding 2269193323Sed if (DestVT == MVT::f64) { 2270193323Sed // do nothing 2271193323Sed Result = Sub; 2272193323Sed } else if (DestVT.bitsLT(MVT::f64)) { 2273193323Sed Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 2274193323Sed DAG.getIntPtrConstant(0)); 2275193323Sed } else if (DestVT.bitsGT(MVT::f64)) { 2276193323Sed Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 2277193323Sed } 2278193323Sed return Result; 2279193323Sed } 2280193323Sed assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 2281208599Srdivacky // Code below here assumes !isSigned without checking again. 2282204792Srdivacky 2283204792Srdivacky // Implementation of unsigned i64 to f64 following the algorithm in 2284204792Srdivacky // __floatundidf in compiler_rt. This implementation has the advantage 2285204792Srdivacky // of performing rounding correctly, both in the default rounding mode 2286204792Srdivacky // and in all alternate rounding modes. 2287204792Srdivacky // TODO: Generalize this for use with other types. 2288204792Srdivacky if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) { 2289204792Srdivacky SDValue TwoP52 = 2290204792Srdivacky DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64); 2291204792Srdivacky SDValue TwoP84PlusTwoP52 = 2292204792Srdivacky DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64); 2293204792Srdivacky SDValue TwoP84 = 2294204792Srdivacky DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64); 2295204792Srdivacky 2296204792Srdivacky SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); 2297204792Srdivacky SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, 2298204792Srdivacky DAG.getConstant(32, MVT::i64)); 2299204792Srdivacky SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); 2300204792Srdivacky SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); 2301218893Sdim SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); 2302218893Sdim SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); 2303210299Sed SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, 2304210299Sed TwoP84PlusTwoP52); 2305204792Srdivacky return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); 2306204792Srdivacky } 2307204792Srdivacky 2308218893Sdim // Implementation of unsigned i64 to f32. 2309208599Srdivacky // TODO: Generalize this for use with other types. 2310208599Srdivacky if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { 2311218893Sdim // For unsigned conversions, convert them to signed conversions using the 2312218893Sdim // algorithm from the x86_64 __floatundidf in compiler_rt. 2313218893Sdim if (!isSigned) { 2314218893Sdim SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); 2315218893Sdim 2316219077Sdim SDValue ShiftConst = 2317219077Sdim DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType())); 2318218893Sdim SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); 2319218893Sdim SDValue AndConst = DAG.getConstant(1, MVT::i64); 2320218893Sdim SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); 2321218893Sdim SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); 2322218893Sdim 2323218893Sdim SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); 2324218893Sdim SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); 2325218893Sdim 2326218893Sdim // TODO: This really should be implemented using a branch rather than a 2327218893Sdim // select. We happen to get lucky and machinesink does the right 2328218893Sdim // thing most of the time. This would be a good candidate for a 2329218893Sdim //pseudo-op, or, even better, for whole-function isel. 2330218893Sdim SDValue SignBitTest = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2331218893Sdim Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); 2332218893Sdim return DAG.getNode(ISD::SELECT, dl, MVT::f32, SignBitTest, Slow, Fast); 2333218893Sdim } 2334218893Sdim 2335218893Sdim // Otherwise, implement the fully general conversion. 2336208599Srdivacky 2337210299Sed SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2338208599Srdivacky DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); 2339208599Srdivacky SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, 2340208599Srdivacky DAG.getConstant(UINT64_C(0x800), MVT::i64)); 2341210299Sed SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2342208599Srdivacky DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); 2343208599Srdivacky SDValue Ne = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2344208599Srdivacky And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); 2345208599Srdivacky SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0); 2346208599Srdivacky SDValue Ge = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), 2347208599Srdivacky Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), 2348218893Sdim ISD::SETUGE); 2349208599Srdivacky SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0); 2350219077Sdim EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType()); 2351208599Srdivacky 2352208599Srdivacky SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, 2353208599Srdivacky DAG.getConstant(32, SHVT)); 2354208599Srdivacky SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); 2355208599Srdivacky SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); 2356208599Srdivacky SDValue TwoP32 = 2357208599Srdivacky DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); 2358208599Srdivacky SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); 2359208599Srdivacky SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); 2360208599Srdivacky SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); 2361208599Srdivacky SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); 2362208599Srdivacky return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, 2363208599Srdivacky DAG.getIntPtrConstant(0)); 2364208599Srdivacky } 2365208599Srdivacky 2366193323Sed SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2367193323Sed 2368193323Sed SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()), 2369193323Sed Op0, DAG.getConstant(0, Op0.getValueType()), 2370193323Sed ISD::SETLT); 2371193323Sed SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 2372193323Sed SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), 2373193323Sed SignSet, Four, Zero); 2374193323Sed 2375193323Sed // If the sign bit of the integer is set, the large number will be treated 2376193323Sed // as a negative number. To counteract this, the dynamic code adds an 2377193323Sed // offset depending on the data type. 2378193323Sed uint64_t FF; 2379198090Srdivacky switch (Op0.getValueType().getSimpleVT().SimpleTy) { 2380234353Sdim default: llvm_unreachable("Unsupported integer type!"); 2381193323Sed case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2382193323Sed case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2383193323Sed case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2384193323Sed case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2385193323Sed } 2386193323Sed if (TLI.isLittleEndian()) FF <<= 32; 2387198090Srdivacky Constant *FudgeFactor = ConstantInt::get( 2388198090Srdivacky Type::getInt64Ty(*DAG.getContext()), FF); 2389193323Sed 2390193323Sed SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 2391193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 2392193323Sed CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); 2393193323Sed Alignment = std::min(Alignment, 4u); 2394193323Sed SDValue FudgeInReg; 2395193323Sed if (DestVT == MVT::f32) 2396193323Sed FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2397218893Sdim MachinePointerInfo::getConstantPool(), 2398234353Sdim false, false, false, Alignment); 2399193323Sed else { 2400234353Sdim SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 2401234353Sdim DAG.getEntryNode(), CPIdx, 2402234353Sdim MachinePointerInfo::getConstantPool(), 2403234353Sdim MVT::f32, false, false, Alignment); 2404234353Sdim HandleSDNode Handle(Load); 2405234353Sdim LegalizeOp(Load.getNode()); 2406234353Sdim FudgeInReg = Handle.getValue(); 2407193323Sed } 2408193323Sed 2409193323Sed return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2410193323Sed} 2411193323Sed 2412193323Sed/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 2413193323Sed/// *INT_TO_FP operation of the specified operand when the target requests that 2414193323Sed/// we promote it. At this point, we know that the result and operand types are 2415193323Sed/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2416193323Sed/// operation that takes a larger input. 2417193323SedSDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 2418198090Srdivacky EVT DestVT, 2419193323Sed bool isSigned, 2420193323Sed DebugLoc dl) { 2421193323Sed // First step, figure out the appropriate *INT_TO_FP operation to use. 2422198090Srdivacky EVT NewInTy = LegalOp.getValueType(); 2423193323Sed 2424193323Sed unsigned OpToUse = 0; 2425193323Sed 2426193323Sed // Scan for the appropriate larger type to use. 2427193323Sed while (1) { 2428198090Srdivacky NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2429193323Sed assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2430193323Sed 2431193323Sed // If the target supports SINT_TO_FP of this type, use it. 2432193323Sed if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { 2433193323Sed OpToUse = ISD::SINT_TO_FP; 2434193323Sed break; 2435193323Sed } 2436193323Sed if (isSigned) continue; 2437193323Sed 2438193323Sed // If the target supports UINT_TO_FP of this type, use it. 2439193323Sed if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { 2440193323Sed OpToUse = ISD::UINT_TO_FP; 2441193323Sed break; 2442193323Sed } 2443193323Sed 2444193323Sed // Otherwise, try a larger type. 2445193323Sed } 2446193323Sed 2447193323Sed // Okay, we found the operation and type to use. Zero extend our input to the 2448193323Sed // desired type then run the operation on it. 2449193323Sed return DAG.getNode(OpToUse, dl, DestVT, 2450193323Sed DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2451193323Sed dl, NewInTy, LegalOp)); 2452193323Sed} 2453193323Sed 2454193323Sed/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 2455193323Sed/// FP_TO_*INT operation of the specified operand when the target requests that 2456193323Sed/// we promote it. At this point, we know that the result and operand types are 2457193323Sed/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2458193323Sed/// operation that returns a larger result. 2459193323SedSDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 2460198090Srdivacky EVT DestVT, 2461193323Sed bool isSigned, 2462193323Sed DebugLoc dl) { 2463193323Sed // First step, figure out the appropriate FP_TO*INT operation to use. 2464198090Srdivacky EVT NewOutTy = DestVT; 2465193323Sed 2466193323Sed unsigned OpToUse = 0; 2467193323Sed 2468193323Sed // Scan for the appropriate larger type to use. 2469193323Sed while (1) { 2470198090Srdivacky NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2471193323Sed assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2472193323Sed 2473193323Sed if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { 2474193323Sed OpToUse = ISD::FP_TO_SINT; 2475193323Sed break; 2476193323Sed } 2477193323Sed 2478193323Sed if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { 2479193323Sed OpToUse = ISD::FP_TO_UINT; 2480193323Sed break; 2481193323Sed } 2482193323Sed 2483193323Sed // Otherwise, try a larger type. 2484193323Sed } 2485193323Sed 2486193323Sed 2487193323Sed // Okay, we found the operation and type to use. 2488193323Sed SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2489193323Sed 2490193323Sed // Truncate the result of the extended FP_TO_*INT operation to the desired 2491193323Sed // size. 2492193323Sed return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2493193323Sed} 2494193323Sed 2495193323Sed/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 2496193323Sed/// 2497193323SedSDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { 2498198090Srdivacky EVT VT = Op.getValueType(); 2499219077Sdim EVT SHVT = TLI.getShiftAmountTy(VT); 2500193323Sed SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2501198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 2502234353Sdim default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2503193323Sed case MVT::i16: 2504193323Sed Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2505193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2506193323Sed return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 2507193323Sed case MVT::i32: 2508193323Sed Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2509193323Sed Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2510193323Sed Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2511193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2512193323Sed Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 2513193323Sed Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 2514193323Sed Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2515193323Sed Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2516193323Sed return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2517193323Sed case MVT::i64: 2518193323Sed Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2519193323Sed Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2520193323Sed Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2521193323Sed Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2522193323Sed Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2523193323Sed Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2524193323Sed Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2525193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2526193323Sed Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 2527193323Sed Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 2528193323Sed Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 2529193323Sed Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 2530193323Sed Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 2531193323Sed Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 2532193323Sed Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2533193323Sed Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2534193323Sed Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2535193323Sed Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2536193323Sed Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2537193323Sed Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2538193323Sed return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2539193323Sed } 2540193323Sed} 2541193323Sed 2542193323Sed/// ExpandBitCount - Expand the specified bitcount instruction into operations. 2543193323Sed/// 2544193323SedSDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 2545193323Sed DebugLoc dl) { 2546193323Sed switch (Opc) { 2547234353Sdim default: llvm_unreachable("Cannot expand this yet!"); 2548193323Sed case ISD::CTPOP: { 2549198090Srdivacky EVT VT = Op.getValueType(); 2550219077Sdim EVT ShVT = TLI.getShiftAmountTy(VT); 2551218893Sdim unsigned Len = VT.getSizeInBits(); 2552218893Sdim 2553218893Sdim assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && 2554218893Sdim "CTPOP not implemented for this type."); 2555218893Sdim 2556218893Sdim // This is the "best" algorithm from 2557218893Sdim // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 2558218893Sdim 2559249423Sdim SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), VT); 2560249423Sdim SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), VT); 2561249423Sdim SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), VT); 2562249423Sdim SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), VT); 2563218893Sdim 2564218893Sdim // v = v - ((v >> 1) & 0x55555555...) 2565218893Sdim Op = DAG.getNode(ISD::SUB, dl, VT, Op, 2566218893Sdim DAG.getNode(ISD::AND, dl, VT, 2567218893Sdim DAG.getNode(ISD::SRL, dl, VT, Op, 2568218893Sdim DAG.getConstant(1, ShVT)), 2569218893Sdim Mask55)); 2570218893Sdim // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) 2571218893Sdim Op = DAG.getNode(ISD::ADD, dl, VT, 2572218893Sdim DAG.getNode(ISD::AND, dl, VT, Op, Mask33), 2573218893Sdim DAG.getNode(ISD::AND, dl, VT, 2574218893Sdim DAG.getNode(ISD::SRL, dl, VT, Op, 2575218893Sdim DAG.getConstant(2, ShVT)), 2576218893Sdim Mask33)); 2577218893Sdim // v = (v + (v >> 4)) & 0x0F0F0F0F... 2578218893Sdim Op = DAG.getNode(ISD::AND, dl, VT, 2579218893Sdim DAG.getNode(ISD::ADD, dl, VT, Op, 2580218893Sdim DAG.getNode(ISD::SRL, dl, VT, Op, 2581218893Sdim DAG.getConstant(4, ShVT))), 2582218893Sdim Mask0F); 2583218893Sdim // v = (v * 0x01010101...) >> (Len - 8) 2584218893Sdim Op = DAG.getNode(ISD::SRL, dl, VT, 2585218893Sdim DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), 2586218893Sdim DAG.getConstant(Len - 8, ShVT)); 2587219077Sdim 2588193323Sed return Op; 2589193323Sed } 2590234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2591234353Sdim // This trivially expands to CTLZ. 2592234353Sdim return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); 2593193323Sed case ISD::CTLZ: { 2594193323Sed // for now, we do this: 2595193323Sed // x = x | (x >> 1); 2596193323Sed // x = x | (x >> 2); 2597193323Sed // ... 2598193323Sed // x = x | (x >>16); 2599193323Sed // x = x | (x >>32); // for 64-bit input 2600193323Sed // return popcount(~x); 2601193323Sed // 2602193323Sed // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 2603198090Srdivacky EVT VT = Op.getValueType(); 2604219077Sdim EVT ShVT = TLI.getShiftAmountTy(VT); 2605193323Sed unsigned len = VT.getSizeInBits(); 2606193323Sed for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 2607193323Sed SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 2608193323Sed Op = DAG.getNode(ISD::OR, dl, VT, Op, 2609193323Sed DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); 2610193323Sed } 2611193323Sed Op = DAG.getNOT(dl, Op, VT); 2612193323Sed return DAG.getNode(ISD::CTPOP, dl, VT, Op); 2613193323Sed } 2614234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2615234353Sdim // This trivially expands to CTTZ. 2616234353Sdim return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); 2617193323Sed case ISD::CTTZ: { 2618193323Sed // for now, we use: { return popcount(~x & (x - 1)); } 2619193323Sed // unless the target has ctlz but not ctpop, in which case we use: 2620193323Sed // { return 32 - nlz(~x & (x-1)); } 2621193323Sed // see also http://www.hackersdelight.org/HDcode/ntz.cc 2622198090Srdivacky EVT VT = Op.getValueType(); 2623193323Sed SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 2624193323Sed DAG.getNOT(dl, Op, VT), 2625193323Sed DAG.getNode(ISD::SUB, dl, VT, Op, 2626193323Sed DAG.getConstant(1, VT))); 2627193323Sed // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 2628193323Sed if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 2629193323Sed TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 2630193323Sed return DAG.getNode(ISD::SUB, dl, VT, 2631193323Sed DAG.getConstant(VT.getSizeInBits(), VT), 2632193323Sed DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 2633193323Sed return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 2634193323Sed } 2635193323Sed } 2636193323Sed} 2637193323Sed 2638210299Sedstd::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { 2639210299Sed unsigned Opc = Node->getOpcode(); 2640210299Sed MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2641210299Sed RTLIB::Libcall LC; 2642210299Sed 2643210299Sed switch (Opc) { 2644210299Sed default: 2645210299Sed llvm_unreachable("Unhandled atomic intrinsic Expand!"); 2646210299Sed case ISD::ATOMIC_SWAP: 2647210299Sed switch (VT.SimpleTy) { 2648210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2649210299Sed case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 2650210299Sed case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 2651210299Sed case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 2652210299Sed case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 2653210299Sed } 2654210299Sed break; 2655210299Sed case ISD::ATOMIC_CMP_SWAP: 2656210299Sed switch (VT.SimpleTy) { 2657210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2658210299Sed case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 2659210299Sed case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 2660210299Sed case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 2661210299Sed case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 2662210299Sed } 2663210299Sed break; 2664210299Sed case ISD::ATOMIC_LOAD_ADD: 2665210299Sed switch (VT.SimpleTy) { 2666210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2667210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 2668210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 2669210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 2670210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 2671210299Sed } 2672210299Sed break; 2673210299Sed case ISD::ATOMIC_LOAD_SUB: 2674210299Sed switch (VT.SimpleTy) { 2675210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2676210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 2677210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 2678210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 2679210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 2680210299Sed } 2681210299Sed break; 2682210299Sed case ISD::ATOMIC_LOAD_AND: 2683210299Sed switch (VT.SimpleTy) { 2684210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2685210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 2686210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 2687210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 2688210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 2689210299Sed } 2690210299Sed break; 2691210299Sed case ISD::ATOMIC_LOAD_OR: 2692210299Sed switch (VT.SimpleTy) { 2693210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2694210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 2695210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 2696210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 2697210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 2698210299Sed } 2699210299Sed break; 2700210299Sed case ISD::ATOMIC_LOAD_XOR: 2701210299Sed switch (VT.SimpleTy) { 2702210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2703210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 2704210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 2705210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 2706210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 2707210299Sed } 2708210299Sed break; 2709210299Sed case ISD::ATOMIC_LOAD_NAND: 2710210299Sed switch (VT.SimpleTy) { 2711210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2712210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 2713210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 2714210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 2715210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 2716210299Sed } 2717210299Sed break; 2718210299Sed } 2719210299Sed 2720210299Sed return ExpandChainLibCall(LC, Node, false); 2721210299Sed} 2722210299Sed 2723234353Sdimvoid SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2724234353Sdim SmallVector<SDValue, 8> Results; 2725193323Sed DebugLoc dl = Node->getDebugLoc(); 2726193323Sed SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2727193323Sed switch (Node->getOpcode()) { 2728193323Sed case ISD::CTPOP: 2729193323Sed case ISD::CTLZ: 2730234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2731193323Sed case ISD::CTTZ: 2732234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2733193323Sed Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); 2734193323Sed Results.push_back(Tmp1); 2735193323Sed break; 2736193323Sed case ISD::BSWAP: 2737193323Sed Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2738193323Sed break; 2739193323Sed case ISD::FRAMEADDR: 2740193323Sed case ISD::RETURNADDR: 2741193323Sed case ISD::FRAME_TO_ARGS_OFFSET: 2742193323Sed Results.push_back(DAG.getConstant(0, Node->getValueType(0))); 2743193323Sed break; 2744193323Sed case ISD::FLT_ROUNDS_: 2745193323Sed Results.push_back(DAG.getConstant(1, Node->getValueType(0))); 2746193323Sed break; 2747193323Sed case ISD::EH_RETURN: 2748193323Sed case ISD::EH_LABEL: 2749193323Sed case ISD::PREFETCH: 2750193323Sed case ISD::VAEND: 2751210299Sed case ISD::EH_SJLJ_LONGJMP: 2752218893Sdim // If the target didn't expand these, there's nothing to do, so just 2753218893Sdim // preserve the chain and be done. 2754193323Sed Results.push_back(Node->getOperand(0)); 2755193323Sed break; 2756210299Sed case ISD::EH_SJLJ_SETJMP: 2757218893Sdim // If the target didn't expand this, just return 'zero' and preserve the 2758218893Sdim // chain. 2759210299Sed Results.push_back(DAG.getConstant(0, MVT::i32)); 2760210299Sed Results.push_back(Node->getOperand(0)); 2761210299Sed break; 2762226633Sdim case ISD::ATOMIC_FENCE: 2763210299Sed case ISD::MEMBARRIER: { 2764210299Sed // If the target didn't lower this, lower it to '__sync_synchronize()' call 2765226633Sdim // FIXME: handle "fence singlethread" more efficiently. 2766210299Sed TargetLowering::ArgListTy Args; 2767239462Sdim TargetLowering:: 2768239462Sdim CallLoweringInfo CLI(Node->getOperand(0), 2769239462Sdim Type::getVoidTy(*DAG.getContext()), 2770218893Sdim false, false, false, false, 0, CallingConv::C, 2771218893Sdim /*isTailCall=*/false, 2772234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2773210299Sed DAG.getExternalSymbol("__sync_synchronize", 2774210299Sed TLI.getPointerTy()), 2775210299Sed Args, DAG, dl); 2776239462Sdim std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 2777239462Sdim 2778210299Sed Results.push_back(CallResult.second); 2779210299Sed break; 2780210299Sed } 2781226633Sdim case ISD::ATOMIC_LOAD: { 2782226633Sdim // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 2783226633Sdim SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); 2784226633Sdim SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 2785226633Sdim cast<AtomicSDNode>(Node)->getMemoryVT(), 2786226633Sdim Node->getOperand(0), 2787226633Sdim Node->getOperand(1), Zero, Zero, 2788226633Sdim cast<AtomicSDNode>(Node)->getMemOperand(), 2789226633Sdim cast<AtomicSDNode>(Node)->getOrdering(), 2790226633Sdim cast<AtomicSDNode>(Node)->getSynchScope()); 2791226633Sdim Results.push_back(Swap.getValue(0)); 2792226633Sdim Results.push_back(Swap.getValue(1)); 2793226633Sdim break; 2794226633Sdim } 2795226633Sdim case ISD::ATOMIC_STORE: { 2796226633Sdim // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 2797226633Sdim SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2798226633Sdim cast<AtomicSDNode>(Node)->getMemoryVT(), 2799226633Sdim Node->getOperand(0), 2800226633Sdim Node->getOperand(1), Node->getOperand(2), 2801226633Sdim cast<AtomicSDNode>(Node)->getMemOperand(), 2802226633Sdim cast<AtomicSDNode>(Node)->getOrdering(), 2803226633Sdim cast<AtomicSDNode>(Node)->getSynchScope()); 2804226633Sdim Results.push_back(Swap.getValue(1)); 2805226633Sdim break; 2806226633Sdim } 2807210299Sed // By default, atomic intrinsics are marked Legal and lowered. Targets 2808210299Sed // which don't support them directly, however, may want libcalls, in which 2809210299Sed // case they mark them Expand, and we get here. 2810210299Sed case ISD::ATOMIC_SWAP: 2811210299Sed case ISD::ATOMIC_LOAD_ADD: 2812210299Sed case ISD::ATOMIC_LOAD_SUB: 2813210299Sed case ISD::ATOMIC_LOAD_AND: 2814210299Sed case ISD::ATOMIC_LOAD_OR: 2815210299Sed case ISD::ATOMIC_LOAD_XOR: 2816210299Sed case ISD::ATOMIC_LOAD_NAND: 2817210299Sed case ISD::ATOMIC_LOAD_MIN: 2818210299Sed case ISD::ATOMIC_LOAD_MAX: 2819210299Sed case ISD::ATOMIC_LOAD_UMIN: 2820210299Sed case ISD::ATOMIC_LOAD_UMAX: 2821210299Sed case ISD::ATOMIC_CMP_SWAP: { 2822210299Sed std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node); 2823210299Sed Results.push_back(Tmp.first); 2824210299Sed Results.push_back(Tmp.second); 2825210299Sed break; 2826210299Sed } 2827193323Sed case ISD::DYNAMIC_STACKALLOC: 2828193323Sed ExpandDYNAMIC_STACKALLOC(Node, Results); 2829193323Sed break; 2830193323Sed case ISD::MERGE_VALUES: 2831193323Sed for (unsigned i = 0; i < Node->getNumValues(); i++) 2832193323Sed Results.push_back(Node->getOperand(i)); 2833193323Sed break; 2834193323Sed case ISD::UNDEF: { 2835198090Srdivacky EVT VT = Node->getValueType(0); 2836193323Sed if (VT.isInteger()) 2837193323Sed Results.push_back(DAG.getConstant(0, VT)); 2838207618Srdivacky else { 2839207618Srdivacky assert(VT.isFloatingPoint() && "Unknown value type!"); 2840193323Sed Results.push_back(DAG.getConstantFP(0, VT)); 2841207618Srdivacky } 2842193323Sed break; 2843193323Sed } 2844193323Sed case ISD::TRAP: { 2845193323Sed // If this operation is not supported, lower it to 'abort()' call 2846193323Sed TargetLowering::ArgListTy Args; 2847239462Sdim TargetLowering:: 2848239462Sdim CallLoweringInfo CLI(Node->getOperand(0), 2849239462Sdim Type::getVoidTy(*DAG.getContext()), 2850218893Sdim false, false, false, false, 0, CallingConv::C, 2851218893Sdim /*isTailCall=*/false, 2852234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2853193323Sed DAG.getExternalSymbol("abort", TLI.getPointerTy()), 2854204642Srdivacky Args, DAG, dl); 2855239462Sdim std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 2856239462Sdim 2857193323Sed Results.push_back(CallResult.second); 2858193323Sed break; 2859193323Sed } 2860193323Sed case ISD::FP_ROUND: 2861218893Sdim case ISD::BITCAST: 2862193323Sed Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 2863193323Sed Node->getValueType(0), dl); 2864193323Sed Results.push_back(Tmp1); 2865193323Sed break; 2866193323Sed case ISD::FP_EXTEND: 2867193323Sed Tmp1 = EmitStackConvert(Node->getOperand(0), 2868193323Sed Node->getOperand(0).getValueType(), 2869193323Sed Node->getValueType(0), dl); 2870193323Sed Results.push_back(Tmp1); 2871193323Sed break; 2872193323Sed case ISD::SIGN_EXTEND_INREG: { 2873193323Sed // NOTE: we could fall back on load/store here too for targets without 2874193323Sed // SAR. However, it is doubtful that any exist. 2875198090Srdivacky EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2876200581Srdivacky EVT VT = Node->getValueType(0); 2877219077Sdim EVT ShiftAmountTy = TLI.getShiftAmountTy(VT); 2878202375Srdivacky if (VT.isVector()) 2879200581Srdivacky ShiftAmountTy = VT; 2880202375Srdivacky unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 2881202375Srdivacky ExtraVT.getScalarType().getSizeInBits(); 2882200581Srdivacky SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy); 2883193323Sed Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 2884193323Sed Node->getOperand(0), ShiftCst); 2885193323Sed Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 2886193323Sed Results.push_back(Tmp1); 2887193323Sed break; 2888193323Sed } 2889193323Sed case ISD::FP_ROUND_INREG: { 2890193323Sed // The only way we can lower this is to turn it into a TRUNCSTORE, 2891221345Sdim // EXTLOAD pair, targeting a temporary location (a stack slot). 2892193323Sed 2893193323Sed // NOTE: there is a choice here between constantly creating new stack 2894193323Sed // slots and always reusing the same one. We currently always create 2895193323Sed // new ones, as reuse may inhibit scheduling. 2896198090Srdivacky EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2897193323Sed Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, 2898193323Sed Node->getValueType(0), dl); 2899193323Sed Results.push_back(Tmp1); 2900193323Sed break; 2901193323Sed } 2902193323Sed case ISD::SINT_TO_FP: 2903193323Sed case ISD::UINT_TO_FP: 2904193323Sed Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, 2905193323Sed Node->getOperand(0), Node->getValueType(0), dl); 2906193323Sed Results.push_back(Tmp1); 2907193323Sed break; 2908193323Sed case ISD::FP_TO_UINT: { 2909193323Sed SDValue True, False; 2910198090Srdivacky EVT VT = Node->getOperand(0).getValueType(); 2911198090Srdivacky EVT NVT = Node->getValueType(0); 2912249423Sdim APFloat apf(DAG.EVTToAPFloatSemantics(VT), 2913249423Sdim APInt::getNullValue(VT.getSizeInBits())); 2914193323Sed APInt x = APInt::getSignBit(NVT.getSizeInBits()); 2915193323Sed (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 2916193323Sed Tmp1 = DAG.getConstantFP(apf, VT); 2917193323Sed Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 2918193323Sed Node->getOperand(0), 2919193323Sed Tmp1, ISD::SETLT); 2920193323Sed True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 2921193323Sed False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 2922193323Sed DAG.getNode(ISD::FSUB, dl, VT, 2923193323Sed Node->getOperand(0), Tmp1)); 2924193323Sed False = DAG.getNode(ISD::XOR, dl, NVT, False, 2925193323Sed DAG.getConstant(x, NVT)); 2926193323Sed Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, True, False); 2927193323Sed Results.push_back(Tmp1); 2928193323Sed break; 2929193323Sed } 2930193323Sed case ISD::VAARG: { 2931193323Sed const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 2932198090Srdivacky EVT VT = Node->getValueType(0); 2933193323Sed Tmp1 = Node->getOperand(0); 2934193323Sed Tmp2 = Node->getOperand(1); 2935210299Sed unsigned Align = Node->getConstantOperandVal(3); 2936210299Sed 2937218893Sdim SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, 2938234353Sdim MachinePointerInfo(V), 2939234353Sdim false, false, false, 0); 2940210299Sed SDValue VAList = VAListLoad; 2941210299Sed 2942210299Sed if (Align > TLI.getMinStackArgumentAlignment()) { 2943210299Sed assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); 2944210299Sed 2945210299Sed VAList = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 2946210299Sed DAG.getConstant(Align - 1, 2947210299Sed TLI.getPointerTy())); 2948210299Sed 2949210299Sed VAList = DAG.getNode(ISD::AND, dl, TLI.getPointerTy(), VAList, 2950218893Sdim DAG.getConstant(-(int64_t)Align, 2951210299Sed TLI.getPointerTy())); 2952210299Sed } 2953210299Sed 2954193323Sed // Increment the pointer, VAList, to the next vaarg 2955193323Sed Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, 2956243830Sdim DAG.getConstant(TLI.getDataLayout()-> 2957207618Srdivacky getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), 2958193323Sed TLI.getPointerTy())); 2959193323Sed // Store the incremented VAList to the legalized pointer 2960218893Sdim Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, 2961218893Sdim MachinePointerInfo(V), false, false, 0); 2962193323Sed // Load the actual argument out of the pointer VAList 2963218893Sdim Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), 2964234353Sdim false, false, false, 0)); 2965193323Sed Results.push_back(Results[0].getValue(1)); 2966193323Sed break; 2967193323Sed } 2968193323Sed case ISD::VACOPY: { 2969193323Sed // This defaults to loading a pointer from the input and storing it to the 2970193323Sed // output, returning the chain. 2971193323Sed const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 2972193323Sed const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 2973193323Sed Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), 2974218893Sdim Node->getOperand(2), MachinePointerInfo(VS), 2975234353Sdim false, false, false, 0); 2976218893Sdim Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 2977218893Sdim MachinePointerInfo(VD), false, false, 0); 2978193323Sed Results.push_back(Tmp1); 2979193323Sed break; 2980193323Sed } 2981193323Sed case ISD::EXTRACT_VECTOR_ELT: 2982193323Sed if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 2983193323Sed // This must be an access of the only element. Return it. 2984218893Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 2985193323Sed Node->getOperand(0)); 2986193323Sed else 2987193323Sed Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 2988193323Sed Results.push_back(Tmp1); 2989193323Sed break; 2990193323Sed case ISD::EXTRACT_SUBVECTOR: 2991193323Sed Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 2992193323Sed break; 2993218893Sdim case ISD::INSERT_SUBVECTOR: 2994218893Sdim Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 2995218893Sdim break; 2996193323Sed case ISD::CONCAT_VECTORS: { 2997193574Sed Results.push_back(ExpandVectorBuildThroughStack(Node)); 2998193323Sed break; 2999193323Sed } 3000193323Sed case ISD::SCALAR_TO_VECTOR: 3001193323Sed Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 3002193323Sed break; 3003193323Sed case ISD::INSERT_VECTOR_ELT: 3004193323Sed Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 3005193323Sed Node->getOperand(1), 3006193323Sed Node->getOperand(2), dl)); 3007193323Sed break; 3008193323Sed case ISD::VECTOR_SHUFFLE: { 3009234353Sdim SmallVector<int, 32> NewMask; 3010234353Sdim ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3011193323Sed 3012198090Srdivacky EVT VT = Node->getValueType(0); 3013198090Srdivacky EVT EltVT = VT.getVectorElementType(); 3014234353Sdim SDValue Op0 = Node->getOperand(0); 3015234353Sdim SDValue Op1 = Node->getOperand(1); 3016234353Sdim if (!TLI.isTypeLegal(EltVT)) { 3017234353Sdim 3018234353Sdim EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 3019234353Sdim 3020234353Sdim // BUILD_VECTOR operands are allowed to be wider than the element type. 3021234353Sdim // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept it 3022234353Sdim if (NewEltVT.bitsLT(EltVT)) { 3023234353Sdim 3024234353Sdim // Convert shuffle node. 3025234353Sdim // If original node was v4i64 and the new EltVT is i32, 3026234353Sdim // cast operands to v8i32 and re-build the mask. 3027234353Sdim 3028234353Sdim // Calculate new VT, the size of the new VT should be equal to original. 3029234353Sdim EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, 3030234353Sdim VT.getSizeInBits()/NewEltVT.getSizeInBits()); 3031234353Sdim assert(NewVT.bitsEq(VT)); 3032234353Sdim 3033234353Sdim // cast operands to new VT 3034234353Sdim Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 3035234353Sdim Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 3036234353Sdim 3037234353Sdim // Convert the shuffle mask 3038234353Sdim unsigned int factor = NewVT.getVectorNumElements()/VT.getVectorNumElements(); 3039234353Sdim 3040234353Sdim // EltVT gets smaller 3041234353Sdim assert(factor > 0); 3042234353Sdim 3043234353Sdim for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 3044234353Sdim if (Mask[i] < 0) { 3045234353Sdim for (unsigned fi = 0; fi < factor; ++fi) 3046234353Sdim NewMask.push_back(Mask[i]); 3047234353Sdim } 3048234353Sdim else { 3049234353Sdim for (unsigned fi = 0; fi < factor; ++fi) 3050234353Sdim NewMask.push_back(Mask[i]*factor+fi); 3051234353Sdim } 3052234353Sdim } 3053234353Sdim Mask = NewMask; 3054234353Sdim VT = NewVT; 3055234353Sdim } 3056234353Sdim EltVT = NewEltVT; 3057234353Sdim } 3058193323Sed unsigned NumElems = VT.getVectorNumElements(); 3059234353Sdim SmallVector<SDValue, 16> Ops; 3060193323Sed for (unsigned i = 0; i != NumElems; ++i) { 3061193323Sed if (Mask[i] < 0) { 3062193323Sed Ops.push_back(DAG.getUNDEF(EltVT)); 3063193323Sed continue; 3064193323Sed } 3065193323Sed unsigned Idx = Mask[i]; 3066193323Sed if (Idx < NumElems) 3067193323Sed Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 3068234353Sdim Op0, 3069193323Sed DAG.getIntPtrConstant(Idx))); 3070193323Sed else 3071193323Sed Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 3072234353Sdim Op1, 3073193323Sed DAG.getIntPtrConstant(Idx - NumElems))); 3074193323Sed } 3075234353Sdim 3076193323Sed Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); 3077234353Sdim // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 3078234353Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 3079193323Sed Results.push_back(Tmp1); 3080193323Sed break; 3081193323Sed } 3082193323Sed case ISD::EXTRACT_ELEMENT: { 3083198090Srdivacky EVT OpTy = Node->getOperand(0).getValueType(); 3084193323Sed if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 3085193323Sed // 1 -> Hi 3086193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 3087193323Sed DAG.getConstant(OpTy.getSizeInBits()/2, 3088219077Sdim TLI.getShiftAmountTy(Node->getOperand(0).getValueType()))); 3089193323Sed Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 3090193323Sed } else { 3091193323Sed // 0 -> Lo 3092193323Sed Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 3093193323Sed Node->getOperand(0)); 3094193323Sed } 3095193323Sed Results.push_back(Tmp1); 3096193323Sed break; 3097193323Sed } 3098193323Sed case ISD::STACKSAVE: 3099193323Sed // Expand to CopyFromReg if the target set 3100193323Sed // StackPointerRegisterToSaveRestore. 3101193323Sed if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3102193323Sed Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 3103193323Sed Node->getValueType(0))); 3104193323Sed Results.push_back(Results[0].getValue(1)); 3105193323Sed } else { 3106193323Sed Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 3107193323Sed Results.push_back(Node->getOperand(0)); 3108193323Sed } 3109193323Sed break; 3110193323Sed case ISD::STACKRESTORE: 3111193323Sed // Expand to CopyToReg if the target set 3112193323Sed // StackPointerRegisterToSaveRestore. 3113193323Sed if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3114193323Sed Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 3115193323Sed Node->getOperand(1))); 3116193323Sed } else { 3117193323Sed Results.push_back(Node->getOperand(0)); 3118193323Sed } 3119193323Sed break; 3120193323Sed case ISD::FCOPYSIGN: 3121193323Sed Results.push_back(ExpandFCOPYSIGN(Node)); 3122193323Sed break; 3123193323Sed case ISD::FNEG: 3124193323Sed // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3125193323Sed Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3126193323Sed Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, 3127193323Sed Node->getOperand(0)); 3128193323Sed Results.push_back(Tmp1); 3129193323Sed break; 3130193323Sed case ISD::FABS: { 3131193323Sed // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3132198090Srdivacky EVT VT = Node->getValueType(0); 3133193323Sed Tmp1 = Node->getOperand(0); 3134193323Sed Tmp2 = DAG.getConstantFP(0.0, VT); 3135193323Sed Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), 3136193323Sed Tmp1, Tmp2, ISD::SETUGT); 3137193323Sed Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 3138193323Sed Tmp1 = DAG.getNode(ISD::SELECT, dl, VT, Tmp2, Tmp1, Tmp3); 3139193323Sed Results.push_back(Tmp1); 3140193323Sed break; 3141193323Sed } 3142193323Sed case ISD::FSQRT: 3143193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3144249423Sdim RTLIB::SQRT_F80, RTLIB::SQRT_F128, 3145249423Sdim RTLIB::SQRT_PPCF128)); 3146193323Sed break; 3147193323Sed case ISD::FSIN: 3148249423Sdim case ISD::FCOS: { 3149249423Sdim EVT VT = Node->getValueType(0); 3150249423Sdim bool isSIN = Node->getOpcode() == ISD::FSIN; 3151249423Sdim // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / 3152249423Sdim // fcos which share the same operand and both are used. 3153249423Sdim if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || 3154249423Sdim canCombineSinCosLibcall(Node, TLI, TM)) 3155249423Sdim && useSinCos(Node)) { 3156249423Sdim SDVTList VTs = DAG.getVTList(VT, VT); 3157249423Sdim Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); 3158249423Sdim if (!isSIN) 3159249423Sdim Tmp1 = Tmp1.getValue(1); 3160249423Sdim Results.push_back(Tmp1); 3161249423Sdim } else if (isSIN) { 3162249423Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 3163249423Sdim RTLIB::SIN_F80, RTLIB::SIN_F128, 3164249423Sdim RTLIB::SIN_PPCF128)); 3165249423Sdim } else { 3166249423Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 3167249423Sdim RTLIB::COS_F80, RTLIB::COS_F128, 3168249423Sdim RTLIB::COS_PPCF128)); 3169249423Sdim } 3170193323Sed break; 3171249423Sdim } 3172249423Sdim case ISD::FSINCOS: 3173249423Sdim // Expand into sincos libcall. 3174249423Sdim ExpandSinCosLibCall(Node, Results); 3175193323Sed break; 3176193323Sed case ISD::FLOG: 3177193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, 3178249423Sdim RTLIB::LOG_F80, RTLIB::LOG_F128, 3179249423Sdim RTLIB::LOG_PPCF128)); 3180193323Sed break; 3181193323Sed case ISD::FLOG2: 3182193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3183249423Sdim RTLIB::LOG2_F80, RTLIB::LOG2_F128, 3184249423Sdim RTLIB::LOG2_PPCF128)); 3185193323Sed break; 3186193323Sed case ISD::FLOG10: 3187193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3188249423Sdim RTLIB::LOG10_F80, RTLIB::LOG10_F128, 3189249423Sdim RTLIB::LOG10_PPCF128)); 3190193323Sed break; 3191193323Sed case ISD::FEXP: 3192193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, 3193249423Sdim RTLIB::EXP_F80, RTLIB::EXP_F128, 3194249423Sdim RTLIB::EXP_PPCF128)); 3195193323Sed break; 3196193323Sed case ISD::FEXP2: 3197193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3198249423Sdim RTLIB::EXP2_F80, RTLIB::EXP2_F128, 3199249423Sdim RTLIB::EXP2_PPCF128)); 3200193323Sed break; 3201193323Sed case ISD::FTRUNC: 3202193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3203249423Sdim RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 3204249423Sdim RTLIB::TRUNC_PPCF128)); 3205193323Sed break; 3206193323Sed case ISD::FFLOOR: 3207193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3208249423Sdim RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 3209249423Sdim RTLIB::FLOOR_PPCF128)); 3210193323Sed break; 3211193323Sed case ISD::FCEIL: 3212193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3213249423Sdim RTLIB::CEIL_F80, RTLIB::CEIL_F128, 3214249423Sdim RTLIB::CEIL_PPCF128)); 3215193323Sed break; 3216193323Sed case ISD::FRINT: 3217193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 3218249423Sdim RTLIB::RINT_F80, RTLIB::RINT_F128, 3219249423Sdim RTLIB::RINT_PPCF128)); 3220193323Sed break; 3221193323Sed case ISD::FNEARBYINT: 3222193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 3223193323Sed RTLIB::NEARBYINT_F64, 3224193323Sed RTLIB::NEARBYINT_F80, 3225249423Sdim RTLIB::NEARBYINT_F128, 3226193323Sed RTLIB::NEARBYINT_PPCF128)); 3227193323Sed break; 3228193323Sed case ISD::FPOWI: 3229193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 3230249423Sdim RTLIB::POWI_F80, RTLIB::POWI_F128, 3231249423Sdim RTLIB::POWI_PPCF128)); 3232193323Sed break; 3233193323Sed case ISD::FPOW: 3234193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, 3235249423Sdim RTLIB::POW_F80, RTLIB::POW_F128, 3236249423Sdim RTLIB::POW_PPCF128)); 3237193323Sed break; 3238193323Sed case ISD::FDIV: 3239193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 3240249423Sdim RTLIB::DIV_F80, RTLIB::DIV_F128, 3241249423Sdim RTLIB::DIV_PPCF128)); 3242193323Sed break; 3243193323Sed case ISD::FREM: 3244193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 3245249423Sdim RTLIB::REM_F80, RTLIB::REM_F128, 3246249423Sdim RTLIB::REM_PPCF128)); 3247193323Sed break; 3248224145Sdim case ISD::FMA: 3249224145Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 3250249423Sdim RTLIB::FMA_F80, RTLIB::FMA_F128, 3251249423Sdim RTLIB::FMA_PPCF128)); 3252224145Sdim break; 3253205218Srdivacky case ISD::FP16_TO_FP32: 3254205218Srdivacky Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 3255205218Srdivacky break; 3256205218Srdivacky case ISD::FP32_TO_FP16: 3257205218Srdivacky Results.push_back(ExpandLibCall(RTLIB::FPROUND_F32_F16, Node, false)); 3258205218Srdivacky break; 3259193323Sed case ISD::ConstantFP: { 3260193323Sed ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3261193323Sed // Check to see if this FP immediate is already legal. 3262193323Sed // If this is a legal constant, turn it into a TargetConstantFP node. 3263234353Sdim if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) 3264234353Sdim Results.push_back(ExpandConstantFP(CFP, true)); 3265193323Sed break; 3266193323Sed } 3267193323Sed case ISD::EHSELECTION: { 3268193323Sed unsigned Reg = TLI.getExceptionSelectorRegister(); 3269193323Sed assert(Reg && "Can't expand to unknown register!"); 3270193323Sed Results.push_back(DAG.getCopyFromReg(Node->getOperand(1), dl, Reg, 3271193323Sed Node->getValueType(0))); 3272193323Sed Results.push_back(Results[0].getValue(1)); 3273193323Sed break; 3274193323Sed } 3275193323Sed case ISD::EXCEPTIONADDR: { 3276234353Sdim unsigned Reg = TLI.getExceptionPointerRegister(); 3277193323Sed assert(Reg && "Can't expand to unknown register!"); 3278193323Sed Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, Reg, 3279193323Sed Node->getValueType(0))); 3280193323Sed Results.push_back(Results[0].getValue(1)); 3281193323Sed break; 3282193323Sed } 3283234353Sdim case ISD::FSUB: { 3284234353Sdim EVT VT = Node->getValueType(0); 3285234353Sdim assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3286234353Sdim TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && 3287234353Sdim "Don't know how to expand this FP subtraction!"); 3288234353Sdim Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3289234353Sdim Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); 3290234353Sdim Results.push_back(Tmp1); 3291234353Sdim break; 3292234353Sdim } 3293193323Sed case ISD::SUB: { 3294198090Srdivacky EVT VT = Node->getValueType(0); 3295193323Sed assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3296193323Sed TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3297193323Sed "Don't know how to expand this subtraction!"); 3298193323Sed Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3299193323Sed DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT)); 3300239462Sdim Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT)); 3301193323Sed Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3302193323Sed break; 3303193323Sed } 3304193323Sed case ISD::UREM: 3305193323Sed case ISD::SREM: { 3306198090Srdivacky EVT VT = Node->getValueType(0); 3307193323Sed bool isSigned = Node->getOpcode() == ISD::SREM; 3308193323Sed unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; 3309193323Sed unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3310193323Sed Tmp2 = Node->getOperand(0); 3311193323Sed Tmp3 = Node->getOperand(1); 3312221345Sdim if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3313221345Sdim (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3314243830Sdim // If div is legal, it's better to do the normal expansion 3315243830Sdim !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) && 3316239462Sdim useDivRem(Node, isSigned, false))) { 3317249423Sdim SDVTList VTs = DAG.getVTList(VT, VT); 3318193323Sed Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); 3319193323Sed } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { 3320193323Sed // X % Y -> X-X/Y*Y 3321193323Sed Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); 3322193323Sed Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); 3323193323Sed Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); 3324221345Sdim } else if (isSigned) 3325199481Srdivacky Tmp1 = ExpandIntLibCall(Node, true, 3326199481Srdivacky RTLIB::SREM_I8, 3327199481Srdivacky RTLIB::SREM_I16, RTLIB::SREM_I32, 3328193323Sed RTLIB::SREM_I64, RTLIB::SREM_I128); 3329221345Sdim else 3330199481Srdivacky Tmp1 = ExpandIntLibCall(Node, false, 3331199481Srdivacky RTLIB::UREM_I8, 3332199481Srdivacky RTLIB::UREM_I16, RTLIB::UREM_I32, 3333193323Sed RTLIB::UREM_I64, RTLIB::UREM_I128); 3334193323Sed Results.push_back(Tmp1); 3335193323Sed break; 3336193323Sed } 3337193323Sed case ISD::UDIV: 3338193323Sed case ISD::SDIV: { 3339193323Sed bool isSigned = Node->getOpcode() == ISD::SDIV; 3340193323Sed unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3341198090Srdivacky EVT VT = Node->getValueType(0); 3342193323Sed SDVTList VTs = DAG.getVTList(VT, VT); 3343221345Sdim if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3344221345Sdim (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3345239462Sdim useDivRem(Node, isSigned, true))) 3346193323Sed Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3347193323Sed Node->getOperand(1)); 3348193323Sed else if (isSigned) 3349199481Srdivacky Tmp1 = ExpandIntLibCall(Node, true, 3350199481Srdivacky RTLIB::SDIV_I8, 3351199481Srdivacky RTLIB::SDIV_I16, RTLIB::SDIV_I32, 3352193323Sed RTLIB::SDIV_I64, RTLIB::SDIV_I128); 3353193323Sed else 3354199481Srdivacky Tmp1 = ExpandIntLibCall(Node, false, 3355199481Srdivacky RTLIB::UDIV_I8, 3356199481Srdivacky RTLIB::UDIV_I16, RTLIB::UDIV_I32, 3357193323Sed RTLIB::UDIV_I64, RTLIB::UDIV_I128); 3358193323Sed Results.push_back(Tmp1); 3359193323Sed break; 3360193323Sed } 3361193323Sed case ISD::MULHU: 3362193323Sed case ISD::MULHS: { 3363193323Sed unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : 3364193323Sed ISD::SMUL_LOHI; 3365198090Srdivacky EVT VT = Node->getValueType(0); 3366193323Sed SDVTList VTs = DAG.getVTList(VT, VT); 3367193323Sed assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && 3368193323Sed "If this wasn't legal, it shouldn't have been created!"); 3369193323Sed Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3370193323Sed Node->getOperand(1)); 3371193323Sed Results.push_back(Tmp1.getValue(1)); 3372193323Sed break; 3373193323Sed } 3374221345Sdim case ISD::SDIVREM: 3375221345Sdim case ISD::UDIVREM: 3376221345Sdim // Expand into divrem libcall 3377221345Sdim ExpandDivRemLibCall(Node, Results); 3378221345Sdim break; 3379193323Sed case ISD::MUL: { 3380198090Srdivacky EVT VT = Node->getValueType(0); 3381193323Sed SDVTList VTs = DAG.getVTList(VT, VT); 3382193323Sed // See if multiply or divide can be lowered using two-result operations. 3383193323Sed // We just need the low half of the multiply; try both the signed 3384193323Sed // and unsigned forms. If the target supports both SMUL_LOHI and 3385193323Sed // UMUL_LOHI, form a preference by checking which forms of plain 3386193323Sed // MULH it supports. 3387193323Sed bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3388193323Sed bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3389193323Sed bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3390193323Sed bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3391193323Sed unsigned OpToUse = 0; 3392193323Sed if (HasSMUL_LOHI && !HasMULHS) { 3393193323Sed OpToUse = ISD::SMUL_LOHI; 3394193323Sed } else if (HasUMUL_LOHI && !HasMULHU) { 3395193323Sed OpToUse = ISD::UMUL_LOHI; 3396193323Sed } else if (HasSMUL_LOHI) { 3397193323Sed OpToUse = ISD::SMUL_LOHI; 3398193323Sed } else if (HasUMUL_LOHI) { 3399193323Sed OpToUse = ISD::UMUL_LOHI; 3400193323Sed } 3401193323Sed if (OpToUse) { 3402193323Sed Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3403193323Sed Node->getOperand(1))); 3404193323Sed break; 3405193323Sed } 3406199481Srdivacky Tmp1 = ExpandIntLibCall(Node, false, 3407199481Srdivacky RTLIB::MUL_I8, 3408199481Srdivacky RTLIB::MUL_I16, RTLIB::MUL_I32, 3409193323Sed RTLIB::MUL_I64, RTLIB::MUL_I128); 3410193323Sed Results.push_back(Tmp1); 3411193323Sed break; 3412193323Sed } 3413193323Sed case ISD::SADDO: 3414193323Sed case ISD::SSUBO: { 3415193323Sed SDValue LHS = Node->getOperand(0); 3416193323Sed SDValue RHS = Node->getOperand(1); 3417193323Sed SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3418193323Sed ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3419193323Sed LHS, RHS); 3420193323Sed Results.push_back(Sum); 3421198090Srdivacky EVT OType = Node->getValueType(1); 3422193323Sed 3423193323Sed SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 3424193323Sed 3425193323Sed // LHSSign -> LHS >= 0 3426193323Sed // RHSSign -> RHS >= 0 3427193323Sed // SumSign -> Sum >= 0 3428193323Sed // 3429193323Sed // Add: 3430193323Sed // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3431193323Sed // Sub: 3432193323Sed // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3433193323Sed // 3434193323Sed SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3435193323Sed SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3436193323Sed SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3437193323Sed Node->getOpcode() == ISD::SADDO ? 3438193323Sed ISD::SETEQ : ISD::SETNE); 3439193323Sed 3440193323Sed SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3441193323Sed SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3442193323Sed 3443193323Sed SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3444193323Sed Results.push_back(Cmp); 3445193323Sed break; 3446193323Sed } 3447193323Sed case ISD::UADDO: 3448193323Sed case ISD::USUBO: { 3449193323Sed SDValue LHS = Node->getOperand(0); 3450193323Sed SDValue RHS = Node->getOperand(1); 3451193323Sed SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 3452193323Sed ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3453193323Sed LHS, RHS); 3454193323Sed Results.push_back(Sum); 3455193323Sed Results.push_back(DAG.getSetCC(dl, Node->getValueType(1), Sum, LHS, 3456193323Sed Node->getOpcode () == ISD::UADDO ? 3457193323Sed ISD::SETULT : ISD::SETUGT)); 3458193323Sed break; 3459193323Sed } 3460194612Sed case ISD::UMULO: 3461194612Sed case ISD::SMULO: { 3462198090Srdivacky EVT VT = Node->getValueType(0); 3463221345Sdim EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); 3464194612Sed SDValue LHS = Node->getOperand(0); 3465194612Sed SDValue RHS = Node->getOperand(1); 3466194612Sed SDValue BottomHalf; 3467194612Sed SDValue TopHalf; 3468201360Srdivacky static const unsigned Ops[2][3] = 3469194612Sed { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, 3470194612Sed { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; 3471194612Sed bool isSigned = Node->getOpcode() == ISD::SMULO; 3472194612Sed if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { 3473194612Sed BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3474194612Sed TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); 3475194612Sed } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { 3476194612Sed BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, 3477194612Sed RHS); 3478194612Sed TopHalf = BottomHalf.getValue(1); 3479218893Sdim } else if (TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(), 3480218893Sdim VT.getSizeInBits() * 2))) { 3481194612Sed LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); 3482194612Sed RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); 3483194612Sed Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 3484194612Sed BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3485194612Sed DAG.getIntPtrConstant(0)); 3486194612Sed TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3487194612Sed DAG.getIntPtrConstant(1)); 3488218893Sdim } else { 3489218893Sdim // We can fall back to a libcall with an illegal type for the MUL if we 3490218893Sdim // have a libcall big enough. 3491218893Sdim // Also, we can fall back to a division in some cases, but that's a big 3492218893Sdim // performance hit in the general case. 3493218893Sdim RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3494218893Sdim if (WideVT == MVT::i16) 3495218893Sdim LC = RTLIB::MUL_I16; 3496218893Sdim else if (WideVT == MVT::i32) 3497218893Sdim LC = RTLIB::MUL_I32; 3498218893Sdim else if (WideVT == MVT::i64) 3499218893Sdim LC = RTLIB::MUL_I64; 3500218893Sdim else if (WideVT == MVT::i128) 3501218893Sdim LC = RTLIB::MUL_I128; 3502218893Sdim assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); 3503223017Sdim 3504223017Sdim // The high part is obtained by SRA'ing all but one of the bits of low 3505221345Sdim // part. 3506221345Sdim unsigned LoSize = VT.getSizeInBits(); 3507221345Sdim SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, 3508221345Sdim DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3509221345Sdim SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, 3510221345Sdim DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3511219077Sdim 3512221345Sdim // Here we're passing the 2 arguments explicitly as 4 arguments that are 3513221345Sdim // pre-lowered to the correct types. This all depends upon WideVT not 3514221345Sdim // being a legal type for the architecture and thus has to be split to 3515221345Sdim // two arguments. 3516221345Sdim SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; 3517221345Sdim SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); 3518221345Sdim BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3519221345Sdim DAG.getIntPtrConstant(0)); 3520221345Sdim TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3521221345Sdim DAG.getIntPtrConstant(1)); 3522234353Sdim // Ret is a node with an illegal type. Because such things are not 3523234353Sdim // generally permitted during this phase of legalization, delete the 3524234353Sdim // node. The above EXTRACT_ELEMENT nodes should have been folded. 3525234353Sdim DAG.DeleteNode(Ret.getNode()); 3526194612Sed } 3527223017Sdim 3528194612Sed if (isSigned) { 3529219077Sdim Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, 3530219077Sdim TLI.getShiftAmountTy(BottomHalf.getValueType())); 3531194612Sed Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); 3532194612Sed TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, Tmp1, 3533194612Sed ISD::SETNE); 3534194612Sed } else { 3535194612Sed TopHalf = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), TopHalf, 3536194612Sed DAG.getConstant(0, VT), ISD::SETNE); 3537194612Sed } 3538194612Sed Results.push_back(BottomHalf); 3539194612Sed Results.push_back(TopHalf); 3540194612Sed break; 3541194612Sed } 3542193323Sed case ISD::BUILD_PAIR: { 3543198090Srdivacky EVT PairTy = Node->getValueType(0); 3544193323Sed Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3545193323Sed Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3546193323Sed Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3547193323Sed DAG.getConstant(PairTy.getSizeInBits()/2, 3548219077Sdim TLI.getShiftAmountTy(PairTy))); 3549193323Sed Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3550193323Sed break; 3551193323Sed } 3552193323Sed case ISD::SELECT: 3553193323Sed Tmp1 = Node->getOperand(0); 3554193323Sed Tmp2 = Node->getOperand(1); 3555193323Sed Tmp3 = Node->getOperand(2); 3556193323Sed if (Tmp1.getOpcode() == ISD::SETCC) { 3557193323Sed Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3558193323Sed Tmp2, Tmp3, 3559193323Sed cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3560193323Sed } else { 3561193323Sed Tmp1 = DAG.getSelectCC(dl, Tmp1, 3562193323Sed DAG.getConstant(0, Tmp1.getValueType()), 3563193323Sed Tmp2, Tmp3, ISD::SETNE); 3564193323Sed } 3565193323Sed Results.push_back(Tmp1); 3566193323Sed break; 3567193323Sed case ISD::BR_JT: { 3568193323Sed SDValue Chain = Node->getOperand(0); 3569193323Sed SDValue Table = Node->getOperand(1); 3570193323Sed SDValue Index = Node->getOperand(2); 3571193323Sed 3572198090Srdivacky EVT PTy = TLI.getPointerTy(); 3573203954Srdivacky 3574243830Sdim const DataLayout &TD = *TLI.getDataLayout(); 3575203954Srdivacky unsigned EntrySize = 3576203954Srdivacky DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3577210299Sed 3578203954Srdivacky Index = DAG.getNode(ISD::MUL, dl, PTy, 3579193323Sed Index, DAG.getConstant(EntrySize, PTy)); 3580193323Sed SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table); 3581193323Sed 3582198090Srdivacky EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3583218893Sdim SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3584218893Sdim MachinePointerInfo::getJumpTable(), MemVT, 3585203954Srdivacky false, false, 0); 3586193323Sed Addr = LD; 3587207618Srdivacky if (TM.getRelocationModel() == Reloc::PIC_) { 3588193323Sed // For PIC, the sequence is: 3589193323Sed // BRIND(load(Jumptable + index) + RelocBase) 3590193323Sed // RelocBase can be JumpTable, GOT or some sort of global base. 3591193323Sed Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3592193323Sed TLI.getPICJumpTableRelocBase(Table, DAG)); 3593193323Sed } 3594193323Sed Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 3595193323Sed Results.push_back(Tmp1); 3596193323Sed break; 3597193323Sed } 3598193323Sed case ISD::BRCOND: 3599193323Sed // Expand brcond's setcc into its constituent parts and create a BR_CC 3600193323Sed // Node. 3601193323Sed Tmp1 = Node->getOperand(0); 3602193323Sed Tmp2 = Node->getOperand(1); 3603193323Sed if (Tmp2.getOpcode() == ISD::SETCC) { 3604193323Sed Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3605193323Sed Tmp1, Tmp2.getOperand(2), 3606193323Sed Tmp2.getOperand(0), Tmp2.getOperand(1), 3607193323Sed Node->getOperand(2)); 3608193323Sed } else { 3609223017Sdim // We test only the i1 bit. Skip the AND if UNDEF. 3610223017Sdim Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : 3611223017Sdim DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3612223017Sdim DAG.getConstant(1, Tmp2.getValueType())); 3613193323Sed Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3614223017Sdim DAG.getCondCode(ISD::SETNE), Tmp3, 3615223017Sdim DAG.getConstant(0, Tmp3.getValueType()), 3616193323Sed Node->getOperand(2)); 3617193323Sed } 3618193323Sed Results.push_back(Tmp1); 3619193323Sed break; 3620193323Sed case ISD::SETCC: { 3621193323Sed Tmp1 = Node->getOperand(0); 3622193323Sed Tmp2 = Node->getOperand(1); 3623193323Sed Tmp3 = Node->getOperand(2); 3624193323Sed LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, Tmp3, dl); 3625193323Sed 3626193323Sed // If we expanded the SETCC into an AND/OR, return the new node 3627193323Sed if (Tmp2.getNode() == 0) { 3628193323Sed Results.push_back(Tmp1); 3629193323Sed break; 3630193323Sed } 3631193323Sed 3632193323Sed // Otherwise, SETCC for the given comparison type must be completely 3633193323Sed // illegal; expand it into a SELECT_CC. 3634198090Srdivacky EVT VT = Node->getValueType(0); 3635249423Sdim int TrueValue; 3636249423Sdim switch (TLI.getBooleanContents(VT.isVector())) { 3637249423Sdim case TargetLowering::ZeroOrOneBooleanContent: 3638249423Sdim case TargetLowering::UndefinedBooleanContent: 3639249423Sdim TrueValue = 1; 3640249423Sdim break; 3641249423Sdim case TargetLowering::ZeroOrNegativeOneBooleanContent: 3642249423Sdim TrueValue = -1; 3643249423Sdim break; 3644249423Sdim } 3645193323Sed Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3646249423Sdim DAG.getConstant(TrueValue, VT), DAG.getConstant(0, VT), 3647249423Sdim Tmp3); 3648193323Sed Results.push_back(Tmp1); 3649193323Sed break; 3650193323Sed } 3651193323Sed case ISD::SELECT_CC: { 3652193323Sed Tmp1 = Node->getOperand(0); // LHS 3653193323Sed Tmp2 = Node->getOperand(1); // RHS 3654193323Sed Tmp3 = Node->getOperand(2); // True 3655193323Sed Tmp4 = Node->getOperand(3); // False 3656193323Sed SDValue CC = Node->getOperand(4); 3657193323Sed 3658193323Sed LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp1.getValueType()), 3659193323Sed Tmp1, Tmp2, CC, dl); 3660193323Sed 3661193323Sed assert(!Tmp2.getNode() && "Can't legalize SELECT_CC with legal condition!"); 3662193323Sed Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 3663193323Sed CC = DAG.getCondCode(ISD::SETNE); 3664193323Sed Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2, 3665193323Sed Tmp3, Tmp4, CC); 3666193323Sed Results.push_back(Tmp1); 3667193323Sed break; 3668193323Sed } 3669193323Sed case ISD::BR_CC: { 3670193323Sed Tmp1 = Node->getOperand(0); // Chain 3671193323Sed Tmp2 = Node->getOperand(2); // LHS 3672193323Sed Tmp3 = Node->getOperand(3); // RHS 3673193323Sed Tmp4 = Node->getOperand(1); // CC 3674193323Sed 3675193323Sed LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp2.getValueType()), 3676193323Sed Tmp2, Tmp3, Tmp4, dl); 3677193323Sed 3678193323Sed assert(!Tmp3.getNode() && "Can't legalize BR_CC with legal condition!"); 3679193323Sed Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 3680193323Sed Tmp4 = DAG.getCondCode(ISD::SETNE); 3681193323Sed Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2, 3682193323Sed Tmp3, Node->getOperand(4)); 3683193323Sed Results.push_back(Tmp1); 3684193323Sed break; 3685193323Sed } 3686234353Sdim case ISD::BUILD_VECTOR: 3687234353Sdim Results.push_back(ExpandBUILD_VECTOR(Node)); 3688234353Sdim break; 3689234353Sdim case ISD::SRA: 3690234353Sdim case ISD::SRL: 3691234353Sdim case ISD::SHL: { 3692234353Sdim // Scalarize vector SRA/SRL/SHL. 3693234353Sdim EVT VT = Node->getValueType(0); 3694234353Sdim assert(VT.isVector() && "Unable to legalize non-vector shift"); 3695234353Sdim assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 3696234353Sdim unsigned NumElem = VT.getVectorNumElements(); 3697234353Sdim 3698234353Sdim SmallVector<SDValue, 8> Scalars; 3699234353Sdim for (unsigned Idx = 0; Idx < NumElem; Idx++) { 3700234353Sdim SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3701234353Sdim VT.getScalarType(), 3702234353Sdim Node->getOperand(0), DAG.getIntPtrConstant(Idx)); 3703234353Sdim SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3704234353Sdim VT.getScalarType(), 3705234353Sdim Node->getOperand(1), DAG.getIntPtrConstant(Idx)); 3706234353Sdim Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 3707234353Sdim VT.getScalarType(), Ex, Sh)); 3708234353Sdim } 3709234353Sdim SDValue Result = 3710234353Sdim DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), 3711234353Sdim &Scalars[0], Scalars.size()); 3712234353Sdim ReplaceNode(SDValue(Node, 0), Result); 3713234353Sdim break; 3714234353Sdim } 3715193323Sed case ISD::GLOBAL_OFFSET_TABLE: 3716193323Sed case ISD::GlobalAddress: 3717193323Sed case ISD::GlobalTLSAddress: 3718193323Sed case ISD::ExternalSymbol: 3719193323Sed case ISD::ConstantPool: 3720193323Sed case ISD::JumpTable: 3721193323Sed case ISD::INTRINSIC_W_CHAIN: 3722193323Sed case ISD::INTRINSIC_WO_CHAIN: 3723193323Sed case ISD::INTRINSIC_VOID: 3724193323Sed // FIXME: Custom lowering for these operations shouldn't return null! 3725193323Sed break; 3726193323Sed } 3727234353Sdim 3728234353Sdim // Replace the original node with the legalized result. 3729234353Sdim if (!Results.empty()) 3730234353Sdim ReplaceNode(Node, Results.data()); 3731193323Sed} 3732234353Sdim 3733234353Sdimvoid SelectionDAGLegalize::PromoteNode(SDNode *Node) { 3734234353Sdim SmallVector<SDValue, 8> Results; 3735249423Sdim MVT OVT = Node->getSimpleValueType(0); 3736193323Sed if (Node->getOpcode() == ISD::UINT_TO_FP || 3737198090Srdivacky Node->getOpcode() == ISD::SINT_TO_FP || 3738198090Srdivacky Node->getOpcode() == ISD::SETCC) { 3739249423Sdim OVT = Node->getOperand(0).getSimpleValueType(); 3740193323Sed } 3741249423Sdim MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3742193323Sed DebugLoc dl = Node->getDebugLoc(); 3743193323Sed SDValue Tmp1, Tmp2, Tmp3; 3744193323Sed switch (Node->getOpcode()) { 3745193323Sed case ISD::CTTZ: 3746234353Sdim case ISD::CTTZ_ZERO_UNDEF: 3747193323Sed case ISD::CTLZ: 3748234353Sdim case ISD::CTLZ_ZERO_UNDEF: 3749193323Sed case ISD::CTPOP: 3750193323Sed // Zero extend the argument. 3751193323Sed Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3752234353Sdim // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 3753234353Sdim // already the correct result. 3754198090Srdivacky Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3755193323Sed if (Node->getOpcode() == ISD::CTTZ) { 3756234353Sdim // FIXME: This should set a bit in the zero extended value instead. 3757198090Srdivacky Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), 3758193323Sed Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 3759193323Sed ISD::SETEQ); 3760193323Sed Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, 3761193323Sed DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3762234353Sdim } else if (Node->getOpcode() == ISD::CTLZ || 3763234353Sdim Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 3764193323Sed // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3765193323Sed Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 3766193323Sed DAG.getConstant(NVT.getSizeInBits() - 3767193323Sed OVT.getSizeInBits(), NVT)); 3768193323Sed } 3769198090Srdivacky Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 3770193323Sed break; 3771193323Sed case ISD::BSWAP: { 3772193323Sed unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3773201360Srdivacky Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3774193323Sed Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 3775193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 3776219077Sdim DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); 3777193323Sed Results.push_back(Tmp1); 3778193323Sed break; 3779193323Sed } 3780193323Sed case ISD::FP_TO_UINT: 3781193323Sed case ISD::FP_TO_SINT: 3782193323Sed Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), 3783193323Sed Node->getOpcode() == ISD::FP_TO_SINT, dl); 3784193323Sed Results.push_back(Tmp1); 3785193323Sed break; 3786193323Sed case ISD::UINT_TO_FP: 3787193323Sed case ISD::SINT_TO_FP: 3788193323Sed Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), 3789193323Sed Node->getOpcode() == ISD::SINT_TO_FP, dl); 3790193323Sed Results.push_back(Tmp1); 3791193323Sed break; 3792234353Sdim case ISD::VAARG: { 3793234353Sdim SDValue Chain = Node->getOperand(0); // Get the chain. 3794234353Sdim SDValue Ptr = Node->getOperand(1); // Get the pointer. 3795234353Sdim 3796234353Sdim unsigned TruncOp; 3797234353Sdim if (OVT.isVector()) { 3798234353Sdim TruncOp = ISD::BITCAST; 3799234353Sdim } else { 3800234353Sdim assert(OVT.isInteger() 3801234353Sdim && "VAARG promotion is supported only for vectors or integer types"); 3802234353Sdim TruncOp = ISD::TRUNCATE; 3803234353Sdim } 3804234353Sdim 3805234353Sdim // Perform the larger operation, then convert back 3806234353Sdim Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 3807234353Sdim Node->getConstantOperandVal(3)); 3808234353Sdim Chain = Tmp1.getValue(1); 3809234353Sdim 3810234353Sdim Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 3811234353Sdim 3812234353Sdim // Modified the chain result - switch anything that used the old chain to 3813234353Sdim // use the new one. 3814234353Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 3815234353Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 3816234353Sdim ReplacedNode(Node); 3817234353Sdim break; 3818234353Sdim } 3819193323Sed case ISD::AND: 3820193323Sed case ISD::OR: 3821198090Srdivacky case ISD::XOR: { 3822198090Srdivacky unsigned ExtOp, TruncOp; 3823198090Srdivacky if (OVT.isVector()) { 3824218893Sdim ExtOp = ISD::BITCAST; 3825218893Sdim TruncOp = ISD::BITCAST; 3826207618Srdivacky } else { 3827207618Srdivacky assert(OVT.isInteger() && "Cannot promote logic operation"); 3828198090Srdivacky ExtOp = ISD::ANY_EXTEND; 3829198090Srdivacky TruncOp = ISD::TRUNCATE; 3830198090Srdivacky } 3831198090Srdivacky // Promote each of the values to the new type. 3832198090Srdivacky Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3833198090Srdivacky Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3834198090Srdivacky // Perform the larger operation, then convert back 3835193323Sed Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3836198090Srdivacky Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 3837193323Sed break; 3838198090Srdivacky } 3839198090Srdivacky case ISD::SELECT: { 3840193323Sed unsigned ExtOp, TruncOp; 3841193323Sed if (Node->getValueType(0).isVector()) { 3842218893Sdim ExtOp = ISD::BITCAST; 3843218893Sdim TruncOp = ISD::BITCAST; 3844193323Sed } else if (Node->getValueType(0).isInteger()) { 3845193323Sed ExtOp = ISD::ANY_EXTEND; 3846193323Sed TruncOp = ISD::TRUNCATE; 3847193323Sed } else { 3848193323Sed ExtOp = ISD::FP_EXTEND; 3849193323Sed TruncOp = ISD::FP_ROUND; 3850193323Sed } 3851193323Sed Tmp1 = Node->getOperand(0); 3852193323Sed // Promote each of the values to the new type. 3853193323Sed Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3854193323Sed Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 3855193323Sed // Perform the larger operation, then round down. 3856193323Sed Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3); 3857193323Sed if (TruncOp != ISD::FP_ROUND) 3858193323Sed Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 3859193323Sed else 3860193323Sed Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 3861193323Sed DAG.getIntPtrConstant(0)); 3862193323Sed Results.push_back(Tmp1); 3863193323Sed break; 3864198090Srdivacky } 3865193323Sed case ISD::VECTOR_SHUFFLE: { 3866234353Sdim ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3867193323Sed 3868193323Sed // Cast the two input vectors. 3869218893Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 3870218893Sdim Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 3871193323Sed 3872193323Sed // Convert the shuffle mask to the right # elements. 3873193323Sed Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 3874218893Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 3875193323Sed Results.push_back(Tmp1); 3876193323Sed break; 3877193323Sed } 3878193323Sed case ISD::SETCC: { 3879198090Srdivacky unsigned ExtOp = ISD::FP_EXTEND; 3880198090Srdivacky if (NVT.isInteger()) { 3881198090Srdivacky ISD::CondCode CCCode = 3882198090Srdivacky cast<CondCodeSDNode>(Node->getOperand(2))->get(); 3883198090Srdivacky ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 3884193323Sed } 3885198090Srdivacky Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3886198090Srdivacky Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3887193323Sed Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3888193323Sed Tmp1, Tmp2, Node->getOperand(2))); 3889193323Sed break; 3890193323Sed } 3891234353Sdim case ISD::FDIV: 3892234353Sdim case ISD::FREM: 3893234353Sdim case ISD::FPOW: { 3894234353Sdim Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 3895234353Sdim Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 3896234353Sdim Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3897234353Sdim Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 3898234353Sdim Tmp3, DAG.getIntPtrConstant(0))); 3899234353Sdim break; 3900193323Sed } 3901234353Sdim case ISD::FLOG2: 3902234353Sdim case ISD::FEXP2: 3903234353Sdim case ISD::FLOG: 3904234353Sdim case ISD::FEXP: { 3905234353Sdim Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 3906234353Sdim Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3907234353Sdim Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 3908234353Sdim Tmp2, DAG.getIntPtrConstant(0))); 3909234353Sdim break; 3910234353Sdim } 3911234353Sdim } 3912234353Sdim 3913234353Sdim // Replace the original node with the legalized result. 3914234353Sdim if (!Results.empty()) 3915234353Sdim ReplaceNode(Node, Results.data()); 3916193323Sed} 3917193323Sed 3918193323Sed// SelectionDAG::Legalize - This is the entry point for the file. 3919193323Sed// 3920223017Sdimvoid SelectionDAG::Legalize() { 3921193323Sed /// run - This is the main entry point to this class. 3922193323Sed /// 3923223017Sdim SelectionDAGLegalize(*this).LegalizeDAG(); 3924193323Sed} 3925