LegalizeDAG.cpp revision 276479
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::Legalize method. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "llvm/ADT/SmallPtrSet.h" 16#include "llvm/ADT/SmallSet.h" 17#include "llvm/ADT/SmallVector.h" 18#include "llvm/ADT/Triple.h" 19#include "llvm/CodeGen/Analysis.h" 20#include "llvm/CodeGen/MachineFunction.h" 21#include "llvm/CodeGen/MachineJumpTableInfo.h" 22#include "llvm/IR/CallingConv.h" 23#include "llvm/IR/Constants.h" 24#include "llvm/IR/DataLayout.h" 25#include "llvm/IR/DebugInfo.h" 26#include "llvm/IR/DerivedTypes.h" 27#include "llvm/IR/Function.h" 28#include "llvm/IR/LLVMContext.h" 29#include "llvm/Support/Debug.h" 30#include "llvm/Support/ErrorHandling.h" 31#include "llvm/Support/MathExtras.h" 32#include "llvm/Support/raw_ostream.h" 33#include "llvm/Target/TargetFrameLowering.h" 34#include "llvm/Target/TargetLowering.h" 35#include "llvm/Target/TargetMachine.h" 36using namespace llvm; 37 38//===----------------------------------------------------------------------===// 39/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 40/// hacks on it until the target machine can handle it. This involves 41/// eliminating value sizes the machine cannot handle (promoting small sizes to 42/// large sizes or splitting up large values into small values) as well as 43/// eliminating operations the machine cannot handle. 44/// 45/// This code also does a small amount of optimization and recognition of idioms 46/// as part of its processing. For example, if a target does not support a 47/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 48/// will attempt merge setcc and brc instructions into brcc's. 49/// 50namespace { 51class SelectionDAGLegalize : public SelectionDAG::DAGUpdateListener { 52 const TargetMachine &TM; 53 const TargetLowering &TLI; 54 SelectionDAG &DAG; 55 56 /// LegalizePosition - The iterator for walking through the node list. 57 SelectionDAG::allnodes_iterator LegalizePosition; 58 59 /// LegalizedNodes - The set of nodes which have already been legalized. 60 SmallPtrSet<SDNode *, 16> LegalizedNodes; 61 62 EVT getSetCCResultType(EVT VT) const { 63 return TLI.getSetCCResultType(*DAG.getContext(), VT); 64 } 65 66 // Libcall insertion helpers. 67 68public: 69 explicit SelectionDAGLegalize(SelectionDAG &DAG); 70 71 void LegalizeDAG(); 72 73private: 74 /// LegalizeOp - Legalizes the given operation. 75 void LegalizeOp(SDNode *Node); 76 77 SDValue OptimizeFloatStore(StoreSDNode *ST); 78 79 void LegalizeLoadOps(SDNode *Node); 80 void LegalizeStoreOps(SDNode *Node); 81 82 /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 83 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 84 /// is necessary to spill the vector being inserted into to memory, perform 85 /// the insert there, and then read the result back. 86 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 87 SDValue Idx, SDLoc dl); 88 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 89 SDValue Idx, SDLoc dl); 90 91 /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 92 /// performs the same shuffe in terms of order or result bytes, but on a type 93 /// whose vector element type is narrower than the original shuffle type. 94 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 95 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl, 96 SDValue N1, SDValue N2, 97 ArrayRef<int> Mask) const; 98 99 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 100 bool &NeedInvert, SDLoc dl); 101 102 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 103 SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, 104 unsigned NumOps, bool isSigned, SDLoc dl); 105 106 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, 107 SDNode *Node, bool isSigned); 108 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 109 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 110 RTLIB::Libcall Call_F128, 111 RTLIB::Libcall Call_PPCF128); 112 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 113 RTLIB::Libcall Call_I8, 114 RTLIB::Libcall Call_I16, 115 RTLIB::Libcall Call_I32, 116 RTLIB::Libcall Call_I64, 117 RTLIB::Libcall Call_I128); 118 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 119 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 120 121 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, SDLoc dl); 122 SDValue ExpandBUILD_VECTOR(SDNode *Node); 123 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 124 void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 125 SmallVectorImpl<SDValue> &Results); 126 SDValue ExpandFCOPYSIGN(SDNode *Node); 127 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, 128 SDLoc dl); 129 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, 130 SDLoc dl); 131 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, 132 SDLoc dl); 133 134 SDValue ExpandBSWAP(SDValue Op, SDLoc dl); 135 SDValue ExpandBitCount(unsigned Opc, SDValue Op, SDLoc dl); 136 137 SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 138 SDValue ExpandInsertToVectorThroughStack(SDValue Op); 139 SDValue ExpandVectorBuildThroughStack(SDNode* Node); 140 141 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 142 143 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); 144 145 void ExpandNode(SDNode *Node); 146 void PromoteNode(SDNode *Node); 147 148 void ForgetNode(SDNode *N) { 149 LegalizedNodes.erase(N); 150 if (LegalizePosition == SelectionDAG::allnodes_iterator(N)) 151 ++LegalizePosition; 152 } 153 154public: 155 // DAGUpdateListener implementation. 156 void NodeDeleted(SDNode *N, SDNode *E) override { 157 ForgetNode(N); 158 } 159 void NodeUpdated(SDNode *N) override {} 160 161 // Node replacement helpers 162 void ReplacedNode(SDNode *N) { 163 if (N->use_empty()) { 164 DAG.RemoveDeadNode(N); 165 } else { 166 ForgetNode(N); 167 } 168 } 169 void ReplaceNode(SDNode *Old, SDNode *New) { 170 DAG.ReplaceAllUsesWith(Old, New); 171 ReplacedNode(Old); 172 } 173 void ReplaceNode(SDValue Old, SDValue New) { 174 DAG.ReplaceAllUsesWith(Old, New); 175 ReplacedNode(Old.getNode()); 176 } 177 void ReplaceNode(SDNode *Old, const SDValue *New) { 178 DAG.ReplaceAllUsesWith(Old, New); 179 ReplacedNode(Old); 180 } 181}; 182} 183 184/// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 185/// performs the same shuffe in terms of order or result bytes, but on a type 186/// whose vector element type is narrower than the original shuffle type. 187/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 188SDValue 189SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl, 190 SDValue N1, SDValue N2, 191 ArrayRef<int> Mask) const { 192 unsigned NumMaskElts = VT.getVectorNumElements(); 193 unsigned NumDestElts = NVT.getVectorNumElements(); 194 unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 195 196 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 197 198 if (NumEltsGrowth == 1) 199 return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); 200 201 SmallVector<int, 8> NewMask; 202 for (unsigned i = 0; i != NumMaskElts; ++i) { 203 int Idx = Mask[i]; 204 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 205 if (Idx < 0) 206 NewMask.push_back(-1); 207 else 208 NewMask.push_back(Idx * NumEltsGrowth + j); 209 } 210 } 211 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 212 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 213 return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); 214} 215 216SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) 217 : SelectionDAG::DAGUpdateListener(dag), 218 TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), 219 DAG(dag) { 220} 221 222void SelectionDAGLegalize::LegalizeDAG() { 223 DAG.AssignTopologicalOrder(); 224 225 // Visit all the nodes. We start in topological order, so that we see 226 // nodes with their original operands intact. Legalization can produce 227 // new nodes which may themselves need to be legalized. Iterate until all 228 // nodes have been legalized. 229 for (;;) { 230 bool AnyLegalized = false; 231 for (LegalizePosition = DAG.allnodes_end(); 232 LegalizePosition != DAG.allnodes_begin(); ) { 233 --LegalizePosition; 234 235 SDNode *N = LegalizePosition; 236 if (LegalizedNodes.insert(N)) { 237 AnyLegalized = true; 238 LegalizeOp(N); 239 } 240 } 241 if (!AnyLegalized) 242 break; 243 244 } 245 246 // Remove dead nodes now. 247 DAG.RemoveDeadNodes(); 248} 249 250/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 251/// a load from the constant pool. 252SDValue 253SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 254 bool Extend = false; 255 SDLoc dl(CFP); 256 257 // If a FP immediate is precise when represented as a float and if the 258 // target can do an extending load from float to double, we put it into 259 // the constant pool as a float, even if it's is statically typed as a 260 // double. This shrinks FP constants and canonicalizes them for targets where 261 // an FP extending load is the same cost as a normal load (such as on the x87 262 // fp stack or PPC FP unit). 263 EVT VT = CFP->getValueType(0); 264 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 265 if (!UseCP) { 266 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 267 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 268 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 269 } 270 271 EVT OrigVT = VT; 272 EVT SVT = VT; 273 while (SVT != MVT::f32 && SVT != MVT::f16) { 274 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 275 if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && 276 // Only do this if the target has a native EXTLOAD instruction from 277 // smaller type. 278 TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 279 TLI.ShouldShrinkFPConstant(OrigVT)) { 280 Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 281 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 282 VT = SVT; 283 Extend = true; 284 } 285 } 286 287 SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 288 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 289 if (Extend) { 290 SDValue Result = 291 DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, 292 DAG.getEntryNode(), 293 CPIdx, MachinePointerInfo::getConstantPool(), 294 VT, false, false, Alignment); 295 return Result; 296 } 297 SDValue Result = 298 DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 299 MachinePointerInfo::getConstantPool(), false, false, false, 300 Alignment); 301 return Result; 302} 303 304/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 305static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 306 const TargetLowering &TLI, 307 SelectionDAGLegalize *DAGLegalize) { 308 assert(ST->getAddressingMode() == ISD::UNINDEXED && 309 "unaligned indexed stores not implemented!"); 310 SDValue Chain = ST->getChain(); 311 SDValue Ptr = ST->getBasePtr(); 312 SDValue Val = ST->getValue(); 313 EVT VT = Val.getValueType(); 314 int Alignment = ST->getAlignment(); 315 unsigned AS = ST->getAddressSpace(); 316 317 SDLoc dl(ST); 318 if (ST->getMemoryVT().isFloatingPoint() || 319 ST->getMemoryVT().isVector()) { 320 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 321 if (TLI.isTypeLegal(intVT)) { 322 // Expand to a bitconvert of the value to the integer type of the 323 // same size, then a (misaligned) int store. 324 // FIXME: Does not handle truncating floating point stores! 325 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); 326 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), 327 ST->isVolatile(), ST->isNonTemporal(), Alignment); 328 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 329 return; 330 } 331 // Do a (aligned) store to a stack slot, then copy from the stack slot 332 // to the final destination using (unaligned) integer loads and stores. 333 EVT StoredVT = ST->getMemoryVT(); 334 MVT RegVT = 335 TLI.getRegisterType(*DAG.getContext(), 336 EVT::getIntegerVT(*DAG.getContext(), 337 StoredVT.getSizeInBits())); 338 unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 339 unsigned RegBytes = RegVT.getSizeInBits() / 8; 340 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 341 342 // Make sure the stack slot is also aligned for the register type. 343 SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 344 345 // Perform the original store, only redirected to the stack slot. 346 SDValue Store = DAG.getTruncStore(Chain, dl, 347 Val, StackPtr, MachinePointerInfo(), 348 StoredVT, false, false, 0); 349 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy(AS)); 350 SmallVector<SDValue, 8> Stores; 351 unsigned Offset = 0; 352 353 // Do all but one copies using the full register width. 354 for (unsigned i = 1; i < NumRegs; i++) { 355 // Load one integer register's worth from the stack slot. 356 SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, 357 MachinePointerInfo(), 358 false, false, false, 0); 359 // Store it to the final location. Remember the store. 360 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 361 ST->getPointerInfo().getWithOffset(Offset), 362 ST->isVolatile(), ST->isNonTemporal(), 363 MinAlign(ST->getAlignment(), Offset))); 364 // Increment the pointers. 365 Offset += RegBytes; 366 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 367 Increment); 368 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 369 } 370 371 // The last store may be partial. Do a truncating store. On big-endian 372 // machines this requires an extending load from the stack slot to ensure 373 // that the bits are in the right place. 374 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 375 8 * (StoredBytes - Offset)); 376 377 // Load from the stack slot. 378 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 379 MachinePointerInfo(), 380 MemVT, false, false, 0); 381 382 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 383 ST->getPointerInfo() 384 .getWithOffset(Offset), 385 MemVT, ST->isVolatile(), 386 ST->isNonTemporal(), 387 MinAlign(ST->getAlignment(), Offset), 388 ST->getTBAAInfo())); 389 // The order of the stores doesn't matter - say it with a TokenFactor. 390 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 391 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 392 return; 393 } 394 assert(ST->getMemoryVT().isInteger() && 395 !ST->getMemoryVT().isVector() && 396 "Unaligned store of unknown type."); 397 // Get the half-size VT 398 EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); 399 int NumBits = NewStoredVT.getSizeInBits(); 400 int IncrementSize = NumBits / 8; 401 402 // Divide the stored value in two parts. 403 SDValue ShiftAmount = DAG.getConstant(NumBits, 404 TLI.getShiftAmountTy(Val.getValueType())); 405 SDValue Lo = Val; 406 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 407 408 // Store the two parts 409 SDValue Store1, Store2; 410 Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 411 ST->getPointerInfo(), NewStoredVT, 412 ST->isVolatile(), ST->isNonTemporal(), Alignment); 413 414 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 415 DAG.getConstant(IncrementSize, TLI.getPointerTy(AS))); 416 Alignment = MinAlign(Alignment, IncrementSize); 417 Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 418 ST->getPointerInfo().getWithOffset(IncrementSize), 419 NewStoredVT, ST->isVolatile(), ST->isNonTemporal(), 420 Alignment, ST->getTBAAInfo()); 421 422 SDValue Result = 423 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 424 DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 425} 426 427/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 428static void 429ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 430 const TargetLowering &TLI, 431 SDValue &ValResult, SDValue &ChainResult) { 432 assert(LD->getAddressingMode() == ISD::UNINDEXED && 433 "unaligned indexed loads not implemented!"); 434 SDValue Chain = LD->getChain(); 435 SDValue Ptr = LD->getBasePtr(); 436 EVT VT = LD->getValueType(0); 437 EVT LoadedVT = LD->getMemoryVT(); 438 SDLoc dl(LD); 439 if (VT.isFloatingPoint() || VT.isVector()) { 440 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); 441 if (TLI.isTypeLegal(intVT) && TLI.isTypeLegal(LoadedVT)) { 442 // Expand to a (misaligned) integer load of the same size, 443 // then bitconvert to floating point or vector. 444 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, 445 LD->getMemOperand()); 446 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); 447 if (LoadedVT != VT) 448 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : 449 ISD::ANY_EXTEND, dl, VT, Result); 450 451 ValResult = Result; 452 ChainResult = Chain; 453 return; 454 } 455 456 // Copy the value to a (aligned) stack slot using (unaligned) integer 457 // loads and stores, then do a (aligned) load from the stack slot. 458 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); 459 unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 460 unsigned RegBytes = RegVT.getSizeInBits() / 8; 461 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 462 463 // Make sure the stack slot is also aligned for the register type. 464 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 465 466 SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 467 SmallVector<SDValue, 8> Stores; 468 SDValue StackPtr = StackBase; 469 unsigned Offset = 0; 470 471 // Do all but one copies using the full register width. 472 for (unsigned i = 1; i < NumRegs; i++) { 473 // Load one integer register's worth from the original location. 474 SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, 475 LD->getPointerInfo().getWithOffset(Offset), 476 LD->isVolatile(), LD->isNonTemporal(), 477 LD->isInvariant(), 478 MinAlign(LD->getAlignment(), Offset), 479 LD->getTBAAInfo()); 480 // Follow the load with a store to the stack slot. Remember the store. 481 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 482 MachinePointerInfo(), false, false, 0)); 483 // Increment the pointers. 484 Offset += RegBytes; 485 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 486 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 487 Increment); 488 } 489 490 // The last copy may be partial. Do an extending load. 491 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 492 8 * (LoadedBytes - Offset)); 493 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 494 LD->getPointerInfo().getWithOffset(Offset), 495 MemVT, LD->isVolatile(), 496 LD->isNonTemporal(), 497 MinAlign(LD->getAlignment(), Offset), 498 LD->getTBAAInfo()); 499 // Follow the load with a store to the stack slot. Remember the store. 500 // On big-endian machines this requires a truncating store to ensure 501 // that the bits end up in the right place. 502 Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 503 MachinePointerInfo(), MemVT, 504 false, false, 0)); 505 506 // The order of the stores doesn't matter - say it with a TokenFactor. 507 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 508 509 // Finally, perform the original load only redirected to the stack slot. 510 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 511 MachinePointerInfo(), LoadedVT, false, false, 0); 512 513 // Callers expect a MERGE_VALUES node. 514 ValResult = Load; 515 ChainResult = TF; 516 return; 517 } 518 assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 519 "Unaligned load of unsupported type."); 520 521 // Compute the new VT that is half the size of the old one. This is an 522 // integer MVT. 523 unsigned NumBits = LoadedVT.getSizeInBits(); 524 EVT NewLoadedVT; 525 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2); 526 NumBits >>= 1; 527 528 unsigned Alignment = LD->getAlignment(); 529 unsigned IncrementSize = NumBits / 8; 530 ISD::LoadExtType HiExtType = LD->getExtensionType(); 531 532 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 533 if (HiExtType == ISD::NON_EXTLOAD) 534 HiExtType = ISD::ZEXTLOAD; 535 536 // Load the value in two parts 537 SDValue Lo, Hi; 538 if (TLI.isLittleEndian()) { 539 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), 540 NewLoadedVT, LD->isVolatile(), 541 LD->isNonTemporal(), Alignment, LD->getTBAAInfo()); 542 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 543 DAG.getConstant(IncrementSize, Ptr.getValueType())); 544 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, 545 LD->getPointerInfo().getWithOffset(IncrementSize), 546 NewLoadedVT, LD->isVolatile(), 547 LD->isNonTemporal(), MinAlign(Alignment, IncrementSize), 548 LD->getTBAAInfo()); 549 } else { 550 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), 551 NewLoadedVT, LD->isVolatile(), 552 LD->isNonTemporal(), Alignment, LD->getTBAAInfo()); 553 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 554 DAG.getConstant(IncrementSize, Ptr.getValueType())); 555 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, 556 LD->getPointerInfo().getWithOffset(IncrementSize), 557 NewLoadedVT, LD->isVolatile(), 558 LD->isNonTemporal(), MinAlign(Alignment, IncrementSize), 559 LD->getTBAAInfo()); 560 } 561 562 // aggregate the two parts 563 SDValue ShiftAmount = DAG.getConstant(NumBits, 564 TLI.getShiftAmountTy(Hi.getValueType())); 565 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 566 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 567 568 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 569 Hi.getValue(1)); 570 571 ValResult = Result; 572 ChainResult = TF; 573} 574 575/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 576/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 577/// is necessary to spill the vector being inserted into to memory, perform 578/// the insert there, and then read the result back. 579SDValue SelectionDAGLegalize:: 580PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 581 SDLoc dl) { 582 SDValue Tmp1 = Vec; 583 SDValue Tmp2 = Val; 584 SDValue Tmp3 = Idx; 585 586 // If the target doesn't support this, we have to spill the input vector 587 // to a temporary stack slot, update the element, then reload it. This is 588 // badness. We could also load the value into a vector register (either 589 // with a "move to register" or "extload into register" instruction, then 590 // permute it into place, if the idx is a constant and if the idx is 591 // supported by the target. 592 EVT VT = Tmp1.getValueType(); 593 EVT EltVT = VT.getVectorElementType(); 594 EVT IdxVT = Tmp3.getValueType(); 595 EVT PtrVT = TLI.getPointerTy(); 596 SDValue StackPtr = DAG.CreateStackTemporary(VT); 597 598 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 599 600 // Store the vector. 601 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 602 MachinePointerInfo::getFixedStack(SPFI), 603 false, false, 0); 604 605 // Truncate or zero extend offset to target pointer type. 606 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 607 Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 608 // Add the offset to the index. 609 unsigned EltSize = EltVT.getSizeInBits()/8; 610 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 611 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 612 // Store the scalar value. 613 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT, 614 false, false, 0); 615 // Load the updated vector. 616 return DAG.getLoad(VT, dl, Ch, StackPtr, 617 MachinePointerInfo::getFixedStack(SPFI), false, false, 618 false, 0); 619} 620 621 622SDValue SelectionDAGLegalize:: 623ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, SDLoc dl) { 624 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 625 // SCALAR_TO_VECTOR requires that the type of the value being inserted 626 // match the element type of the vector being created, except for 627 // integers in which case the inserted value can be over width. 628 EVT EltVT = Vec.getValueType().getVectorElementType(); 629 if (Val.getValueType() == EltVT || 630 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 631 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 632 Vec.getValueType(), Val); 633 634 unsigned NumElts = Vec.getValueType().getVectorNumElements(); 635 // We generate a shuffle of InVec and ScVec, so the shuffle mask 636 // should be 0,1,2,3,4,5... with the appropriate element replaced with 637 // elt 0 of the RHS. 638 SmallVector<int, 8> ShufOps; 639 for (unsigned i = 0; i != NumElts; ++i) 640 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 641 642 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, 643 &ShufOps[0]); 644 } 645 } 646 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 647} 648 649SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 650 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 651 // FIXME: We shouldn't do this for TargetConstantFP's. 652 // FIXME: move this to the DAG Combiner! Note that we can't regress due 653 // to phase ordering between legalized code and the dag combiner. This 654 // probably means that we need to integrate dag combiner and legalizer 655 // together. 656 // We generally can't do this one for long doubles. 657 SDValue Chain = ST->getChain(); 658 SDValue Ptr = ST->getBasePtr(); 659 unsigned Alignment = ST->getAlignment(); 660 bool isVolatile = ST->isVolatile(); 661 bool isNonTemporal = ST->isNonTemporal(); 662 const MDNode *TBAAInfo = ST->getTBAAInfo(); 663 SDLoc dl(ST); 664 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 665 if (CFP->getValueType(0) == MVT::f32 && 666 TLI.isTypeLegal(MVT::i32)) { 667 SDValue Con = DAG.getConstant(CFP->getValueAPF(). 668 bitcastToAPInt().zextOrTrunc(32), 669 MVT::i32); 670 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 671 isVolatile, isNonTemporal, Alignment, TBAAInfo); 672 } 673 674 if (CFP->getValueType(0) == MVT::f64) { 675 // If this target supports 64-bit registers, do a single 64-bit store. 676 if (TLI.isTypeLegal(MVT::i64)) { 677 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 678 zextOrTrunc(64), MVT::i64); 679 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 680 isVolatile, isNonTemporal, Alignment, TBAAInfo); 681 } 682 683 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 684 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 685 // stores. If the target supports neither 32- nor 64-bits, this 686 // xform is certainly not worth it. 687 const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 688 SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32); 689 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 690 if (TLI.isBigEndian()) std::swap(Lo, Hi); 691 692 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile, 693 isNonTemporal, Alignment, TBAAInfo); 694 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 695 DAG.getConstant(4, Ptr.getValueType())); 696 Hi = DAG.getStore(Chain, dl, Hi, Ptr, 697 ST->getPointerInfo().getWithOffset(4), 698 isVolatile, isNonTemporal, MinAlign(Alignment, 4U), 699 TBAAInfo); 700 701 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 702 } 703 } 704 } 705 return SDValue(nullptr, 0); 706} 707 708void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { 709 StoreSDNode *ST = cast<StoreSDNode>(Node); 710 SDValue Chain = ST->getChain(); 711 SDValue Ptr = ST->getBasePtr(); 712 SDLoc dl(Node); 713 714 unsigned Alignment = ST->getAlignment(); 715 bool isVolatile = ST->isVolatile(); 716 bool isNonTemporal = ST->isNonTemporal(); 717 const MDNode *TBAAInfo = ST->getTBAAInfo(); 718 719 if (!ST->isTruncatingStore()) { 720 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 721 ReplaceNode(ST, OptStore); 722 return; 723 } 724 725 { 726 SDValue Value = ST->getValue(); 727 MVT VT = Value.getSimpleValueType(); 728 switch (TLI.getOperationAction(ISD::STORE, VT)) { 729 default: llvm_unreachable("This action is not supported yet!"); 730 case TargetLowering::Legal: { 731 // If this is an unaligned store and the target doesn't support it, 732 // expand it. 733 unsigned AS = ST->getAddressSpace(); 734 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT(), AS)) { 735 Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 736 unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 737 if (ST->getAlignment() < ABIAlignment) 738 ExpandUnalignedStore(cast<StoreSDNode>(Node), 739 DAG, TLI, this); 740 } 741 break; 742 } 743 case TargetLowering::Custom: { 744 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 745 if (Res.getNode()) 746 ReplaceNode(SDValue(Node, 0), Res); 747 return; 748 } 749 case TargetLowering::Promote: { 750 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); 751 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 752 "Can only promote stores to same size type"); 753 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); 754 SDValue Result = 755 DAG.getStore(Chain, dl, Value, Ptr, 756 ST->getPointerInfo(), isVolatile, 757 isNonTemporal, Alignment, TBAAInfo); 758 ReplaceNode(SDValue(Node, 0), Result); 759 break; 760 } 761 } 762 return; 763 } 764 } else { 765 SDValue Value = ST->getValue(); 766 767 EVT StVT = ST->getMemoryVT(); 768 unsigned StWidth = StVT.getSizeInBits(); 769 770 if (StWidth != StVT.getStoreSizeInBits()) { 771 // Promote to a byte-sized store with upper bits zero if not 772 // storing an integral number of bytes. For example, promote 773 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 774 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), 775 StVT.getStoreSizeInBits()); 776 Value = DAG.getZeroExtendInReg(Value, dl, StVT); 777 SDValue Result = 778 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 779 NVT, isVolatile, isNonTemporal, Alignment, 780 TBAAInfo); 781 ReplaceNode(SDValue(Node, 0), Result); 782 } else if (StWidth & (StWidth - 1)) { 783 // If not storing a power-of-2 number of bits, expand as two stores. 784 assert(!StVT.isVector() && "Unsupported truncstore!"); 785 unsigned RoundWidth = 1 << Log2_32(StWidth); 786 assert(RoundWidth < StWidth); 787 unsigned ExtraWidth = StWidth - RoundWidth; 788 assert(ExtraWidth < RoundWidth); 789 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 790 "Store size not an integral number of bytes!"); 791 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 792 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 793 SDValue Lo, Hi; 794 unsigned IncrementSize; 795 796 if (TLI.isLittleEndian()) { 797 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 798 // Store the bottom RoundWidth bits. 799 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 800 RoundVT, 801 isVolatile, isNonTemporal, Alignment, 802 TBAAInfo); 803 804 // Store the remaining ExtraWidth bits. 805 IncrementSize = RoundWidth / 8; 806 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 807 DAG.getConstant(IncrementSize, Ptr.getValueType())); 808 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 809 DAG.getConstant(RoundWidth, 810 TLI.getShiftAmountTy(Value.getValueType()))); 811 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, 812 ST->getPointerInfo().getWithOffset(IncrementSize), 813 ExtraVT, isVolatile, isNonTemporal, 814 MinAlign(Alignment, IncrementSize), TBAAInfo); 815 } else { 816 // Big endian - avoid unaligned stores. 817 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 818 // Store the top RoundWidth bits. 819 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 820 DAG.getConstant(ExtraWidth, 821 TLI.getShiftAmountTy(Value.getValueType()))); 822 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), 823 RoundVT, isVolatile, isNonTemporal, Alignment, 824 TBAAInfo); 825 826 // Store the remaining ExtraWidth bits. 827 IncrementSize = RoundWidth / 8; 828 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 829 DAG.getConstant(IncrementSize, Ptr.getValueType())); 830 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, 831 ST->getPointerInfo().getWithOffset(IncrementSize), 832 ExtraVT, isVolatile, isNonTemporal, 833 MinAlign(Alignment, IncrementSize), TBAAInfo); 834 } 835 836 // The order of the stores doesn't matter. 837 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 838 ReplaceNode(SDValue(Node, 0), Result); 839 } else { 840 switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(), 841 StVT.getSimpleVT())) { 842 default: llvm_unreachable("This action is not supported yet!"); 843 case TargetLowering::Legal: { 844 unsigned AS = ST->getAddressSpace(); 845 // If this is an unaligned store and the target doesn't support it, 846 // expand it. 847 if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT(), AS)) { 848 Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 849 unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 850 if (ST->getAlignment() < ABIAlignment) 851 ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); 852 } 853 break; 854 } 855 case TargetLowering::Custom: { 856 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 857 if (Res.getNode()) 858 ReplaceNode(SDValue(Node, 0), Res); 859 return; 860 } 861 case TargetLowering::Expand: 862 assert(!StVT.isVector() && 863 "Vector Stores are handled in LegalizeVectorOps"); 864 865 // TRUNCSTORE:i16 i32 -> STORE i16 866 assert(TLI.isTypeLegal(StVT) && 867 "Do not know how to expand this store!"); 868 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); 869 SDValue Result = 870 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 871 isVolatile, isNonTemporal, Alignment, TBAAInfo); 872 ReplaceNode(SDValue(Node, 0), Result); 873 break; 874 } 875 } 876 } 877} 878 879void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { 880 LoadSDNode *LD = cast<LoadSDNode>(Node); 881 SDValue Chain = LD->getChain(); // The chain. 882 SDValue Ptr = LD->getBasePtr(); // The base pointer. 883 SDValue Value; // The value returned by the load op. 884 SDLoc dl(Node); 885 886 ISD::LoadExtType ExtType = LD->getExtensionType(); 887 if (ExtType == ISD::NON_EXTLOAD) { 888 MVT VT = Node->getSimpleValueType(0); 889 SDValue RVal = SDValue(Node, 0); 890 SDValue RChain = SDValue(Node, 1); 891 892 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 893 default: llvm_unreachable("This action is not supported yet!"); 894 case TargetLowering::Legal: { 895 unsigned AS = LD->getAddressSpace(); 896 // If this is an unaligned load and the target doesn't support it, 897 // expand it. 898 if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT(), AS)) { 899 Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 900 unsigned ABIAlignment = 901 TLI.getDataLayout()->getABITypeAlignment(Ty); 902 if (LD->getAlignment() < ABIAlignment){ 903 ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain); 904 } 905 } 906 break; 907 } 908 case TargetLowering::Custom: { 909 SDValue Res = TLI.LowerOperation(RVal, DAG); 910 if (Res.getNode()) { 911 RVal = Res; 912 RChain = Res.getValue(1); 913 } 914 break; 915 } 916 case TargetLowering::Promote: { 917 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 918 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 919 "Can only promote loads to same size type"); 920 921 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); 922 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); 923 RChain = Res.getValue(1); 924 break; 925 } 926 } 927 if (RChain.getNode() != Node) { 928 assert(RVal.getNode() != Node && "Load must be completely replaced"); 929 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); 930 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); 931 ReplacedNode(Node); 932 } 933 return; 934 } 935 936 EVT SrcVT = LD->getMemoryVT(); 937 unsigned SrcWidth = SrcVT.getSizeInBits(); 938 unsigned Alignment = LD->getAlignment(); 939 bool isVolatile = LD->isVolatile(); 940 bool isNonTemporal = LD->isNonTemporal(); 941 const MDNode *TBAAInfo = LD->getTBAAInfo(); 942 943 if (SrcWidth != SrcVT.getStoreSizeInBits() && 944 // Some targets pretend to have an i1 loading operation, and actually 945 // load an i8. This trick is correct for ZEXTLOAD because the top 7 946 // bits are guaranteed to be zero; it helps the optimizers understand 947 // that these bits are zero. It is also useful for EXTLOAD, since it 948 // tells the optimizers that those bits are undefined. It would be 949 // nice to have an effective generic way of getting these benefits... 950 // Until such a way is found, don't insist on promoting i1 here. 951 (SrcVT != MVT::i1 || 952 TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 953 // Promote to a byte-sized load if not loading an integral number of 954 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 955 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 956 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 957 SDValue Ch; 958 959 // The extra bits are guaranteed to be zero, since we stored them that 960 // way. A zext load from NVT thus automatically gives zext from SrcVT. 961 962 ISD::LoadExtType NewExtType = 963 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 964 965 SDValue Result = 966 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 967 Chain, Ptr, LD->getPointerInfo(), 968 NVT, isVolatile, isNonTemporal, Alignment, TBAAInfo); 969 970 Ch = Result.getValue(1); // The chain. 971 972 if (ExtType == ISD::SEXTLOAD) 973 // Having the top bits zero doesn't help when sign extending. 974 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 975 Result.getValueType(), 976 Result, DAG.getValueType(SrcVT)); 977 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 978 // All the top bits are guaranteed to be zero - inform the optimizers. 979 Result = DAG.getNode(ISD::AssertZext, dl, 980 Result.getValueType(), Result, 981 DAG.getValueType(SrcVT)); 982 983 Value = Result; 984 Chain = Ch; 985 } else if (SrcWidth & (SrcWidth - 1)) { 986 // If not loading a power-of-2 number of bits, expand as two loads. 987 assert(!SrcVT.isVector() && "Unsupported extload!"); 988 unsigned RoundWidth = 1 << Log2_32(SrcWidth); 989 assert(RoundWidth < SrcWidth); 990 unsigned ExtraWidth = SrcWidth - RoundWidth; 991 assert(ExtraWidth < RoundWidth); 992 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 993 "Load size not an integral number of bytes!"); 994 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 995 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 996 SDValue Lo, Hi, Ch; 997 unsigned IncrementSize; 998 999 if (TLI.isLittleEndian()) { 1000 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 1001 // Load the bottom RoundWidth bits. 1002 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), 1003 Chain, Ptr, 1004 LD->getPointerInfo(), RoundVT, isVolatile, 1005 isNonTemporal, Alignment, TBAAInfo); 1006 1007 // Load the remaining ExtraWidth bits. 1008 IncrementSize = RoundWidth / 8; 1009 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1010 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1011 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 1012 LD->getPointerInfo().getWithOffset(IncrementSize), 1013 ExtraVT, isVolatile, isNonTemporal, 1014 MinAlign(Alignment, IncrementSize), TBAAInfo); 1015 1016 // Build a factor node to remember that this load is independent of 1017 // the other one. 1018 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1019 Hi.getValue(1)); 1020 1021 // Move the top bits to the right place. 1022 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1023 DAG.getConstant(RoundWidth, 1024 TLI.getShiftAmountTy(Hi.getValueType()))); 1025 1026 // Join the hi and lo parts. 1027 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1028 } else { 1029 // Big endian - avoid unaligned loads. 1030 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 1031 // Load the top RoundWidth bits. 1032 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 1033 LD->getPointerInfo(), RoundVT, isVolatile, 1034 isNonTemporal, Alignment, TBAAInfo); 1035 1036 // Load the remaining ExtraWidth bits. 1037 IncrementSize = RoundWidth / 8; 1038 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1039 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1040 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, 1041 dl, Node->getValueType(0), Chain, Ptr, 1042 LD->getPointerInfo().getWithOffset(IncrementSize), 1043 ExtraVT, isVolatile, isNonTemporal, 1044 MinAlign(Alignment, IncrementSize), TBAAInfo); 1045 1046 // Build a factor node to remember that this load is independent of 1047 // the other one. 1048 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1049 Hi.getValue(1)); 1050 1051 // Move the top bits to the right place. 1052 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1053 DAG.getConstant(ExtraWidth, 1054 TLI.getShiftAmountTy(Hi.getValueType()))); 1055 1056 // Join the hi and lo parts. 1057 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1058 } 1059 1060 Chain = Ch; 1061 } else { 1062 bool isCustom = false; 1063 switch (TLI.getLoadExtAction(ExtType, SrcVT.getSimpleVT())) { 1064 default: llvm_unreachable("This action is not supported yet!"); 1065 case TargetLowering::Custom: 1066 isCustom = true; 1067 // FALLTHROUGH 1068 case TargetLowering::Legal: { 1069 Value = SDValue(Node, 0); 1070 Chain = SDValue(Node, 1); 1071 1072 if (isCustom) { 1073 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1074 if (Res.getNode()) { 1075 Value = Res; 1076 Chain = Res.getValue(1); 1077 } 1078 } else { 1079 // If this is an unaligned load and the target doesn't support 1080 // it, expand it. 1081 EVT MemVT = LD->getMemoryVT(); 1082 unsigned AS = LD->getAddressSpace(); 1083 if (!TLI.allowsUnalignedMemoryAccesses(MemVT, AS)) { 1084 Type *Ty = 1085 LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1086 unsigned ABIAlignment = 1087 TLI.getDataLayout()->getABITypeAlignment(Ty); 1088 if (LD->getAlignment() < ABIAlignment){ 1089 ExpandUnalignedLoad(cast<LoadSDNode>(Node), 1090 DAG, TLI, Value, Chain); 1091 } 1092 } 1093 } 1094 break; 1095 } 1096 case TargetLowering::Expand: 1097 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && 1098 TLI.isTypeLegal(SrcVT)) { 1099 SDValue Load = DAG.getLoad(SrcVT, dl, Chain, Ptr, 1100 LD->getMemOperand()); 1101 unsigned ExtendOp; 1102 switch (ExtType) { 1103 case ISD::EXTLOAD: 1104 ExtendOp = (SrcVT.isFloatingPoint() ? 1105 ISD::FP_EXTEND : ISD::ANY_EXTEND); 1106 break; 1107 case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break; 1108 case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break; 1109 default: llvm_unreachable("Unexpected extend load type!"); 1110 } 1111 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 1112 Chain = Load.getValue(1); 1113 break; 1114 } 1115 1116 assert(!SrcVT.isVector() && 1117 "Vector Loads are handled in LegalizeVectorOps"); 1118 1119 // FIXME: This does not work for vectors on most targets. Sign- 1120 // and zero-extend operations are currently folded into extending 1121 // loads, whether they are legal or not, and then we end up here 1122 // without any support for legalizing them. 1123 assert(ExtType != ISD::EXTLOAD && 1124 "EXTLOAD should always be supported!"); 1125 // Turn the unsupported load into an EXTLOAD followed by an 1126 // explicit zero/sign extend inreg. 1127 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, 1128 Node->getValueType(0), 1129 Chain, Ptr, SrcVT, 1130 LD->getMemOperand()); 1131 SDValue ValRes; 1132 if (ExtType == ISD::SEXTLOAD) 1133 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 1134 Result.getValueType(), 1135 Result, DAG.getValueType(SrcVT)); 1136 else 1137 ValRes = DAG.getZeroExtendInReg(Result, dl, 1138 SrcVT.getScalarType()); 1139 Value = ValRes; 1140 Chain = Result.getValue(1); 1141 break; 1142 } 1143 } 1144 1145 // Since loads produce two values, make sure to remember that we legalized 1146 // both of them. 1147 if (Chain.getNode() != Node) { 1148 assert(Value.getNode() != Node && "Load must be completely replaced"); 1149 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); 1150 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 1151 ReplacedNode(Node); 1152 } 1153} 1154 1155/// LegalizeOp - Return a legal replacement for the given operation, with 1156/// all legal operands. 1157void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 1158 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 1159 return; 1160 1161 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1162 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 1163 TargetLowering::TypeLegal && 1164 "Unexpected illegal type!"); 1165 1166 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1167 assert((TLI.getTypeAction(*DAG.getContext(), 1168 Node->getOperand(i).getValueType()) == 1169 TargetLowering::TypeLegal || 1170 Node->getOperand(i).getOpcode() == ISD::TargetConstant) && 1171 "Unexpected illegal type!"); 1172 1173 // Figure out the correct action; the way to query this varies by opcode 1174 TargetLowering::LegalizeAction Action = TargetLowering::Legal; 1175 bool SimpleFinishLegalizing = true; 1176 switch (Node->getOpcode()) { 1177 case ISD::INTRINSIC_W_CHAIN: 1178 case ISD::INTRINSIC_WO_CHAIN: 1179 case ISD::INTRINSIC_VOID: 1180 case ISD::STACKSAVE: 1181 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1182 break; 1183 case ISD::VAARG: 1184 Action = TLI.getOperationAction(Node->getOpcode(), 1185 Node->getValueType(0)); 1186 if (Action != TargetLowering::Promote) 1187 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1188 break; 1189 case ISD::FP_TO_FP16: 1190 case ISD::SINT_TO_FP: 1191 case ISD::UINT_TO_FP: 1192 case ISD::EXTRACT_VECTOR_ELT: 1193 Action = TLI.getOperationAction(Node->getOpcode(), 1194 Node->getOperand(0).getValueType()); 1195 break; 1196 case ISD::FP_ROUND_INREG: 1197 case ISD::SIGN_EXTEND_INREG: { 1198 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1199 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 1200 break; 1201 } 1202 case ISD::ATOMIC_STORE: { 1203 Action = TLI.getOperationAction(Node->getOpcode(), 1204 Node->getOperand(2).getValueType()); 1205 break; 1206 } 1207 case ISD::SELECT_CC: 1208 case ISD::SETCC: 1209 case ISD::BR_CC: { 1210 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 1211 Node->getOpcode() == ISD::SETCC ? 2 : 1; 1212 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; 1213 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); 1214 ISD::CondCode CCCode = 1215 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 1216 Action = TLI.getCondCodeAction(CCCode, OpVT); 1217 if (Action == TargetLowering::Legal) { 1218 if (Node->getOpcode() == ISD::SELECT_CC) 1219 Action = TLI.getOperationAction(Node->getOpcode(), 1220 Node->getValueType(0)); 1221 else 1222 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 1223 } 1224 break; 1225 } 1226 case ISD::LOAD: 1227 case ISD::STORE: 1228 // FIXME: Model these properly. LOAD and STORE are complicated, and 1229 // STORE expects the unlegalized operand in some cases. 1230 SimpleFinishLegalizing = false; 1231 break; 1232 case ISD::CALLSEQ_START: 1233 case ISD::CALLSEQ_END: 1234 // FIXME: This shouldn't be necessary. These nodes have special properties 1235 // dealing with the recursive nature of legalization. Removing this 1236 // special case should be done as part of making LegalizeDAG non-recursive. 1237 SimpleFinishLegalizing = false; 1238 break; 1239 case ISD::EXTRACT_ELEMENT: 1240 case ISD::FLT_ROUNDS_: 1241 case ISD::SADDO: 1242 case ISD::SSUBO: 1243 case ISD::UADDO: 1244 case ISD::USUBO: 1245 case ISD::SMULO: 1246 case ISD::UMULO: 1247 case ISD::FPOWI: 1248 case ISD::MERGE_VALUES: 1249 case ISD::EH_RETURN: 1250 case ISD::FRAME_TO_ARGS_OFFSET: 1251 case ISD::EH_SJLJ_SETJMP: 1252 case ISD::EH_SJLJ_LONGJMP: 1253 // These operations lie about being legal: when they claim to be legal, 1254 // they should actually be expanded. 1255 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1256 if (Action == TargetLowering::Legal) 1257 Action = TargetLowering::Expand; 1258 break; 1259 case ISD::INIT_TRAMPOLINE: 1260 case ISD::ADJUST_TRAMPOLINE: 1261 case ISD::FRAMEADDR: 1262 case ISD::RETURNADDR: 1263 // These operations lie about being legal: when they claim to be legal, 1264 // they should actually be custom-lowered. 1265 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1266 if (Action == TargetLowering::Legal) 1267 Action = TargetLowering::Custom; 1268 break; 1269 case ISD::READ_REGISTER: 1270 case ISD::WRITE_REGISTER: 1271 // Named register is legal in the DAG, but blocked by register name 1272 // selection if not implemented by target (to chose the correct register) 1273 // They'll be converted to Copy(To/From)Reg. 1274 Action = TargetLowering::Legal; 1275 break; 1276 case ISD::DEBUGTRAP: 1277 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1278 if (Action == TargetLowering::Expand) { 1279 // replace ISD::DEBUGTRAP with ISD::TRAP 1280 SDValue NewVal; 1281 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), 1282 Node->getOperand(0)); 1283 ReplaceNode(Node, NewVal.getNode()); 1284 LegalizeOp(NewVal.getNode()); 1285 return; 1286 } 1287 break; 1288 1289 default: 1290 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1291 Action = TargetLowering::Legal; 1292 } else { 1293 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1294 } 1295 break; 1296 } 1297 1298 if (SimpleFinishLegalizing) { 1299 SDNode *NewNode = Node; 1300 switch (Node->getOpcode()) { 1301 default: break; 1302 case ISD::SHL: 1303 case ISD::SRL: 1304 case ISD::SRA: 1305 case ISD::ROTL: 1306 case ISD::ROTR: 1307 // Legalizing shifts/rotates requires adjusting the shift amount 1308 // to the appropriate width. 1309 if (!Node->getOperand(1).getValueType().isVector()) { 1310 SDValue SAO = 1311 DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1312 Node->getOperand(1)); 1313 HandleSDNode Handle(SAO); 1314 LegalizeOp(SAO.getNode()); 1315 NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1316 Handle.getValue()); 1317 } 1318 break; 1319 case ISD::SRL_PARTS: 1320 case ISD::SRA_PARTS: 1321 case ISD::SHL_PARTS: 1322 // Legalizing shifts/rotates requires adjusting the shift amount 1323 // to the appropriate width. 1324 if (!Node->getOperand(2).getValueType().isVector()) { 1325 SDValue SAO = 1326 DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1327 Node->getOperand(2)); 1328 HandleSDNode Handle(SAO); 1329 LegalizeOp(SAO.getNode()); 1330 NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1331 Node->getOperand(1), 1332 Handle.getValue()); 1333 } 1334 break; 1335 } 1336 1337 if (NewNode != Node) { 1338 DAG.ReplaceAllUsesWith(Node, NewNode); 1339 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1340 DAG.TransferDbgValues(SDValue(Node, i), SDValue(NewNode, i)); 1341 ReplacedNode(Node); 1342 Node = NewNode; 1343 } 1344 switch (Action) { 1345 case TargetLowering::Legal: 1346 return; 1347 case TargetLowering::Custom: { 1348 // FIXME: The handling for custom lowering with multiple results is 1349 // a complete mess. 1350 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1351 if (Res.getNode()) { 1352 SmallVector<SDValue, 8> ResultVals; 1353 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 1354 if (e == 1) 1355 ResultVals.push_back(Res); 1356 else 1357 ResultVals.push_back(Res.getValue(i)); 1358 } 1359 if (Res.getNode() != Node || Res.getResNo() != 0) { 1360 DAG.ReplaceAllUsesWith(Node, ResultVals.data()); 1361 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1362 DAG.TransferDbgValues(SDValue(Node, i), ResultVals[i]); 1363 ReplacedNode(Node); 1364 } 1365 return; 1366 } 1367 } 1368 // FALL THROUGH 1369 case TargetLowering::Expand: 1370 ExpandNode(Node); 1371 return; 1372 case TargetLowering::Promote: 1373 PromoteNode(Node); 1374 return; 1375 } 1376 } 1377 1378 switch (Node->getOpcode()) { 1379 default: 1380#ifndef NDEBUG 1381 dbgs() << "NODE: "; 1382 Node->dump( &DAG); 1383 dbgs() << "\n"; 1384#endif 1385 llvm_unreachable("Do not know how to legalize this operator!"); 1386 1387 case ISD::CALLSEQ_START: 1388 case ISD::CALLSEQ_END: 1389 break; 1390 case ISD::LOAD: { 1391 return LegalizeLoadOps(Node); 1392 } 1393 case ISD::STORE: { 1394 return LegalizeStoreOps(Node); 1395 } 1396 } 1397} 1398 1399SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1400 SDValue Vec = Op.getOperand(0); 1401 SDValue Idx = Op.getOperand(1); 1402 SDLoc dl(Op); 1403 1404 // Before we generate a new store to a temporary stack slot, see if there is 1405 // already one that we can use. There often is because when we scalarize 1406 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole 1407 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in 1408 // the vector. If all are expanded here, we don't want one store per vector 1409 // element. 1410 SDValue StackPtr, Ch; 1411 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), 1412 UE = Vec.getNode()->use_end(); UI != UE; ++UI) { 1413 SDNode *User = *UI; 1414 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { 1415 if (ST->isIndexed() || ST->isTruncatingStore() || 1416 ST->getValue() != Vec) 1417 continue; 1418 1419 // Make sure that nothing else could have stored into the destination of 1420 // this store. 1421 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) 1422 continue; 1423 1424 StackPtr = ST->getBasePtr(); 1425 Ch = SDValue(ST, 0); 1426 break; 1427 } 1428 } 1429 1430 if (!Ch.getNode()) { 1431 // Store the value to a temporary stack slot, then LOAD the returned part. 1432 StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1433 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1434 MachinePointerInfo(), false, false, 0); 1435 } 1436 1437 // Add the offset to the index. 1438 unsigned EltSize = 1439 Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1440 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1441 DAG.getConstant(EltSize, Idx.getValueType())); 1442 1443 Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); 1444 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 1445 1446 if (Op.getValueType().isVector()) 1447 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), 1448 false, false, false, 0); 1449 return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1450 MachinePointerInfo(), 1451 Vec.getValueType().getVectorElementType(), 1452 false, false, 0); 1453} 1454 1455SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1456 assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1457 1458 SDValue Vec = Op.getOperand(0); 1459 SDValue Part = Op.getOperand(1); 1460 SDValue Idx = Op.getOperand(2); 1461 SDLoc dl(Op); 1462 1463 // Store the value to a temporary stack slot, then LOAD the returned part. 1464 1465 SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1466 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1467 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1468 1469 // First store the whole vector. 1470 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1471 false, false, 0); 1472 1473 // Then store the inserted part. 1474 1475 // Add the offset to the index. 1476 unsigned EltSize = 1477 Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1478 1479 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1480 DAG.getConstant(EltSize, Idx.getValueType())); 1481 Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); 1482 1483 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1484 StackPtr); 1485 1486 // Store the subvector. 1487 Ch = DAG.getStore(DAG.getEntryNode(), dl, Part, SubStackPtr, 1488 MachinePointerInfo(), false, false, 0); 1489 1490 // Finally, load the updated vector. 1491 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo, 1492 false, false, false, 0); 1493} 1494 1495SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1496 // We can't handle this case efficiently. Allocate a sufficiently 1497 // aligned object on the stack, store each element into it, then load 1498 // the result as a vector. 1499 // Create the stack frame object. 1500 EVT VT = Node->getValueType(0); 1501 EVT EltVT = VT.getVectorElementType(); 1502 SDLoc dl(Node); 1503 SDValue FIPtr = DAG.CreateStackTemporary(VT); 1504 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1505 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1506 1507 // Emit a store of each element to the stack slot. 1508 SmallVector<SDValue, 8> Stores; 1509 unsigned TypeByteSize = EltVT.getSizeInBits() / 8; 1510 // Store (in the right endianness) the elements to memory. 1511 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1512 // Ignore undef elements. 1513 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1514 1515 unsigned Offset = TypeByteSize*i; 1516 1517 SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 1518 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 1519 1520 // If the destination vector element type is narrower than the source 1521 // element type, only store the bits necessary. 1522 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { 1523 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1524 Node->getOperand(i), Idx, 1525 PtrInfo.getWithOffset(Offset), 1526 EltVT, false, false, 0)); 1527 } else 1528 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, 1529 Node->getOperand(i), Idx, 1530 PtrInfo.getWithOffset(Offset), 1531 false, false, 0)); 1532 } 1533 1534 SDValue StoreChain; 1535 if (!Stores.empty()) // Not all undef elements? 1536 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 1537 else 1538 StoreChain = DAG.getEntryNode(); 1539 1540 // Result is a load from the stack slot. 1541 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, 1542 false, false, false, 0); 1543} 1544 1545SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) { 1546 SDLoc dl(Node); 1547 SDValue Tmp1 = Node->getOperand(0); 1548 SDValue Tmp2 = Node->getOperand(1); 1549 1550 // Get the sign bit of the RHS. First obtain a value that has the same 1551 // sign as the sign bit, i.e. negative if and only if the sign bit is 1. 1552 SDValue SignBit; 1553 EVT FloatVT = Tmp2.getValueType(); 1554 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits()); 1555 if (TLI.isTypeLegal(IVT)) { 1556 // Convert to an integer with the same sign bit. 1557 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); 1558 } else { 1559 // Store the float to memory, then load the sign part out as an integer. 1560 MVT LoadTy = TLI.getPointerTy(); 1561 // First create a temporary that is aligned for both the load and store. 1562 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1563 // Then store the float to it. 1564 SDValue Ch = 1565 DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(), 1566 false, false, 0); 1567 if (TLI.isBigEndian()) { 1568 assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1569 // Load out a legal integer with the same sign bit as the float. 1570 SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(), 1571 false, false, false, 0); 1572 } else { // Little endian 1573 SDValue LoadPtr = StackPtr; 1574 // The float may be wider than the integer we are going to load. Advance 1575 // the pointer so that the loaded integer will contain the sign bit. 1576 unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits(); 1577 unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8; 1578 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), LoadPtr, 1579 DAG.getConstant(ByteOffset, LoadPtr.getValueType())); 1580 // Load a legal integer containing the sign bit. 1581 SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(), 1582 false, false, false, 0); 1583 // Move the sign bit to the top bit of the loaded integer. 1584 unsigned BitShift = LoadTy.getSizeInBits() - 1585 (FloatVT.getSizeInBits() - 8 * ByteOffset); 1586 assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?"); 1587 if (BitShift) 1588 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, 1589 DAG.getConstant(BitShift, 1590 TLI.getShiftAmountTy(SignBit.getValueType()))); 1591 } 1592 } 1593 // Now get the sign bit proper, by seeing whether the value is negative. 1594 SignBit = DAG.getSetCC(dl, getSetCCResultType(SignBit.getValueType()), 1595 SignBit, DAG.getConstant(0, SignBit.getValueType()), 1596 ISD::SETLT); 1597 // Get the absolute value of the result. 1598 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 1599 // Select between the nabs and abs value based on the sign bit of 1600 // the input. 1601 return DAG.getSelect(dl, AbsVal.getValueType(), SignBit, 1602 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), 1603 AbsVal); 1604} 1605 1606void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1607 SmallVectorImpl<SDValue> &Results) { 1608 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1609 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1610 " not tell us which reg is the stack pointer!"); 1611 SDLoc dl(Node); 1612 EVT VT = Node->getValueType(0); 1613 SDValue Tmp1 = SDValue(Node, 0); 1614 SDValue Tmp2 = SDValue(Node, 1); 1615 SDValue Tmp3 = Node->getOperand(2); 1616 SDValue Chain = Tmp1.getOperand(0); 1617 1618 // Chain the dynamic stack allocation so that it doesn't modify the stack 1619 // pointer when other instructions are using the stack. 1620 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true), 1621 SDLoc(Node)); 1622 1623 SDValue Size = Tmp2.getOperand(1); 1624 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1625 Chain = SP.getValue(1); 1626 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1627 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 1628 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1629 if (Align > StackAlign) 1630 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, 1631 DAG.getConstant(-(uint64_t)Align, VT)); 1632 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1633 1634 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1635 DAG.getIntPtrConstant(0, true), SDValue(), 1636 SDLoc(Node)); 1637 1638 Results.push_back(Tmp1); 1639 Results.push_back(Tmp2); 1640} 1641 1642/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 1643/// condition code CC on the current target. 1644/// 1645/// If the SETCC has been legalized using AND / OR, then the legalized node 1646/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert 1647/// will be set to false. 1648/// 1649/// If the SETCC has been legalized by using getSetCCSwappedOperands(), 1650/// then the values of LHS and RHS will be swapped, CC will be set to the 1651/// new condition, and NeedInvert will be set to false. 1652/// 1653/// If the SETCC has been legalized using the inverse condcode, then LHS and 1654/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert 1655/// will be set to true. The caller must invert the result of the SETCC with 1656/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect 1657/// of a true/false result. 1658/// 1659/// \returns true if the SetCC has been legalized, false if it hasn't. 1660bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, 1661 SDValue &LHS, SDValue &RHS, 1662 SDValue &CC, 1663 bool &NeedInvert, 1664 SDLoc dl) { 1665 MVT OpVT = LHS.getSimpleValueType(); 1666 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1667 NeedInvert = false; 1668 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1669 default: llvm_unreachable("Unknown condition code action!"); 1670 case TargetLowering::Legal: 1671 // Nothing to do. 1672 break; 1673 case TargetLowering::Expand: { 1674 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); 1675 if (TLI.isCondCodeLegal(InvCC, OpVT)) { 1676 std::swap(LHS, RHS); 1677 CC = DAG.getCondCode(InvCC); 1678 return true; 1679 } 1680 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1681 unsigned Opc = 0; 1682 switch (CCCode) { 1683 default: llvm_unreachable("Don't know how to expand this condition!"); 1684 case ISD::SETO: 1685 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) 1686 == TargetLowering::Legal 1687 && "If SETO is expanded, SETOEQ must be legal!"); 1688 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; 1689 case ISD::SETUO: 1690 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) 1691 == TargetLowering::Legal 1692 && "If SETUO is expanded, SETUNE must be legal!"); 1693 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; 1694 case ISD::SETOEQ: 1695 case ISD::SETOGT: 1696 case ISD::SETOGE: 1697 case ISD::SETOLT: 1698 case ISD::SETOLE: 1699 case ISD::SETONE: 1700 case ISD::SETUEQ: 1701 case ISD::SETUNE: 1702 case ISD::SETUGT: 1703 case ISD::SETUGE: 1704 case ISD::SETULT: 1705 case ISD::SETULE: 1706 // If we are floating point, assign and break, otherwise fall through. 1707 if (!OpVT.isInteger()) { 1708 // We can use the 4th bit to tell if we are the unordered 1709 // or ordered version of the opcode. 1710 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1711 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; 1712 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); 1713 break; 1714 } 1715 // Fallthrough if we are unsigned integer. 1716 case ISD::SETLE: 1717 case ISD::SETGT: 1718 case ISD::SETGE: 1719 case ISD::SETLT: 1720 // We only support using the inverted operation, which is computed above 1721 // and not a different manner of supporting expanding these cases. 1722 llvm_unreachable("Don't know how to expand this condition!"); 1723 case ISD::SETNE: 1724 case ISD::SETEQ: 1725 // Try inverting the result of the inverse condition. 1726 InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ; 1727 if (TLI.isCondCodeLegal(InvCC, OpVT)) { 1728 CC = DAG.getCondCode(InvCC); 1729 NeedInvert = true; 1730 return true; 1731 } 1732 // If inverting the condition didn't work then we have no means to expand 1733 // the condition. 1734 llvm_unreachable("Don't know how to expand this condition!"); 1735 } 1736 1737 SDValue SetCC1, SetCC2; 1738 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { 1739 // If we aren't the ordered or unorder operation, 1740 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). 1741 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 1742 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 1743 } else { 1744 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) 1745 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); 1746 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); 1747 } 1748 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1749 RHS = SDValue(); 1750 CC = SDValue(); 1751 return true; 1752 } 1753 } 1754 return false; 1755} 1756 1757/// EmitStackConvert - Emit a store/load combination to the stack. This stores 1758/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1759/// a load from the stack slot to DestVT, extending it if needed. 1760/// The resultant code need not be legal. 1761SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 1762 EVT SlotVT, 1763 EVT DestVT, 1764 SDLoc dl) { 1765 // Create the stack frame object. 1766 unsigned SrcAlign = 1767 TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType(). 1768 getTypeForEVT(*DAG.getContext())); 1769 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 1770 1771 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1772 int SPFI = StackPtrFI->getIndex(); 1773 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); 1774 1775 unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 1776 unsigned SlotSize = SlotVT.getSizeInBits(); 1777 unsigned DestSize = DestVT.getSizeInBits(); 1778 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1779 unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType); 1780 1781 // Emit a store to the stack slot. Use a truncstore if the input value is 1782 // later than DestVT. 1783 SDValue Store; 1784 1785 if (SrcSize > SlotSize) 1786 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1787 PtrInfo, SlotVT, false, false, SrcAlign); 1788 else { 1789 assert(SrcSize == SlotSize && "Invalid store"); 1790 Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1791 PtrInfo, false, false, SrcAlign); 1792 } 1793 1794 // Result is a load from the stack slot. 1795 if (SlotSize == DestSize) 1796 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, 1797 false, false, false, DestAlign); 1798 1799 assert(SlotSize < DestSize && "Unknown extension!"); 1800 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, 1801 PtrInfo, SlotVT, false, false, DestAlign); 1802} 1803 1804SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1805 SDLoc dl(Node); 1806 // Create a vector sized/aligned stack slot, store the value to element #0, 1807 // then load the whole vector back out. 1808 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1809 1810 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1811 int SPFI = StackPtrFI->getIndex(); 1812 1813 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), 1814 StackPtr, 1815 MachinePointerInfo::getFixedStack(SPFI), 1816 Node->getValueType(0).getVectorElementType(), 1817 false, false, 0); 1818 return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 1819 MachinePointerInfo::getFixedStack(SPFI), 1820 false, false, false, 0); 1821} 1822 1823static bool 1824ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, 1825 const TargetLowering &TLI, SDValue &Res) { 1826 unsigned NumElems = Node->getNumOperands(); 1827 SDLoc dl(Node); 1828 EVT VT = Node->getValueType(0); 1829 1830 // Try to group the scalars into pairs, shuffle the pairs together, then 1831 // shuffle the pairs of pairs together, etc. until the vector has 1832 // been built. This will work only if all of the necessary shuffle masks 1833 // are legal. 1834 1835 // We do this in two phases; first to check the legality of the shuffles, 1836 // and next, assuming that all shuffles are legal, to create the new nodes. 1837 for (int Phase = 0; Phase < 2; ++Phase) { 1838 SmallVector<std::pair<SDValue, SmallVector<int, 16> >, 16> IntermedVals, 1839 NewIntermedVals; 1840 for (unsigned i = 0; i < NumElems; ++i) { 1841 SDValue V = Node->getOperand(i); 1842 if (V.getOpcode() == ISD::UNDEF) 1843 continue; 1844 1845 SDValue Vec; 1846 if (Phase) 1847 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); 1848 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); 1849 } 1850 1851 while (IntermedVals.size() > 2) { 1852 NewIntermedVals.clear(); 1853 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { 1854 // This vector and the next vector are shuffled together (simply to 1855 // append the one to the other). 1856 SmallVector<int, 16> ShuffleVec(NumElems, -1); 1857 1858 SmallVector<int, 16> FinalIndices; 1859 FinalIndices.reserve(IntermedVals[i].second.size() + 1860 IntermedVals[i+1].second.size()); 1861 1862 int k = 0; 1863 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; 1864 ++j, ++k) { 1865 ShuffleVec[k] = j; 1866 FinalIndices.push_back(IntermedVals[i].second[j]); 1867 } 1868 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; 1869 ++j, ++k) { 1870 ShuffleVec[k] = NumElems + j; 1871 FinalIndices.push_back(IntermedVals[i+1].second[j]); 1872 } 1873 1874 SDValue Shuffle; 1875 if (Phase) 1876 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, 1877 IntermedVals[i+1].first, 1878 ShuffleVec.data()); 1879 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) 1880 return false; 1881 NewIntermedVals.push_back(std::make_pair(Shuffle, FinalIndices)); 1882 } 1883 1884 // If we had an odd number of defined values, then append the last 1885 // element to the array of new vectors. 1886 if ((IntermedVals.size() & 1) != 0) 1887 NewIntermedVals.push_back(IntermedVals.back()); 1888 1889 IntermedVals.swap(NewIntermedVals); 1890 } 1891 1892 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && 1893 "Invalid number of intermediate vectors"); 1894 SDValue Vec1 = IntermedVals[0].first; 1895 SDValue Vec2; 1896 if (IntermedVals.size() > 1) 1897 Vec2 = IntermedVals[1].first; 1898 else if (Phase) 1899 Vec2 = DAG.getUNDEF(VT); 1900 1901 SmallVector<int, 16> ShuffleVec(NumElems, -1); 1902 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) 1903 ShuffleVec[IntermedVals[0].second[i]] = i; 1904 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) 1905 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; 1906 1907 if (Phase) 1908 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); 1909 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) 1910 return false; 1911 } 1912 1913 return true; 1914} 1915 1916/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 1917/// support the operation, but do support the resultant vector type. 1918SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1919 unsigned NumElems = Node->getNumOperands(); 1920 SDValue Value1, Value2; 1921 SDLoc dl(Node); 1922 EVT VT = Node->getValueType(0); 1923 EVT OpVT = Node->getOperand(0).getValueType(); 1924 EVT EltVT = VT.getVectorElementType(); 1925 1926 // If the only non-undef value is the low element, turn this into a 1927 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 1928 bool isOnlyLowElement = true; 1929 bool MoreThanTwoValues = false; 1930 bool isConstant = true; 1931 for (unsigned i = 0; i < NumElems; ++i) { 1932 SDValue V = Node->getOperand(i); 1933 if (V.getOpcode() == ISD::UNDEF) 1934 continue; 1935 if (i > 0) 1936 isOnlyLowElement = false; 1937 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 1938 isConstant = false; 1939 1940 if (!Value1.getNode()) { 1941 Value1 = V; 1942 } else if (!Value2.getNode()) { 1943 if (V != Value1) 1944 Value2 = V; 1945 } else if (V != Value1 && V != Value2) { 1946 MoreThanTwoValues = true; 1947 } 1948 } 1949 1950 if (!Value1.getNode()) 1951 return DAG.getUNDEF(VT); 1952 1953 if (isOnlyLowElement) 1954 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 1955 1956 // If all elements are constants, create a load from the constant pool. 1957 if (isConstant) { 1958 SmallVector<Constant*, 16> CV; 1959 for (unsigned i = 0, e = NumElems; i != e; ++i) { 1960 if (ConstantFPSDNode *V = 1961 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 1962 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 1963 } else if (ConstantSDNode *V = 1964 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 1965 if (OpVT==EltVT) 1966 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 1967 else { 1968 // If OpVT and EltVT don't match, EltVT is not legal and the 1969 // element values have been promoted/truncated earlier. Undo this; 1970 // we don't want a v16i8 to become a v16i32 for example. 1971 const ConstantInt *CI = V->getConstantIntValue(); 1972 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 1973 CI->getZExtValue())); 1974 } 1975 } else { 1976 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 1977 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 1978 CV.push_back(UndefValue::get(OpNTy)); 1979 } 1980 } 1981 Constant *CP = ConstantVector::get(CV); 1982 SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 1983 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 1984 return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 1985 MachinePointerInfo::getConstantPool(), 1986 false, false, false, Alignment); 1987 } 1988 1989 SmallSet<SDValue, 16> DefinedValues; 1990 for (unsigned i = 0; i < NumElems; ++i) { 1991 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) 1992 continue; 1993 DefinedValues.insert(Node->getOperand(i)); 1994 } 1995 1996 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { 1997 if (!MoreThanTwoValues) { 1998 SmallVector<int, 8> ShuffleVec(NumElems, -1); 1999 for (unsigned i = 0; i < NumElems; ++i) { 2000 SDValue V = Node->getOperand(i); 2001 if (V.getOpcode() == ISD::UNDEF) 2002 continue; 2003 ShuffleVec[i] = V == Value1 ? 0 : NumElems; 2004 } 2005 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 2006 // Get the splatted value into the low element of a vector register. 2007 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 2008 SDValue Vec2; 2009 if (Value2.getNode()) 2010 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 2011 else 2012 Vec2 = DAG.getUNDEF(VT); 2013 2014 // Return shuffle(LowValVec, undef, <0,0,0,0>) 2015 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); 2016 } 2017 } else { 2018 SDValue Res; 2019 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) 2020 return Res; 2021 } 2022 } 2023 2024 // Otherwise, we can't handle this case efficiently. 2025 return ExpandVectorBuildThroughStack(Node); 2026} 2027 2028// ExpandLibCall - Expand a node into a call to a libcall. If the result value 2029// does not fit into a register, return the lo part and set the hi part to the 2030// by-reg argument. If it does fit into a single register, return the result 2031// and leave the Hi part unset. 2032SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 2033 bool isSigned) { 2034 TargetLowering::ArgListTy Args; 2035 TargetLowering::ArgListEntry Entry; 2036 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 2037 EVT ArgVT = Node->getOperand(i).getValueType(); 2038 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2039 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 2040 Entry.isSExt = isSigned; 2041 Entry.isZExt = !isSigned; 2042 Args.push_back(Entry); 2043 } 2044 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2045 TLI.getPointerTy()); 2046 2047 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 2048 2049 // By default, the input chain to this libcall is the entry node of the 2050 // function. If the libcall is going to be emitted as a tail call then 2051 // TLI.isUsedByReturnOnly will change it to the right chain if the return 2052 // node which is being folded has a non-entry input chain. 2053 SDValue InChain = DAG.getEntryNode(); 2054 2055 // isTailCall may be true since the callee does not reference caller stack 2056 // frame. Check if it's in the right position. 2057 SDValue TCChain = InChain; 2058 bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain); 2059 if (isTailCall) 2060 InChain = TCChain; 2061 2062 TargetLowering::CallLoweringInfo CLI(DAG); 2063 CLI.setDebugLoc(SDLoc(Node)).setChain(InChain) 2064 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) 2065 .setTailCall(isTailCall).setSExtResult(isSigned).setZExtResult(!isSigned); 2066 2067 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2068 2069 if (!CallInfo.second.getNode()) 2070 // It's a tailcall, return the chain (which is the DAG root). 2071 return DAG.getRoot(); 2072 2073 return CallInfo.first; 2074} 2075 2076/// ExpandLibCall - Generate a libcall taking the given operands as arguments 2077/// and returning a result of type RetVT. 2078SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, 2079 const SDValue *Ops, unsigned NumOps, 2080 bool isSigned, SDLoc dl) { 2081 TargetLowering::ArgListTy Args; 2082 Args.reserve(NumOps); 2083 2084 TargetLowering::ArgListEntry Entry; 2085 for (unsigned i = 0; i != NumOps; ++i) { 2086 Entry.Node = Ops[i]; 2087 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); 2088 Entry.isSExt = isSigned; 2089 Entry.isZExt = !isSigned; 2090 Args.push_back(Entry); 2091 } 2092 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2093 TLI.getPointerTy()); 2094 2095 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2096 2097 TargetLowering::CallLoweringInfo CLI(DAG); 2098 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode()) 2099 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) 2100 .setSExtResult(isSigned).setZExtResult(!isSigned); 2101 2102 std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); 2103 2104 return CallInfo.first; 2105} 2106 2107// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to 2108// ExpandLibCall except that the first operand is the in-chain. 2109std::pair<SDValue, SDValue> 2110SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, 2111 SDNode *Node, 2112 bool isSigned) { 2113 SDValue InChain = Node->getOperand(0); 2114 2115 TargetLowering::ArgListTy Args; 2116 TargetLowering::ArgListEntry Entry; 2117 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { 2118 EVT ArgVT = Node->getOperand(i).getValueType(); 2119 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2120 Entry.Node = Node->getOperand(i); 2121 Entry.Ty = ArgTy; 2122 Entry.isSExt = isSigned; 2123 Entry.isZExt = !isSigned; 2124 Args.push_back(Entry); 2125 } 2126 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2127 TLI.getPointerTy()); 2128 2129 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 2130 2131 TargetLowering::CallLoweringInfo CLI(DAG); 2132 CLI.setDebugLoc(SDLoc(Node)).setChain(InChain) 2133 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) 2134 .setSExtResult(isSigned).setZExtResult(!isSigned); 2135 2136 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2137 2138 return CallInfo; 2139} 2140 2141SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 2142 RTLIB::Libcall Call_F32, 2143 RTLIB::Libcall Call_F64, 2144 RTLIB::Libcall Call_F80, 2145 RTLIB::Libcall Call_F128, 2146 RTLIB::Libcall Call_PPCF128) { 2147 RTLIB::Libcall LC; 2148 switch (Node->getSimpleValueType(0).SimpleTy) { 2149 default: llvm_unreachable("Unexpected request for libcall!"); 2150 case MVT::f32: LC = Call_F32; break; 2151 case MVT::f64: LC = Call_F64; break; 2152 case MVT::f80: LC = Call_F80; break; 2153 case MVT::f128: LC = Call_F128; break; 2154 case MVT::ppcf128: LC = Call_PPCF128; break; 2155 } 2156 return ExpandLibCall(LC, Node, false); 2157} 2158 2159SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 2160 RTLIB::Libcall Call_I8, 2161 RTLIB::Libcall Call_I16, 2162 RTLIB::Libcall Call_I32, 2163 RTLIB::Libcall Call_I64, 2164 RTLIB::Libcall Call_I128) { 2165 RTLIB::Libcall LC; 2166 switch (Node->getSimpleValueType(0).SimpleTy) { 2167 default: llvm_unreachable("Unexpected request for libcall!"); 2168 case MVT::i8: LC = Call_I8; break; 2169 case MVT::i16: LC = Call_I16; break; 2170 case MVT::i32: LC = Call_I32; break; 2171 case MVT::i64: LC = Call_I64; break; 2172 case MVT::i128: LC = Call_I128; break; 2173 } 2174 return ExpandLibCall(LC, Node, isSigned); 2175} 2176 2177/// isDivRemLibcallAvailable - Return true if divmod libcall is available. 2178static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, 2179 const TargetLowering &TLI) { 2180 RTLIB::Libcall LC; 2181 switch (Node->getSimpleValueType(0).SimpleTy) { 2182 default: llvm_unreachable("Unexpected request for libcall!"); 2183 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2184 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2185 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2186 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2187 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2188 } 2189 2190 return TLI.getLibcallName(LC) != nullptr; 2191} 2192 2193/// useDivRem - Only issue divrem libcall if both quotient and remainder are 2194/// needed. 2195static bool useDivRem(SDNode *Node, bool isSigned, bool isDIV) { 2196 // The other use might have been replaced with a divrem already. 2197 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 2198 unsigned OtherOpcode = 0; 2199 if (isSigned) 2200 OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; 2201 else 2202 OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; 2203 2204 SDValue Op0 = Node->getOperand(0); 2205 SDValue Op1 = Node->getOperand(1); 2206 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2207 UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2208 SDNode *User = *UI; 2209 if (User == Node) 2210 continue; 2211 if ((User->getOpcode() == OtherOpcode || User->getOpcode() == DivRemOpc) && 2212 User->getOperand(0) == Op0 && 2213 User->getOperand(1) == Op1) 2214 return true; 2215 } 2216 return false; 2217} 2218 2219/// ExpandDivRemLibCall - Issue libcalls to __{u}divmod to compute div / rem 2220/// pairs. 2221void 2222SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 2223 SmallVectorImpl<SDValue> &Results) { 2224 unsigned Opcode = Node->getOpcode(); 2225 bool isSigned = Opcode == ISD::SDIVREM; 2226 2227 RTLIB::Libcall LC; 2228 switch (Node->getSimpleValueType(0).SimpleTy) { 2229 default: llvm_unreachable("Unexpected request for libcall!"); 2230 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2231 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2232 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2233 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2234 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2235 } 2236 2237 // The input chain to this libcall is the entry node of the function. 2238 // Legalizing the call will automatically add the previous call to the 2239 // dependence. 2240 SDValue InChain = DAG.getEntryNode(); 2241 2242 EVT RetVT = Node->getValueType(0); 2243 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2244 2245 TargetLowering::ArgListTy Args; 2246 TargetLowering::ArgListEntry Entry; 2247 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 2248 EVT ArgVT = Node->getOperand(i).getValueType(); 2249 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2250 Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 2251 Entry.isSExt = isSigned; 2252 Entry.isZExt = !isSigned; 2253 Args.push_back(Entry); 2254 } 2255 2256 // Also pass the return address of the remainder. 2257 SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 2258 Entry.Node = FIPtr; 2259 Entry.Ty = RetTy->getPointerTo(); 2260 Entry.isSExt = isSigned; 2261 Entry.isZExt = !isSigned; 2262 Args.push_back(Entry); 2263 2264 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2265 TLI.getPointerTy()); 2266 2267 SDLoc dl(Node); 2268 TargetLowering::CallLoweringInfo CLI(DAG); 2269 CLI.setDebugLoc(dl).setChain(InChain) 2270 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0) 2271 .setSExtResult(isSigned).setZExtResult(!isSigned); 2272 2273 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2274 2275 // Remainder is loaded back from the stack frame. 2276 SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, 2277 MachinePointerInfo(), false, false, false, 0); 2278 Results.push_back(CallInfo.first); 2279 Results.push_back(Rem); 2280} 2281 2282/// isSinCosLibcallAvailable - Return true if sincos libcall is available. 2283static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { 2284 RTLIB::Libcall LC; 2285 switch (Node->getSimpleValueType(0).SimpleTy) { 2286 default: llvm_unreachable("Unexpected request for libcall!"); 2287 case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2288 case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2289 case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2290 case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2291 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2292 } 2293 return TLI.getLibcallName(LC) != nullptr; 2294} 2295 2296/// canCombineSinCosLibcall - Return true if sincos libcall is available and 2297/// can be used to combine sin and cos. 2298static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, 2299 const TargetMachine &TM) { 2300 if (!isSinCosLibcallAvailable(Node, TLI)) 2301 return false; 2302 // GNU sin/cos functions set errno while sincos does not. Therefore 2303 // combining sin and cos is only safe if unsafe-fpmath is enabled. 2304 bool isGNU = Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU; 2305 if (isGNU && !TM.Options.UnsafeFPMath) 2306 return false; 2307 return true; 2308} 2309 2310/// useSinCos - Only issue sincos libcall if both sin and cos are 2311/// needed. 2312static bool useSinCos(SDNode *Node) { 2313 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN 2314 ? ISD::FCOS : ISD::FSIN; 2315 2316 SDValue Op0 = Node->getOperand(0); 2317 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2318 UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2319 SDNode *User = *UI; 2320 if (User == Node) 2321 continue; 2322 // The other user might have been turned into sincos already. 2323 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) 2324 return true; 2325 } 2326 return false; 2327} 2328 2329/// ExpandSinCosLibCall - Issue libcalls to sincos to compute sin / cos 2330/// pairs. 2331void 2332SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, 2333 SmallVectorImpl<SDValue> &Results) { 2334 RTLIB::Libcall LC; 2335 switch (Node->getSimpleValueType(0).SimpleTy) { 2336 default: llvm_unreachable("Unexpected request for libcall!"); 2337 case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2338 case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2339 case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2340 case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2341 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2342 } 2343 2344 // The input chain to this libcall is the entry node of the function. 2345 // Legalizing the call will automatically add the previous call to the 2346 // dependence. 2347 SDValue InChain = DAG.getEntryNode(); 2348 2349 EVT RetVT = Node->getValueType(0); 2350 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2351 2352 TargetLowering::ArgListTy Args; 2353 TargetLowering::ArgListEntry Entry; 2354 2355 // Pass the argument. 2356 Entry.Node = Node->getOperand(0); 2357 Entry.Ty = RetTy; 2358 Entry.isSExt = false; 2359 Entry.isZExt = false; 2360 Args.push_back(Entry); 2361 2362 // Pass the return address of sin. 2363 SDValue SinPtr = DAG.CreateStackTemporary(RetVT); 2364 Entry.Node = SinPtr; 2365 Entry.Ty = RetTy->getPointerTo(); 2366 Entry.isSExt = false; 2367 Entry.isZExt = false; 2368 Args.push_back(Entry); 2369 2370 // Also pass the return address of the cos. 2371 SDValue CosPtr = DAG.CreateStackTemporary(RetVT); 2372 Entry.Node = CosPtr; 2373 Entry.Ty = RetTy->getPointerTo(); 2374 Entry.isSExt = false; 2375 Entry.isZExt = false; 2376 Args.push_back(Entry); 2377 2378 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2379 TLI.getPointerTy()); 2380 2381 SDLoc dl(Node); 2382 TargetLowering::CallLoweringInfo CLI(DAG); 2383 CLI.setDebugLoc(dl).setChain(InChain) 2384 .setCallee(TLI.getLibcallCallingConv(LC), 2385 Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args), 0); 2386 2387 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2388 2389 Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, 2390 MachinePointerInfo(), false, false, false, 0)); 2391 Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, 2392 MachinePointerInfo(), false, false, false, 0)); 2393} 2394 2395/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 2396/// INT_TO_FP operation of the specified operand when the target requests that 2397/// we expand it. At this point, we know that the result and operand types are 2398/// legal for the target. 2399SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 2400 SDValue Op0, 2401 EVT DestVT, 2402 SDLoc dl) { 2403 if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { 2404 // simple 32-bit [signed|unsigned] integer to float/double expansion 2405 2406 // Get the stack frame index of a 8 byte buffer. 2407 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2408 2409 // word offset constant for Hi/Lo address computation 2410 SDValue WordOff = DAG.getConstant(sizeof(int), StackSlot.getValueType()); 2411 // set up Hi and Lo (into buffer) address based on endian 2412 SDValue Hi = StackSlot; 2413 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), 2414 StackSlot, WordOff); 2415 if (TLI.isLittleEndian()) 2416 std::swap(Hi, Lo); 2417 2418 // if signed map to unsigned space 2419 SDValue Op0Mapped; 2420 if (isSigned) { 2421 // constant used to invert sign bit (signed to unsigned mapping) 2422 SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 2423 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 2424 } else { 2425 Op0Mapped = Op0; 2426 } 2427 // store the lo of the constructed double - based on integer input 2428 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 2429 Op0Mapped, Lo, MachinePointerInfo(), 2430 false, false, 0); 2431 // initial hi portion of constructed double 2432 SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 2433 // store the hi of the constructed double - biased exponent 2434 SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi, 2435 MachinePointerInfo(), 2436 false, false, 0); 2437 // load the constructed double 2438 SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, 2439 MachinePointerInfo(), false, false, false, 0); 2440 // FP constant to bias correct the final result 2441 SDValue Bias = DAG.getConstantFP(isSigned ? 2442 BitsToDouble(0x4330000080000000ULL) : 2443 BitsToDouble(0x4330000000000000ULL), 2444 MVT::f64); 2445 // subtract the bias 2446 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2447 // final result 2448 SDValue Result; 2449 // handle final rounding 2450 if (DestVT == MVT::f64) { 2451 // do nothing 2452 Result = Sub; 2453 } else if (DestVT.bitsLT(MVT::f64)) { 2454 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 2455 DAG.getIntPtrConstant(0)); 2456 } else if (DestVT.bitsGT(MVT::f64)) { 2457 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 2458 } 2459 return Result; 2460 } 2461 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 2462 // Code below here assumes !isSigned without checking again. 2463 2464 // Implementation of unsigned i64 to f64 following the algorithm in 2465 // __floatundidf in compiler_rt. This implementation has the advantage 2466 // of performing rounding correctly, both in the default rounding mode 2467 // and in all alternate rounding modes. 2468 // TODO: Generalize this for use with other types. 2469 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) { 2470 SDValue TwoP52 = 2471 DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64); 2472 SDValue TwoP84PlusTwoP52 = 2473 DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64); 2474 SDValue TwoP84 = 2475 DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64); 2476 2477 SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); 2478 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, 2479 DAG.getConstant(32, MVT::i64)); 2480 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); 2481 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); 2482 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); 2483 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); 2484 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, 2485 TwoP84PlusTwoP52); 2486 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); 2487 } 2488 2489 // Implementation of unsigned i64 to f32. 2490 // TODO: Generalize this for use with other types. 2491 if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { 2492 // For unsigned conversions, convert them to signed conversions using the 2493 // algorithm from the x86_64 __floatundidf in compiler_rt. 2494 if (!isSigned) { 2495 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); 2496 2497 SDValue ShiftConst = 2498 DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType())); 2499 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); 2500 SDValue AndConst = DAG.getConstant(1, MVT::i64); 2501 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); 2502 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); 2503 2504 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); 2505 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); 2506 2507 // TODO: This really should be implemented using a branch rather than a 2508 // select. We happen to get lucky and machinesink does the right 2509 // thing most of the time. This would be a good candidate for a 2510 //pseudo-op, or, even better, for whole-function isel. 2511 SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 2512 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); 2513 return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast); 2514 } 2515 2516 // Otherwise, implement the fully general conversion. 2517 2518 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2519 DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); 2520 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, 2521 DAG.getConstant(UINT64_C(0x800), MVT::i64)); 2522 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2523 DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); 2524 SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 2525 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); 2526 SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0); 2527 SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 2528 Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), 2529 ISD::SETUGE); 2530 SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0); 2531 EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType()); 2532 2533 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, 2534 DAG.getConstant(32, SHVT)); 2535 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); 2536 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); 2537 SDValue TwoP32 = 2538 DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); 2539 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); 2540 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); 2541 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); 2542 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); 2543 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, 2544 DAG.getIntPtrConstant(0)); 2545 } 2546 2547 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2548 2549 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()), 2550 Op0, DAG.getConstant(0, Op0.getValueType()), 2551 ISD::SETLT); 2552 SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 2553 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), 2554 SignSet, Four, Zero); 2555 2556 // If the sign bit of the integer is set, the large number will be treated 2557 // as a negative number. To counteract this, the dynamic code adds an 2558 // offset depending on the data type. 2559 uint64_t FF; 2560 switch (Op0.getSimpleValueType().SimpleTy) { 2561 default: llvm_unreachable("Unsupported integer type!"); 2562 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2563 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2564 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2565 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2566 } 2567 if (TLI.isLittleEndian()) FF <<= 32; 2568 Constant *FudgeFactor = ConstantInt::get( 2569 Type::getInt64Ty(*DAG.getContext()), FF); 2570 2571 SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 2572 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 2573 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); 2574 Alignment = std::min(Alignment, 4u); 2575 SDValue FudgeInReg; 2576 if (DestVT == MVT::f32) 2577 FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2578 MachinePointerInfo::getConstantPool(), 2579 false, false, false, Alignment); 2580 else { 2581 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 2582 DAG.getEntryNode(), CPIdx, 2583 MachinePointerInfo::getConstantPool(), 2584 MVT::f32, false, false, Alignment); 2585 HandleSDNode Handle(Load); 2586 LegalizeOp(Load.getNode()); 2587 FudgeInReg = Handle.getValue(); 2588 } 2589 2590 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2591} 2592 2593/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 2594/// *INT_TO_FP operation of the specified operand when the target requests that 2595/// we promote it. At this point, we know that the result and operand types are 2596/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2597/// operation that takes a larger input. 2598SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 2599 EVT DestVT, 2600 bool isSigned, 2601 SDLoc dl) { 2602 // First step, figure out the appropriate *INT_TO_FP operation to use. 2603 EVT NewInTy = LegalOp.getValueType(); 2604 2605 unsigned OpToUse = 0; 2606 2607 // Scan for the appropriate larger type to use. 2608 while (1) { 2609 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2610 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2611 2612 // If the target supports SINT_TO_FP of this type, use it. 2613 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { 2614 OpToUse = ISD::SINT_TO_FP; 2615 break; 2616 } 2617 if (isSigned) continue; 2618 2619 // If the target supports UINT_TO_FP of this type, use it. 2620 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { 2621 OpToUse = ISD::UINT_TO_FP; 2622 break; 2623 } 2624 2625 // Otherwise, try a larger type. 2626 } 2627 2628 // Okay, we found the operation and type to use. Zero extend our input to the 2629 // desired type then run the operation on it. 2630 return DAG.getNode(OpToUse, dl, DestVT, 2631 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2632 dl, NewInTy, LegalOp)); 2633} 2634 2635/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 2636/// FP_TO_*INT operation of the specified operand when the target requests that 2637/// we promote it. At this point, we know that the result and operand types are 2638/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2639/// operation that returns a larger result. 2640SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 2641 EVT DestVT, 2642 bool isSigned, 2643 SDLoc dl) { 2644 // First step, figure out the appropriate FP_TO*INT operation to use. 2645 EVT NewOutTy = DestVT; 2646 2647 unsigned OpToUse = 0; 2648 2649 // Scan for the appropriate larger type to use. 2650 while (1) { 2651 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2652 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2653 2654 // A larger signed type can hold all unsigned values of the requested type, 2655 // so using FP_TO_SINT is valid 2656 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { 2657 OpToUse = ISD::FP_TO_SINT; 2658 break; 2659 } 2660 2661 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. 2662 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { 2663 OpToUse = ISD::FP_TO_UINT; 2664 break; 2665 } 2666 2667 // Otherwise, try a larger type. 2668 } 2669 2670 2671 // Okay, we found the operation and type to use. 2672 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2673 2674 // Truncate the result of the extended FP_TO_*INT operation to the desired 2675 // size. 2676 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2677} 2678 2679/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 2680/// 2681SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, SDLoc dl) { 2682 EVT VT = Op.getValueType(); 2683 EVT SHVT = TLI.getShiftAmountTy(VT); 2684 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2685 switch (VT.getSimpleVT().SimpleTy) { 2686 default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2687 case MVT::i16: 2688 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2689 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2690 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 2691 case MVT::i32: 2692 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2693 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2694 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2695 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2696 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 2697 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 2698 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2699 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2700 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2701 case MVT::i64: 2702 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2703 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2704 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2705 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2706 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2707 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2708 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2709 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2710 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 2711 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 2712 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 2713 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 2714 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 2715 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 2716 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2717 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2718 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2719 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2720 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2721 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2722 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2723 } 2724} 2725 2726/// ExpandBitCount - Expand the specified bitcount instruction into operations. 2727/// 2728SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 2729 SDLoc dl) { 2730 switch (Opc) { 2731 default: llvm_unreachable("Cannot expand this yet!"); 2732 case ISD::CTPOP: { 2733 EVT VT = Op.getValueType(); 2734 EVT ShVT = TLI.getShiftAmountTy(VT); 2735 unsigned Len = VT.getSizeInBits(); 2736 2737 assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && 2738 "CTPOP not implemented for this type."); 2739 2740 // This is the "best" algorithm from 2741 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 2742 2743 SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), VT); 2744 SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), VT); 2745 SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), VT); 2746 SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), VT); 2747 2748 // v = v - ((v >> 1) & 0x55555555...) 2749 Op = DAG.getNode(ISD::SUB, dl, VT, Op, 2750 DAG.getNode(ISD::AND, dl, VT, 2751 DAG.getNode(ISD::SRL, dl, VT, Op, 2752 DAG.getConstant(1, ShVT)), 2753 Mask55)); 2754 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) 2755 Op = DAG.getNode(ISD::ADD, dl, VT, 2756 DAG.getNode(ISD::AND, dl, VT, Op, Mask33), 2757 DAG.getNode(ISD::AND, dl, VT, 2758 DAG.getNode(ISD::SRL, dl, VT, Op, 2759 DAG.getConstant(2, ShVT)), 2760 Mask33)); 2761 // v = (v + (v >> 4)) & 0x0F0F0F0F... 2762 Op = DAG.getNode(ISD::AND, dl, VT, 2763 DAG.getNode(ISD::ADD, dl, VT, Op, 2764 DAG.getNode(ISD::SRL, dl, VT, Op, 2765 DAG.getConstant(4, ShVT))), 2766 Mask0F); 2767 // v = (v * 0x01010101...) >> (Len - 8) 2768 Op = DAG.getNode(ISD::SRL, dl, VT, 2769 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), 2770 DAG.getConstant(Len - 8, ShVT)); 2771 2772 return Op; 2773 } 2774 case ISD::CTLZ_ZERO_UNDEF: 2775 // This trivially expands to CTLZ. 2776 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); 2777 case ISD::CTLZ: { 2778 // for now, we do this: 2779 // x = x | (x >> 1); 2780 // x = x | (x >> 2); 2781 // ... 2782 // x = x | (x >>16); 2783 // x = x | (x >>32); // for 64-bit input 2784 // return popcount(~x); 2785 // 2786 // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 2787 EVT VT = Op.getValueType(); 2788 EVT ShVT = TLI.getShiftAmountTy(VT); 2789 unsigned len = VT.getSizeInBits(); 2790 for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 2791 SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 2792 Op = DAG.getNode(ISD::OR, dl, VT, Op, 2793 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); 2794 } 2795 Op = DAG.getNOT(dl, Op, VT); 2796 return DAG.getNode(ISD::CTPOP, dl, VT, Op); 2797 } 2798 case ISD::CTTZ_ZERO_UNDEF: 2799 // This trivially expands to CTTZ. 2800 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); 2801 case ISD::CTTZ: { 2802 // for now, we use: { return popcount(~x & (x - 1)); } 2803 // unless the target has ctlz but not ctpop, in which case we use: 2804 // { return 32 - nlz(~x & (x-1)); } 2805 // see also http://www.hackersdelight.org/HDcode/ntz.cc 2806 EVT VT = Op.getValueType(); 2807 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 2808 DAG.getNOT(dl, Op, VT), 2809 DAG.getNode(ISD::SUB, dl, VT, Op, 2810 DAG.getConstant(1, VT))); 2811 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 2812 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 2813 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 2814 return DAG.getNode(ISD::SUB, dl, VT, 2815 DAG.getConstant(VT.getSizeInBits(), VT), 2816 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 2817 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 2818 } 2819 } 2820} 2821 2822std::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { 2823 unsigned Opc = Node->getOpcode(); 2824 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2825 RTLIB::Libcall LC; 2826 2827 switch (Opc) { 2828 default: 2829 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 2830 case ISD::ATOMIC_SWAP: 2831 switch (VT.SimpleTy) { 2832 default: llvm_unreachable("Unexpected value type for atomic!"); 2833 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 2834 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 2835 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 2836 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 2837 case MVT::i128:LC = RTLIB::SYNC_LOCK_TEST_AND_SET_16;break; 2838 } 2839 break; 2840 case ISD::ATOMIC_CMP_SWAP: 2841 switch (VT.SimpleTy) { 2842 default: llvm_unreachable("Unexpected value type for atomic!"); 2843 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 2844 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 2845 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 2846 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 2847 case MVT::i128:LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16;break; 2848 } 2849 break; 2850 case ISD::ATOMIC_LOAD_ADD: 2851 switch (VT.SimpleTy) { 2852 default: llvm_unreachable("Unexpected value type for atomic!"); 2853 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 2854 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 2855 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 2856 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 2857 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_ADD_16;break; 2858 } 2859 break; 2860 case ISD::ATOMIC_LOAD_SUB: 2861 switch (VT.SimpleTy) { 2862 default: llvm_unreachable("Unexpected value type for atomic!"); 2863 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 2864 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 2865 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 2866 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 2867 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_SUB_16;break; 2868 } 2869 break; 2870 case ISD::ATOMIC_LOAD_AND: 2871 switch (VT.SimpleTy) { 2872 default: llvm_unreachable("Unexpected value type for atomic!"); 2873 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 2874 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 2875 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 2876 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 2877 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_AND_16;break; 2878 } 2879 break; 2880 case ISD::ATOMIC_LOAD_OR: 2881 switch (VT.SimpleTy) { 2882 default: llvm_unreachable("Unexpected value type for atomic!"); 2883 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 2884 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 2885 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 2886 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 2887 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_OR_16;break; 2888 } 2889 break; 2890 case ISD::ATOMIC_LOAD_XOR: 2891 switch (VT.SimpleTy) { 2892 default: llvm_unreachable("Unexpected value type for atomic!"); 2893 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 2894 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 2895 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 2896 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 2897 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_XOR_16;break; 2898 } 2899 break; 2900 case ISD::ATOMIC_LOAD_NAND: 2901 switch (VT.SimpleTy) { 2902 default: llvm_unreachable("Unexpected value type for atomic!"); 2903 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 2904 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 2905 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 2906 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 2907 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_NAND_16;break; 2908 } 2909 break; 2910 case ISD::ATOMIC_LOAD_MAX: 2911 switch (VT.SimpleTy) { 2912 default: llvm_unreachable("Unexpected value type for atomic!"); 2913 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MAX_1; break; 2914 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MAX_2; break; 2915 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MAX_4; break; 2916 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MAX_8; break; 2917 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MAX_16;break; 2918 } 2919 break; 2920 case ISD::ATOMIC_LOAD_UMAX: 2921 switch (VT.SimpleTy) { 2922 default: llvm_unreachable("Unexpected value type for atomic!"); 2923 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMAX_1; break; 2924 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMAX_2; break; 2925 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMAX_4; break; 2926 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMAX_8; break; 2927 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMAX_16;break; 2928 } 2929 break; 2930 case ISD::ATOMIC_LOAD_MIN: 2931 switch (VT.SimpleTy) { 2932 default: llvm_unreachable("Unexpected value type for atomic!"); 2933 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MIN_1; break; 2934 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MIN_2; break; 2935 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MIN_4; break; 2936 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MIN_8; break; 2937 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MIN_16;break; 2938 } 2939 break; 2940 case ISD::ATOMIC_LOAD_UMIN: 2941 switch (VT.SimpleTy) { 2942 default: llvm_unreachable("Unexpected value type for atomic!"); 2943 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMIN_1; break; 2944 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMIN_2; break; 2945 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMIN_4; break; 2946 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMIN_8; break; 2947 case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMIN_16;break; 2948 } 2949 break; 2950 } 2951 2952 return ExpandChainLibCall(LC, Node, false); 2953} 2954 2955void SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2956 SmallVector<SDValue, 8> Results; 2957 SDLoc dl(Node); 2958 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2959 bool NeedInvert; 2960 switch (Node->getOpcode()) { 2961 case ISD::CTPOP: 2962 case ISD::CTLZ: 2963 case ISD::CTLZ_ZERO_UNDEF: 2964 case ISD::CTTZ: 2965 case ISD::CTTZ_ZERO_UNDEF: 2966 Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); 2967 Results.push_back(Tmp1); 2968 break; 2969 case ISD::BSWAP: 2970 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2971 break; 2972 case ISD::FRAMEADDR: 2973 case ISD::RETURNADDR: 2974 case ISD::FRAME_TO_ARGS_OFFSET: 2975 Results.push_back(DAG.getConstant(0, Node->getValueType(0))); 2976 break; 2977 case ISD::FLT_ROUNDS_: 2978 Results.push_back(DAG.getConstant(1, Node->getValueType(0))); 2979 break; 2980 case ISD::EH_RETURN: 2981 case ISD::EH_LABEL: 2982 case ISD::PREFETCH: 2983 case ISD::VAEND: 2984 case ISD::EH_SJLJ_LONGJMP: 2985 // If the target didn't expand these, there's nothing to do, so just 2986 // preserve the chain and be done. 2987 Results.push_back(Node->getOperand(0)); 2988 break; 2989 case ISD::EH_SJLJ_SETJMP: 2990 // If the target didn't expand this, just return 'zero' and preserve the 2991 // chain. 2992 Results.push_back(DAG.getConstant(0, MVT::i32)); 2993 Results.push_back(Node->getOperand(0)); 2994 break; 2995 case ISD::ATOMIC_FENCE: { 2996 // If the target didn't lower this, lower it to '__sync_synchronize()' call 2997 // FIXME: handle "fence singlethread" more efficiently. 2998 TargetLowering::ArgListTy Args; 2999 3000 TargetLowering::CallLoweringInfo CLI(DAG); 3001 CLI.setDebugLoc(dl).setChain(Node->getOperand(0)) 3002 .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3003 DAG.getExternalSymbol("__sync_synchronize", 3004 TLI.getPointerTy()), std::move(Args), 0); 3005 3006 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 3007 3008 Results.push_back(CallResult.second); 3009 break; 3010 } 3011 case ISD::ATOMIC_LOAD: { 3012 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 3013 SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); 3014 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 3015 SDValue Swap = DAG.getAtomicCmpSwap( 3016 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, 3017 Node->getOperand(0), Node->getOperand(1), Zero, Zero, 3018 cast<AtomicSDNode>(Node)->getMemOperand(), 3019 cast<AtomicSDNode>(Node)->getOrdering(), 3020 cast<AtomicSDNode>(Node)->getOrdering(), 3021 cast<AtomicSDNode>(Node)->getSynchScope()); 3022 Results.push_back(Swap.getValue(0)); 3023 Results.push_back(Swap.getValue(1)); 3024 break; 3025 } 3026 case ISD::ATOMIC_STORE: { 3027 // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 3028 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 3029 cast<AtomicSDNode>(Node)->getMemoryVT(), 3030 Node->getOperand(0), 3031 Node->getOperand(1), Node->getOperand(2), 3032 cast<AtomicSDNode>(Node)->getMemOperand(), 3033 cast<AtomicSDNode>(Node)->getOrdering(), 3034 cast<AtomicSDNode>(Node)->getSynchScope()); 3035 Results.push_back(Swap.getValue(1)); 3036 break; 3037 } 3038 // By default, atomic intrinsics are marked Legal and lowered. Targets 3039 // which don't support them directly, however, may want libcalls, in which 3040 // case they mark them Expand, and we get here. 3041 case ISD::ATOMIC_SWAP: 3042 case ISD::ATOMIC_LOAD_ADD: 3043 case ISD::ATOMIC_LOAD_SUB: 3044 case ISD::ATOMIC_LOAD_AND: 3045 case ISD::ATOMIC_LOAD_OR: 3046 case ISD::ATOMIC_LOAD_XOR: 3047 case ISD::ATOMIC_LOAD_NAND: 3048 case ISD::ATOMIC_LOAD_MIN: 3049 case ISD::ATOMIC_LOAD_MAX: 3050 case ISD::ATOMIC_LOAD_UMIN: 3051 case ISD::ATOMIC_LOAD_UMAX: 3052 case ISD::ATOMIC_CMP_SWAP: { 3053 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node); 3054 Results.push_back(Tmp.first); 3055 Results.push_back(Tmp.second); 3056 break; 3057 } 3058 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 3059 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and 3060 // splits out the success value as a comparison. Expanding the resulting 3061 // ATOMIC_CMP_SWAP will produce a libcall. 3062 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 3063 SDValue Res = DAG.getAtomicCmpSwap( 3064 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, 3065 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), 3066 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(), 3067 cast<AtomicSDNode>(Node)->getSuccessOrdering(), 3068 cast<AtomicSDNode>(Node)->getFailureOrdering(), 3069 cast<AtomicSDNode>(Node)->getSynchScope()); 3070 3071 SDValue Success = DAG.getSetCC(SDLoc(Node), Node->getValueType(1), 3072 Res, Node->getOperand(2), ISD::SETEQ); 3073 3074 Results.push_back(Res.getValue(0)); 3075 Results.push_back(Success); 3076 Results.push_back(Res.getValue(1)); 3077 break; 3078 } 3079 case ISD::DYNAMIC_STACKALLOC: 3080 ExpandDYNAMIC_STACKALLOC(Node, Results); 3081 break; 3082 case ISD::MERGE_VALUES: 3083 for (unsigned i = 0; i < Node->getNumValues(); i++) 3084 Results.push_back(Node->getOperand(i)); 3085 break; 3086 case ISD::UNDEF: { 3087 EVT VT = Node->getValueType(0); 3088 if (VT.isInteger()) 3089 Results.push_back(DAG.getConstant(0, VT)); 3090 else { 3091 assert(VT.isFloatingPoint() && "Unknown value type!"); 3092 Results.push_back(DAG.getConstantFP(0, VT)); 3093 } 3094 break; 3095 } 3096 case ISD::TRAP: { 3097 // If this operation is not supported, lower it to 'abort()' call 3098 TargetLowering::ArgListTy Args; 3099 TargetLowering::CallLoweringInfo CLI(DAG); 3100 CLI.setDebugLoc(dl).setChain(Node->getOperand(0)) 3101 .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3102 DAG.getExternalSymbol("abort", TLI.getPointerTy()), 3103 std::move(Args), 0); 3104 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 3105 3106 Results.push_back(CallResult.second); 3107 break; 3108 } 3109 case ISD::FP_ROUND: 3110 case ISD::BITCAST: 3111 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 3112 Node->getValueType(0), dl); 3113 Results.push_back(Tmp1); 3114 break; 3115 case ISD::FP_EXTEND: 3116 Tmp1 = EmitStackConvert(Node->getOperand(0), 3117 Node->getOperand(0).getValueType(), 3118 Node->getValueType(0), dl); 3119 Results.push_back(Tmp1); 3120 break; 3121 case ISD::SIGN_EXTEND_INREG: { 3122 // NOTE: we could fall back on load/store here too for targets without 3123 // SAR. However, it is doubtful that any exist. 3124 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 3125 EVT VT = Node->getValueType(0); 3126 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT); 3127 if (VT.isVector()) 3128 ShiftAmountTy = VT; 3129 unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 3130 ExtraVT.getScalarType().getSizeInBits(); 3131 SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy); 3132 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 3133 Node->getOperand(0), ShiftCst); 3134 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 3135 Results.push_back(Tmp1); 3136 break; 3137 } 3138 case ISD::FP_ROUND_INREG: { 3139 // The only way we can lower this is to turn it into a TRUNCSTORE, 3140 // EXTLOAD pair, targeting a temporary location (a stack slot). 3141 3142 // NOTE: there is a choice here between constantly creating new stack 3143 // slots and always reusing the same one. We currently always create 3144 // new ones, as reuse may inhibit scheduling. 3145 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 3146 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, 3147 Node->getValueType(0), dl); 3148 Results.push_back(Tmp1); 3149 break; 3150 } 3151 case ISD::SINT_TO_FP: 3152 case ISD::UINT_TO_FP: 3153 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, 3154 Node->getOperand(0), Node->getValueType(0), dl); 3155 Results.push_back(Tmp1); 3156 break; 3157 case ISD::FP_TO_SINT: 3158 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) 3159 Results.push_back(Tmp1); 3160 break; 3161 case ISD::FP_TO_UINT: { 3162 SDValue True, False; 3163 EVT VT = Node->getOperand(0).getValueType(); 3164 EVT NVT = Node->getValueType(0); 3165 APFloat apf(DAG.EVTToAPFloatSemantics(VT), 3166 APInt::getNullValue(VT.getSizeInBits())); 3167 APInt x = APInt::getSignBit(NVT.getSizeInBits()); 3168 (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 3169 Tmp1 = DAG.getConstantFP(apf, VT); 3170 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT), 3171 Node->getOperand(0), 3172 Tmp1, ISD::SETLT); 3173 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 3174 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 3175 DAG.getNode(ISD::FSUB, dl, VT, 3176 Node->getOperand(0), Tmp1)); 3177 False = DAG.getNode(ISD::XOR, dl, NVT, False, 3178 DAG.getConstant(x, NVT)); 3179 Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False); 3180 Results.push_back(Tmp1); 3181 break; 3182 } 3183 case ISD::VAARG: { 3184 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3185 EVT VT = Node->getValueType(0); 3186 Tmp1 = Node->getOperand(0); 3187 Tmp2 = Node->getOperand(1); 3188 unsigned Align = Node->getConstantOperandVal(3); 3189 3190 SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, 3191 MachinePointerInfo(V), 3192 false, false, false, 0); 3193 SDValue VAList = VAListLoad; 3194 3195 if (Align > TLI.getMinStackArgumentAlignment()) { 3196 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); 3197 3198 VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 3199 DAG.getConstant(Align - 1, 3200 VAList.getValueType())); 3201 3202 VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList, 3203 DAG.getConstant(-(int64_t)Align, 3204 VAList.getValueType())); 3205 } 3206 3207 // Increment the pointer, VAList, to the next vaarg 3208 Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 3209 DAG.getConstant(TLI.getDataLayout()-> 3210 getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), 3211 VAList.getValueType())); 3212 // Store the incremented VAList to the legalized pointer 3213 Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, 3214 MachinePointerInfo(V), false, false, 0); 3215 // Load the actual argument out of the pointer VAList 3216 Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), 3217 false, false, false, 0)); 3218 Results.push_back(Results[0].getValue(1)); 3219 break; 3220 } 3221 case ISD::VACOPY: { 3222 // This defaults to loading a pointer from the input and storing it to the 3223 // output, returning the chain. 3224 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 3225 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 3226 Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), 3227 Node->getOperand(2), MachinePointerInfo(VS), 3228 false, false, false, 0); 3229 Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 3230 MachinePointerInfo(VD), false, false, 0); 3231 Results.push_back(Tmp1); 3232 break; 3233 } 3234 case ISD::EXTRACT_VECTOR_ELT: 3235 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 3236 // This must be an access of the only element. Return it. 3237 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 3238 Node->getOperand(0)); 3239 else 3240 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 3241 Results.push_back(Tmp1); 3242 break; 3243 case ISD::EXTRACT_SUBVECTOR: 3244 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 3245 break; 3246 case ISD::INSERT_SUBVECTOR: 3247 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 3248 break; 3249 case ISD::CONCAT_VECTORS: { 3250 Results.push_back(ExpandVectorBuildThroughStack(Node)); 3251 break; 3252 } 3253 case ISD::SCALAR_TO_VECTOR: 3254 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 3255 break; 3256 case ISD::INSERT_VECTOR_ELT: 3257 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 3258 Node->getOperand(1), 3259 Node->getOperand(2), dl)); 3260 break; 3261 case ISD::VECTOR_SHUFFLE: { 3262 SmallVector<int, 32> NewMask; 3263 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3264 3265 EVT VT = Node->getValueType(0); 3266 EVT EltVT = VT.getVectorElementType(); 3267 SDValue Op0 = Node->getOperand(0); 3268 SDValue Op1 = Node->getOperand(1); 3269 if (!TLI.isTypeLegal(EltVT)) { 3270 3271 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 3272 3273 // BUILD_VECTOR operands are allowed to be wider than the element type. 3274 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept 3275 // it. 3276 if (NewEltVT.bitsLT(EltVT)) { 3277 3278 // Convert shuffle node. 3279 // If original node was v4i64 and the new EltVT is i32, 3280 // cast operands to v8i32 and re-build the mask. 3281 3282 // Calculate new VT, the size of the new VT should be equal to original. 3283 EVT NewVT = 3284 EVT::getVectorVT(*DAG.getContext(), NewEltVT, 3285 VT.getSizeInBits() / NewEltVT.getSizeInBits()); 3286 assert(NewVT.bitsEq(VT)); 3287 3288 // cast operands to new VT 3289 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 3290 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 3291 3292 // Convert the shuffle mask 3293 unsigned int factor = 3294 NewVT.getVectorNumElements()/VT.getVectorNumElements(); 3295 3296 // EltVT gets smaller 3297 assert(factor > 0); 3298 3299 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 3300 if (Mask[i] < 0) { 3301 for (unsigned fi = 0; fi < factor; ++fi) 3302 NewMask.push_back(Mask[i]); 3303 } 3304 else { 3305 for (unsigned fi = 0; fi < factor; ++fi) 3306 NewMask.push_back(Mask[i]*factor+fi); 3307 } 3308 } 3309 Mask = NewMask; 3310 VT = NewVT; 3311 } 3312 EltVT = NewEltVT; 3313 } 3314 unsigned NumElems = VT.getVectorNumElements(); 3315 SmallVector<SDValue, 16> Ops; 3316 for (unsigned i = 0; i != NumElems; ++i) { 3317 if (Mask[i] < 0) { 3318 Ops.push_back(DAG.getUNDEF(EltVT)); 3319 continue; 3320 } 3321 unsigned Idx = Mask[i]; 3322 if (Idx < NumElems) 3323 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 3324 Op0, 3325 DAG.getConstant(Idx, TLI.getVectorIdxTy()))); 3326 else 3327 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 3328 Op1, 3329 DAG.getConstant(Idx - NumElems, 3330 TLI.getVectorIdxTy()))); 3331 } 3332 3333 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 3334 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 3335 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 3336 Results.push_back(Tmp1); 3337 break; 3338 } 3339 case ISD::EXTRACT_ELEMENT: { 3340 EVT OpTy = Node->getOperand(0).getValueType(); 3341 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 3342 // 1 -> Hi 3343 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 3344 DAG.getConstant(OpTy.getSizeInBits()/2, 3345 TLI.getShiftAmountTy(Node->getOperand(0).getValueType()))); 3346 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 3347 } else { 3348 // 0 -> Lo 3349 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 3350 Node->getOperand(0)); 3351 } 3352 Results.push_back(Tmp1); 3353 break; 3354 } 3355 case ISD::STACKSAVE: 3356 // Expand to CopyFromReg if the target set 3357 // StackPointerRegisterToSaveRestore. 3358 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3359 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 3360 Node->getValueType(0))); 3361 Results.push_back(Results[0].getValue(1)); 3362 } else { 3363 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 3364 Results.push_back(Node->getOperand(0)); 3365 } 3366 break; 3367 case ISD::STACKRESTORE: 3368 // Expand to CopyToReg if the target set 3369 // StackPointerRegisterToSaveRestore. 3370 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3371 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 3372 Node->getOperand(1))); 3373 } else { 3374 Results.push_back(Node->getOperand(0)); 3375 } 3376 break; 3377 case ISD::FCOPYSIGN: 3378 Results.push_back(ExpandFCOPYSIGN(Node)); 3379 break; 3380 case ISD::FNEG: 3381 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3382 Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3383 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, 3384 Node->getOperand(0)); 3385 Results.push_back(Tmp1); 3386 break; 3387 case ISD::FABS: { 3388 // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3389 EVT VT = Node->getValueType(0); 3390 Tmp1 = Node->getOperand(0); 3391 Tmp2 = DAG.getConstantFP(0.0, VT); 3392 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(Tmp1.getValueType()), 3393 Tmp1, Tmp2, ISD::SETUGT); 3394 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 3395 Tmp1 = DAG.getSelect(dl, VT, Tmp2, Tmp1, Tmp3); 3396 Results.push_back(Tmp1); 3397 break; 3398 } 3399 case ISD::FSQRT: 3400 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3401 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 3402 RTLIB::SQRT_PPCF128)); 3403 break; 3404 case ISD::FSIN: 3405 case ISD::FCOS: { 3406 EVT VT = Node->getValueType(0); 3407 bool isSIN = Node->getOpcode() == ISD::FSIN; 3408 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / 3409 // fcos which share the same operand and both are used. 3410 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || 3411 canCombineSinCosLibcall(Node, TLI, TM)) 3412 && useSinCos(Node)) { 3413 SDVTList VTs = DAG.getVTList(VT, VT); 3414 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); 3415 if (!isSIN) 3416 Tmp1 = Tmp1.getValue(1); 3417 Results.push_back(Tmp1); 3418 } else if (isSIN) { 3419 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 3420 RTLIB::SIN_F80, RTLIB::SIN_F128, 3421 RTLIB::SIN_PPCF128)); 3422 } else { 3423 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 3424 RTLIB::COS_F80, RTLIB::COS_F128, 3425 RTLIB::COS_PPCF128)); 3426 } 3427 break; 3428 } 3429 case ISD::FSINCOS: 3430 // Expand into sincos libcall. 3431 ExpandSinCosLibCall(Node, Results); 3432 break; 3433 case ISD::FLOG: 3434 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, 3435 RTLIB::LOG_F80, RTLIB::LOG_F128, 3436 RTLIB::LOG_PPCF128)); 3437 break; 3438 case ISD::FLOG2: 3439 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3440 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 3441 RTLIB::LOG2_PPCF128)); 3442 break; 3443 case ISD::FLOG10: 3444 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3445 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 3446 RTLIB::LOG10_PPCF128)); 3447 break; 3448 case ISD::FEXP: 3449 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, 3450 RTLIB::EXP_F80, RTLIB::EXP_F128, 3451 RTLIB::EXP_PPCF128)); 3452 break; 3453 case ISD::FEXP2: 3454 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3455 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 3456 RTLIB::EXP2_PPCF128)); 3457 break; 3458 case ISD::FTRUNC: 3459 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3460 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 3461 RTLIB::TRUNC_PPCF128)); 3462 break; 3463 case ISD::FFLOOR: 3464 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3465 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 3466 RTLIB::FLOOR_PPCF128)); 3467 break; 3468 case ISD::FCEIL: 3469 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3470 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 3471 RTLIB::CEIL_PPCF128)); 3472 break; 3473 case ISD::FRINT: 3474 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 3475 RTLIB::RINT_F80, RTLIB::RINT_F128, 3476 RTLIB::RINT_PPCF128)); 3477 break; 3478 case ISD::FNEARBYINT: 3479 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 3480 RTLIB::NEARBYINT_F64, 3481 RTLIB::NEARBYINT_F80, 3482 RTLIB::NEARBYINT_F128, 3483 RTLIB::NEARBYINT_PPCF128)); 3484 break; 3485 case ISD::FROUND: 3486 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, 3487 RTLIB::ROUND_F64, 3488 RTLIB::ROUND_F80, 3489 RTLIB::ROUND_F128, 3490 RTLIB::ROUND_PPCF128)); 3491 break; 3492 case ISD::FPOWI: 3493 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 3494 RTLIB::POWI_F80, RTLIB::POWI_F128, 3495 RTLIB::POWI_PPCF128)); 3496 break; 3497 case ISD::FPOW: 3498 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, 3499 RTLIB::POW_F80, RTLIB::POW_F128, 3500 RTLIB::POW_PPCF128)); 3501 break; 3502 case ISD::FDIV: 3503 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 3504 RTLIB::DIV_F80, RTLIB::DIV_F128, 3505 RTLIB::DIV_PPCF128)); 3506 break; 3507 case ISD::FREM: 3508 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 3509 RTLIB::REM_F80, RTLIB::REM_F128, 3510 RTLIB::REM_PPCF128)); 3511 break; 3512 case ISD::FMA: 3513 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 3514 RTLIB::FMA_F80, RTLIB::FMA_F128, 3515 RTLIB::FMA_PPCF128)); 3516 break; 3517 case ISD::FP16_TO_FP: { 3518 if (Node->getValueType(0) == MVT::f32) { 3519 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 3520 break; 3521 } 3522 3523 // We can extend to types bigger than f32 in two steps without changing the 3524 // result. Since "f16 -> f32" is much more commonly available, give CodeGen 3525 // the option of emitting that before resorting to a libcall. 3526 SDValue Res = 3527 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); 3528 Results.push_back( 3529 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); 3530 break; 3531 } 3532 case ISD::FP_TO_FP16: { 3533 RTLIB::Libcall LC = 3534 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); 3535 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); 3536 Results.push_back(ExpandLibCall(LC, Node, false)); 3537 break; 3538 } 3539 case ISD::ConstantFP: { 3540 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3541 // Check to see if this FP immediate is already legal. 3542 // If this is a legal constant, turn it into a TargetConstantFP node. 3543 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) 3544 Results.push_back(ExpandConstantFP(CFP, true)); 3545 break; 3546 } 3547 case ISD::FSUB: { 3548 EVT VT = Node->getValueType(0); 3549 assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3550 TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && 3551 "Don't know how to expand this FP subtraction!"); 3552 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3553 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); 3554 Results.push_back(Tmp1); 3555 break; 3556 } 3557 case ISD::SUB: { 3558 EVT VT = Node->getValueType(0); 3559 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3560 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3561 "Don't know how to expand this subtraction!"); 3562 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3563 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT)); 3564 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT)); 3565 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3566 break; 3567 } 3568 case ISD::UREM: 3569 case ISD::SREM: { 3570 EVT VT = Node->getValueType(0); 3571 bool isSigned = Node->getOpcode() == ISD::SREM; 3572 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; 3573 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3574 Tmp2 = Node->getOperand(0); 3575 Tmp3 = Node->getOperand(1); 3576 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3577 (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3578 // If div is legal, it's better to do the normal expansion 3579 !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) && 3580 useDivRem(Node, isSigned, false))) { 3581 SDVTList VTs = DAG.getVTList(VT, VT); 3582 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); 3583 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { 3584 // X % Y -> X-X/Y*Y 3585 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); 3586 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); 3587 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); 3588 } else if (isSigned) 3589 Tmp1 = ExpandIntLibCall(Node, true, 3590 RTLIB::SREM_I8, 3591 RTLIB::SREM_I16, RTLIB::SREM_I32, 3592 RTLIB::SREM_I64, RTLIB::SREM_I128); 3593 else 3594 Tmp1 = ExpandIntLibCall(Node, false, 3595 RTLIB::UREM_I8, 3596 RTLIB::UREM_I16, RTLIB::UREM_I32, 3597 RTLIB::UREM_I64, RTLIB::UREM_I128); 3598 Results.push_back(Tmp1); 3599 break; 3600 } 3601 case ISD::UDIV: 3602 case ISD::SDIV: { 3603 bool isSigned = Node->getOpcode() == ISD::SDIV; 3604 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3605 EVT VT = Node->getValueType(0); 3606 SDVTList VTs = DAG.getVTList(VT, VT); 3607 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3608 (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3609 useDivRem(Node, isSigned, true))) 3610 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3611 Node->getOperand(1)); 3612 else if (isSigned) 3613 Tmp1 = ExpandIntLibCall(Node, true, 3614 RTLIB::SDIV_I8, 3615 RTLIB::SDIV_I16, RTLIB::SDIV_I32, 3616 RTLIB::SDIV_I64, RTLIB::SDIV_I128); 3617 else 3618 Tmp1 = ExpandIntLibCall(Node, false, 3619 RTLIB::UDIV_I8, 3620 RTLIB::UDIV_I16, RTLIB::UDIV_I32, 3621 RTLIB::UDIV_I64, RTLIB::UDIV_I128); 3622 Results.push_back(Tmp1); 3623 break; 3624 } 3625 case ISD::MULHU: 3626 case ISD::MULHS: { 3627 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : 3628 ISD::SMUL_LOHI; 3629 EVT VT = Node->getValueType(0); 3630 SDVTList VTs = DAG.getVTList(VT, VT); 3631 assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && 3632 "If this wasn't legal, it shouldn't have been created!"); 3633 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3634 Node->getOperand(1)); 3635 Results.push_back(Tmp1.getValue(1)); 3636 break; 3637 } 3638 case ISD::SDIVREM: 3639 case ISD::UDIVREM: 3640 // Expand into divrem libcall 3641 ExpandDivRemLibCall(Node, Results); 3642 break; 3643 case ISD::MUL: { 3644 EVT VT = Node->getValueType(0); 3645 SDVTList VTs = DAG.getVTList(VT, VT); 3646 // See if multiply or divide can be lowered using two-result operations. 3647 // We just need the low half of the multiply; try both the signed 3648 // and unsigned forms. If the target supports both SMUL_LOHI and 3649 // UMUL_LOHI, form a preference by checking which forms of plain 3650 // MULH it supports. 3651 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3652 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3653 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3654 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3655 unsigned OpToUse = 0; 3656 if (HasSMUL_LOHI && !HasMULHS) { 3657 OpToUse = ISD::SMUL_LOHI; 3658 } else if (HasUMUL_LOHI && !HasMULHU) { 3659 OpToUse = ISD::UMUL_LOHI; 3660 } else if (HasSMUL_LOHI) { 3661 OpToUse = ISD::SMUL_LOHI; 3662 } else if (HasUMUL_LOHI) { 3663 OpToUse = ISD::UMUL_LOHI; 3664 } 3665 if (OpToUse) { 3666 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3667 Node->getOperand(1))); 3668 break; 3669 } 3670 3671 SDValue Lo, Hi; 3672 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); 3673 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && 3674 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && 3675 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && 3676 TLI.isOperationLegalOrCustom(ISD::OR, VT) && 3677 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) { 3678 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); 3679 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); 3680 SDValue Shift = DAG.getConstant(HalfType.getSizeInBits(), 3681 TLI.getShiftAmountTy(HalfType)); 3682 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); 3683 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); 3684 break; 3685 } 3686 3687 Tmp1 = ExpandIntLibCall(Node, false, 3688 RTLIB::MUL_I8, 3689 RTLIB::MUL_I16, RTLIB::MUL_I32, 3690 RTLIB::MUL_I64, RTLIB::MUL_I128); 3691 Results.push_back(Tmp1); 3692 break; 3693 } 3694 case ISD::SADDO: 3695 case ISD::SSUBO: { 3696 SDValue LHS = Node->getOperand(0); 3697 SDValue RHS = Node->getOperand(1); 3698 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3699 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3700 LHS, RHS); 3701 Results.push_back(Sum); 3702 EVT ResultType = Node->getValueType(1); 3703 EVT OType = getSetCCResultType(Node->getValueType(0)); 3704 3705 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 3706 3707 // LHSSign -> LHS >= 0 3708 // RHSSign -> RHS >= 0 3709 // SumSign -> Sum >= 0 3710 // 3711 // Add: 3712 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3713 // Sub: 3714 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3715 // 3716 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3717 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3718 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3719 Node->getOpcode() == ISD::SADDO ? 3720 ISD::SETEQ : ISD::SETNE); 3721 3722 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3723 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3724 3725 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3726 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType)); 3727 break; 3728 } 3729 case ISD::UADDO: 3730 case ISD::USUBO: { 3731 SDValue LHS = Node->getOperand(0); 3732 SDValue RHS = Node->getOperand(1); 3733 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 3734 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3735 LHS, RHS); 3736 Results.push_back(Sum); 3737 3738 EVT ResultType = Node->getValueType(1); 3739 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); 3740 ISD::CondCode CC 3741 = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT; 3742 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); 3743 3744 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType)); 3745 break; 3746 } 3747 case ISD::UMULO: 3748 case ISD::SMULO: { 3749 EVT VT = Node->getValueType(0); 3750 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); 3751 SDValue LHS = Node->getOperand(0); 3752 SDValue RHS = Node->getOperand(1); 3753 SDValue BottomHalf; 3754 SDValue TopHalf; 3755 static const unsigned Ops[2][3] = 3756 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, 3757 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; 3758 bool isSigned = Node->getOpcode() == ISD::SMULO; 3759 if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { 3760 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3761 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); 3762 } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { 3763 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, 3764 RHS); 3765 TopHalf = BottomHalf.getValue(1); 3766 } else if (TLI.isTypeLegal(WideVT)) { 3767 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); 3768 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); 3769 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 3770 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3771 DAG.getIntPtrConstant(0)); 3772 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3773 DAG.getIntPtrConstant(1)); 3774 } else { 3775 // We can fall back to a libcall with an illegal type for the MUL if we 3776 // have a libcall big enough. 3777 // Also, we can fall back to a division in some cases, but that's a big 3778 // performance hit in the general case. 3779 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3780 if (WideVT == MVT::i16) 3781 LC = RTLIB::MUL_I16; 3782 else if (WideVT == MVT::i32) 3783 LC = RTLIB::MUL_I32; 3784 else if (WideVT == MVT::i64) 3785 LC = RTLIB::MUL_I64; 3786 else if (WideVT == MVT::i128) 3787 LC = RTLIB::MUL_I128; 3788 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); 3789 3790 // The high part is obtained by SRA'ing all but one of the bits of low 3791 // part. 3792 unsigned LoSize = VT.getSizeInBits(); 3793 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, 3794 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3795 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, 3796 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3797 3798 // Here we're passing the 2 arguments explicitly as 4 arguments that are 3799 // pre-lowered to the correct types. This all depends upon WideVT not 3800 // being a legal type for the architecture and thus has to be split to 3801 // two arguments. 3802 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; 3803 SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); 3804 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3805 DAG.getIntPtrConstant(0)); 3806 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3807 DAG.getIntPtrConstant(1)); 3808 // Ret is a node with an illegal type. Because such things are not 3809 // generally permitted during this phase of legalization, delete the 3810 // node. The above EXTRACT_ELEMENT nodes should have been folded. 3811 DAG.DeleteNode(Ret.getNode()); 3812 } 3813 3814 if (isSigned) { 3815 Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, 3816 TLI.getShiftAmountTy(BottomHalf.getValueType())); 3817 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); 3818 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1, 3819 ISD::SETNE); 3820 } else { 3821 TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, 3822 DAG.getConstant(0, VT), ISD::SETNE); 3823 } 3824 Results.push_back(BottomHalf); 3825 Results.push_back(TopHalf); 3826 break; 3827 } 3828 case ISD::BUILD_PAIR: { 3829 EVT PairTy = Node->getValueType(0); 3830 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3831 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3832 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3833 DAG.getConstant(PairTy.getSizeInBits()/2, 3834 TLI.getShiftAmountTy(PairTy))); 3835 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3836 break; 3837 } 3838 case ISD::SELECT: 3839 Tmp1 = Node->getOperand(0); 3840 Tmp2 = Node->getOperand(1); 3841 Tmp3 = Node->getOperand(2); 3842 if (Tmp1.getOpcode() == ISD::SETCC) { 3843 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3844 Tmp2, Tmp3, 3845 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3846 } else { 3847 Tmp1 = DAG.getSelectCC(dl, Tmp1, 3848 DAG.getConstant(0, Tmp1.getValueType()), 3849 Tmp2, Tmp3, ISD::SETNE); 3850 } 3851 Results.push_back(Tmp1); 3852 break; 3853 case ISD::BR_JT: { 3854 SDValue Chain = Node->getOperand(0); 3855 SDValue Table = Node->getOperand(1); 3856 SDValue Index = Node->getOperand(2); 3857 3858 EVT PTy = TLI.getPointerTy(); 3859 3860 const DataLayout &TD = *TLI.getDataLayout(); 3861 unsigned EntrySize = 3862 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3863 3864 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), 3865 Index, DAG.getConstant(EntrySize, Index.getValueType())); 3866 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), 3867 Index, Table); 3868 3869 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3870 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3871 MachinePointerInfo::getJumpTable(), MemVT, 3872 false, false, 0); 3873 Addr = LD; 3874 if (TM.getRelocationModel() == Reloc::PIC_) { 3875 // For PIC, the sequence is: 3876 // BRIND(load(Jumptable + index) + RelocBase) 3877 // RelocBase can be JumpTable, GOT or some sort of global base. 3878 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3879 TLI.getPICJumpTableRelocBase(Table, DAG)); 3880 } 3881 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 3882 Results.push_back(Tmp1); 3883 break; 3884 } 3885 case ISD::BRCOND: 3886 // Expand brcond's setcc into its constituent parts and create a BR_CC 3887 // Node. 3888 Tmp1 = Node->getOperand(0); 3889 Tmp2 = Node->getOperand(1); 3890 if (Tmp2.getOpcode() == ISD::SETCC) { 3891 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3892 Tmp1, Tmp2.getOperand(2), 3893 Tmp2.getOperand(0), Tmp2.getOperand(1), 3894 Node->getOperand(2)); 3895 } else { 3896 // We test only the i1 bit. Skip the AND if UNDEF. 3897 Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : 3898 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3899 DAG.getConstant(1, Tmp2.getValueType())); 3900 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3901 DAG.getCondCode(ISD::SETNE), Tmp3, 3902 DAG.getConstant(0, Tmp3.getValueType()), 3903 Node->getOperand(2)); 3904 } 3905 Results.push_back(Tmp1); 3906 break; 3907 case ISD::SETCC: { 3908 Tmp1 = Node->getOperand(0); 3909 Tmp2 = Node->getOperand(1); 3910 Tmp3 = Node->getOperand(2); 3911 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, 3912 Tmp3, NeedInvert, dl); 3913 3914 if (Legalized) { 3915 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3916 // condition code, create a new SETCC node. 3917 if (Tmp3.getNode()) 3918 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3919 Tmp1, Tmp2, Tmp3); 3920 3921 // If we expanded the SETCC by inverting the condition code, then wrap 3922 // the existing SETCC in a NOT to restore the intended condition. 3923 if (NeedInvert) 3924 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); 3925 3926 Results.push_back(Tmp1); 3927 break; 3928 } 3929 3930 // Otherwise, SETCC for the given comparison type must be completely 3931 // illegal; expand it into a SELECT_CC. 3932 EVT VT = Node->getValueType(0); 3933 int TrueValue; 3934 switch (TLI.getBooleanContents(Tmp1->getValueType(0))) { 3935 case TargetLowering::ZeroOrOneBooleanContent: 3936 case TargetLowering::UndefinedBooleanContent: 3937 TrueValue = 1; 3938 break; 3939 case TargetLowering::ZeroOrNegativeOneBooleanContent: 3940 TrueValue = -1; 3941 break; 3942 } 3943 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3944 DAG.getConstant(TrueValue, VT), DAG.getConstant(0, VT), 3945 Tmp3); 3946 Results.push_back(Tmp1); 3947 break; 3948 } 3949 case ISD::SELECT_CC: { 3950 Tmp1 = Node->getOperand(0); // LHS 3951 Tmp2 = Node->getOperand(1); // RHS 3952 Tmp3 = Node->getOperand(2); // True 3953 Tmp4 = Node->getOperand(3); // False 3954 EVT VT = Node->getValueType(0); 3955 SDValue CC = Node->getOperand(4); 3956 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); 3957 3958 if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) { 3959 // If the condition code is legal, then we need to expand this 3960 // node using SETCC and SELECT. 3961 EVT CmpVT = Tmp1.getValueType(); 3962 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && 3963 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " 3964 "expanded."); 3965 EVT CCVT = TLI.getSetCCResultType(*DAG.getContext(), CmpVT); 3966 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC); 3967 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); 3968 break; 3969 } 3970 3971 // SELECT_CC is legal, so the condition code must not be. 3972 bool Legalized = false; 3973 // Try to legalize by inverting the condition. This is for targets that 3974 // might support an ordered version of a condition, but not the unordered 3975 // version (or vice versa). 3976 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, 3977 Tmp1.getValueType().isInteger()); 3978 if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) { 3979 // Use the new condition code and swap true and false 3980 Legalized = true; 3981 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); 3982 } else { 3983 // If The inverse is not legal, then try to swap the arguments using 3984 // the inverse condition code. 3985 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); 3986 if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) { 3987 // The swapped inverse condition is legal, so swap true and false, 3988 // lhs and rhs. 3989 Legalized = true; 3990 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); 3991 } 3992 } 3993 3994 if (!Legalized) { 3995 Legalized = LegalizeSetCCCondCode( 3996 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, 3997 dl); 3998 3999 assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); 4000 4001 // If we expanded the SETCC by inverting the condition code, then swap 4002 // the True/False operands to match. 4003 if (NeedInvert) 4004 std::swap(Tmp3, Tmp4); 4005 4006 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 4007 // condition code, create a new SELECT_CC node. 4008 if (CC.getNode()) { 4009 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), 4010 Tmp1, Tmp2, Tmp3, Tmp4, CC); 4011 } else { 4012 Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 4013 CC = DAG.getCondCode(ISD::SETNE); 4014 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, 4015 Tmp2, Tmp3, Tmp4, CC); 4016 } 4017 } 4018 Results.push_back(Tmp1); 4019 break; 4020 } 4021 case ISD::BR_CC: { 4022 Tmp1 = Node->getOperand(0); // Chain 4023 Tmp2 = Node->getOperand(2); // LHS 4024 Tmp3 = Node->getOperand(3); // RHS 4025 Tmp4 = Node->getOperand(1); // CC 4026 4027 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( 4028 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); 4029 (void)Legalized; 4030 assert(Legalized && "Can't legalize BR_CC with legal condition!"); 4031 4032 // If we expanded the SETCC by inverting the condition code, then wrap 4033 // the existing SETCC in a NOT to restore the intended condition. 4034 if (NeedInvert) 4035 Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0)); 4036 4037 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC 4038 // node. 4039 if (Tmp4.getNode()) { 4040 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, 4041 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); 4042 } else { 4043 Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 4044 Tmp4 = DAG.getCondCode(ISD::SETNE); 4045 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, 4046 Tmp2, Tmp3, Node->getOperand(4)); 4047 } 4048 Results.push_back(Tmp1); 4049 break; 4050 } 4051 case ISD::BUILD_VECTOR: 4052 Results.push_back(ExpandBUILD_VECTOR(Node)); 4053 break; 4054 case ISD::SRA: 4055 case ISD::SRL: 4056 case ISD::SHL: { 4057 // Scalarize vector SRA/SRL/SHL. 4058 EVT VT = Node->getValueType(0); 4059 assert(VT.isVector() && "Unable to legalize non-vector shift"); 4060 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 4061 unsigned NumElem = VT.getVectorNumElements(); 4062 4063 SmallVector<SDValue, 8> Scalars; 4064 for (unsigned Idx = 0; Idx < NumElem; Idx++) { 4065 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4066 VT.getScalarType(), 4067 Node->getOperand(0), DAG.getConstant(Idx, 4068 TLI.getVectorIdxTy())); 4069 SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 4070 VT.getScalarType(), 4071 Node->getOperand(1), DAG.getConstant(Idx, 4072 TLI.getVectorIdxTy())); 4073 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 4074 VT.getScalarType(), Ex, Sh)); 4075 } 4076 SDValue Result = 4077 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Scalars); 4078 ReplaceNode(SDValue(Node, 0), Result); 4079 break; 4080 } 4081 case ISD::GLOBAL_OFFSET_TABLE: 4082 case ISD::GlobalAddress: 4083 case ISD::GlobalTLSAddress: 4084 case ISD::ExternalSymbol: 4085 case ISD::ConstantPool: 4086 case ISD::JumpTable: 4087 case ISD::INTRINSIC_W_CHAIN: 4088 case ISD::INTRINSIC_WO_CHAIN: 4089 case ISD::INTRINSIC_VOID: 4090 // FIXME: Custom lowering for these operations shouldn't return null! 4091 break; 4092 } 4093 4094 // Replace the original node with the legalized result. 4095 if (!Results.empty()) 4096 ReplaceNode(Node, Results.data()); 4097} 4098 4099void SelectionDAGLegalize::PromoteNode(SDNode *Node) { 4100 SmallVector<SDValue, 8> Results; 4101 MVT OVT = Node->getSimpleValueType(0); 4102 if (Node->getOpcode() == ISD::UINT_TO_FP || 4103 Node->getOpcode() == ISD::SINT_TO_FP || 4104 Node->getOpcode() == ISD::SETCC) { 4105 OVT = Node->getOperand(0).getSimpleValueType(); 4106 } 4107 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 4108 SDLoc dl(Node); 4109 SDValue Tmp1, Tmp2, Tmp3; 4110 switch (Node->getOpcode()) { 4111 case ISD::CTTZ: 4112 case ISD::CTTZ_ZERO_UNDEF: 4113 case ISD::CTLZ: 4114 case ISD::CTLZ_ZERO_UNDEF: 4115 case ISD::CTPOP: 4116 // Zero extend the argument. 4117 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4118 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 4119 // already the correct result. 4120 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4121 if (Node->getOpcode() == ISD::CTTZ) { 4122 // FIXME: This should set a bit in the zero extended value instead. 4123 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), 4124 Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 4125 ISD::SETEQ); 4126 Tmp1 = DAG.getSelect(dl, NVT, Tmp2, 4127 DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 4128 } else if (Node->getOpcode() == ISD::CTLZ || 4129 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 4130 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 4131 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 4132 DAG.getConstant(NVT.getSizeInBits() - 4133 OVT.getSizeInBits(), NVT)); 4134 } 4135 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4136 break; 4137 case ISD::BSWAP: { 4138 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 4139 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4140 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 4141 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 4142 DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); 4143 Results.push_back(Tmp1); 4144 break; 4145 } 4146 case ISD::FP_TO_UINT: 4147 case ISD::FP_TO_SINT: 4148 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), 4149 Node->getOpcode() == ISD::FP_TO_SINT, dl); 4150 Results.push_back(Tmp1); 4151 break; 4152 case ISD::UINT_TO_FP: 4153 case ISD::SINT_TO_FP: 4154 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), 4155 Node->getOpcode() == ISD::SINT_TO_FP, dl); 4156 Results.push_back(Tmp1); 4157 break; 4158 case ISD::VAARG: { 4159 SDValue Chain = Node->getOperand(0); // Get the chain. 4160 SDValue Ptr = Node->getOperand(1); // Get the pointer. 4161 4162 unsigned TruncOp; 4163 if (OVT.isVector()) { 4164 TruncOp = ISD::BITCAST; 4165 } else { 4166 assert(OVT.isInteger() 4167 && "VAARG promotion is supported only for vectors or integer types"); 4168 TruncOp = ISD::TRUNCATE; 4169 } 4170 4171 // Perform the larger operation, then convert back 4172 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 4173 Node->getConstantOperandVal(3)); 4174 Chain = Tmp1.getValue(1); 4175 4176 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 4177 4178 // Modified the chain result - switch anything that used the old chain to 4179 // use the new one. 4180 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 4181 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 4182 ReplacedNode(Node); 4183 break; 4184 } 4185 case ISD::AND: 4186 case ISD::OR: 4187 case ISD::XOR: { 4188 unsigned ExtOp, TruncOp; 4189 if (OVT.isVector()) { 4190 ExtOp = ISD::BITCAST; 4191 TruncOp = ISD::BITCAST; 4192 } else { 4193 assert(OVT.isInteger() && "Cannot promote logic operation"); 4194 ExtOp = ISD::ANY_EXTEND; 4195 TruncOp = ISD::TRUNCATE; 4196 } 4197 // Promote each of the values to the new type. 4198 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4199 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4200 // Perform the larger operation, then convert back 4201 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4202 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 4203 break; 4204 } 4205 case ISD::SELECT: { 4206 unsigned ExtOp, TruncOp; 4207 if (Node->getValueType(0).isVector() || 4208 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { 4209 ExtOp = ISD::BITCAST; 4210 TruncOp = ISD::BITCAST; 4211 } else if (Node->getValueType(0).isInteger()) { 4212 ExtOp = ISD::ANY_EXTEND; 4213 TruncOp = ISD::TRUNCATE; 4214 } else { 4215 ExtOp = ISD::FP_EXTEND; 4216 TruncOp = ISD::FP_ROUND; 4217 } 4218 Tmp1 = Node->getOperand(0); 4219 // Promote each of the values to the new type. 4220 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4221 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 4222 // Perform the larger operation, then round down. 4223 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); 4224 if (TruncOp != ISD::FP_ROUND) 4225 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 4226 else 4227 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 4228 DAG.getIntPtrConstant(0)); 4229 Results.push_back(Tmp1); 4230 break; 4231 } 4232 case ISD::VECTOR_SHUFFLE: { 4233 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 4234 4235 // Cast the two input vectors. 4236 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 4237 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 4238 4239 // Convert the shuffle mask to the right # elements. 4240 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 4241 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 4242 Results.push_back(Tmp1); 4243 break; 4244 } 4245 case ISD::SETCC: { 4246 unsigned ExtOp = ISD::FP_EXTEND; 4247 if (NVT.isInteger()) { 4248 ISD::CondCode CCCode = 4249 cast<CondCodeSDNode>(Node->getOperand(2))->get(); 4250 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4251 } 4252 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4253 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4254 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 4255 Tmp1, Tmp2, Node->getOperand(2))); 4256 break; 4257 } 4258 case ISD::FDIV: 4259 case ISD::FREM: 4260 case ISD::FPOW: { 4261 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4262 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 4263 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4264 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4265 Tmp3, DAG.getIntPtrConstant(0))); 4266 break; 4267 } 4268 case ISD::FLOG2: 4269 case ISD::FEXP2: 4270 case ISD::FLOG: 4271 case ISD::FEXP: { 4272 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4273 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4274 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4275 Tmp2, DAG.getIntPtrConstant(0))); 4276 break; 4277 } 4278 } 4279 4280 // Replace the original node with the legalized result. 4281 if (!Results.empty()) 4282 ReplaceNode(Node, Results.data()); 4283} 4284 4285// SelectionDAG::Legalize - This is the entry point for the file. 4286// 4287void SelectionDAG::Legalize() { 4288 /// run - This is the main entry point to this class. 4289 /// 4290 SelectionDAGLegalize(*this).LegalizeDAG(); 4291} 4292