LegalizeDAG.cpp revision 353358
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file implements the SelectionDAG::Legalize method. 10// 11//===----------------------------------------------------------------------===// 12 13#include "llvm/ADT/APFloat.h" 14#include "llvm/ADT/APInt.h" 15#include "llvm/ADT/ArrayRef.h" 16#include "llvm/ADT/SetVector.h" 17#include "llvm/ADT/SmallPtrSet.h" 18#include "llvm/ADT/SmallSet.h" 19#include "llvm/ADT/SmallVector.h" 20#include "llvm/CodeGen/ISDOpcodes.h" 21#include "llvm/CodeGen/MachineFunction.h" 22#include "llvm/CodeGen/MachineJumpTableInfo.h" 23#include "llvm/CodeGen/MachineMemOperand.h" 24#include "llvm/CodeGen/RuntimeLibcalls.h" 25#include "llvm/CodeGen/SelectionDAG.h" 26#include "llvm/CodeGen/SelectionDAGNodes.h" 27#include "llvm/CodeGen/TargetFrameLowering.h" 28#include "llvm/CodeGen/TargetLowering.h" 29#include "llvm/CodeGen/TargetSubtargetInfo.h" 30#include "llvm/CodeGen/ValueTypes.h" 31#include "llvm/IR/CallingConv.h" 32#include "llvm/IR/Constants.h" 33#include "llvm/IR/DataLayout.h" 34#include "llvm/IR/DerivedTypes.h" 35#include "llvm/IR/Function.h" 36#include "llvm/IR/Metadata.h" 37#include "llvm/IR/Type.h" 38#include "llvm/Support/Casting.h" 39#include "llvm/Support/Compiler.h" 40#include "llvm/Support/Debug.h" 41#include "llvm/Support/ErrorHandling.h" 42#include "llvm/Support/MachineValueType.h" 43#include "llvm/Support/MathExtras.h" 44#include "llvm/Support/raw_ostream.h" 45#include "llvm/Target/TargetMachine.h" 46#include "llvm/Target/TargetOptions.h" 47#include <algorithm> 48#include <cassert> 49#include <cstdint> 50#include <tuple> 51#include <utility> 52 53using namespace llvm; 54 55#define DEBUG_TYPE "legalizedag" 56 57namespace { 58 59/// Keeps track of state when getting the sign of a floating-point value as an 60/// integer. 61struct FloatSignAsInt { 62 EVT FloatVT; 63 SDValue Chain; 64 SDValue FloatPtr; 65 SDValue IntPtr; 66 MachinePointerInfo IntPointerInfo; 67 MachinePointerInfo FloatPointerInfo; 68 SDValue IntValue; 69 APInt SignMask; 70 uint8_t SignBit; 71}; 72 73//===----------------------------------------------------------------------===// 74/// This takes an arbitrary SelectionDAG as input and 75/// hacks on it until the target machine can handle it. This involves 76/// eliminating value sizes the machine cannot handle (promoting small sizes to 77/// large sizes or splitting up large values into small values) as well as 78/// eliminating operations the machine cannot handle. 79/// 80/// This code also does a small amount of optimization and recognition of idioms 81/// as part of its processing. For example, if a target does not support a 82/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 83/// will attempt merge setcc and brc instructions into brcc's. 84class SelectionDAGLegalize { 85 const TargetMachine &TM; 86 const TargetLowering &TLI; 87 SelectionDAG &DAG; 88 89 /// The set of nodes which have already been legalized. We hold a 90 /// reference to it in order to update as necessary on node deletion. 91 SmallPtrSetImpl<SDNode *> &LegalizedNodes; 92 93 /// A set of all the nodes updated during legalization. 94 SmallSetVector<SDNode *, 16> *UpdatedNodes; 95 96 EVT getSetCCResultType(EVT VT) const { 97 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); 98 } 99 100 // Libcall insertion helpers. 101 102public: 103 SelectionDAGLegalize(SelectionDAG &DAG, 104 SmallPtrSetImpl<SDNode *> &LegalizedNodes, 105 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) 106 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), 107 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} 108 109 /// Legalizes the given operation. 110 void LegalizeOp(SDNode *Node); 111 112private: 113 SDValue OptimizeFloatStore(StoreSDNode *ST); 114 115 void LegalizeLoadOps(SDNode *Node); 116 void LegalizeStoreOps(SDNode *Node); 117 118 /// Some targets cannot handle a variable 119 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 120 /// is necessary to spill the vector being inserted into to memory, perform 121 /// the insert there, and then read the result back. 122 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 123 const SDLoc &dl); 124 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, 125 const SDLoc &dl); 126 127 /// Return a vector shuffle operation which 128 /// performs the same shuffe in terms of order or result bytes, but on a type 129 /// whose vector element type is narrower than the original shuffle type. 130 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 131 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, 132 SDValue N1, SDValue N2, 133 ArrayRef<int> Mask) const; 134 135 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 136 bool &NeedInvert, const SDLoc &dl); 137 138 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 139 140 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, 141 SDNode *Node, bool isSigned); 142 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 143 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 144 RTLIB::Libcall Call_F128, 145 RTLIB::Libcall Call_PPCF128); 146 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 147 RTLIB::Libcall Call_I8, 148 RTLIB::Libcall Call_I16, 149 RTLIB::Libcall Call_I32, 150 RTLIB::Libcall Call_I64, 151 RTLIB::Libcall Call_I128); 152 SDValue ExpandArgFPLibCall(SDNode *Node, 153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, 154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, 155 RTLIB::Libcall Call_PPCF128); 156 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 157 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 158 159 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, 160 const SDLoc &dl); 161 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, 162 const SDLoc &dl, SDValue ChainIn); 163 SDValue ExpandBUILD_VECTOR(SDNode *Node); 164 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 165 void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 166 SmallVectorImpl<SDValue> &Results); 167 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, 168 SDValue Value) const; 169 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, 170 SDValue NewIntValue) const; 171 SDValue ExpandFCOPYSIGN(SDNode *Node) const; 172 SDValue ExpandFABS(SDNode *Node) const; 173 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT, 174 const SDLoc &dl); 175 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, 176 const SDLoc &dl); 177 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, 178 const SDLoc &dl); 179 180 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl); 181 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl); 182 183 SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 184 SDValue ExpandInsertToVectorThroughStack(SDValue Op); 185 SDValue ExpandVectorBuildThroughStack(SDNode* Node); 186 187 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 188 SDValue ExpandConstant(ConstantSDNode *CP); 189 190 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall 191 bool ExpandNode(SDNode *Node); 192 void ConvertNodeToLibcall(SDNode *Node); 193 void PromoteNode(SDNode *Node); 194 195public: 196 // Node replacement helpers 197 198 void ReplacedNode(SDNode *N) { 199 LegalizedNodes.erase(N); 200 if (UpdatedNodes) 201 UpdatedNodes->insert(N); 202 } 203 204 void ReplaceNode(SDNode *Old, SDNode *New) { 205 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); 206 dbgs() << " with: "; New->dump(&DAG)); 207 208 assert(Old->getNumValues() == New->getNumValues() && 209 "Replacing one node with another that produces a different number " 210 "of values!"); 211 DAG.ReplaceAllUsesWith(Old, New); 212 if (UpdatedNodes) 213 UpdatedNodes->insert(New); 214 ReplacedNode(Old); 215 } 216 217 void ReplaceNode(SDValue Old, SDValue New) { 218 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); 219 dbgs() << " with: "; New->dump(&DAG)); 220 221 DAG.ReplaceAllUsesWith(Old, New); 222 if (UpdatedNodes) 223 UpdatedNodes->insert(New.getNode()); 224 ReplacedNode(Old.getNode()); 225 } 226 227 void ReplaceNode(SDNode *Old, const SDValue *New) { 228 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); 229 230 DAG.ReplaceAllUsesWith(Old, New); 231 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { 232 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: "); 233 New[i]->dump(&DAG)); 234 if (UpdatedNodes) 235 UpdatedNodes->insert(New[i].getNode()); 236 } 237 ReplacedNode(Old); 238 } 239}; 240 241} // end anonymous namespace 242 243/// Return a vector shuffle operation which 244/// performs the same shuffle in terms of order or result bytes, but on a type 245/// whose vector element type is narrower than the original shuffle type. 246/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 247SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( 248 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, 249 ArrayRef<int> Mask) const { 250 unsigned NumMaskElts = VT.getVectorNumElements(); 251 unsigned NumDestElts = NVT.getVectorNumElements(); 252 unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 253 254 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 255 256 if (NumEltsGrowth == 1) 257 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); 258 259 SmallVector<int, 8> NewMask; 260 for (unsigned i = 0; i != NumMaskElts; ++i) { 261 int Idx = Mask[i]; 262 for (unsigned j = 0; j != NumEltsGrowth; ++j) { 263 if (Idx < 0) 264 NewMask.push_back(-1); 265 else 266 NewMask.push_back(Idx * NumEltsGrowth + j); 267 } 268 } 269 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 270 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 271 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); 272} 273 274/// Expands the ConstantFP node to an integer constant or 275/// a load from the constant pool. 276SDValue 277SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 278 bool Extend = false; 279 SDLoc dl(CFP); 280 281 // If a FP immediate is precise when represented as a float and if the 282 // target can do an extending load from float to double, we put it into 283 // the constant pool as a float, even if it's is statically typed as a 284 // double. This shrinks FP constants and canonicalizes them for targets where 285 // an FP extending load is the same cost as a normal load (such as on the x87 286 // fp stack or PPC FP unit). 287 EVT VT = CFP->getValueType(0); 288 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 289 if (!UseCP) { 290 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 291 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, 292 (VT == MVT::f64) ? MVT::i64 : MVT::i32); 293 } 294 295 APFloat APF = CFP->getValueAPF(); 296 EVT OrigVT = VT; 297 EVT SVT = VT; 298 299 // We don't want to shrink SNaNs. Converting the SNaN back to its real type 300 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). 301 if (!APF.isSignaling()) { 302 while (SVT != MVT::f32 && SVT != MVT::f16) { 303 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 304 if (ConstantFPSDNode::isValueValidForType(SVT, APF) && 305 // Only do this if the target has a native EXTLOAD instruction from 306 // smaller type. 307 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && 308 TLI.ShouldShrinkFPConstant(OrigVT)) { 309 Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 310 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 311 VT = SVT; 312 Extend = true; 313 } 314 } 315 } 316 317 SDValue CPIdx = 318 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); 319 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 320 if (Extend) { 321 SDValue Result = DAG.getExtLoad( 322 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, 323 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, 324 Alignment); 325 return Result; 326 } 327 SDValue Result = DAG.getLoad( 328 OrigVT, dl, DAG.getEntryNode(), CPIdx, 329 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); 330 return Result; 331} 332 333/// Expands the Constant node to a load from the constant pool. 334SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { 335 SDLoc dl(CP); 336 EVT VT = CP->getValueType(0); 337 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), 338 TLI.getPointerTy(DAG.getDataLayout())); 339 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 340 SDValue Result = DAG.getLoad( 341 VT, dl, DAG.getEntryNode(), CPIdx, 342 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); 343 return Result; 344} 345 346/// Some target cannot handle a variable insertion index for the 347/// INSERT_VECTOR_ELT instruction. In this case, it 348/// is necessary to spill the vector being inserted into to memory, perform 349/// the insert there, and then read the result back. 350SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, 351 SDValue Val, 352 SDValue Idx, 353 const SDLoc &dl) { 354 SDValue Tmp1 = Vec; 355 SDValue Tmp2 = Val; 356 SDValue Tmp3 = Idx; 357 358 // If the target doesn't support this, we have to spill the input vector 359 // to a temporary stack slot, update the element, then reload it. This is 360 // badness. We could also load the value into a vector register (either 361 // with a "move to register" or "extload into register" instruction, then 362 // permute it into place, if the idx is a constant and if the idx is 363 // supported by the target. 364 EVT VT = Tmp1.getValueType(); 365 EVT EltVT = VT.getVectorElementType(); 366 SDValue StackPtr = DAG.CreateStackTemporary(VT); 367 368 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 369 370 // Store the vector. 371 SDValue Ch = DAG.getStore( 372 DAG.getEntryNode(), dl, Tmp1, StackPtr, 373 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); 374 375 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); 376 377 // Store the scalar value. 378 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT); 379 // Load the updated vector. 380 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( 381 DAG.getMachineFunction(), SPFI)); 382} 383 384SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 385 SDValue Idx, 386 const SDLoc &dl) { 387 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 388 // SCALAR_TO_VECTOR requires that the type of the value being inserted 389 // match the element type of the vector being created, except for 390 // integers in which case the inserted value can be over width. 391 EVT EltVT = Vec.getValueType().getVectorElementType(); 392 if (Val.getValueType() == EltVT || 393 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 394 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 395 Vec.getValueType(), Val); 396 397 unsigned NumElts = Vec.getValueType().getVectorNumElements(); 398 // We generate a shuffle of InVec and ScVec, so the shuffle mask 399 // should be 0,1,2,3,4,5... with the appropriate element replaced with 400 // elt 0 of the RHS. 401 SmallVector<int, 8> ShufOps; 402 for (unsigned i = 0; i != NumElts; ++i) 403 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 404 405 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); 406 } 407 } 408 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 409} 410 411SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 412 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n"); 413 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 414 // FIXME: We shouldn't do this for TargetConstantFP's. 415 // FIXME: move this to the DAG Combiner! Note that we can't regress due 416 // to phase ordering between legalized code and the dag combiner. This 417 // probably means that we need to integrate dag combiner and legalizer 418 // together. 419 // We generally can't do this one for long doubles. 420 SDValue Chain = ST->getChain(); 421 SDValue Ptr = ST->getBasePtr(); 422 unsigned Alignment = ST->getAlignment(); 423 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 424 AAMDNodes AAInfo = ST->getAAInfo(); 425 SDLoc dl(ST); 426 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 427 if (CFP->getValueType(0) == MVT::f32 && 428 TLI.isTypeLegal(MVT::i32)) { 429 SDValue Con = DAG.getConstant(CFP->getValueAPF(). 430 bitcastToAPInt().zextOrTrunc(32), 431 SDLoc(CFP), MVT::i32); 432 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment, 433 MMOFlags, AAInfo); 434 } 435 436 if (CFP->getValueType(0) == MVT::f64) { 437 // If this target supports 64-bit registers, do a single 64-bit store. 438 if (TLI.isTypeLegal(MVT::i64)) { 439 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 440 zextOrTrunc(64), SDLoc(CFP), MVT::i64); 441 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 442 Alignment, MMOFlags, AAInfo); 443 } 444 445 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 446 // Otherwise, if the target supports 32-bit registers, use 2 32-bit 447 // stores. If the target supports neither 32- nor 64-bits, this 448 // xform is certainly not worth it. 449 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); 450 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); 451 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); 452 if (DAG.getDataLayout().isBigEndian()) 453 std::swap(Lo, Hi); 454 455 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment, 456 MMOFlags, AAInfo); 457 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 458 DAG.getConstant(4, dl, Ptr.getValueType())); 459 Hi = DAG.getStore(Chain, dl, Hi, Ptr, 460 ST->getPointerInfo().getWithOffset(4), 461 MinAlign(Alignment, 4U), MMOFlags, AAInfo); 462 463 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 464 } 465 } 466 } 467 return SDValue(nullptr, 0); 468} 469 470void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { 471 StoreSDNode *ST = cast<StoreSDNode>(Node); 472 SDValue Chain = ST->getChain(); 473 SDValue Ptr = ST->getBasePtr(); 474 SDLoc dl(Node); 475 476 unsigned Alignment = ST->getAlignment(); 477 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 478 AAMDNodes AAInfo = ST->getAAInfo(); 479 480 if (!ST->isTruncatingStore()) { 481 LLVM_DEBUG(dbgs() << "Legalizing store operation\n"); 482 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 483 ReplaceNode(ST, OptStore); 484 return; 485 } 486 487 SDValue Value = ST->getValue(); 488 MVT VT = Value.getSimpleValueType(); 489 switch (TLI.getOperationAction(ISD::STORE, VT)) { 490 default: llvm_unreachable("This action is not supported yet!"); 491 case TargetLowering::Legal: { 492 // If this is an unaligned store and the target doesn't support it, 493 // expand it. 494 EVT MemVT = ST->getMemoryVT(); 495 const DataLayout &DL = DAG.getDataLayout(); 496 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, 497 *ST->getMemOperand())) { 498 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n"); 499 SDValue Result = TLI.expandUnalignedStore(ST, DAG); 500 ReplaceNode(SDValue(ST, 0), Result); 501 } else 502 LLVM_DEBUG(dbgs() << "Legal store\n"); 503 break; 504 } 505 case TargetLowering::Custom: { 506 LLVM_DEBUG(dbgs() << "Trying custom lowering\n"); 507 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 508 if (Res && Res != SDValue(Node, 0)) 509 ReplaceNode(SDValue(Node, 0), Res); 510 return; 511 } 512 case TargetLowering::Promote: { 513 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); 514 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 515 "Can only promote stores to same size type"); 516 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); 517 SDValue Result = 518 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 519 Alignment, MMOFlags, AAInfo); 520 ReplaceNode(SDValue(Node, 0), Result); 521 break; 522 } 523 } 524 return; 525 } 526 527 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n"); 528 SDValue Value = ST->getValue(); 529 EVT StVT = ST->getMemoryVT(); 530 unsigned StWidth = StVT.getSizeInBits(); 531 auto &DL = DAG.getDataLayout(); 532 533 if (StWidth != StVT.getStoreSizeInBits()) { 534 // Promote to a byte-sized store with upper bits zero if not 535 // storing an integral number of bytes. For example, promote 536 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 537 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), 538 StVT.getStoreSizeInBits()); 539 Value = DAG.getZeroExtendInReg(Value, dl, StVT); 540 SDValue Result = 541 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, 542 Alignment, MMOFlags, AAInfo); 543 ReplaceNode(SDValue(Node, 0), Result); 544 } else if (StWidth & (StWidth - 1)) { 545 // If not storing a power-of-2 number of bits, expand as two stores. 546 assert(!StVT.isVector() && "Unsupported truncstore!"); 547 unsigned LogStWidth = Log2_32(StWidth); 548 assert(LogStWidth < 32); 549 unsigned RoundWidth = 1 << LogStWidth; 550 assert(RoundWidth < StWidth); 551 unsigned ExtraWidth = StWidth - RoundWidth; 552 assert(ExtraWidth < RoundWidth); 553 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 554 "Store size not an integral number of bytes!"); 555 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 556 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 557 SDValue Lo, Hi; 558 unsigned IncrementSize; 559 560 if (DL.isLittleEndian()) { 561 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 562 // Store the bottom RoundWidth bits. 563 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 564 RoundVT, Alignment, MMOFlags, AAInfo); 565 566 // Store the remaining ExtraWidth bits. 567 IncrementSize = RoundWidth / 8; 568 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 569 DAG.getConstant(IncrementSize, dl, 570 Ptr.getValueType())); 571 Hi = DAG.getNode( 572 ISD::SRL, dl, Value.getValueType(), Value, 573 DAG.getConstant(RoundWidth, dl, 574 TLI.getShiftAmountTy(Value.getValueType(), DL))); 575 Hi = DAG.getTruncStore( 576 Chain, dl, Hi, Ptr, 577 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, 578 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 579 } else { 580 // Big endian - avoid unaligned stores. 581 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 582 // Store the top RoundWidth bits. 583 Hi = DAG.getNode( 584 ISD::SRL, dl, Value.getValueType(), Value, 585 DAG.getConstant(ExtraWidth, dl, 586 TLI.getShiftAmountTy(Value.getValueType(), DL))); 587 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), 588 RoundVT, Alignment, MMOFlags, AAInfo); 589 590 // Store the remaining ExtraWidth bits. 591 IncrementSize = RoundWidth / 8; 592 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 593 DAG.getConstant(IncrementSize, dl, 594 Ptr.getValueType())); 595 Lo = DAG.getTruncStore( 596 Chain, dl, Value, Ptr, 597 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, 598 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 599 } 600 601 // The order of the stores doesn't matter. 602 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 603 ReplaceNode(SDValue(Node, 0), Result); 604 } else { 605 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { 606 default: llvm_unreachable("This action is not supported yet!"); 607 case TargetLowering::Legal: { 608 EVT MemVT = ST->getMemoryVT(); 609 // If this is an unaligned store and the target doesn't support it, 610 // expand it. 611 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, 612 *ST->getMemOperand())) { 613 SDValue Result = TLI.expandUnalignedStore(ST, DAG); 614 ReplaceNode(SDValue(ST, 0), Result); 615 } 616 break; 617 } 618 case TargetLowering::Custom: { 619 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 620 if (Res && Res != SDValue(Node, 0)) 621 ReplaceNode(SDValue(Node, 0), Res); 622 return; 623 } 624 case TargetLowering::Expand: 625 assert(!StVT.isVector() && 626 "Vector Stores are handled in LegalizeVectorOps"); 627 628 SDValue Result; 629 630 // TRUNCSTORE:i16 i32 -> STORE i16 631 if (TLI.isTypeLegal(StVT)) { 632 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); 633 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 634 Alignment, MMOFlags, AAInfo); 635 } else { 636 // The in-memory type isn't legal. Truncate to the type it would promote 637 // to, and then do a truncstore. 638 Value = DAG.getNode(ISD::TRUNCATE, dl, 639 TLI.getTypeToTransformTo(*DAG.getContext(), StVT), 640 Value); 641 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 642 StVT, Alignment, MMOFlags, AAInfo); 643 } 644 645 ReplaceNode(SDValue(Node, 0), Result); 646 break; 647 } 648 } 649} 650 651void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { 652 LoadSDNode *LD = cast<LoadSDNode>(Node); 653 SDValue Chain = LD->getChain(); // The chain. 654 SDValue Ptr = LD->getBasePtr(); // The base pointer. 655 SDValue Value; // The value returned by the load op. 656 SDLoc dl(Node); 657 658 ISD::LoadExtType ExtType = LD->getExtensionType(); 659 if (ExtType == ISD::NON_EXTLOAD) { 660 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n"); 661 MVT VT = Node->getSimpleValueType(0); 662 SDValue RVal = SDValue(Node, 0); 663 SDValue RChain = SDValue(Node, 1); 664 665 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 666 default: llvm_unreachable("This action is not supported yet!"); 667 case TargetLowering::Legal: { 668 EVT MemVT = LD->getMemoryVT(); 669 const DataLayout &DL = DAG.getDataLayout(); 670 // If this is an unaligned load and the target doesn't support it, 671 // expand it. 672 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, 673 *LD->getMemOperand())) { 674 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); 675 } 676 break; 677 } 678 case TargetLowering::Custom: 679 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { 680 RVal = Res; 681 RChain = Res.getValue(1); 682 } 683 break; 684 685 case TargetLowering::Promote: { 686 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 687 assert(NVT.getSizeInBits() == VT.getSizeInBits() && 688 "Can only promote loads to same size type"); 689 690 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); 691 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); 692 RChain = Res.getValue(1); 693 break; 694 } 695 } 696 if (RChain.getNode() != Node) { 697 assert(RVal.getNode() != Node && "Load must be completely replaced"); 698 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); 699 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); 700 if (UpdatedNodes) { 701 UpdatedNodes->insert(RVal.getNode()); 702 UpdatedNodes->insert(RChain.getNode()); 703 } 704 ReplacedNode(Node); 705 } 706 return; 707 } 708 709 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n"); 710 EVT SrcVT = LD->getMemoryVT(); 711 unsigned SrcWidth = SrcVT.getSizeInBits(); 712 unsigned Alignment = LD->getAlignment(); 713 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 714 AAMDNodes AAInfo = LD->getAAInfo(); 715 716 if (SrcWidth != SrcVT.getStoreSizeInBits() && 717 // Some targets pretend to have an i1 loading operation, and actually 718 // load an i8. This trick is correct for ZEXTLOAD because the top 7 719 // bits are guaranteed to be zero; it helps the optimizers understand 720 // that these bits are zero. It is also useful for EXTLOAD, since it 721 // tells the optimizers that those bits are undefined. It would be 722 // nice to have an effective generic way of getting these benefits... 723 // Until such a way is found, don't insist on promoting i1 here. 724 (SrcVT != MVT::i1 || 725 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == 726 TargetLowering::Promote)) { 727 // Promote to a byte-sized load if not loading an integral number of 728 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 729 unsigned NewWidth = SrcVT.getStoreSizeInBits(); 730 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 731 SDValue Ch; 732 733 // The extra bits are guaranteed to be zero, since we stored them that 734 // way. A zext load from NVT thus automatically gives zext from SrcVT. 735 736 ISD::LoadExtType NewExtType = 737 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 738 739 SDValue Result = 740 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr, 741 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo); 742 743 Ch = Result.getValue(1); // The chain. 744 745 if (ExtType == ISD::SEXTLOAD) 746 // Having the top bits zero doesn't help when sign extending. 747 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 748 Result.getValueType(), 749 Result, DAG.getValueType(SrcVT)); 750 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 751 // All the top bits are guaranteed to be zero - inform the optimizers. 752 Result = DAG.getNode(ISD::AssertZext, dl, 753 Result.getValueType(), Result, 754 DAG.getValueType(SrcVT)); 755 756 Value = Result; 757 Chain = Ch; 758 } else if (SrcWidth & (SrcWidth - 1)) { 759 // If not loading a power-of-2 number of bits, expand as two loads. 760 assert(!SrcVT.isVector() && "Unsupported extload!"); 761 unsigned LogSrcWidth = Log2_32(SrcWidth); 762 assert(LogSrcWidth < 32); 763 unsigned RoundWidth = 1 << LogSrcWidth; 764 assert(RoundWidth < SrcWidth); 765 unsigned ExtraWidth = SrcWidth - RoundWidth; 766 assert(ExtraWidth < RoundWidth); 767 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 768 "Load size not an integral number of bytes!"); 769 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 770 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 771 SDValue Lo, Hi, Ch; 772 unsigned IncrementSize; 773 auto &DL = DAG.getDataLayout(); 774 775 if (DL.isLittleEndian()) { 776 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 777 // Load the bottom RoundWidth bits. 778 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, 779 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, 780 AAInfo); 781 782 // Load the remaining ExtraWidth bits. 783 IncrementSize = RoundWidth / 8; 784 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 785 DAG.getConstant(IncrementSize, dl, 786 Ptr.getValueType())); 787 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 788 LD->getPointerInfo().getWithOffset(IncrementSize), 789 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, 790 AAInfo); 791 792 // Build a factor node to remember that this load is independent of 793 // the other one. 794 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 795 Hi.getValue(1)); 796 797 // Move the top bits to the right place. 798 Hi = DAG.getNode( 799 ISD::SHL, dl, Hi.getValueType(), Hi, 800 DAG.getConstant(RoundWidth, dl, 801 TLI.getShiftAmountTy(Hi.getValueType(), DL))); 802 803 // Join the hi and lo parts. 804 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 805 } else { 806 // Big endian - avoid unaligned loads. 807 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 808 // Load the top RoundWidth bits. 809 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 810 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, 811 AAInfo); 812 813 // Load the remaining ExtraWidth bits. 814 IncrementSize = RoundWidth / 8; 815 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 816 DAG.getConstant(IncrementSize, dl, 817 Ptr.getValueType())); 818 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, 819 LD->getPointerInfo().getWithOffset(IncrementSize), 820 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, 821 AAInfo); 822 823 // Build a factor node to remember that this load is independent of 824 // the other one. 825 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 826 Hi.getValue(1)); 827 828 // Move the top bits to the right place. 829 Hi = DAG.getNode( 830 ISD::SHL, dl, Hi.getValueType(), Hi, 831 DAG.getConstant(ExtraWidth, dl, 832 TLI.getShiftAmountTy(Hi.getValueType(), DL))); 833 834 // Join the hi and lo parts. 835 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 836 } 837 838 Chain = Ch; 839 } else { 840 bool isCustom = false; 841 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), 842 SrcVT.getSimpleVT())) { 843 default: llvm_unreachable("This action is not supported yet!"); 844 case TargetLowering::Custom: 845 isCustom = true; 846 LLVM_FALLTHROUGH; 847 case TargetLowering::Legal: 848 Value = SDValue(Node, 0); 849 Chain = SDValue(Node, 1); 850 851 if (isCustom) { 852 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { 853 Value = Res; 854 Chain = Res.getValue(1); 855 } 856 } else { 857 // If this is an unaligned load and the target doesn't support it, 858 // expand it. 859 EVT MemVT = LD->getMemoryVT(); 860 const DataLayout &DL = DAG.getDataLayout(); 861 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, 862 *LD->getMemOperand())) { 863 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); 864 } 865 } 866 break; 867 868 case TargetLowering::Expand: { 869 EVT DestVT = Node->getValueType(0); 870 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { 871 // If the source type is not legal, see if there is a legal extload to 872 // an intermediate type that we can then extend further. 873 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); 874 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? 875 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) { 876 // If we are loading a legal type, this is a non-extload followed by a 877 // full extend. 878 ISD::LoadExtType MidExtType = 879 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; 880 881 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, 882 SrcVT, LD->getMemOperand()); 883 unsigned ExtendOp = 884 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); 885 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 886 Chain = Load.getValue(1); 887 break; 888 } 889 890 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the 891 // normal undefined upper bits behavior to allow using an in-reg extend 892 // with the illegal FP type, so load as an integer and do the 893 // from-integer conversion. 894 if (SrcVT.getScalarType() == MVT::f16) { 895 EVT ISrcVT = SrcVT.changeTypeToInteger(); 896 EVT IDestVT = DestVT.changeTypeToInteger(); 897 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); 898 899 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT, 900 Chain, Ptr, ISrcVT, 901 LD->getMemOperand()); 902 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); 903 Chain = Result.getValue(1); 904 break; 905 } 906 } 907 908 assert(!SrcVT.isVector() && 909 "Vector Loads are handled in LegalizeVectorOps"); 910 911 // FIXME: This does not work for vectors on most targets. Sign- 912 // and zero-extend operations are currently folded into extending 913 // loads, whether they are legal or not, and then we end up here 914 // without any support for legalizing them. 915 assert(ExtType != ISD::EXTLOAD && 916 "EXTLOAD should always be supported!"); 917 // Turn the unsupported load into an EXTLOAD followed by an 918 // explicit zero/sign extend inreg. 919 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, 920 Node->getValueType(0), 921 Chain, Ptr, SrcVT, 922 LD->getMemOperand()); 923 SDValue ValRes; 924 if (ExtType == ISD::SEXTLOAD) 925 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 926 Result.getValueType(), 927 Result, DAG.getValueType(SrcVT)); 928 else 929 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); 930 Value = ValRes; 931 Chain = Result.getValue(1); 932 break; 933 } 934 } 935 } 936 937 // Since loads produce two values, make sure to remember that we legalized 938 // both of them. 939 if (Chain.getNode() != Node) { 940 assert(Value.getNode() != Node && "Load must be completely replaced"); 941 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); 942 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 943 if (UpdatedNodes) { 944 UpdatedNodes->insert(Value.getNode()); 945 UpdatedNodes->insert(Chain.getNode()); 946 } 947 ReplacedNode(Node); 948 } 949} 950 951/// Return a legal replacement for the given operation, with all legal operands. 952void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 953 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); 954 955 // Allow illegal target nodes and illegal registers. 956 if (Node->getOpcode() == ISD::TargetConstant || 957 Node->getOpcode() == ISD::Register) 958 return; 959 960#ifndef NDEBUG 961 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 962 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 963 TargetLowering::TypeLegal || 964 TLI.isTypeLegal(Node->getValueType(i))) && 965 "Unexpected illegal type!"); 966 967 for (const SDValue &Op : Node->op_values()) 968 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == 969 TargetLowering::TypeLegal || 970 TLI.isTypeLegal(Op.getValueType()) || 971 Op.getOpcode() == ISD::TargetConstant || 972 Op.getOpcode() == ISD::Register) && 973 "Unexpected illegal type!"); 974#endif 975 976 // Figure out the correct action; the way to query this varies by opcode 977 TargetLowering::LegalizeAction Action = TargetLowering::Legal; 978 bool SimpleFinishLegalizing = true; 979 switch (Node->getOpcode()) { 980 case ISD::INTRINSIC_W_CHAIN: 981 case ISD::INTRINSIC_WO_CHAIN: 982 case ISD::INTRINSIC_VOID: 983 case ISD::STACKSAVE: 984 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 985 break; 986 case ISD::GET_DYNAMIC_AREA_OFFSET: 987 Action = TLI.getOperationAction(Node->getOpcode(), 988 Node->getValueType(0)); 989 break; 990 case ISD::VAARG: 991 Action = TLI.getOperationAction(Node->getOpcode(), 992 Node->getValueType(0)); 993 if (Action != TargetLowering::Promote) 994 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 995 break; 996 case ISD::FP_TO_FP16: 997 case ISD::SINT_TO_FP: 998 case ISD::UINT_TO_FP: 999 case ISD::EXTRACT_VECTOR_ELT: 1000 case ISD::LROUND: 1001 case ISD::LLROUND: 1002 case ISD::LRINT: 1003 case ISD::LLRINT: 1004 Action = TLI.getOperationAction(Node->getOpcode(), 1005 Node->getOperand(0).getValueType()); 1006 break; 1007 case ISD::FP_ROUND_INREG: 1008 case ISD::SIGN_EXTEND_INREG: { 1009 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1010 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 1011 break; 1012 } 1013 case ISD::ATOMIC_STORE: 1014 Action = TLI.getOperationAction(Node->getOpcode(), 1015 Node->getOperand(2).getValueType()); 1016 break; 1017 case ISD::SELECT_CC: 1018 case ISD::SETCC: 1019 case ISD::BR_CC: { 1020 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 1021 Node->getOpcode() == ISD::SETCC ? 2 : 1; 1022 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; 1023 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); 1024 ISD::CondCode CCCode = 1025 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 1026 Action = TLI.getCondCodeAction(CCCode, OpVT); 1027 if (Action == TargetLowering::Legal) { 1028 if (Node->getOpcode() == ISD::SELECT_CC) 1029 Action = TLI.getOperationAction(Node->getOpcode(), 1030 Node->getValueType(0)); 1031 else 1032 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 1033 } 1034 break; 1035 } 1036 case ISD::LOAD: 1037 case ISD::STORE: 1038 // FIXME: Model these properly. LOAD and STORE are complicated, and 1039 // STORE expects the unlegalized operand in some cases. 1040 SimpleFinishLegalizing = false; 1041 break; 1042 case ISD::CALLSEQ_START: 1043 case ISD::CALLSEQ_END: 1044 // FIXME: This shouldn't be necessary. These nodes have special properties 1045 // dealing with the recursive nature of legalization. Removing this 1046 // special case should be done as part of making LegalizeDAG non-recursive. 1047 SimpleFinishLegalizing = false; 1048 break; 1049 case ISD::EXTRACT_ELEMENT: 1050 case ISD::FLT_ROUNDS_: 1051 case ISD::MERGE_VALUES: 1052 case ISD::EH_RETURN: 1053 case ISD::FRAME_TO_ARGS_OFFSET: 1054 case ISD::EH_DWARF_CFA: 1055 case ISD::EH_SJLJ_SETJMP: 1056 case ISD::EH_SJLJ_LONGJMP: 1057 case ISD::EH_SJLJ_SETUP_DISPATCH: 1058 // These operations lie about being legal: when they claim to be legal, 1059 // they should actually be expanded. 1060 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1061 if (Action == TargetLowering::Legal) 1062 Action = TargetLowering::Expand; 1063 break; 1064 case ISD::INIT_TRAMPOLINE: 1065 case ISD::ADJUST_TRAMPOLINE: 1066 case ISD::FRAMEADDR: 1067 case ISD::RETURNADDR: 1068 case ISD::ADDROFRETURNADDR: 1069 case ISD::SPONENTRY: 1070 // These operations lie about being legal: when they claim to be legal, 1071 // they should actually be custom-lowered. 1072 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1073 if (Action == TargetLowering::Legal) 1074 Action = TargetLowering::Custom; 1075 break; 1076 case ISD::READCYCLECOUNTER: 1077 // READCYCLECOUNTER returns an i64, even if type legalization might have 1078 // expanded that to several smaller types. 1079 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); 1080 break; 1081 case ISD::READ_REGISTER: 1082 case ISD::WRITE_REGISTER: 1083 // Named register is legal in the DAG, but blocked by register name 1084 // selection if not implemented by target (to chose the correct register) 1085 // They'll be converted to Copy(To/From)Reg. 1086 Action = TargetLowering::Legal; 1087 break; 1088 case ISD::DEBUGTRAP: 1089 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1090 if (Action == TargetLowering::Expand) { 1091 // replace ISD::DEBUGTRAP with ISD::TRAP 1092 SDValue NewVal; 1093 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), 1094 Node->getOperand(0)); 1095 ReplaceNode(Node, NewVal.getNode()); 1096 LegalizeOp(NewVal.getNode()); 1097 return; 1098 } 1099 break; 1100 case ISD::STRICT_FADD: 1101 case ISD::STRICT_FSUB: 1102 case ISD::STRICT_FMUL: 1103 case ISD::STRICT_FDIV: 1104 case ISD::STRICT_FREM: 1105 case ISD::STRICT_FSQRT: 1106 case ISD::STRICT_FMA: 1107 case ISD::STRICT_FPOW: 1108 case ISD::STRICT_FPOWI: 1109 case ISD::STRICT_FSIN: 1110 case ISD::STRICT_FCOS: 1111 case ISD::STRICT_FEXP: 1112 case ISD::STRICT_FEXP2: 1113 case ISD::STRICT_FLOG: 1114 case ISD::STRICT_FLOG10: 1115 case ISD::STRICT_FLOG2: 1116 case ISD::STRICT_FRINT: 1117 case ISD::STRICT_FNEARBYINT: 1118 case ISD::STRICT_FMAXNUM: 1119 case ISD::STRICT_FMINNUM: 1120 case ISD::STRICT_FCEIL: 1121 case ISD::STRICT_FFLOOR: 1122 case ISD::STRICT_FROUND: 1123 case ISD::STRICT_FTRUNC: 1124 case ISD::STRICT_FP_ROUND: 1125 case ISD::STRICT_FP_EXTEND: 1126 // These pseudo-ops get legalized as if they were their non-strict 1127 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT 1128 // is also legal, but if ISD::FSQRT requires expansion then so does 1129 // ISD::STRICT_FSQRT. 1130 Action = TLI.getStrictFPOperationAction(Node->getOpcode(), 1131 Node->getValueType(0)); 1132 break; 1133 case ISD::SADDSAT: 1134 case ISD::UADDSAT: 1135 case ISD::SSUBSAT: 1136 case ISD::USUBSAT: { 1137 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1138 break; 1139 } 1140 case ISD::SMULFIX: 1141 case ISD::SMULFIXSAT: 1142 case ISD::UMULFIX: { 1143 unsigned Scale = Node->getConstantOperandVal(2); 1144 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), 1145 Node->getValueType(0), Scale); 1146 break; 1147 } 1148 case ISD::MSCATTER: 1149 Action = TLI.getOperationAction(Node->getOpcode(), 1150 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); 1151 break; 1152 case ISD::MSTORE: 1153 Action = TLI.getOperationAction(Node->getOpcode(), 1154 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); 1155 break; 1156 case ISD::VECREDUCE_FADD: 1157 case ISD::VECREDUCE_FMUL: 1158 case ISD::VECREDUCE_ADD: 1159 case ISD::VECREDUCE_MUL: 1160 case ISD::VECREDUCE_AND: 1161 case ISD::VECREDUCE_OR: 1162 case ISD::VECREDUCE_XOR: 1163 case ISD::VECREDUCE_SMAX: 1164 case ISD::VECREDUCE_SMIN: 1165 case ISD::VECREDUCE_UMAX: 1166 case ISD::VECREDUCE_UMIN: 1167 case ISD::VECREDUCE_FMAX: 1168 case ISD::VECREDUCE_FMIN: 1169 Action = TLI.getOperationAction( 1170 Node->getOpcode(), Node->getOperand(0).getValueType()); 1171 break; 1172 default: 1173 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1174 Action = TargetLowering::Legal; 1175 } else { 1176 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1177 } 1178 break; 1179 } 1180 1181 if (SimpleFinishLegalizing) { 1182 SDNode *NewNode = Node; 1183 switch (Node->getOpcode()) { 1184 default: break; 1185 case ISD::SHL: 1186 case ISD::SRL: 1187 case ISD::SRA: 1188 case ISD::ROTL: 1189 case ISD::ROTR: { 1190 // Legalizing shifts/rotates requires adjusting the shift amount 1191 // to the appropriate width. 1192 SDValue Op0 = Node->getOperand(0); 1193 SDValue Op1 = Node->getOperand(1); 1194 if (!Op1.getValueType().isVector()) { 1195 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); 1196 // The getShiftAmountOperand() may create a new operand node or 1197 // return the existing one. If new operand is created we need 1198 // to update the parent node. 1199 // Do not try to legalize SAO here! It will be automatically legalized 1200 // in the next round. 1201 if (SAO != Op1) 1202 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); 1203 } 1204 } 1205 break; 1206 case ISD::FSHL: 1207 case ISD::FSHR: 1208 case ISD::SRL_PARTS: 1209 case ISD::SRA_PARTS: 1210 case ISD::SHL_PARTS: { 1211 // Legalizing shifts/rotates requires adjusting the shift amount 1212 // to the appropriate width. 1213 SDValue Op0 = Node->getOperand(0); 1214 SDValue Op1 = Node->getOperand(1); 1215 SDValue Op2 = Node->getOperand(2); 1216 if (!Op2.getValueType().isVector()) { 1217 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); 1218 // The getShiftAmountOperand() may create a new operand node or 1219 // return the existing one. If new operand is created we need 1220 // to update the parent node. 1221 if (SAO != Op2) 1222 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); 1223 } 1224 break; 1225 } 1226 } 1227 1228 if (NewNode != Node) { 1229 ReplaceNode(Node, NewNode); 1230 Node = NewNode; 1231 } 1232 switch (Action) { 1233 case TargetLowering::Legal: 1234 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n"); 1235 return; 1236 case TargetLowering::Custom: 1237 LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); 1238 // FIXME: The handling for custom lowering with multiple results is 1239 // a complete mess. 1240 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { 1241 if (!(Res.getNode() != Node || Res.getResNo() != 0)) 1242 return; 1243 1244 if (Node->getNumValues() == 1) { 1245 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); 1246 // We can just directly replace this node with the lowered value. 1247 ReplaceNode(SDValue(Node, 0), Res); 1248 return; 1249 } 1250 1251 SmallVector<SDValue, 8> ResultVals; 1252 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1253 ResultVals.push_back(Res.getValue(i)); 1254 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); 1255 ReplaceNode(Node, ResultVals.data()); 1256 return; 1257 } 1258 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n"); 1259 LLVM_FALLTHROUGH; 1260 case TargetLowering::Expand: 1261 if (ExpandNode(Node)) 1262 return; 1263 LLVM_FALLTHROUGH; 1264 case TargetLowering::LibCall: 1265 ConvertNodeToLibcall(Node); 1266 return; 1267 case TargetLowering::Promote: 1268 PromoteNode(Node); 1269 return; 1270 } 1271 } 1272 1273 switch (Node->getOpcode()) { 1274 default: 1275#ifndef NDEBUG 1276 dbgs() << "NODE: "; 1277 Node->dump( &DAG); 1278 dbgs() << "\n"; 1279#endif 1280 llvm_unreachable("Do not know how to legalize this operator!"); 1281 1282 case ISD::CALLSEQ_START: 1283 case ISD::CALLSEQ_END: 1284 break; 1285 case ISD::LOAD: 1286 return LegalizeLoadOps(Node); 1287 case ISD::STORE: 1288 return LegalizeStoreOps(Node); 1289 } 1290} 1291 1292SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1293 SDValue Vec = Op.getOperand(0); 1294 SDValue Idx = Op.getOperand(1); 1295 SDLoc dl(Op); 1296 1297 // Before we generate a new store to a temporary stack slot, see if there is 1298 // already one that we can use. There often is because when we scalarize 1299 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole 1300 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in 1301 // the vector. If all are expanded here, we don't want one store per vector 1302 // element. 1303 1304 // Caches for hasPredecessorHelper 1305 SmallPtrSet<const SDNode *, 32> Visited; 1306 SmallVector<const SDNode *, 16> Worklist; 1307 Visited.insert(Op.getNode()); 1308 Worklist.push_back(Idx.getNode()); 1309 SDValue StackPtr, Ch; 1310 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), 1311 UE = Vec.getNode()->use_end(); UI != UE; ++UI) { 1312 SDNode *User = *UI; 1313 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { 1314 if (ST->isIndexed() || ST->isTruncatingStore() || 1315 ST->getValue() != Vec) 1316 continue; 1317 1318 // Make sure that nothing else could have stored into the destination of 1319 // this store. 1320 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) 1321 continue; 1322 1323 // If the index is dependent on the store we will introduce a cycle when 1324 // creating the load (the load uses the index, and by replacing the chain 1325 // we will make the index dependent on the load). Also, the store might be 1326 // dependent on the extractelement and introduce a cycle when creating 1327 // the load. 1328 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) || 1329 ST->hasPredecessor(Op.getNode())) 1330 continue; 1331 1332 StackPtr = ST->getBasePtr(); 1333 Ch = SDValue(ST, 0); 1334 break; 1335 } 1336 } 1337 1338 EVT VecVT = Vec.getValueType(); 1339 1340 if (!Ch.getNode()) { 1341 // Store the value to a temporary stack slot, then LOAD the returned part. 1342 StackPtr = DAG.CreateStackTemporary(VecVT); 1343 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1344 MachinePointerInfo()); 1345 } 1346 1347 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1348 1349 SDValue NewLoad; 1350 1351 if (Op.getValueType().isVector()) 1352 NewLoad = 1353 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); 1354 else 1355 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1356 MachinePointerInfo(), 1357 VecVT.getVectorElementType()); 1358 1359 // Replace the chain going out of the store, by the one out of the load. 1360 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); 1361 1362 // We introduced a cycle though, so update the loads operands, making sure 1363 // to use the original store's chain as an incoming chain. 1364 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), 1365 NewLoad->op_end()); 1366 NewLoadOperands[0] = Ch; 1367 NewLoad = 1368 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); 1369 return NewLoad; 1370} 1371 1372SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1373 assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1374 1375 SDValue Vec = Op.getOperand(0); 1376 SDValue Part = Op.getOperand(1); 1377 SDValue Idx = Op.getOperand(2); 1378 SDLoc dl(Op); 1379 1380 // Store the value to a temporary stack slot, then LOAD the returned part. 1381 EVT VecVT = Vec.getValueType(); 1382 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1383 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1384 MachinePointerInfo PtrInfo = 1385 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 1386 1387 // First store the whole vector. 1388 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); 1389 1390 // Then store the inserted part. 1391 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1392 1393 // Store the subvector. 1394 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo()); 1395 1396 // Finally, load the updated vector. 1397 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); 1398} 1399 1400SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1401 // We can't handle this case efficiently. Allocate a sufficiently 1402 // aligned object on the stack, store each element into it, then load 1403 // the result as a vector. 1404 // Create the stack frame object. 1405 EVT VT = Node->getValueType(0); 1406 EVT EltVT = VT.getVectorElementType(); 1407 SDLoc dl(Node); 1408 SDValue FIPtr = DAG.CreateStackTemporary(VT); 1409 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1410 MachinePointerInfo PtrInfo = 1411 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 1412 1413 // Emit a store of each element to the stack slot. 1414 SmallVector<SDValue, 8> Stores; 1415 unsigned TypeByteSize = EltVT.getSizeInBits() / 8; 1416 assert(TypeByteSize > 0 && "Vector element type too small for stack store!"); 1417 // Store (in the right endianness) the elements to memory. 1418 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1419 // Ignore undef elements. 1420 if (Node->getOperand(i).isUndef()) continue; 1421 1422 unsigned Offset = TypeByteSize*i; 1423 1424 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType()); 1425 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 1426 1427 // If the destination vector element type is narrower than the source 1428 // element type, only store the bits necessary. 1429 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { 1430 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1431 Node->getOperand(i), Idx, 1432 PtrInfo.getWithOffset(Offset), EltVT)); 1433 } else 1434 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), 1435 Idx, PtrInfo.getWithOffset(Offset))); 1436 } 1437 1438 SDValue StoreChain; 1439 if (!Stores.empty()) // Not all undef elements? 1440 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 1441 else 1442 StoreChain = DAG.getEntryNode(); 1443 1444 // Result is a load from the stack slot. 1445 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); 1446} 1447 1448/// Bitcast a floating-point value to an integer value. Only bitcast the part 1449/// containing the sign bit if the target has no integer value capable of 1450/// holding all bits of the floating-point value. 1451void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, 1452 const SDLoc &DL, 1453 SDValue Value) const { 1454 EVT FloatVT = Value.getValueType(); 1455 unsigned NumBits = FloatVT.getSizeInBits(); 1456 State.FloatVT = FloatVT; 1457 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); 1458 // Convert to an integer of the same size. 1459 if (TLI.isTypeLegal(IVT)) { 1460 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); 1461 State.SignMask = APInt::getSignMask(NumBits); 1462 State.SignBit = NumBits - 1; 1463 return; 1464 } 1465 1466 auto &DataLayout = DAG.getDataLayout(); 1467 // Store the float to memory, then load the sign part out as an integer. 1468 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); 1469 // First create a temporary that is aligned for both the load and store. 1470 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1471 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1472 // Then store the float to it. 1473 State.FloatPtr = StackPtr; 1474 MachineFunction &MF = DAG.getMachineFunction(); 1475 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); 1476 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, 1477 State.FloatPointerInfo); 1478 1479 SDValue IntPtr; 1480 if (DataLayout.isBigEndian()) { 1481 assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1482 // Load out a legal integer with the same sign bit as the float. 1483 IntPtr = StackPtr; 1484 State.IntPointerInfo = State.FloatPointerInfo; 1485 } else { 1486 // Advance the pointer so that the loaded byte will contain the sign bit. 1487 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1; 1488 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr, 1489 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType())); 1490 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, 1491 ByteOffset); 1492 } 1493 1494 State.IntPtr = IntPtr; 1495 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, 1496 State.IntPointerInfo, MVT::i8); 1497 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7); 1498 State.SignBit = 7; 1499} 1500 1501/// Replace the integer value produced by getSignAsIntValue() with a new value 1502/// and cast the result back to a floating-point type. 1503SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, 1504 const SDLoc &DL, 1505 SDValue NewIntValue) const { 1506 if (!State.Chain) 1507 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); 1508 1509 // Override the part containing the sign bit in the value stored on the stack. 1510 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, 1511 State.IntPointerInfo, MVT::i8); 1512 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, 1513 State.FloatPointerInfo); 1514} 1515 1516SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { 1517 SDLoc DL(Node); 1518 SDValue Mag = Node->getOperand(0); 1519 SDValue Sign = Node->getOperand(1); 1520 1521 // Get sign bit into an integer value. 1522 FloatSignAsInt SignAsInt; 1523 getSignAsIntValue(SignAsInt, DL, Sign); 1524 1525 EVT IntVT = SignAsInt.IntValue.getValueType(); 1526 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); 1527 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, 1528 SignMask); 1529 1530 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) 1531 EVT FloatVT = Mag.getValueType(); 1532 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && 1533 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { 1534 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); 1535 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); 1536 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, 1537 DAG.getConstant(0, DL, IntVT), ISD::SETNE); 1538 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); 1539 } 1540 1541 // Transform Mag value to integer, and clear the sign bit. 1542 FloatSignAsInt MagAsInt; 1543 getSignAsIntValue(MagAsInt, DL, Mag); 1544 EVT MagVT = MagAsInt.IntValue.getValueType(); 1545 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); 1546 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, 1547 ClearSignMask); 1548 1549 // Get the signbit at the right position for MagAsInt. 1550 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; 1551 EVT ShiftVT = IntVT; 1552 if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) { 1553 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); 1554 ShiftVT = MagVT; 1555 } 1556 if (ShiftAmount > 0) { 1557 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); 1558 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); 1559 } else if (ShiftAmount < 0) { 1560 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); 1561 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); 1562 } 1563 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) { 1564 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); 1565 } 1566 1567 // Store the part with the modified sign and convert back to float. 1568 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); 1569 return modifySignAsInt(MagAsInt, DL, CopiedSign); 1570} 1571 1572SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { 1573 SDLoc DL(Node); 1574 SDValue Value = Node->getOperand(0); 1575 1576 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal. 1577 EVT FloatVT = Value.getValueType(); 1578 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { 1579 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); 1580 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); 1581 } 1582 1583 // Transform value to integer, clear the sign bit and transform back. 1584 FloatSignAsInt ValueAsInt; 1585 getSignAsIntValue(ValueAsInt, DL, Value); 1586 EVT IntVT = ValueAsInt.IntValue.getValueType(); 1587 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); 1588 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, 1589 ClearSignMask); 1590 return modifySignAsInt(ValueAsInt, DL, ClearedSign); 1591} 1592 1593void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1594 SmallVectorImpl<SDValue> &Results) { 1595 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1596 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1597 " not tell us which reg is the stack pointer!"); 1598 SDLoc dl(Node); 1599 EVT VT = Node->getValueType(0); 1600 SDValue Tmp1 = SDValue(Node, 0); 1601 SDValue Tmp2 = SDValue(Node, 1); 1602 SDValue Tmp3 = Node->getOperand(2); 1603 SDValue Chain = Tmp1.getOperand(0); 1604 1605 // Chain the dynamic stack allocation so that it doesn't modify the stack 1606 // pointer when other instructions are using the stack. 1607 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); 1608 1609 SDValue Size = Tmp2.getOperand(1); 1610 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1611 Chain = SP.getValue(1); 1612 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1613 unsigned StackAlign = 1614 DAG.getSubtarget().getFrameLowering()->getStackAlignment(); 1615 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1616 if (Align > StackAlign) 1617 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, 1618 DAG.getConstant(-(uint64_t)Align, dl, VT)); 1619 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1620 1621 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), 1622 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); 1623 1624 Results.push_back(Tmp1); 1625 Results.push_back(Tmp2); 1626} 1627 1628/// Legalize a SETCC with given LHS and RHS and condition code CC on the current 1629/// target. 1630/// 1631/// If the SETCC has been legalized using AND / OR, then the legalized node 1632/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert 1633/// will be set to false. 1634/// 1635/// If the SETCC has been legalized by using getSetCCSwappedOperands(), 1636/// then the values of LHS and RHS will be swapped, CC will be set to the 1637/// new condition, and NeedInvert will be set to false. 1638/// 1639/// If the SETCC has been legalized using the inverse condcode, then LHS and 1640/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert 1641/// will be set to true. The caller must invert the result of the SETCC with 1642/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect 1643/// of a true/false result. 1644/// 1645/// \returns true if the SetCC has been legalized, false if it hasn't. 1646bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, 1647 SDValue &RHS, SDValue &CC, 1648 bool &NeedInvert, 1649 const SDLoc &dl) { 1650 MVT OpVT = LHS.getSimpleValueType(); 1651 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1652 NeedInvert = false; 1653 bool NeedSwap = false; 1654 switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1655 default: llvm_unreachable("Unknown condition code action!"); 1656 case TargetLowering::Legal: 1657 // Nothing to do. 1658 break; 1659 case TargetLowering::Expand: { 1660 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); 1661 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { 1662 std::swap(LHS, RHS); 1663 CC = DAG.getCondCode(InvCC); 1664 return true; 1665 } 1666 // Swapping operands didn't work. Try inverting the condition. 1667 InvCC = getSetCCInverse(CCCode, OpVT.isInteger()); 1668 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { 1669 // If inverting the condition is not enough, try swapping operands 1670 // on top of it. 1671 InvCC = ISD::getSetCCSwappedOperands(InvCC); 1672 NeedSwap = true; 1673 } 1674 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { 1675 CC = DAG.getCondCode(InvCC); 1676 NeedInvert = true; 1677 if (NeedSwap) 1678 std::swap(LHS, RHS); 1679 return true; 1680 } 1681 1682 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1683 unsigned Opc = 0; 1684 switch (CCCode) { 1685 default: llvm_unreachable("Don't know how to expand this condition!"); 1686 case ISD::SETO: 1687 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) 1688 && "If SETO is expanded, SETOEQ must be legal!"); 1689 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; 1690 case ISD::SETUO: 1691 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT) 1692 && "If SETUO is expanded, SETUNE must be legal!"); 1693 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; 1694 case ISD::SETOEQ: 1695 case ISD::SETOGT: 1696 case ISD::SETOGE: 1697 case ISD::SETOLT: 1698 case ISD::SETOLE: 1699 case ISD::SETONE: 1700 case ISD::SETUEQ: 1701 case ISD::SETUNE: 1702 case ISD::SETUGT: 1703 case ISD::SETUGE: 1704 case ISD::SETULT: 1705 case ISD::SETULE: 1706 // If we are floating point, assign and break, otherwise fall through. 1707 if (!OpVT.isInteger()) { 1708 // We can use the 4th bit to tell if we are the unordered 1709 // or ordered version of the opcode. 1710 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1711 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; 1712 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); 1713 break; 1714 } 1715 // Fallthrough if we are unsigned integer. 1716 LLVM_FALLTHROUGH; 1717 case ISD::SETLE: 1718 case ISD::SETGT: 1719 case ISD::SETGE: 1720 case ISD::SETLT: 1721 case ISD::SETNE: 1722 case ISD::SETEQ: 1723 // If all combinations of inverting the condition and swapping operands 1724 // didn't work then we have no means to expand the condition. 1725 llvm_unreachable("Don't know how to expand this condition!"); 1726 } 1727 1728 SDValue SetCC1, SetCC2; 1729 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { 1730 // If we aren't the ordered or unorder operation, 1731 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). 1732 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 1733 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 1734 } else { 1735 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) 1736 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); 1737 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); 1738 } 1739 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1740 RHS = SDValue(); 1741 CC = SDValue(); 1742 return true; 1743 } 1744 } 1745 return false; 1746} 1747 1748/// Emit a store/load combination to the stack. This stores 1749/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1750/// a load from the stack slot to DestVT, extending it if needed. 1751/// The resultant code need not be legal. 1752SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, 1753 EVT DestVT, const SDLoc &dl) { 1754 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); 1755} 1756 1757SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, 1758 EVT DestVT, const SDLoc &dl, 1759 SDValue Chain) { 1760 // Create the stack frame object. 1761 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment( 1762 SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); 1763 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 1764 1765 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1766 int SPFI = StackPtrFI->getIndex(); 1767 MachinePointerInfo PtrInfo = 1768 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); 1769 1770 unsigned SrcSize = SrcOp.getValueSizeInBits(); 1771 unsigned SlotSize = SlotVT.getSizeInBits(); 1772 unsigned DestSize = DestVT.getSizeInBits(); 1773 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1774 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType); 1775 1776 // Emit a store to the stack slot. Use a truncstore if the input value is 1777 // later than DestVT. 1778 SDValue Store; 1779 1780 if (SrcSize > SlotSize) 1781 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, 1782 SlotVT, SrcAlign); 1783 else { 1784 assert(SrcSize == SlotSize && "Invalid store"); 1785 Store = 1786 DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); 1787 } 1788 1789 // Result is a load from the stack slot. 1790 if (SlotSize == DestSize) 1791 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); 1792 1793 assert(SlotSize < DestSize && "Unknown extension!"); 1794 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, 1795 DestAlign); 1796} 1797 1798SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1799 SDLoc dl(Node); 1800 // Create a vector sized/aligned stack slot, store the value to element #0, 1801 // then load the whole vector back out. 1802 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1803 1804 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1805 int SPFI = StackPtrFI->getIndex(); 1806 1807 SDValue Ch = DAG.getTruncStore( 1808 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, 1809 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), 1810 Node->getValueType(0).getVectorElementType()); 1811 return DAG.getLoad( 1812 Node->getValueType(0), dl, Ch, StackPtr, 1813 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); 1814} 1815 1816static bool 1817ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, 1818 const TargetLowering &TLI, SDValue &Res) { 1819 unsigned NumElems = Node->getNumOperands(); 1820 SDLoc dl(Node); 1821 EVT VT = Node->getValueType(0); 1822 1823 // Try to group the scalars into pairs, shuffle the pairs together, then 1824 // shuffle the pairs of pairs together, etc. until the vector has 1825 // been built. This will work only if all of the necessary shuffle masks 1826 // are legal. 1827 1828 // We do this in two phases; first to check the legality of the shuffles, 1829 // and next, assuming that all shuffles are legal, to create the new nodes. 1830 for (int Phase = 0; Phase < 2; ++Phase) { 1831 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, 1832 NewIntermedVals; 1833 for (unsigned i = 0; i < NumElems; ++i) { 1834 SDValue V = Node->getOperand(i); 1835 if (V.isUndef()) 1836 continue; 1837 1838 SDValue Vec; 1839 if (Phase) 1840 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); 1841 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); 1842 } 1843 1844 while (IntermedVals.size() > 2) { 1845 NewIntermedVals.clear(); 1846 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { 1847 // This vector and the next vector are shuffled together (simply to 1848 // append the one to the other). 1849 SmallVector<int, 16> ShuffleVec(NumElems, -1); 1850 1851 SmallVector<int, 16> FinalIndices; 1852 FinalIndices.reserve(IntermedVals[i].second.size() + 1853 IntermedVals[i+1].second.size()); 1854 1855 int k = 0; 1856 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; 1857 ++j, ++k) { 1858 ShuffleVec[k] = j; 1859 FinalIndices.push_back(IntermedVals[i].second[j]); 1860 } 1861 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; 1862 ++j, ++k) { 1863 ShuffleVec[k] = NumElems + j; 1864 FinalIndices.push_back(IntermedVals[i+1].second[j]); 1865 } 1866 1867 SDValue Shuffle; 1868 if (Phase) 1869 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, 1870 IntermedVals[i+1].first, 1871 ShuffleVec); 1872 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) 1873 return false; 1874 NewIntermedVals.push_back( 1875 std::make_pair(Shuffle, std::move(FinalIndices))); 1876 } 1877 1878 // If we had an odd number of defined values, then append the last 1879 // element to the array of new vectors. 1880 if ((IntermedVals.size() & 1) != 0) 1881 NewIntermedVals.push_back(IntermedVals.back()); 1882 1883 IntermedVals.swap(NewIntermedVals); 1884 } 1885 1886 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && 1887 "Invalid number of intermediate vectors"); 1888 SDValue Vec1 = IntermedVals[0].first; 1889 SDValue Vec2; 1890 if (IntermedVals.size() > 1) 1891 Vec2 = IntermedVals[1].first; 1892 else if (Phase) 1893 Vec2 = DAG.getUNDEF(VT); 1894 1895 SmallVector<int, 16> ShuffleVec(NumElems, -1); 1896 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) 1897 ShuffleVec[IntermedVals[0].second[i]] = i; 1898 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) 1899 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; 1900 1901 if (Phase) 1902 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); 1903 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) 1904 return false; 1905 } 1906 1907 return true; 1908} 1909 1910/// Expand a BUILD_VECTOR node on targets that don't 1911/// support the operation, but do support the resultant vector type. 1912SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1913 unsigned NumElems = Node->getNumOperands(); 1914 SDValue Value1, Value2; 1915 SDLoc dl(Node); 1916 EVT VT = Node->getValueType(0); 1917 EVT OpVT = Node->getOperand(0).getValueType(); 1918 EVT EltVT = VT.getVectorElementType(); 1919 1920 // If the only non-undef value is the low element, turn this into a 1921 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 1922 bool isOnlyLowElement = true; 1923 bool MoreThanTwoValues = false; 1924 bool isConstant = true; 1925 for (unsigned i = 0; i < NumElems; ++i) { 1926 SDValue V = Node->getOperand(i); 1927 if (V.isUndef()) 1928 continue; 1929 if (i > 0) 1930 isOnlyLowElement = false; 1931 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 1932 isConstant = false; 1933 1934 if (!Value1.getNode()) { 1935 Value1 = V; 1936 } else if (!Value2.getNode()) { 1937 if (V != Value1) 1938 Value2 = V; 1939 } else if (V != Value1 && V != Value2) { 1940 MoreThanTwoValues = true; 1941 } 1942 } 1943 1944 if (!Value1.getNode()) 1945 return DAG.getUNDEF(VT); 1946 1947 if (isOnlyLowElement) 1948 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 1949 1950 // If all elements are constants, create a load from the constant pool. 1951 if (isConstant) { 1952 SmallVector<Constant*, 16> CV; 1953 for (unsigned i = 0, e = NumElems; i != e; ++i) { 1954 if (ConstantFPSDNode *V = 1955 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 1956 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 1957 } else if (ConstantSDNode *V = 1958 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 1959 if (OpVT==EltVT) 1960 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 1961 else { 1962 // If OpVT and EltVT don't match, EltVT is not legal and the 1963 // element values have been promoted/truncated earlier. Undo this; 1964 // we don't want a v16i8 to become a v16i32 for example. 1965 const ConstantInt *CI = V->getConstantIntValue(); 1966 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 1967 CI->getZExtValue())); 1968 } 1969 } else { 1970 assert(Node->getOperand(i).isUndef()); 1971 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 1972 CV.push_back(UndefValue::get(OpNTy)); 1973 } 1974 } 1975 Constant *CP = ConstantVector::get(CV); 1976 SDValue CPIdx = 1977 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); 1978 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 1979 return DAG.getLoad( 1980 VT, dl, DAG.getEntryNode(), CPIdx, 1981 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), 1982 Alignment); 1983 } 1984 1985 SmallSet<SDValue, 16> DefinedValues; 1986 for (unsigned i = 0; i < NumElems; ++i) { 1987 if (Node->getOperand(i).isUndef()) 1988 continue; 1989 DefinedValues.insert(Node->getOperand(i)); 1990 } 1991 1992 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { 1993 if (!MoreThanTwoValues) { 1994 SmallVector<int, 8> ShuffleVec(NumElems, -1); 1995 for (unsigned i = 0; i < NumElems; ++i) { 1996 SDValue V = Node->getOperand(i); 1997 if (V.isUndef()) 1998 continue; 1999 ShuffleVec[i] = V == Value1 ? 0 : NumElems; 2000 } 2001 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 2002 // Get the splatted value into the low element of a vector register. 2003 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 2004 SDValue Vec2; 2005 if (Value2.getNode()) 2006 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 2007 else 2008 Vec2 = DAG.getUNDEF(VT); 2009 2010 // Return shuffle(LowValVec, undef, <0,0,0,0>) 2011 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); 2012 } 2013 } else { 2014 SDValue Res; 2015 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) 2016 return Res; 2017 } 2018 } 2019 2020 // Otherwise, we can't handle this case efficiently. 2021 return ExpandVectorBuildThroughStack(Node); 2022} 2023 2024// Expand a node into a call to a libcall. If the result value 2025// does not fit into a register, return the lo part and set the hi part to the 2026// by-reg argument. If it does fit into a single register, return the result 2027// and leave the Hi part unset. 2028SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 2029 bool isSigned) { 2030 TargetLowering::ArgListTy Args; 2031 TargetLowering::ArgListEntry Entry; 2032 for (const SDValue &Op : Node->op_values()) { 2033 EVT ArgVT = Op.getValueType(); 2034 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2035 Entry.Node = Op; 2036 Entry.Ty = ArgTy; 2037 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); 2038 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); 2039 Args.push_back(Entry); 2040 } 2041 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2042 TLI.getPointerTy(DAG.getDataLayout())); 2043 2044 EVT RetVT = Node->getValueType(0); 2045 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2046 2047 // By default, the input chain to this libcall is the entry node of the 2048 // function. If the libcall is going to be emitted as a tail call then 2049 // TLI.isUsedByReturnOnly will change it to the right chain if the return 2050 // node which is being folded has a non-entry input chain. 2051 SDValue InChain = DAG.getEntryNode(); 2052 2053 // isTailCall may be true since the callee does not reference caller stack 2054 // frame. Check if it's in the right position and that the return types match. 2055 SDValue TCChain = InChain; 2056 const Function &F = DAG.getMachineFunction().getFunction(); 2057 bool isTailCall = 2058 TLI.isInTailCallPosition(DAG, Node, TCChain) && 2059 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); 2060 if (isTailCall) 2061 InChain = TCChain; 2062 2063 TargetLowering::CallLoweringInfo CLI(DAG); 2064 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned); 2065 CLI.setDebugLoc(SDLoc(Node)) 2066 .setChain(InChain) 2067 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, 2068 std::move(Args)) 2069 .setTailCall(isTailCall) 2070 .setSExtResult(signExtend) 2071 .setZExtResult(!signExtend) 2072 .setIsPostTypeLegalization(true); 2073 2074 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2075 2076 if (!CallInfo.second.getNode()) { 2077 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump()); 2078 // It's a tailcall, return the chain (which is the DAG root). 2079 return DAG.getRoot(); 2080 } 2081 2082 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump()); 2083 return CallInfo.first; 2084} 2085 2086// Expand a node into a call to a libcall. Similar to 2087// ExpandLibCall except that the first operand is the in-chain. 2088std::pair<SDValue, SDValue> 2089SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, 2090 SDNode *Node, 2091 bool isSigned) { 2092 SDValue InChain = Node->getOperand(0); 2093 2094 TargetLowering::ArgListTy Args; 2095 TargetLowering::ArgListEntry Entry; 2096 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { 2097 EVT ArgVT = Node->getOperand(i).getValueType(); 2098 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2099 Entry.Node = Node->getOperand(i); 2100 Entry.Ty = ArgTy; 2101 Entry.IsSExt = isSigned; 2102 Entry.IsZExt = !isSigned; 2103 Args.push_back(Entry); 2104 } 2105 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2106 TLI.getPointerTy(DAG.getDataLayout())); 2107 2108 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 2109 2110 TargetLowering::CallLoweringInfo CLI(DAG); 2111 CLI.setDebugLoc(SDLoc(Node)) 2112 .setChain(InChain) 2113 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, 2114 std::move(Args)) 2115 .setSExtResult(isSigned) 2116 .setZExtResult(!isSigned); 2117 2118 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2119 2120 return CallInfo; 2121} 2122 2123SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 2124 RTLIB::Libcall Call_F32, 2125 RTLIB::Libcall Call_F64, 2126 RTLIB::Libcall Call_F80, 2127 RTLIB::Libcall Call_F128, 2128 RTLIB::Libcall Call_PPCF128) { 2129 if (Node->isStrictFPOpcode()) 2130 Node = DAG.mutateStrictFPToFP(Node); 2131 2132 RTLIB::Libcall LC; 2133 switch (Node->getSimpleValueType(0).SimpleTy) { 2134 default: llvm_unreachable("Unexpected request for libcall!"); 2135 case MVT::f32: LC = Call_F32; break; 2136 case MVT::f64: LC = Call_F64; break; 2137 case MVT::f80: LC = Call_F80; break; 2138 case MVT::f128: LC = Call_F128; break; 2139 case MVT::ppcf128: LC = Call_PPCF128; break; 2140 } 2141 return ExpandLibCall(LC, Node, false); 2142} 2143 2144SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 2145 RTLIB::Libcall Call_I8, 2146 RTLIB::Libcall Call_I16, 2147 RTLIB::Libcall Call_I32, 2148 RTLIB::Libcall Call_I64, 2149 RTLIB::Libcall Call_I128) { 2150 RTLIB::Libcall LC; 2151 switch (Node->getSimpleValueType(0).SimpleTy) { 2152 default: llvm_unreachable("Unexpected request for libcall!"); 2153 case MVT::i8: LC = Call_I8; break; 2154 case MVT::i16: LC = Call_I16; break; 2155 case MVT::i32: LC = Call_I32; break; 2156 case MVT::i64: LC = Call_I64; break; 2157 case MVT::i128: LC = Call_I128; break; 2158 } 2159 return ExpandLibCall(LC, Node, isSigned); 2160} 2161 2162/// Expand the node to a libcall based on first argument type (for instance 2163/// lround and its variant). 2164SDValue SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, 2165 RTLIB::Libcall Call_F32, 2166 RTLIB::Libcall Call_F64, 2167 RTLIB::Libcall Call_F80, 2168 RTLIB::Libcall Call_F128, 2169 RTLIB::Libcall Call_PPCF128) { 2170 RTLIB::Libcall LC; 2171 switch (Node->getOperand(0).getValueType().getSimpleVT().SimpleTy) { 2172 default: llvm_unreachable("Unexpected request for libcall!"); 2173 case MVT::f32: LC = Call_F32; break; 2174 case MVT::f64: LC = Call_F64; break; 2175 case MVT::f80: LC = Call_F80; break; 2176 case MVT::f128: LC = Call_F128; break; 2177 case MVT::ppcf128: LC = Call_PPCF128; break; 2178 } 2179 2180 return ExpandLibCall(LC, Node, false); 2181} 2182 2183/// Issue libcalls to __{u}divmod to compute div / rem pairs. 2184void 2185SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 2186 SmallVectorImpl<SDValue> &Results) { 2187 unsigned Opcode = Node->getOpcode(); 2188 bool isSigned = Opcode == ISD::SDIVREM; 2189 2190 RTLIB::Libcall LC; 2191 switch (Node->getSimpleValueType(0).SimpleTy) { 2192 default: llvm_unreachable("Unexpected request for libcall!"); 2193 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2194 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2195 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2196 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2197 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2198 } 2199 2200 // The input chain to this libcall is the entry node of the function. 2201 // Legalizing the call will automatically add the previous call to the 2202 // dependence. 2203 SDValue InChain = DAG.getEntryNode(); 2204 2205 EVT RetVT = Node->getValueType(0); 2206 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2207 2208 TargetLowering::ArgListTy Args; 2209 TargetLowering::ArgListEntry Entry; 2210 for (const SDValue &Op : Node->op_values()) { 2211 EVT ArgVT = Op.getValueType(); 2212 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2213 Entry.Node = Op; 2214 Entry.Ty = ArgTy; 2215 Entry.IsSExt = isSigned; 2216 Entry.IsZExt = !isSigned; 2217 Args.push_back(Entry); 2218 } 2219 2220 // Also pass the return address of the remainder. 2221 SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 2222 Entry.Node = FIPtr; 2223 Entry.Ty = RetTy->getPointerTo(); 2224 Entry.IsSExt = isSigned; 2225 Entry.IsZExt = !isSigned; 2226 Args.push_back(Entry); 2227 2228 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2229 TLI.getPointerTy(DAG.getDataLayout())); 2230 2231 SDLoc dl(Node); 2232 TargetLowering::CallLoweringInfo CLI(DAG); 2233 CLI.setDebugLoc(dl) 2234 .setChain(InChain) 2235 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, 2236 std::move(Args)) 2237 .setSExtResult(isSigned) 2238 .setZExtResult(!isSigned); 2239 2240 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2241 2242 // Remainder is loaded back from the stack frame. 2243 SDValue Rem = 2244 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); 2245 Results.push_back(CallInfo.first); 2246 Results.push_back(Rem); 2247} 2248 2249/// Return true if sincos libcall is available. 2250static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { 2251 RTLIB::Libcall LC; 2252 switch (Node->getSimpleValueType(0).SimpleTy) { 2253 default: llvm_unreachable("Unexpected request for libcall!"); 2254 case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2255 case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2256 case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2257 case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2258 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2259 } 2260 return TLI.getLibcallName(LC) != nullptr; 2261} 2262 2263/// Only issue sincos libcall if both sin and cos are needed. 2264static bool useSinCos(SDNode *Node) { 2265 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN 2266 ? ISD::FCOS : ISD::FSIN; 2267 2268 SDValue Op0 = Node->getOperand(0); 2269 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2270 UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2271 SDNode *User = *UI; 2272 if (User == Node) 2273 continue; 2274 // The other user might have been turned into sincos already. 2275 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) 2276 return true; 2277 } 2278 return false; 2279} 2280 2281/// Issue libcalls to sincos to compute sin / cos pairs. 2282void 2283SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, 2284 SmallVectorImpl<SDValue> &Results) { 2285 RTLIB::Libcall LC; 2286 switch (Node->getSimpleValueType(0).SimpleTy) { 2287 default: llvm_unreachable("Unexpected request for libcall!"); 2288 case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2289 case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2290 case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2291 case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2292 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2293 } 2294 2295 // The input chain to this libcall is the entry node of the function. 2296 // Legalizing the call will automatically add the previous call to the 2297 // dependence. 2298 SDValue InChain = DAG.getEntryNode(); 2299 2300 EVT RetVT = Node->getValueType(0); 2301 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2302 2303 TargetLowering::ArgListTy Args; 2304 TargetLowering::ArgListEntry Entry; 2305 2306 // Pass the argument. 2307 Entry.Node = Node->getOperand(0); 2308 Entry.Ty = RetTy; 2309 Entry.IsSExt = false; 2310 Entry.IsZExt = false; 2311 Args.push_back(Entry); 2312 2313 // Pass the return address of sin. 2314 SDValue SinPtr = DAG.CreateStackTemporary(RetVT); 2315 Entry.Node = SinPtr; 2316 Entry.Ty = RetTy->getPointerTo(); 2317 Entry.IsSExt = false; 2318 Entry.IsZExt = false; 2319 Args.push_back(Entry); 2320 2321 // Also pass the return address of the cos. 2322 SDValue CosPtr = DAG.CreateStackTemporary(RetVT); 2323 Entry.Node = CosPtr; 2324 Entry.Ty = RetTy->getPointerTo(); 2325 Entry.IsSExt = false; 2326 Entry.IsZExt = false; 2327 Args.push_back(Entry); 2328 2329 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2330 TLI.getPointerTy(DAG.getDataLayout())); 2331 2332 SDLoc dl(Node); 2333 TargetLowering::CallLoweringInfo CLI(DAG); 2334 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( 2335 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, 2336 std::move(Args)); 2337 2338 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2339 2340 Results.push_back( 2341 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); 2342 Results.push_back( 2343 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); 2344} 2345 2346/// This function is responsible for legalizing a 2347/// INT_TO_FP operation of the specified operand when the target requests that 2348/// we expand it. At this point, we know that the result and operand types are 2349/// legal for the target. 2350SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, 2351 EVT DestVT, 2352 const SDLoc &dl) { 2353 EVT SrcVT = Op0.getValueType(); 2354 2355 // TODO: Should any fast-math-flags be set for the created nodes? 2356 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n"); 2357 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { 2358 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " 2359 "expansion\n"); 2360 2361 // Get the stack frame index of a 8 byte buffer. 2362 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2363 2364 // word offset constant for Hi/Lo address computation 2365 SDValue WordOff = DAG.getConstant(sizeof(int), dl, 2366 StackSlot.getValueType()); 2367 // set up Hi and Lo (into buffer) address based on endian 2368 SDValue Hi = StackSlot; 2369 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), 2370 StackSlot, WordOff); 2371 if (DAG.getDataLayout().isLittleEndian()) 2372 std::swap(Hi, Lo); 2373 2374 // if signed map to unsigned space 2375 SDValue Op0Mapped; 2376 if (isSigned) { 2377 // constant used to invert sign bit (signed to unsigned mapping) 2378 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32); 2379 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 2380 } else { 2381 Op0Mapped = Op0; 2382 } 2383 // store the lo of the constructed double - based on integer input 2384 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo, 2385 MachinePointerInfo()); 2386 // initial hi portion of constructed double 2387 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32); 2388 // store the hi of the constructed double - biased exponent 2389 SDValue Store2 = 2390 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo()); 2391 // load the constructed double 2392 SDValue Load = 2393 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo()); 2394 // FP constant to bias correct the final result 2395 SDValue Bias = DAG.getConstantFP(isSigned ? 2396 BitsToDouble(0x4330000080000000ULL) : 2397 BitsToDouble(0x4330000000000000ULL), 2398 dl, MVT::f64); 2399 // subtract the bias 2400 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2401 // final result 2402 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); 2403 return Result; 2404 } 2405 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 2406 // Code below here assumes !isSigned without checking again. 2407 2408 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2409 2410 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, 2411 DAG.getConstant(0, dl, SrcVT), ISD::SETLT); 2412 SDValue Zero = DAG.getIntPtrConstant(0, dl), 2413 Four = DAG.getIntPtrConstant(4, dl); 2414 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), 2415 SignSet, Four, Zero); 2416 2417 // If the sign bit of the integer is set, the large number will be treated 2418 // as a negative number. To counteract this, the dynamic code adds an 2419 // offset depending on the data type. 2420 uint64_t FF; 2421 switch (SrcVT.getSimpleVT().SimpleTy) { 2422 default: llvm_unreachable("Unsupported integer type!"); 2423 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2424 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2425 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2426 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2427 } 2428 if (DAG.getDataLayout().isLittleEndian()) 2429 FF <<= 32; 2430 Constant *FudgeFactor = ConstantInt::get( 2431 Type::getInt64Ty(*DAG.getContext()), FF); 2432 2433 SDValue CPIdx = 2434 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); 2435 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 2436 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); 2437 Alignment = std::min(Alignment, 4u); 2438 SDValue FudgeInReg; 2439 if (DestVT == MVT::f32) 2440 FudgeInReg = DAG.getLoad( 2441 MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2442 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), 2443 Alignment); 2444 else { 2445 SDValue Load = DAG.getExtLoad( 2446 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, 2447 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, 2448 Alignment); 2449 HandleSDNode Handle(Load); 2450 LegalizeOp(Load.getNode()); 2451 FudgeInReg = Handle.getValue(); 2452 } 2453 2454 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2455} 2456 2457/// This function is responsible for legalizing a 2458/// *INT_TO_FP operation of the specified operand when the target requests that 2459/// we promote it. At this point, we know that the result and operand types are 2460/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2461/// operation that takes a larger input. 2462SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, 2463 bool isSigned, 2464 const SDLoc &dl) { 2465 // First step, figure out the appropriate *INT_TO_FP operation to use. 2466 EVT NewInTy = LegalOp.getValueType(); 2467 2468 unsigned OpToUse = 0; 2469 2470 // Scan for the appropriate larger type to use. 2471 while (true) { 2472 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2473 assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2474 2475 // If the target supports SINT_TO_FP of this type, use it. 2476 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { 2477 OpToUse = ISD::SINT_TO_FP; 2478 break; 2479 } 2480 if (isSigned) continue; 2481 2482 // If the target supports UINT_TO_FP of this type, use it. 2483 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { 2484 OpToUse = ISD::UINT_TO_FP; 2485 break; 2486 } 2487 2488 // Otherwise, try a larger type. 2489 } 2490 2491 // Okay, we found the operation and type to use. Zero extend our input to the 2492 // desired type then run the operation on it. 2493 return DAG.getNode(OpToUse, dl, DestVT, 2494 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2495 dl, NewInTy, LegalOp)); 2496} 2497 2498/// This function is responsible for legalizing a 2499/// FP_TO_*INT operation of the specified operand when the target requests that 2500/// we promote it. At this point, we know that the result and operand types are 2501/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2502/// operation that returns a larger result. 2503SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, 2504 bool isSigned, 2505 const SDLoc &dl) { 2506 // First step, figure out the appropriate FP_TO*INT operation to use. 2507 EVT NewOutTy = DestVT; 2508 2509 unsigned OpToUse = 0; 2510 2511 // Scan for the appropriate larger type to use. 2512 while (true) { 2513 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2514 assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2515 2516 // A larger signed type can hold all unsigned values of the requested type, 2517 // so using FP_TO_SINT is valid 2518 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { 2519 OpToUse = ISD::FP_TO_SINT; 2520 break; 2521 } 2522 2523 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. 2524 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { 2525 OpToUse = ISD::FP_TO_UINT; 2526 break; 2527 } 2528 2529 // Otherwise, try a larger type. 2530 } 2531 2532 // Okay, we found the operation and type to use. 2533 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2534 2535 // Truncate the result of the extended FP_TO_*INT operation to the desired 2536 // size. 2537 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2538} 2539 2540/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. 2541SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { 2542 EVT VT = Op.getValueType(); 2543 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2544 unsigned Sz = VT.getScalarSizeInBits(); 2545 2546 SDValue Tmp, Tmp2, Tmp3; 2547 2548 // If we can, perform BSWAP first and then the mask+swap the i4, then i2 2549 // and finally the i1 pairs. 2550 // TODO: We can easily support i4/i2 legal types if any target ever does. 2551 if (Sz >= 8 && isPowerOf2_32(Sz)) { 2552 // Create the masks - repeating the pattern every byte. 2553 APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0)); 2554 APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC)); 2555 APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA)); 2556 APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F)); 2557 APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33)); 2558 APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55)); 2559 2560 // BSWAP if the type is wider than a single byte. 2561 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); 2562 2563 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) 2564 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); 2565 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); 2566 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT)); 2567 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); 2568 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 2569 2570 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) 2571 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); 2572 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); 2573 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT)); 2574 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); 2575 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 2576 2577 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) 2578 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); 2579 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); 2580 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT)); 2581 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); 2582 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 2583 return Tmp; 2584 } 2585 2586 Tmp = DAG.getConstant(0, dl, VT); 2587 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) { 2588 if (I < J) 2589 Tmp2 = 2590 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); 2591 else 2592 Tmp2 = 2593 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); 2594 2595 APInt Shift(Sz, 1); 2596 Shift <<= J; 2597 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); 2598 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); 2599 } 2600 2601 return Tmp; 2602} 2603 2604/// Open code the operations for BSWAP of the specified operation. 2605SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { 2606 EVT VT = Op.getValueType(); 2607 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2608 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2609 switch (VT.getSimpleVT().getScalarType().SimpleTy) { 2610 default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2611 case MVT::i16: 2612 // Use a rotate by 8. This can be further expanded if necessary. 2613 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2614 case MVT::i32: 2615 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2616 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2617 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2618 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2619 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, 2620 DAG.getConstant(0xFF0000, dl, VT)); 2621 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); 2622 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2623 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2624 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2625 case MVT::i64: 2626 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); 2627 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); 2628 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2629 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2630 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); 2631 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); 2632 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); 2633 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); 2634 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, 2635 DAG.getConstant(255ULL<<48, dl, VT)); 2636 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, 2637 DAG.getConstant(255ULL<<40, dl, VT)); 2638 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, 2639 DAG.getConstant(255ULL<<32, dl, VT)); 2640 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, 2641 DAG.getConstant(255ULL<<24, dl, VT)); 2642 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, 2643 DAG.getConstant(255ULL<<16, dl, VT)); 2644 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, 2645 DAG.getConstant(255ULL<<8 , dl, VT)); 2646 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2647 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2648 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2649 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2650 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2651 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2652 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2653 } 2654} 2655 2656bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2657 LLVM_DEBUG(dbgs() << "Trying to expand node\n"); 2658 SmallVector<SDValue, 8> Results; 2659 SDLoc dl(Node); 2660 SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2661 bool NeedInvert; 2662 switch (Node->getOpcode()) { 2663 case ISD::ABS: 2664 if (TLI.expandABS(Node, Tmp1, DAG)) 2665 Results.push_back(Tmp1); 2666 break; 2667 case ISD::CTPOP: 2668 if (TLI.expandCTPOP(Node, Tmp1, DAG)) 2669 Results.push_back(Tmp1); 2670 break; 2671 case ISD::CTLZ: 2672 case ISD::CTLZ_ZERO_UNDEF: 2673 if (TLI.expandCTLZ(Node, Tmp1, DAG)) 2674 Results.push_back(Tmp1); 2675 break; 2676 case ISD::CTTZ: 2677 case ISD::CTTZ_ZERO_UNDEF: 2678 if (TLI.expandCTTZ(Node, Tmp1, DAG)) 2679 Results.push_back(Tmp1); 2680 break; 2681 case ISD::BITREVERSE: 2682 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl)); 2683 break; 2684 case ISD::BSWAP: 2685 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2686 break; 2687 case ISD::FRAMEADDR: 2688 case ISD::RETURNADDR: 2689 case ISD::FRAME_TO_ARGS_OFFSET: 2690 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); 2691 break; 2692 case ISD::EH_DWARF_CFA: { 2693 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, 2694 TLI.getPointerTy(DAG.getDataLayout())); 2695 SDValue Offset = DAG.getNode(ISD::ADD, dl, 2696 CfaArg.getValueType(), 2697 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, 2698 CfaArg.getValueType()), 2699 CfaArg); 2700 SDValue FA = DAG.getNode( 2701 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), 2702 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); 2703 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), 2704 FA, Offset)); 2705 break; 2706 } 2707 case ISD::FLT_ROUNDS_: 2708 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); 2709 break; 2710 case ISD::EH_RETURN: 2711 case ISD::EH_LABEL: 2712 case ISD::PREFETCH: 2713 case ISD::VAEND: 2714 case ISD::EH_SJLJ_LONGJMP: 2715 // If the target didn't expand these, there's nothing to do, so just 2716 // preserve the chain and be done. 2717 Results.push_back(Node->getOperand(0)); 2718 break; 2719 case ISD::READCYCLECOUNTER: 2720 // If the target didn't expand this, just return 'zero' and preserve the 2721 // chain. 2722 Results.append(Node->getNumValues() - 1, 2723 DAG.getConstant(0, dl, Node->getValueType(0))); 2724 Results.push_back(Node->getOperand(0)); 2725 break; 2726 case ISD::EH_SJLJ_SETJMP: 2727 // If the target didn't expand this, just return 'zero' and preserve the 2728 // chain. 2729 Results.push_back(DAG.getConstant(0, dl, MVT::i32)); 2730 Results.push_back(Node->getOperand(0)); 2731 break; 2732 case ISD::ATOMIC_LOAD: { 2733 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 2734 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); 2735 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 2736 SDValue Swap = DAG.getAtomicCmpSwap( 2737 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, 2738 Node->getOperand(0), Node->getOperand(1), Zero, Zero, 2739 cast<AtomicSDNode>(Node)->getMemOperand()); 2740 Results.push_back(Swap.getValue(0)); 2741 Results.push_back(Swap.getValue(1)); 2742 break; 2743 } 2744 case ISD::ATOMIC_STORE: { 2745 // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 2746 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2747 cast<AtomicSDNode>(Node)->getMemoryVT(), 2748 Node->getOperand(0), 2749 Node->getOperand(1), Node->getOperand(2), 2750 cast<AtomicSDNode>(Node)->getMemOperand()); 2751 Results.push_back(Swap.getValue(1)); 2752 break; 2753 } 2754 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 2755 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and 2756 // splits out the success value as a comparison. Expanding the resulting 2757 // ATOMIC_CMP_SWAP will produce a libcall. 2758 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); 2759 SDValue Res = DAG.getAtomicCmpSwap( 2760 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, 2761 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), 2762 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); 2763 2764 SDValue ExtRes = Res; 2765 SDValue LHS = Res; 2766 SDValue RHS = Node->getOperand(1); 2767 2768 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); 2769 EVT OuterType = Node->getValueType(0); 2770 switch (TLI.getExtendForAtomicOps()) { 2771 case ISD::SIGN_EXTEND: 2772 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, 2773 DAG.getValueType(AtomicType)); 2774 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, 2775 Node->getOperand(2), DAG.getValueType(AtomicType)); 2776 ExtRes = LHS; 2777 break; 2778 case ISD::ZERO_EXTEND: 2779 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, 2780 DAG.getValueType(AtomicType)); 2781 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); 2782 ExtRes = LHS; 2783 break; 2784 case ISD::ANY_EXTEND: 2785 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); 2786 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); 2787 break; 2788 default: 2789 llvm_unreachable("Invalid atomic op extension"); 2790 } 2791 2792 SDValue Success = 2793 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); 2794 2795 Results.push_back(ExtRes.getValue(0)); 2796 Results.push_back(Success); 2797 Results.push_back(Res.getValue(1)); 2798 break; 2799 } 2800 case ISD::DYNAMIC_STACKALLOC: 2801 ExpandDYNAMIC_STACKALLOC(Node, Results); 2802 break; 2803 case ISD::MERGE_VALUES: 2804 for (unsigned i = 0; i < Node->getNumValues(); i++) 2805 Results.push_back(Node->getOperand(i)); 2806 break; 2807 case ISD::UNDEF: { 2808 EVT VT = Node->getValueType(0); 2809 if (VT.isInteger()) 2810 Results.push_back(DAG.getConstant(0, dl, VT)); 2811 else { 2812 assert(VT.isFloatingPoint() && "Unknown value type!"); 2813 Results.push_back(DAG.getConstantFP(0, dl, VT)); 2814 } 2815 break; 2816 } 2817 case ISD::STRICT_FP_ROUND: 2818 Tmp1 = EmitStackConvert(Node->getOperand(1), 2819 Node->getValueType(0), 2820 Node->getValueType(0), dl, Node->getOperand(0)); 2821 ReplaceNode(Node, Tmp1.getNode()); 2822 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n"); 2823 return true; 2824 case ISD::FP_ROUND: 2825 case ISD::BITCAST: 2826 Tmp1 = EmitStackConvert(Node->getOperand(0), 2827 Node->getValueType(0), 2828 Node->getValueType(0), dl); 2829 Results.push_back(Tmp1); 2830 break; 2831 case ISD::STRICT_FP_EXTEND: 2832 Tmp1 = EmitStackConvert(Node->getOperand(1), 2833 Node->getOperand(1).getValueType(), 2834 Node->getValueType(0), dl, Node->getOperand(0)); 2835 ReplaceNode(Node, Tmp1.getNode()); 2836 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n"); 2837 return true; 2838 case ISD::FP_EXTEND: 2839 Tmp1 = EmitStackConvert(Node->getOperand(0), 2840 Node->getOperand(0).getValueType(), 2841 Node->getValueType(0), dl); 2842 Results.push_back(Tmp1); 2843 break; 2844 case ISD::SIGN_EXTEND_INREG: { 2845 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2846 EVT VT = Node->getValueType(0); 2847 2848 // An in-register sign-extend of a boolean is a negation: 2849 // 'true' (1) sign-extended is -1. 2850 // 'false' (0) sign-extended is 0. 2851 // However, we must mask the high bits of the source operand because the 2852 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. 2853 2854 // TODO: Do this for vectors too? 2855 if (ExtraVT.getSizeInBits() == 1) { 2856 SDValue One = DAG.getConstant(1, dl, VT); 2857 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); 2858 SDValue Zero = DAG.getConstant(0, dl, VT); 2859 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); 2860 Results.push_back(Neg); 2861 break; 2862 } 2863 2864 // NOTE: we could fall back on load/store here too for targets without 2865 // SRA. However, it is doubtful that any exist. 2866 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2867 unsigned BitsDiff = VT.getScalarSizeInBits() - 2868 ExtraVT.getScalarSizeInBits(); 2869 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); 2870 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 2871 Node->getOperand(0), ShiftCst); 2872 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 2873 Results.push_back(Tmp1); 2874 break; 2875 } 2876 case ISD::FP_ROUND_INREG: { 2877 // The only way we can lower this is to turn it into a TRUNCSTORE, 2878 // EXTLOAD pair, targeting a temporary location (a stack slot). 2879 2880 // NOTE: there is a choice here between constantly creating new stack 2881 // slots and always reusing the same one. We currently always create 2882 // new ones, as reuse may inhibit scheduling. 2883 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2884 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, 2885 Node->getValueType(0), dl); 2886 Results.push_back(Tmp1); 2887 break; 2888 } 2889 case ISD::UINT_TO_FP: 2890 if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) { 2891 Results.push_back(Tmp1); 2892 break; 2893 } 2894 LLVM_FALLTHROUGH; 2895 case ISD::SINT_TO_FP: 2896 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, 2897 Node->getOperand(0), Node->getValueType(0), dl); 2898 Results.push_back(Tmp1); 2899 break; 2900 case ISD::FP_TO_SINT: 2901 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) 2902 Results.push_back(Tmp1); 2903 break; 2904 case ISD::FP_TO_UINT: 2905 if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG)) 2906 Results.push_back(Tmp1); 2907 break; 2908 case ISD::LROUND: 2909 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, 2910 RTLIB::LROUND_F64, RTLIB::LROUND_F80, 2911 RTLIB::LROUND_F128, 2912 RTLIB::LROUND_PPCF128)); 2913 break; 2914 case ISD::LLROUND: 2915 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, 2916 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80, 2917 RTLIB::LLROUND_F128, 2918 RTLIB::LLROUND_PPCF128)); 2919 break; 2920 case ISD::LRINT: 2921 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, 2922 RTLIB::LRINT_F64, RTLIB::LRINT_F80, 2923 RTLIB::LRINT_F128, 2924 RTLIB::LRINT_PPCF128)); 2925 break; 2926 case ISD::LLRINT: 2927 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, 2928 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80, 2929 RTLIB::LLRINT_F128, 2930 RTLIB::LLRINT_PPCF128)); 2931 break; 2932 case ISD::VAARG: 2933 Results.push_back(DAG.expandVAArg(Node)); 2934 Results.push_back(Results[0].getValue(1)); 2935 break; 2936 case ISD::VACOPY: 2937 Results.push_back(DAG.expandVACopy(Node)); 2938 break; 2939 case ISD::EXTRACT_VECTOR_ELT: 2940 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 2941 // This must be an access of the only element. Return it. 2942 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 2943 Node->getOperand(0)); 2944 else 2945 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 2946 Results.push_back(Tmp1); 2947 break; 2948 case ISD::EXTRACT_SUBVECTOR: 2949 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 2950 break; 2951 case ISD::INSERT_SUBVECTOR: 2952 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 2953 break; 2954 case ISD::CONCAT_VECTORS: 2955 Results.push_back(ExpandVectorBuildThroughStack(Node)); 2956 break; 2957 case ISD::SCALAR_TO_VECTOR: 2958 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 2959 break; 2960 case ISD::INSERT_VECTOR_ELT: 2961 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 2962 Node->getOperand(1), 2963 Node->getOperand(2), dl)); 2964 break; 2965 case ISD::VECTOR_SHUFFLE: { 2966 SmallVector<int, 32> NewMask; 2967 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 2968 2969 EVT VT = Node->getValueType(0); 2970 EVT EltVT = VT.getVectorElementType(); 2971 SDValue Op0 = Node->getOperand(0); 2972 SDValue Op1 = Node->getOperand(1); 2973 if (!TLI.isTypeLegal(EltVT)) { 2974 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 2975 2976 // BUILD_VECTOR operands are allowed to be wider than the element type. 2977 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept 2978 // it. 2979 if (NewEltVT.bitsLT(EltVT)) { 2980 // Convert shuffle node. 2981 // If original node was v4i64 and the new EltVT is i32, 2982 // cast operands to v8i32 and re-build the mask. 2983 2984 // Calculate new VT, the size of the new VT should be equal to original. 2985 EVT NewVT = 2986 EVT::getVectorVT(*DAG.getContext(), NewEltVT, 2987 VT.getSizeInBits() / NewEltVT.getSizeInBits()); 2988 assert(NewVT.bitsEq(VT)); 2989 2990 // cast operands to new VT 2991 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 2992 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 2993 2994 // Convert the shuffle mask 2995 unsigned int factor = 2996 NewVT.getVectorNumElements()/VT.getVectorNumElements(); 2997 2998 // EltVT gets smaller 2999 assert(factor > 0); 3000 3001 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 3002 if (Mask[i] < 0) { 3003 for (unsigned fi = 0; fi < factor; ++fi) 3004 NewMask.push_back(Mask[i]); 3005 } 3006 else { 3007 for (unsigned fi = 0; fi < factor; ++fi) 3008 NewMask.push_back(Mask[i]*factor+fi); 3009 } 3010 } 3011 Mask = NewMask; 3012 VT = NewVT; 3013 } 3014 EltVT = NewEltVT; 3015 } 3016 unsigned NumElems = VT.getVectorNumElements(); 3017 SmallVector<SDValue, 16> Ops; 3018 for (unsigned i = 0; i != NumElems; ++i) { 3019 if (Mask[i] < 0) { 3020 Ops.push_back(DAG.getUNDEF(EltVT)); 3021 continue; 3022 } 3023 unsigned Idx = Mask[i]; 3024 if (Idx < NumElems) 3025 Ops.push_back(DAG.getNode( 3026 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, 3027 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); 3028 else 3029 Ops.push_back(DAG.getNode( 3030 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, 3031 DAG.getConstant(Idx - NumElems, dl, 3032 TLI.getVectorIdxTy(DAG.getDataLayout())))); 3033 } 3034 3035 Tmp1 = DAG.getBuildVector(VT, dl, Ops); 3036 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 3037 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 3038 Results.push_back(Tmp1); 3039 break; 3040 } 3041 case ISD::EXTRACT_ELEMENT: { 3042 EVT OpTy = Node->getOperand(0).getValueType(); 3043 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 3044 // 1 -> Hi 3045 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 3046 DAG.getConstant(OpTy.getSizeInBits() / 2, dl, 3047 TLI.getShiftAmountTy( 3048 Node->getOperand(0).getValueType(), 3049 DAG.getDataLayout()))); 3050 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 3051 } else { 3052 // 0 -> Lo 3053 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 3054 Node->getOperand(0)); 3055 } 3056 Results.push_back(Tmp1); 3057 break; 3058 } 3059 case ISD::STACKSAVE: 3060 // Expand to CopyFromReg if the target set 3061 // StackPointerRegisterToSaveRestore. 3062 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3063 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 3064 Node->getValueType(0))); 3065 Results.push_back(Results[0].getValue(1)); 3066 } else { 3067 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 3068 Results.push_back(Node->getOperand(0)); 3069 } 3070 break; 3071 case ISD::STACKRESTORE: 3072 // Expand to CopyToReg if the target set 3073 // StackPointerRegisterToSaveRestore. 3074 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3075 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 3076 Node->getOperand(1))); 3077 } else { 3078 Results.push_back(Node->getOperand(0)); 3079 } 3080 break; 3081 case ISD::GET_DYNAMIC_AREA_OFFSET: 3082 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); 3083 Results.push_back(Results[0].getValue(0)); 3084 break; 3085 case ISD::FCOPYSIGN: 3086 Results.push_back(ExpandFCOPYSIGN(Node)); 3087 break; 3088 case ISD::FNEG: 3089 // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3090 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0)); 3091 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB. 3092 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, 3093 Node->getOperand(0)); 3094 Results.push_back(Tmp1); 3095 break; 3096 case ISD::FABS: 3097 Results.push_back(ExpandFABS(Node)); 3098 break; 3099 case ISD::SMIN: 3100 case ISD::SMAX: 3101 case ISD::UMIN: 3102 case ISD::UMAX: { 3103 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B 3104 ISD::CondCode Pred; 3105 switch (Node->getOpcode()) { 3106 default: llvm_unreachable("How did we get here?"); 3107 case ISD::SMAX: Pred = ISD::SETGT; break; 3108 case ISD::SMIN: Pred = ISD::SETLT; break; 3109 case ISD::UMAX: Pred = ISD::SETUGT; break; 3110 case ISD::UMIN: Pred = ISD::SETULT; break; 3111 } 3112 Tmp1 = Node->getOperand(0); 3113 Tmp2 = Node->getOperand(1); 3114 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); 3115 Results.push_back(Tmp1); 3116 break; 3117 } 3118 case ISD::FMINNUM: 3119 case ISD::FMAXNUM: { 3120 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) 3121 Results.push_back(Expanded); 3122 break; 3123 } 3124 case ISD::FSIN: 3125 case ISD::FCOS: { 3126 EVT VT = Node->getValueType(0); 3127 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / 3128 // fcos which share the same operand and both are used. 3129 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || 3130 isSinCosLibcallAvailable(Node, TLI)) 3131 && useSinCos(Node)) { 3132 SDVTList VTs = DAG.getVTList(VT, VT); 3133 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); 3134 if (Node->getOpcode() == ISD::FCOS) 3135 Tmp1 = Tmp1.getValue(1); 3136 Results.push_back(Tmp1); 3137 } 3138 break; 3139 } 3140 case ISD::FMAD: 3141 llvm_unreachable("Illegal fmad should never be formed"); 3142 3143 case ISD::FP16_TO_FP: 3144 if (Node->getValueType(0) != MVT::f32) { 3145 // We can extend to types bigger than f32 in two steps without changing 3146 // the result. Since "f16 -> f32" is much more commonly available, give 3147 // CodeGen the option of emitting that before resorting to a libcall. 3148 SDValue Res = 3149 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); 3150 Results.push_back( 3151 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); 3152 } 3153 break; 3154 case ISD::FP_TO_FP16: 3155 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n"); 3156 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { 3157 SDValue Op = Node->getOperand(0); 3158 MVT SVT = Op.getSimpleValueType(); 3159 if ((SVT == MVT::f64 || SVT == MVT::f80) && 3160 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { 3161 // Under fastmath, we can expand this node into a fround followed by 3162 // a float-half conversion. 3163 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, 3164 DAG.getIntPtrConstant(0, dl)); 3165 Results.push_back( 3166 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); 3167 } 3168 } 3169 break; 3170 case ISD::ConstantFP: { 3171 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3172 // Check to see if this FP immediate is already legal. 3173 // If this is a legal constant, turn it into a TargetConstantFP node. 3174 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), 3175 DAG.getMachineFunction().getFunction().hasOptSize())) 3176 Results.push_back(ExpandConstantFP(CFP, true)); 3177 break; 3178 } 3179 case ISD::Constant: { 3180 ConstantSDNode *CP = cast<ConstantSDNode>(Node); 3181 Results.push_back(ExpandConstant(CP)); 3182 break; 3183 } 3184 case ISD::FSUB: { 3185 EVT VT = Node->getValueType(0); 3186 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3187 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { 3188 const SDNodeFlags Flags = Node->getFlags(); 3189 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3190 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); 3191 Results.push_back(Tmp1); 3192 } 3193 break; 3194 } 3195 case ISD::SUB: { 3196 EVT VT = Node->getValueType(0); 3197 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3198 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3199 "Don't know how to expand this subtraction!"); 3200 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3201 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, 3202 VT)); 3203 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); 3204 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3205 break; 3206 } 3207 case ISD::UREM: 3208 case ISD::SREM: { 3209 EVT VT = Node->getValueType(0); 3210 bool isSigned = Node->getOpcode() == ISD::SREM; 3211 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; 3212 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3213 Tmp2 = Node->getOperand(0); 3214 Tmp3 = Node->getOperand(1); 3215 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { 3216 SDVTList VTs = DAG.getVTList(VT, VT); 3217 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); 3218 Results.push_back(Tmp1); 3219 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { 3220 // X % Y -> X-X/Y*Y 3221 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); 3222 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); 3223 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); 3224 Results.push_back(Tmp1); 3225 } 3226 break; 3227 } 3228 case ISD::UDIV: 3229 case ISD::SDIV: { 3230 bool isSigned = Node->getOpcode() == ISD::SDIV; 3231 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3232 EVT VT = Node->getValueType(0); 3233 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { 3234 SDVTList VTs = DAG.getVTList(VT, VT); 3235 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3236 Node->getOperand(1)); 3237 Results.push_back(Tmp1); 3238 } 3239 break; 3240 } 3241 case ISD::MULHU: 3242 case ISD::MULHS: { 3243 unsigned ExpandOpcode = 3244 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; 3245 EVT VT = Node->getValueType(0); 3246 SDVTList VTs = DAG.getVTList(VT, VT); 3247 3248 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3249 Node->getOperand(1)); 3250 Results.push_back(Tmp1.getValue(1)); 3251 break; 3252 } 3253 case ISD::UMUL_LOHI: 3254 case ISD::SMUL_LOHI: { 3255 SDValue LHS = Node->getOperand(0); 3256 SDValue RHS = Node->getOperand(1); 3257 MVT VT = LHS.getSimpleValueType(); 3258 unsigned MULHOpcode = 3259 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; 3260 3261 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { 3262 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); 3263 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); 3264 break; 3265 } 3266 3267 SmallVector<SDValue, 4> Halves; 3268 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); 3269 assert(TLI.isTypeLegal(HalfType)); 3270 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves, 3271 HalfType, DAG, 3272 TargetLowering::MulExpansionKind::Always)) { 3273 for (unsigned i = 0; i < 2; ++i) { 3274 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); 3275 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); 3276 SDValue Shift = DAG.getConstant( 3277 HalfType.getScalarSizeInBits(), dl, 3278 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); 3279 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); 3280 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); 3281 } 3282 break; 3283 } 3284 break; 3285 } 3286 case ISD::MUL: { 3287 EVT VT = Node->getValueType(0); 3288 SDVTList VTs = DAG.getVTList(VT, VT); 3289 // See if multiply or divide can be lowered using two-result operations. 3290 // We just need the low half of the multiply; try both the signed 3291 // and unsigned forms. If the target supports both SMUL_LOHI and 3292 // UMUL_LOHI, form a preference by checking which forms of plain 3293 // MULH it supports. 3294 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3295 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3296 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3297 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3298 unsigned OpToUse = 0; 3299 if (HasSMUL_LOHI && !HasMULHS) { 3300 OpToUse = ISD::SMUL_LOHI; 3301 } else if (HasUMUL_LOHI && !HasMULHU) { 3302 OpToUse = ISD::UMUL_LOHI; 3303 } else if (HasSMUL_LOHI) { 3304 OpToUse = ISD::SMUL_LOHI; 3305 } else if (HasUMUL_LOHI) { 3306 OpToUse = ISD::UMUL_LOHI; 3307 } 3308 if (OpToUse) { 3309 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3310 Node->getOperand(1))); 3311 break; 3312 } 3313 3314 SDValue Lo, Hi; 3315 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); 3316 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && 3317 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && 3318 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && 3319 TLI.isOperationLegalOrCustom(ISD::OR, VT) && 3320 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, 3321 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { 3322 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); 3323 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); 3324 SDValue Shift = 3325 DAG.getConstant(HalfType.getSizeInBits(), dl, 3326 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); 3327 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); 3328 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); 3329 } 3330 break; 3331 } 3332 case ISD::FSHL: 3333 case ISD::FSHR: 3334 if (TLI.expandFunnelShift(Node, Tmp1, DAG)) 3335 Results.push_back(Tmp1); 3336 break; 3337 case ISD::ROTL: 3338 case ISD::ROTR: 3339 if (TLI.expandROT(Node, Tmp1, DAG)) 3340 Results.push_back(Tmp1); 3341 break; 3342 case ISD::SADDSAT: 3343 case ISD::UADDSAT: 3344 case ISD::SSUBSAT: 3345 case ISD::USUBSAT: 3346 Results.push_back(TLI.expandAddSubSat(Node, DAG)); 3347 break; 3348 case ISD::SMULFIX: 3349 case ISD::SMULFIXSAT: 3350 case ISD::UMULFIX: 3351 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); 3352 break; 3353 case ISD::ADDCARRY: 3354 case ISD::SUBCARRY: { 3355 SDValue LHS = Node->getOperand(0); 3356 SDValue RHS = Node->getOperand(1); 3357 SDValue Carry = Node->getOperand(2); 3358 3359 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; 3360 3361 // Initial add of the 2 operands. 3362 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; 3363 EVT VT = LHS.getValueType(); 3364 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); 3365 3366 // Initial check for overflow. 3367 EVT CarryType = Node->getValueType(1); 3368 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); 3369 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; 3370 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); 3371 3372 // Add of the sum and the carry. 3373 SDValue CarryExt = 3374 DAG.getZeroExtendInReg(DAG.getZExtOrTrunc(Carry, dl, VT), dl, MVT::i1); 3375 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); 3376 3377 // Second check for overflow. If we are adding, we can only overflow if the 3378 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0. 3379 // If we are subtracting, we can only overflow if the initial sum is 0 and 3380 // the carry is set, resulting in a new sum of all 1s. 3381 SDValue Zero = DAG.getConstant(0, dl, VT); 3382 SDValue Overflow2 = 3383 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) 3384 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); 3385 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, 3386 DAG.getZExtOrTrunc(Carry, dl, SetCCType)); 3387 3388 SDValue ResultCarry = 3389 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); 3390 3391 Results.push_back(Sum2); 3392 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); 3393 break; 3394 } 3395 case ISD::SADDO: 3396 case ISD::SSUBO: { 3397 SDValue Result, Overflow; 3398 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); 3399 Results.push_back(Result); 3400 Results.push_back(Overflow); 3401 break; 3402 } 3403 case ISD::UADDO: 3404 case ISD::USUBO: { 3405 SDValue Result, Overflow; 3406 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); 3407 Results.push_back(Result); 3408 Results.push_back(Overflow); 3409 break; 3410 } 3411 case ISD::UMULO: 3412 case ISD::SMULO: { 3413 SDValue Result, Overflow; 3414 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { 3415 Results.push_back(Result); 3416 Results.push_back(Overflow); 3417 } 3418 break; 3419 } 3420 case ISD::BUILD_PAIR: { 3421 EVT PairTy = Node->getValueType(0); 3422 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3423 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3424 Tmp2 = DAG.getNode( 3425 ISD::SHL, dl, PairTy, Tmp2, 3426 DAG.getConstant(PairTy.getSizeInBits() / 2, dl, 3427 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); 3428 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3429 break; 3430 } 3431 case ISD::SELECT: 3432 Tmp1 = Node->getOperand(0); 3433 Tmp2 = Node->getOperand(1); 3434 Tmp3 = Node->getOperand(2); 3435 if (Tmp1.getOpcode() == ISD::SETCC) { 3436 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3437 Tmp2, Tmp3, 3438 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3439 } else { 3440 Tmp1 = DAG.getSelectCC(dl, Tmp1, 3441 DAG.getConstant(0, dl, Tmp1.getValueType()), 3442 Tmp2, Tmp3, ISD::SETNE); 3443 } 3444 Tmp1->setFlags(Node->getFlags()); 3445 Results.push_back(Tmp1); 3446 break; 3447 case ISD::BR_JT: { 3448 SDValue Chain = Node->getOperand(0); 3449 SDValue Table = Node->getOperand(1); 3450 SDValue Index = Node->getOperand(2); 3451 3452 const DataLayout &TD = DAG.getDataLayout(); 3453 EVT PTy = TLI.getPointerTy(TD); 3454 3455 unsigned EntrySize = 3456 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3457 3458 // For power-of-two jumptable entry sizes convert multiplication to a shift. 3459 // This transformation needs to be done here since otherwise the MIPS 3460 // backend will end up emitting a three instruction multiply sequence 3461 // instead of a single shift and MSP430 will call a runtime function. 3462 if (llvm::isPowerOf2_32(EntrySize)) 3463 Index = DAG.getNode( 3464 ISD::SHL, dl, Index.getValueType(), Index, 3465 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); 3466 else 3467 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, 3468 DAG.getConstant(EntrySize, dl, Index.getValueType())); 3469 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), 3470 Index, Table); 3471 3472 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3473 SDValue LD = DAG.getExtLoad( 3474 ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3475 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); 3476 Addr = LD; 3477 if (TLI.isJumpTableRelative()) { 3478 // For PIC, the sequence is: 3479 // BRIND(load(Jumptable + index) + RelocBase) 3480 // RelocBase can be JumpTable, GOT or some sort of global base. 3481 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3482 TLI.getPICJumpTableRelocBase(Table, DAG)); 3483 } 3484 3485 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); 3486 Results.push_back(Tmp1); 3487 break; 3488 } 3489 case ISD::BRCOND: 3490 // Expand brcond's setcc into its constituent parts and create a BR_CC 3491 // Node. 3492 Tmp1 = Node->getOperand(0); 3493 Tmp2 = Node->getOperand(1); 3494 if (Tmp2.getOpcode() == ISD::SETCC) { 3495 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3496 Tmp1, Tmp2.getOperand(2), 3497 Tmp2.getOperand(0), Tmp2.getOperand(1), 3498 Node->getOperand(2)); 3499 } else { 3500 // We test only the i1 bit. Skip the AND if UNDEF or another AND. 3501 if (Tmp2.isUndef() || 3502 (Tmp2.getOpcode() == ISD::AND && 3503 isa<ConstantSDNode>(Tmp2.getOperand(1)) && 3504 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1)) 3505 Tmp3 = Tmp2; 3506 else 3507 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3508 DAG.getConstant(1, dl, Tmp2.getValueType())); 3509 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3510 DAG.getCondCode(ISD::SETNE), Tmp3, 3511 DAG.getConstant(0, dl, Tmp3.getValueType()), 3512 Node->getOperand(2)); 3513 } 3514 Results.push_back(Tmp1); 3515 break; 3516 case ISD::SETCC: { 3517 Tmp1 = Node->getOperand(0); 3518 Tmp2 = Node->getOperand(1); 3519 Tmp3 = Node->getOperand(2); 3520 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, 3521 Tmp3, NeedInvert, dl); 3522 3523 if (Legalized) { 3524 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3525 // condition code, create a new SETCC node. 3526 if (Tmp3.getNode()) 3527 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3528 Tmp1, Tmp2, Tmp3, Node->getFlags()); 3529 3530 // If we expanded the SETCC by inverting the condition code, then wrap 3531 // the existing SETCC in a NOT to restore the intended condition. 3532 if (NeedInvert) 3533 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); 3534 3535 Results.push_back(Tmp1); 3536 break; 3537 } 3538 3539 // Otherwise, SETCC for the given comparison type must be completely 3540 // illegal; expand it into a SELECT_CC. 3541 EVT VT = Node->getValueType(0); 3542 int TrueValue; 3543 switch (TLI.getBooleanContents(Tmp1.getValueType())) { 3544 case TargetLowering::ZeroOrOneBooleanContent: 3545 case TargetLowering::UndefinedBooleanContent: 3546 TrueValue = 1; 3547 break; 3548 case TargetLowering::ZeroOrNegativeOneBooleanContent: 3549 TrueValue = -1; 3550 break; 3551 } 3552 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3553 DAG.getConstant(TrueValue, dl, VT), 3554 DAG.getConstant(0, dl, VT), 3555 Tmp3); 3556 Tmp1->setFlags(Node->getFlags()); 3557 Results.push_back(Tmp1); 3558 break; 3559 } 3560 case ISD::SELECT_CC: { 3561 Tmp1 = Node->getOperand(0); // LHS 3562 Tmp2 = Node->getOperand(1); // RHS 3563 Tmp3 = Node->getOperand(2); // True 3564 Tmp4 = Node->getOperand(3); // False 3565 EVT VT = Node->getValueType(0); 3566 SDValue CC = Node->getOperand(4); 3567 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); 3568 3569 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) { 3570 // If the condition code is legal, then we need to expand this 3571 // node using SETCC and SELECT. 3572 EVT CmpVT = Tmp1.getValueType(); 3573 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && 3574 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " 3575 "expanded."); 3576 EVT CCVT = getSetCCResultType(CmpVT); 3577 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); 3578 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); 3579 break; 3580 } 3581 3582 // SELECT_CC is legal, so the condition code must not be. 3583 bool Legalized = false; 3584 // Try to legalize by inverting the condition. This is for targets that 3585 // might support an ordered version of a condition, but not the unordered 3586 // version (or vice versa). 3587 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, 3588 Tmp1.getValueType().isInteger()); 3589 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { 3590 // Use the new condition code and swap true and false 3591 Legalized = true; 3592 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); 3593 Tmp1->setFlags(Node->getFlags()); 3594 } else { 3595 // If The inverse is not legal, then try to swap the arguments using 3596 // the inverse condition code. 3597 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); 3598 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) { 3599 // The swapped inverse condition is legal, so swap true and false, 3600 // lhs and rhs. 3601 Legalized = true; 3602 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); 3603 Tmp1->setFlags(Node->getFlags()); 3604 } 3605 } 3606 3607 if (!Legalized) { 3608 Legalized = LegalizeSetCCCondCode( 3609 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, 3610 dl); 3611 3612 assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); 3613 3614 // If we expanded the SETCC by inverting the condition code, then swap 3615 // the True/False operands to match. 3616 if (NeedInvert) 3617 std::swap(Tmp3, Tmp4); 3618 3619 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3620 // condition code, create a new SELECT_CC node. 3621 if (CC.getNode()) { 3622 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), 3623 Tmp1, Tmp2, Tmp3, Tmp4, CC); 3624 } else { 3625 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); 3626 CC = DAG.getCondCode(ISD::SETNE); 3627 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, 3628 Tmp2, Tmp3, Tmp4, CC); 3629 } 3630 Tmp1->setFlags(Node->getFlags()); 3631 } 3632 Results.push_back(Tmp1); 3633 break; 3634 } 3635 case ISD::BR_CC: { 3636 Tmp1 = Node->getOperand(0); // Chain 3637 Tmp2 = Node->getOperand(2); // LHS 3638 Tmp3 = Node->getOperand(3); // RHS 3639 Tmp4 = Node->getOperand(1); // CC 3640 3641 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( 3642 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); 3643 (void)Legalized; 3644 assert(Legalized && "Can't legalize BR_CC with legal condition!"); 3645 3646 assert(!NeedInvert && "Don't know how to invert BR_CC!"); 3647 3648 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC 3649 // node. 3650 if (Tmp4.getNode()) { 3651 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, 3652 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); 3653 } else { 3654 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); 3655 Tmp4 = DAG.getCondCode(ISD::SETNE); 3656 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, 3657 Tmp2, Tmp3, Node->getOperand(4)); 3658 } 3659 Results.push_back(Tmp1); 3660 break; 3661 } 3662 case ISD::BUILD_VECTOR: 3663 Results.push_back(ExpandBUILD_VECTOR(Node)); 3664 break; 3665 case ISD::SRA: 3666 case ISD::SRL: 3667 case ISD::SHL: { 3668 // Scalarize vector SRA/SRL/SHL. 3669 EVT VT = Node->getValueType(0); 3670 assert(VT.isVector() && "Unable to legalize non-vector shift"); 3671 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 3672 unsigned NumElem = VT.getVectorNumElements(); 3673 3674 SmallVector<SDValue, 8> Scalars; 3675 for (unsigned Idx = 0; Idx < NumElem; Idx++) { 3676 SDValue Ex = DAG.getNode( 3677 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0), 3678 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3679 SDValue Sh = DAG.getNode( 3680 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1), 3681 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3682 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 3683 VT.getScalarType(), Ex, Sh)); 3684 } 3685 3686 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); 3687 ReplaceNode(SDValue(Node, 0), Result); 3688 break; 3689 } 3690 case ISD::VECREDUCE_FADD: 3691 case ISD::VECREDUCE_FMUL: 3692 case ISD::VECREDUCE_ADD: 3693 case ISD::VECREDUCE_MUL: 3694 case ISD::VECREDUCE_AND: 3695 case ISD::VECREDUCE_OR: 3696 case ISD::VECREDUCE_XOR: 3697 case ISD::VECREDUCE_SMAX: 3698 case ISD::VECREDUCE_SMIN: 3699 case ISD::VECREDUCE_UMAX: 3700 case ISD::VECREDUCE_UMIN: 3701 case ISD::VECREDUCE_FMAX: 3702 case ISD::VECREDUCE_FMIN: 3703 Results.push_back(TLI.expandVecReduce(Node, DAG)); 3704 break; 3705 case ISD::GLOBAL_OFFSET_TABLE: 3706 case ISD::GlobalAddress: 3707 case ISD::GlobalTLSAddress: 3708 case ISD::ExternalSymbol: 3709 case ISD::ConstantPool: 3710 case ISD::JumpTable: 3711 case ISD::INTRINSIC_W_CHAIN: 3712 case ISD::INTRINSIC_WO_CHAIN: 3713 case ISD::INTRINSIC_VOID: 3714 // FIXME: Custom lowering for these operations shouldn't return null! 3715 break; 3716 } 3717 3718 // Replace the original node with the legalized result. 3719 if (Results.empty()) { 3720 LLVM_DEBUG(dbgs() << "Cannot expand node\n"); 3721 return false; 3722 } 3723 3724 LLVM_DEBUG(dbgs() << "Successfully expanded node\n"); 3725 ReplaceNode(Node, Results.data()); 3726 return true; 3727} 3728 3729void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { 3730 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n"); 3731 SmallVector<SDValue, 8> Results; 3732 SDLoc dl(Node); 3733 // FIXME: Check flags on the node to see if we can use a finite call. 3734 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath; 3735 unsigned Opc = Node->getOpcode(); 3736 switch (Opc) { 3737 case ISD::ATOMIC_FENCE: { 3738 // If the target didn't lower this, lower it to '__sync_synchronize()' call 3739 // FIXME: handle "fence singlethread" more efficiently. 3740 TargetLowering::ArgListTy Args; 3741 3742 TargetLowering::CallLoweringInfo CLI(DAG); 3743 CLI.setDebugLoc(dl) 3744 .setChain(Node->getOperand(0)) 3745 .setLibCallee( 3746 CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3747 DAG.getExternalSymbol("__sync_synchronize", 3748 TLI.getPointerTy(DAG.getDataLayout())), 3749 std::move(Args)); 3750 3751 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 3752 3753 Results.push_back(CallResult.second); 3754 break; 3755 } 3756 // By default, atomic intrinsics are marked Legal and lowered. Targets 3757 // which don't support them directly, however, may want libcalls, in which 3758 // case they mark them Expand, and we get here. 3759 case ISD::ATOMIC_SWAP: 3760 case ISD::ATOMIC_LOAD_ADD: 3761 case ISD::ATOMIC_LOAD_SUB: 3762 case ISD::ATOMIC_LOAD_AND: 3763 case ISD::ATOMIC_LOAD_CLR: 3764 case ISD::ATOMIC_LOAD_OR: 3765 case ISD::ATOMIC_LOAD_XOR: 3766 case ISD::ATOMIC_LOAD_NAND: 3767 case ISD::ATOMIC_LOAD_MIN: 3768 case ISD::ATOMIC_LOAD_MAX: 3769 case ISD::ATOMIC_LOAD_UMIN: 3770 case ISD::ATOMIC_LOAD_UMAX: 3771 case ISD::ATOMIC_CMP_SWAP: { 3772 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 3773 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); 3774 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); 3775 3776 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false); 3777 Results.push_back(Tmp.first); 3778 Results.push_back(Tmp.second); 3779 break; 3780 } 3781 case ISD::TRAP: { 3782 // If this operation is not supported, lower it to 'abort()' call 3783 TargetLowering::ArgListTy Args; 3784 TargetLowering::CallLoweringInfo CLI(DAG); 3785 CLI.setDebugLoc(dl) 3786 .setChain(Node->getOperand(0)) 3787 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3788 DAG.getExternalSymbol( 3789 "abort", TLI.getPointerTy(DAG.getDataLayout())), 3790 std::move(Args)); 3791 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 3792 3793 Results.push_back(CallResult.second); 3794 break; 3795 } 3796 case ISD::FMINNUM: 3797 case ISD::STRICT_FMINNUM: 3798 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, 3799 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 3800 RTLIB::FMIN_PPCF128)); 3801 break; 3802 case ISD::FMAXNUM: 3803 case ISD::STRICT_FMAXNUM: 3804 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, 3805 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 3806 RTLIB::FMAX_PPCF128)); 3807 break; 3808 case ISD::FSQRT: 3809 case ISD::STRICT_FSQRT: 3810 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3811 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 3812 RTLIB::SQRT_PPCF128)); 3813 break; 3814 case ISD::FCBRT: 3815 Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, 3816 RTLIB::CBRT_F80, RTLIB::CBRT_F128, 3817 RTLIB::CBRT_PPCF128)); 3818 break; 3819 case ISD::FSIN: 3820 case ISD::STRICT_FSIN: 3821 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 3822 RTLIB::SIN_F80, RTLIB::SIN_F128, 3823 RTLIB::SIN_PPCF128)); 3824 break; 3825 case ISD::FCOS: 3826 case ISD::STRICT_FCOS: 3827 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 3828 RTLIB::COS_F80, RTLIB::COS_F128, 3829 RTLIB::COS_PPCF128)); 3830 break; 3831 case ISD::FSINCOS: 3832 // Expand into sincos libcall. 3833 ExpandSinCosLibCall(Node, Results); 3834 break; 3835 case ISD::FLOG: 3836 case ISD::STRICT_FLOG: 3837 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite)) 3838 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32, 3839 RTLIB::LOG_FINITE_F64, 3840 RTLIB::LOG_FINITE_F80, 3841 RTLIB::LOG_FINITE_F128, 3842 RTLIB::LOG_FINITE_PPCF128)); 3843 else 3844 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, 3845 RTLIB::LOG_F80, RTLIB::LOG_F128, 3846 RTLIB::LOG_PPCF128)); 3847 break; 3848 case ISD::FLOG2: 3849 case ISD::STRICT_FLOG2: 3850 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite)) 3851 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32, 3852 RTLIB::LOG2_FINITE_F64, 3853 RTLIB::LOG2_FINITE_F80, 3854 RTLIB::LOG2_FINITE_F128, 3855 RTLIB::LOG2_FINITE_PPCF128)); 3856 else 3857 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3858 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 3859 RTLIB::LOG2_PPCF128)); 3860 break; 3861 case ISD::FLOG10: 3862 case ISD::STRICT_FLOG10: 3863 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite)) 3864 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32, 3865 RTLIB::LOG10_FINITE_F64, 3866 RTLIB::LOG10_FINITE_F80, 3867 RTLIB::LOG10_FINITE_F128, 3868 RTLIB::LOG10_FINITE_PPCF128)); 3869 else 3870 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3871 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 3872 RTLIB::LOG10_PPCF128)); 3873 break; 3874 case ISD::FEXP: 3875 case ISD::STRICT_FEXP: 3876 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite)) 3877 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32, 3878 RTLIB::EXP_FINITE_F64, 3879 RTLIB::EXP_FINITE_F80, 3880 RTLIB::EXP_FINITE_F128, 3881 RTLIB::EXP_FINITE_PPCF128)); 3882 else 3883 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, 3884 RTLIB::EXP_F80, RTLIB::EXP_F128, 3885 RTLIB::EXP_PPCF128)); 3886 break; 3887 case ISD::FEXP2: 3888 case ISD::STRICT_FEXP2: 3889 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite)) 3890 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32, 3891 RTLIB::EXP2_FINITE_F64, 3892 RTLIB::EXP2_FINITE_F80, 3893 RTLIB::EXP2_FINITE_F128, 3894 RTLIB::EXP2_FINITE_PPCF128)); 3895 else 3896 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3897 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 3898 RTLIB::EXP2_PPCF128)); 3899 break; 3900 case ISD::FTRUNC: 3901 case ISD::STRICT_FTRUNC: 3902 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3903 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 3904 RTLIB::TRUNC_PPCF128)); 3905 break; 3906 case ISD::FFLOOR: 3907 case ISD::STRICT_FFLOOR: 3908 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3909 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 3910 RTLIB::FLOOR_PPCF128)); 3911 break; 3912 case ISD::FCEIL: 3913 case ISD::STRICT_FCEIL: 3914 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3915 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 3916 RTLIB::CEIL_PPCF128)); 3917 break; 3918 case ISD::FRINT: 3919 case ISD::STRICT_FRINT: 3920 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 3921 RTLIB::RINT_F80, RTLIB::RINT_F128, 3922 RTLIB::RINT_PPCF128)); 3923 break; 3924 case ISD::FNEARBYINT: 3925 case ISD::STRICT_FNEARBYINT: 3926 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 3927 RTLIB::NEARBYINT_F64, 3928 RTLIB::NEARBYINT_F80, 3929 RTLIB::NEARBYINT_F128, 3930 RTLIB::NEARBYINT_PPCF128)); 3931 break; 3932 case ISD::FROUND: 3933 case ISD::STRICT_FROUND: 3934 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, 3935 RTLIB::ROUND_F64, 3936 RTLIB::ROUND_F80, 3937 RTLIB::ROUND_F128, 3938 RTLIB::ROUND_PPCF128)); 3939 break; 3940 case ISD::FPOWI: 3941 case ISD::STRICT_FPOWI: 3942 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 3943 RTLIB::POWI_F80, RTLIB::POWI_F128, 3944 RTLIB::POWI_PPCF128)); 3945 break; 3946 case ISD::FPOW: 3947 case ISD::STRICT_FPOW: 3948 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite)) 3949 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32, 3950 RTLIB::POW_FINITE_F64, 3951 RTLIB::POW_FINITE_F80, 3952 RTLIB::POW_FINITE_F128, 3953 RTLIB::POW_FINITE_PPCF128)); 3954 else 3955 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, 3956 RTLIB::POW_F80, RTLIB::POW_F128, 3957 RTLIB::POW_PPCF128)); 3958 break; 3959 case ISD::FDIV: 3960 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 3961 RTLIB::DIV_F80, RTLIB::DIV_F128, 3962 RTLIB::DIV_PPCF128)); 3963 break; 3964 case ISD::FREM: 3965 case ISD::STRICT_FREM: 3966 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 3967 RTLIB::REM_F80, RTLIB::REM_F128, 3968 RTLIB::REM_PPCF128)); 3969 break; 3970 case ISD::FMA: 3971 case ISD::STRICT_FMA: 3972 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 3973 RTLIB::FMA_F80, RTLIB::FMA_F128, 3974 RTLIB::FMA_PPCF128)); 3975 break; 3976 case ISD::FADD: 3977 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, 3978 RTLIB::ADD_F80, RTLIB::ADD_F128, 3979 RTLIB::ADD_PPCF128)); 3980 break; 3981 case ISD::FMUL: 3982 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, 3983 RTLIB::MUL_F80, RTLIB::MUL_F128, 3984 RTLIB::MUL_PPCF128)); 3985 break; 3986 case ISD::FP16_TO_FP: 3987 if (Node->getValueType(0) == MVT::f32) { 3988 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 3989 } 3990 break; 3991 case ISD::FP_TO_FP16: { 3992 RTLIB::Libcall LC = 3993 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); 3994 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); 3995 Results.push_back(ExpandLibCall(LC, Node, false)); 3996 break; 3997 } 3998 case ISD::FSUB: 3999 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, 4000 RTLIB::SUB_F80, RTLIB::SUB_F128, 4001 RTLIB::SUB_PPCF128)); 4002 break; 4003 case ISD::SREM: 4004 Results.push_back(ExpandIntLibCall(Node, true, 4005 RTLIB::SREM_I8, 4006 RTLIB::SREM_I16, RTLIB::SREM_I32, 4007 RTLIB::SREM_I64, RTLIB::SREM_I128)); 4008 break; 4009 case ISD::UREM: 4010 Results.push_back(ExpandIntLibCall(Node, false, 4011 RTLIB::UREM_I8, 4012 RTLIB::UREM_I16, RTLIB::UREM_I32, 4013 RTLIB::UREM_I64, RTLIB::UREM_I128)); 4014 break; 4015 case ISD::SDIV: 4016 Results.push_back(ExpandIntLibCall(Node, true, 4017 RTLIB::SDIV_I8, 4018 RTLIB::SDIV_I16, RTLIB::SDIV_I32, 4019 RTLIB::SDIV_I64, RTLIB::SDIV_I128)); 4020 break; 4021 case ISD::UDIV: 4022 Results.push_back(ExpandIntLibCall(Node, false, 4023 RTLIB::UDIV_I8, 4024 RTLIB::UDIV_I16, RTLIB::UDIV_I32, 4025 RTLIB::UDIV_I64, RTLIB::UDIV_I128)); 4026 break; 4027 case ISD::SDIVREM: 4028 case ISD::UDIVREM: 4029 // Expand into divrem libcall 4030 ExpandDivRemLibCall(Node, Results); 4031 break; 4032 case ISD::MUL: 4033 Results.push_back(ExpandIntLibCall(Node, false, 4034 RTLIB::MUL_I8, 4035 RTLIB::MUL_I16, RTLIB::MUL_I32, 4036 RTLIB::MUL_I64, RTLIB::MUL_I128)); 4037 break; 4038 case ISD::CTLZ_ZERO_UNDEF: 4039 switch (Node->getSimpleValueType(0).SimpleTy) { 4040 default: 4041 llvm_unreachable("LibCall explicitly requested, but not available"); 4042 case MVT::i32: 4043 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); 4044 break; 4045 case MVT::i64: 4046 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); 4047 break; 4048 case MVT::i128: 4049 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); 4050 break; 4051 } 4052 break; 4053 } 4054 4055 // Replace the original node with the legalized result. 4056 if (!Results.empty()) { 4057 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n"); 4058 ReplaceNode(Node, Results.data()); 4059 } else 4060 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n"); 4061} 4062 4063// Determine the vector type to use in place of an original scalar element when 4064// promoting equally sized vectors. 4065static MVT getPromotedVectorElementType(const TargetLowering &TLI, 4066 MVT EltVT, MVT NewEltVT) { 4067 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); 4068 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); 4069 assert(TLI.isTypeLegal(MidVT) && "unexpected"); 4070 return MidVT; 4071} 4072 4073void SelectionDAGLegalize::PromoteNode(SDNode *Node) { 4074 LLVM_DEBUG(dbgs() << "Trying to promote node\n"); 4075 SmallVector<SDValue, 8> Results; 4076 MVT OVT = Node->getSimpleValueType(0); 4077 if (Node->getOpcode() == ISD::UINT_TO_FP || 4078 Node->getOpcode() == ISD::SINT_TO_FP || 4079 Node->getOpcode() == ISD::SETCC || 4080 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || 4081 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { 4082 OVT = Node->getOperand(0).getSimpleValueType(); 4083 } 4084 if (Node->getOpcode() == ISD::BR_CC) 4085 OVT = Node->getOperand(2).getSimpleValueType(); 4086 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 4087 SDLoc dl(Node); 4088 SDValue Tmp1, Tmp2, Tmp3; 4089 switch (Node->getOpcode()) { 4090 case ISD::CTTZ: 4091 case ISD::CTTZ_ZERO_UNDEF: 4092 case ISD::CTLZ: 4093 case ISD::CTLZ_ZERO_UNDEF: 4094 case ISD::CTPOP: 4095 // Zero extend the argument. 4096 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4097 if (Node->getOpcode() == ISD::CTTZ) { 4098 // The count is the same in the promoted type except if the original 4099 // value was zero. This can be handled by setting the bit just off 4100 // the top of the original type. 4101 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), 4102 OVT.getSizeInBits()); 4103 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, 4104 DAG.getConstant(TopBit, dl, NVT)); 4105 } 4106 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 4107 // already the correct result. 4108 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4109 if (Node->getOpcode() == ISD::CTLZ || 4110 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 4111 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 4112 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 4113 DAG.getConstant(NVT.getSizeInBits() - 4114 OVT.getSizeInBits(), dl, NVT)); 4115 } 4116 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4117 break; 4118 case ISD::BITREVERSE: 4119 case ISD::BSWAP: { 4120 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 4121 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 4122 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4123 Tmp1 = DAG.getNode( 4124 ISD::SRL, dl, NVT, Tmp1, 4125 DAG.getConstant(DiffBits, dl, 4126 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 4127 4128 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4129 break; 4130 } 4131 case ISD::FP_TO_UINT: 4132 case ISD::FP_TO_SINT: 4133 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), 4134 Node->getOpcode() == ISD::FP_TO_SINT, dl); 4135 Results.push_back(Tmp1); 4136 break; 4137 case ISD::UINT_TO_FP: 4138 case ISD::SINT_TO_FP: 4139 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), 4140 Node->getOpcode() == ISD::SINT_TO_FP, dl); 4141 Results.push_back(Tmp1); 4142 break; 4143 case ISD::VAARG: { 4144 SDValue Chain = Node->getOperand(0); // Get the chain. 4145 SDValue Ptr = Node->getOperand(1); // Get the pointer. 4146 4147 unsigned TruncOp; 4148 if (OVT.isVector()) { 4149 TruncOp = ISD::BITCAST; 4150 } else { 4151 assert(OVT.isInteger() 4152 && "VAARG promotion is supported only for vectors or integer types"); 4153 TruncOp = ISD::TRUNCATE; 4154 } 4155 4156 // Perform the larger operation, then convert back 4157 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 4158 Node->getConstantOperandVal(3)); 4159 Chain = Tmp1.getValue(1); 4160 4161 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 4162 4163 // Modified the chain result - switch anything that used the old chain to 4164 // use the new one. 4165 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 4166 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 4167 if (UpdatedNodes) { 4168 UpdatedNodes->insert(Tmp2.getNode()); 4169 UpdatedNodes->insert(Chain.getNode()); 4170 } 4171 ReplacedNode(Node); 4172 break; 4173 } 4174 case ISD::MUL: 4175 case ISD::SDIV: 4176 case ISD::SREM: 4177 case ISD::UDIV: 4178 case ISD::UREM: 4179 case ISD::AND: 4180 case ISD::OR: 4181 case ISD::XOR: { 4182 unsigned ExtOp, TruncOp; 4183 if (OVT.isVector()) { 4184 ExtOp = ISD::BITCAST; 4185 TruncOp = ISD::BITCAST; 4186 } else { 4187 assert(OVT.isInteger() && "Cannot promote logic operation"); 4188 4189 switch (Node->getOpcode()) { 4190 default: 4191 ExtOp = ISD::ANY_EXTEND; 4192 break; 4193 case ISD::SDIV: 4194 case ISD::SREM: 4195 ExtOp = ISD::SIGN_EXTEND; 4196 break; 4197 case ISD::UDIV: 4198 case ISD::UREM: 4199 ExtOp = ISD::ZERO_EXTEND; 4200 break; 4201 } 4202 TruncOp = ISD::TRUNCATE; 4203 } 4204 // Promote each of the values to the new type. 4205 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4206 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4207 // Perform the larger operation, then convert back 4208 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4209 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 4210 break; 4211 } 4212 case ISD::UMUL_LOHI: 4213 case ISD::SMUL_LOHI: { 4214 // Promote to a multiply in a wider integer type. 4215 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND 4216 : ISD::SIGN_EXTEND; 4217 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4218 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4219 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); 4220 4221 auto &DL = DAG.getDataLayout(); 4222 unsigned OriginalSize = OVT.getScalarSizeInBits(); 4223 Tmp2 = DAG.getNode( 4224 ISD::SRL, dl, NVT, Tmp1, 4225 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); 4226 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 4227 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); 4228 break; 4229 } 4230 case ISD::SELECT: { 4231 unsigned ExtOp, TruncOp; 4232 if (Node->getValueType(0).isVector() || 4233 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { 4234 ExtOp = ISD::BITCAST; 4235 TruncOp = ISD::BITCAST; 4236 } else if (Node->getValueType(0).isInteger()) { 4237 ExtOp = ISD::ANY_EXTEND; 4238 TruncOp = ISD::TRUNCATE; 4239 } else { 4240 ExtOp = ISD::FP_EXTEND; 4241 TruncOp = ISD::FP_ROUND; 4242 } 4243 Tmp1 = Node->getOperand(0); 4244 // Promote each of the values to the new type. 4245 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4246 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 4247 // Perform the larger operation, then round down. 4248 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); 4249 Tmp1->setFlags(Node->getFlags()); 4250 if (TruncOp != ISD::FP_ROUND) 4251 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 4252 else 4253 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 4254 DAG.getIntPtrConstant(0, dl)); 4255 Results.push_back(Tmp1); 4256 break; 4257 } 4258 case ISD::VECTOR_SHUFFLE: { 4259 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 4260 4261 // Cast the two input vectors. 4262 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 4263 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 4264 4265 // Convert the shuffle mask to the right # elements. 4266 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 4267 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 4268 Results.push_back(Tmp1); 4269 break; 4270 } 4271 case ISD::SETCC: { 4272 unsigned ExtOp = ISD::FP_EXTEND; 4273 if (NVT.isInteger()) { 4274 ISD::CondCode CCCode = 4275 cast<CondCodeSDNode>(Node->getOperand(2))->get(); 4276 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4277 } 4278 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4279 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4280 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, 4281 Tmp2, Node->getOperand(2), Node->getFlags())); 4282 break; 4283 } 4284 case ISD::BR_CC: { 4285 unsigned ExtOp = ISD::FP_EXTEND; 4286 if (NVT.isInteger()) { 4287 ISD::CondCode CCCode = 4288 cast<CondCodeSDNode>(Node->getOperand(1))->get(); 4289 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4290 } 4291 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 4292 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); 4293 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), 4294 Node->getOperand(0), Node->getOperand(1), 4295 Tmp1, Tmp2, Node->getOperand(4))); 4296 break; 4297 } 4298 case ISD::FADD: 4299 case ISD::FSUB: 4300 case ISD::FMUL: 4301 case ISD::FDIV: 4302 case ISD::FREM: 4303 case ISD::FMINNUM: 4304 case ISD::FMAXNUM: 4305 case ISD::FPOW: 4306 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4307 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 4308 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, 4309 Node->getFlags()); 4310 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4311 Tmp3, DAG.getIntPtrConstant(0, dl))); 4312 break; 4313 case ISD::FMA: 4314 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4315 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 4316 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); 4317 Results.push_back( 4318 DAG.getNode(ISD::FP_ROUND, dl, OVT, 4319 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), 4320 DAG.getIntPtrConstant(0, dl))); 4321 break; 4322 case ISD::FCOPYSIGN: 4323 case ISD::FPOWI: { 4324 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4325 Tmp2 = Node->getOperand(1); 4326 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4327 4328 // fcopysign doesn't change anything but the sign bit, so 4329 // (fp_round (fcopysign (fpext a), b)) 4330 // is as precise as 4331 // (fp_round (fpext a)) 4332 // which is a no-op. Mark it as a TRUNCating FP_ROUND. 4333 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); 4334 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4335 Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); 4336 break; 4337 } 4338 case ISD::FFLOOR: 4339 case ISD::FCEIL: 4340 case ISD::FRINT: 4341 case ISD::FNEARBYINT: 4342 case ISD::FROUND: 4343 case ISD::FTRUNC: 4344 case ISD::FNEG: 4345 case ISD::FSQRT: 4346 case ISD::FSIN: 4347 case ISD::FCOS: 4348 case ISD::FLOG: 4349 case ISD::FLOG2: 4350 case ISD::FLOG10: 4351 case ISD::FABS: 4352 case ISD::FEXP: 4353 case ISD::FEXP2: 4354 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4355 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4356 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4357 Tmp2, DAG.getIntPtrConstant(0, dl))); 4358 break; 4359 case ISD::BUILD_VECTOR: { 4360 MVT EltVT = OVT.getVectorElementType(); 4361 MVT NewEltVT = NVT.getVectorElementType(); 4362 4363 // Handle bitcasts to a different vector type with the same total bit size 4364 // 4365 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32 4366 // => 4367 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y)) 4368 4369 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && 4370 "Invalid promote type for build_vector"); 4371 assert(NewEltVT.bitsLT(EltVT) && "not handled"); 4372 4373 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 4374 4375 SmallVector<SDValue, 8> NewOps; 4376 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { 4377 SDValue Op = Node->getOperand(I); 4378 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); 4379 } 4380 4381 SDLoc SL(Node); 4382 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); 4383 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); 4384 Results.push_back(CvtVec); 4385 break; 4386 } 4387 case ISD::EXTRACT_VECTOR_ELT: { 4388 MVT EltVT = OVT.getVectorElementType(); 4389 MVT NewEltVT = NVT.getVectorElementType(); 4390 4391 // Handle bitcasts to a different vector type with the same total bit size. 4392 // 4393 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32 4394 // => 4395 // v4i32:castx = bitcast x:v2i64 4396 // 4397 // i64 = bitcast 4398 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), 4399 // (i32 (extract_vector_elt castx, (2 * y + 1))) 4400 // 4401 4402 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && 4403 "Invalid promote type for extract_vector_elt"); 4404 assert(NewEltVT.bitsLT(EltVT) && "not handled"); 4405 4406 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 4407 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); 4408 4409 SDValue Idx = Node->getOperand(1); 4410 EVT IdxVT = Idx.getValueType(); 4411 SDLoc SL(Node); 4412 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); 4413 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); 4414 4415 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); 4416 4417 SmallVector<SDValue, 8> NewOps; 4418 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { 4419 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); 4420 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); 4421 4422 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, 4423 CastVec, TmpIdx); 4424 NewOps.push_back(Elt); 4425 } 4426 4427 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); 4428 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); 4429 break; 4430 } 4431 case ISD::INSERT_VECTOR_ELT: { 4432 MVT EltVT = OVT.getVectorElementType(); 4433 MVT NewEltVT = NVT.getVectorElementType(); 4434 4435 // Handle bitcasts to a different vector type with the same total bit size 4436 // 4437 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32 4438 // => 4439 // v4i32:castx = bitcast x:v2i64 4440 // v2i32:casty = bitcast y:i64 4441 // 4442 // v2i64 = bitcast 4443 // (v4i32 insert_vector_elt 4444 // (v4i32 insert_vector_elt v4i32:castx, 4445 // (extract_vector_elt casty, 0), 2 * z), 4446 // (extract_vector_elt casty, 1), (2 * z + 1)) 4447 4448 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && 4449 "Invalid promote type for insert_vector_elt"); 4450 assert(NewEltVT.bitsLT(EltVT) && "not handled"); 4451 4452 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 4453 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); 4454 4455 SDValue Val = Node->getOperand(1); 4456 SDValue Idx = Node->getOperand(2); 4457 EVT IdxVT = Idx.getValueType(); 4458 SDLoc SL(Node); 4459 4460 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); 4461 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); 4462 4463 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); 4464 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); 4465 4466 SDValue NewVec = CastVec; 4467 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { 4468 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); 4469 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); 4470 4471 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, 4472 CastVal, IdxOffset); 4473 4474 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, 4475 NewVec, Elt, InEltIdx); 4476 } 4477 4478 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); 4479 break; 4480 } 4481 case ISD::SCALAR_TO_VECTOR: { 4482 MVT EltVT = OVT.getVectorElementType(); 4483 MVT NewEltVT = NVT.getVectorElementType(); 4484 4485 // Handle bitcasts to different vector type with the same total bit size. 4486 // 4487 // e.g. v2i64 = scalar_to_vector x:i64 4488 // => 4489 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef) 4490 // 4491 4492 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); 4493 SDValue Val = Node->getOperand(0); 4494 SDLoc SL(Node); 4495 4496 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); 4497 SDValue Undef = DAG.getUNDEF(MidVT); 4498 4499 SmallVector<SDValue, 8> NewElts; 4500 NewElts.push_back(CastVal); 4501 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I) 4502 NewElts.push_back(Undef); 4503 4504 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); 4505 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); 4506 Results.push_back(CvtVec); 4507 break; 4508 } 4509 case ISD::ATOMIC_SWAP: { 4510 AtomicSDNode *AM = cast<AtomicSDNode>(Node); 4511 SDLoc SL(Node); 4512 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); 4513 assert(NVT.getSizeInBits() == OVT.getSizeInBits() && 4514 "unexpected promotion type"); 4515 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && 4516 "unexpected atomic_swap with illegal type"); 4517 4518 SDValue NewAtomic 4519 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, 4520 DAG.getVTList(NVT, MVT::Other), 4521 { AM->getChain(), AM->getBasePtr(), CastVal }, 4522 AM->getMemOperand()); 4523 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); 4524 Results.push_back(NewAtomic.getValue(1)); 4525 break; 4526 } 4527 } 4528 4529 // Replace the original node with the legalized result. 4530 if (!Results.empty()) { 4531 LLVM_DEBUG(dbgs() << "Successfully promoted node\n"); 4532 ReplaceNode(Node, Results.data()); 4533 } else 4534 LLVM_DEBUG(dbgs() << "Could not promote node\n"); 4535} 4536 4537/// This is the entry point for the file. 4538void SelectionDAG::Legalize() { 4539 AssignTopologicalOrder(); 4540 4541 SmallPtrSet<SDNode *, 16> LegalizedNodes; 4542 // Use a delete listener to remove nodes which were deleted during 4543 // legalization from LegalizeNodes. This is needed to handle the situation 4544 // where a new node is allocated by the object pool to the same address of a 4545 // previously deleted node. 4546 DAGNodeDeletedListener DeleteListener( 4547 *this, 4548 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }); 4549 4550 SelectionDAGLegalize Legalizer(*this, LegalizedNodes); 4551 4552 // Visit all the nodes. We start in topological order, so that we see 4553 // nodes with their original operands intact. Legalization can produce 4554 // new nodes which may themselves need to be legalized. Iterate until all 4555 // nodes have been legalized. 4556 while (true) { 4557 bool AnyLegalized = false; 4558 for (auto NI = allnodes_end(); NI != allnodes_begin();) { 4559 --NI; 4560 4561 SDNode *N = &*NI; 4562 if (N->use_empty() && N != getRoot().getNode()) { 4563 ++NI; 4564 DeleteNode(N); 4565 continue; 4566 } 4567 4568 if (LegalizedNodes.insert(N).second) { 4569 AnyLegalized = true; 4570 Legalizer.LegalizeOp(N); 4571 4572 if (N->use_empty() && N != getRoot().getNode()) { 4573 ++NI; 4574 DeleteNode(N); 4575 } 4576 } 4577 } 4578 if (!AnyLegalized) 4579 break; 4580 4581 } 4582 4583 // Remove dead nodes now. 4584 RemoveDeadNodes(); 4585} 4586 4587bool SelectionDAG::LegalizeOp(SDNode *N, 4588 SmallSetVector<SDNode *, 16> &UpdatedNodes) { 4589 SmallPtrSet<SDNode *, 16> LegalizedNodes; 4590 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); 4591 4592 // Directly insert the node in question, and legalize it. This will recurse 4593 // as needed through operands. 4594 LegalizedNodes.insert(N); 4595 Legalizer.LegalizeOp(N); 4596 4597 return LegalizedNodes.count(N); 4598} 4599