1193323Sed//===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===// 2193323Sed// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6193323Sed// 7193323Sed//===----------------------------------------------------------------------===// 8193323Sed// 9193323Sed// This file implements generic type expansion and splitting for LegalizeTypes. 10193323Sed// The routines here perform legalization when the details of the type (such as 11193323Sed// whether it is an integer or a float) do not matter. 12193323Sed// Expansion is the act of changing a computation in an illegal type to be a 13198090Srdivacky// computation in two identical registers of a smaller type. The Lo/Hi part 14198090Srdivacky// is required to be stored first in memory on little/big-endian machines. 15193323Sed// Splitting is the act of changing a computation in an illegal type to be a 16193323Sed// computation in two not necessarily identical registers of a smaller type. 17198090Srdivacky// There are no requirements on how the type is represented in memory. 18193323Sed// 19193323Sed//===----------------------------------------------------------------------===// 20193323Sed 21193323Sed#include "LegalizeTypes.h" 22249423Sdim#include "llvm/IR/DataLayout.h" 23193323Sedusing namespace llvm; 24193323Sed 25276479Sdim#define DEBUG_TYPE "legalize-types" 26276479Sdim 27193323Sed//===----------------------------------------------------------------------===// 28193323Sed// Generic Result Expansion. 29193323Sed//===----------------------------------------------------------------------===// 30193323Sed 31193323Sed// These routines assume that the Lo/Hi part is stored first in memory on 32193323Sed// little/big-endian machines, followed by the Hi/Lo part. This means that 33193323Sed// they cannot be used as is on vectors, for which Lo is always stored first. 34226633Sdimvoid DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 35226633Sdim SDValue &Lo, SDValue &Hi) { 36226633Sdim SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 37226633Sdim GetExpandedOp(Op, Lo, Hi); 38226633Sdim} 39193323Sed 40218893Sdimvoid DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { 41198090Srdivacky EVT OutVT = N->getValueType(0); 42198090Srdivacky EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 43193323Sed SDValue InOp = N->getOperand(0); 44198090Srdivacky EVT InVT = InOp.getValueType(); 45261991Sdim SDLoc dl(N); 46193323Sed 47193323Sed // Handle some special cases efficiently. 48193323Sed switch (getTypeAction(InVT)) { 49223017Sdim case TargetLowering::TypeLegal: 50223017Sdim case TargetLowering::TypePromoteInteger: 51193323Sed break; 52288943Sdim case TargetLowering::TypePromoteFloat: 53288943Sdim llvm_unreachable("Bitcast of a promotion-needing float should never need" 54288943Sdim "expansion"); 55360784Sdim case TargetLowering::TypeSoftenFloat: 56360784Sdim SplitInteger(GetSoftenedFloat(InOp), Lo, Hi); 57218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 58218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 59193323Sed return; 60223017Sdim case TargetLowering::TypeExpandInteger: 61288943Sdim case TargetLowering::TypeExpandFloat: { 62288943Sdim auto &DL = DAG.getDataLayout(); 63193323Sed // Convert the expanded pieces of the input. 64193323Sed GetExpandedOp(InOp, Lo, Hi); 65288943Sdim if (TLI.hasBigEndianPartOrdering(InVT, DL) != 66288943Sdim TLI.hasBigEndianPartOrdering(OutVT, DL)) 67276479Sdim std::swap(Lo, Hi); 68218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 69218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 70193323Sed return; 71288943Sdim } 72223017Sdim case TargetLowering::TypeSplitVector: 73193323Sed GetSplitVector(InOp, Lo, Hi); 74288943Sdim if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 75198090Srdivacky std::swap(Lo, Hi); 76218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 77218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 78198090Srdivacky return; 79223017Sdim case TargetLowering::TypeScalarizeVector: 80193323Sed // Convert the element instead. 81193323Sed SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi); 82218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 83218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 84193323Sed return; 85223017Sdim case TargetLowering::TypeWidenVector: { 86218893Sdim assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST"); 87193323Sed InOp = GetWidenedVector(InOp); 88261991Sdim EVT LoVT, HiVT; 89276479Sdim std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT); 90276479Sdim std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT); 91288943Sdim if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 92193323Sed std::swap(Lo, Hi); 93218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo); 94218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi); 95193323Sed return; 96193323Sed } 97193323Sed } 98193323Sed 99193724Sed if (InVT.isVector() && OutVT.isInteger()) { 100218893Sdim // Handle cases like i64 = BITCAST v1i64 on x86, where the operand 101193724Sed // is legal but the result is not. 102243830Sdim unsigned NumElems = 2; 103243830Sdim EVT ElemVT = NOutVT; 104243830Sdim EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 105193724Sed 106243830Sdim // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>. 107243830Sdim while (!isTypeLegal(NVT)) { 108243830Sdim unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2; 109243830Sdim // If the element size is smaller than byte, bail. 110243830Sdim if (NewSizeInBits < 8) 111243830Sdim break; 112243830Sdim NumElems *= 2; 113243830Sdim ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits); 114243830Sdim NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems); 115243830Sdim } 116243830Sdim 117193724Sed if (isTypeLegal(NVT)) { 118218893Sdim SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp); 119193724Sed 120243830Sdim SmallVector<SDValue, 8> Vals; 121243830Sdim for (unsigned i = 0; i < NumElems; ++i) 122288943Sdim Vals.push_back(DAG.getNode( 123288943Sdim ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, CastInOp, 124288943Sdim DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); 125243830Sdim 126243830Sdim // Build Lo, Hi pair by pairing extracted elements if needed. 127243830Sdim unsigned Slot = 0; 128243830Sdim for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) { 129243830Sdim // Each iteration will BUILD_PAIR two nodes and append the result until 130243830Sdim // there are only two nodes left, i.e. Lo and Hi. 131243830Sdim SDValue LHS = Vals[Slot]; 132243830Sdim SDValue RHS = Vals[Slot + 1]; 133243830Sdim 134288943Sdim if (DAG.getDataLayout().isBigEndian()) 135243830Sdim std::swap(LHS, RHS); 136243830Sdim 137314564Sdim Vals.push_back(DAG.getNode( 138314564Sdim ISD::BUILD_PAIR, dl, 139314564Sdim EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1), 140314564Sdim LHS, RHS)); 141243830Sdim } 142243830Sdim Lo = Vals[Slot++]; 143243830Sdim Hi = Vals[Slot++]; 144243830Sdim 145288943Sdim if (DAG.getDataLayout().isBigEndian()) 146193724Sed std::swap(Lo, Hi); 147198090Srdivacky 148193724Sed return; 149193724Sed } 150193724Sed } 151193724Sed 152193323Sed // Lower the bit-convert to a store/load from the stack. 153193323Sed assert(NOutVT.isByteSized() && "Expanded type not byte sized!"); 154193323Sed 155193323Sed // Create the stack frame object. Make sure it is aligned for both 156193323Sed // the source and expanded destination types. 157288943Sdim unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment( 158288943Sdim NOutVT.getTypeForEVT(*DAG.getContext())); 159193323Sed SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment); 160193323Sed int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 161296417Sdim MachinePointerInfo PtrInfo = 162296417Sdim MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); 163193323Sed 164193323Sed // Emit a store to the stack slot. 165309124Sdim SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo); 166193323Sed 167193323Sed // Load the first half from the stack slot. 168309124Sdim Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo); 169193323Sed 170193323Sed // Increment the pointer to the other half. 171193323Sed unsigned IncrementSize = NOutVT.getSizeInBits() / 8; 172360784Sdim StackPtr = DAG.getMemBasePlusOffset(StackPtr, IncrementSize, dl); 173193323Sed 174193323Sed // Load the second half from the stack slot. 175218893Sdim Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, 176309124Sdim PtrInfo.getWithOffset(IncrementSize), 177309124Sdim MinAlign(Alignment, IncrementSize)); 178193323Sed 179193323Sed // Handle endianness of the load. 180288943Sdim if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout())) 181193323Sed std::swap(Lo, Hi); 182193323Sed} 183193323Sed 184193323Sedvoid DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo, 185193323Sed SDValue &Hi) { 186193323Sed // Return the operands. 187193323Sed Lo = N->getOperand(0); 188193323Sed Hi = N->getOperand(1); 189193323Sed} 190193323Sed 191193323Sedvoid DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo, 192193323Sed SDValue &Hi) { 193193323Sed GetExpandedOp(N->getOperand(0), Lo, Hi); 194193323Sed SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? 195193323Sed Hi : Lo; 196193323Sed 197193323Sed assert(Part.getValueType() == N->getValueType(0) && 198193323Sed "Type twice as big as expanded type not itself expanded!"); 199193323Sed 200193323Sed GetPairElements(Part, Lo, Hi); 201193323Sed} 202193323Sed 203193323Sedvoid DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, 204193323Sed SDValue &Hi) { 205193323Sed SDValue OldVec = N->getOperand(0); 206193323Sed unsigned OldElts = OldVec.getValueType().getVectorNumElements(); 207239462Sdim EVT OldEltVT = OldVec.getValueType().getVectorElementType(); 208261991Sdim SDLoc dl(N); 209193323Sed 210193323Sed // Convert to a vector of the expanded element type, for example 211193323Sed // <3 x i64> -> <6 x i32>. 212198090Srdivacky EVT OldVT = N->getValueType(0); 213198090Srdivacky EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 214193323Sed 215239462Sdim if (OldVT != OldEltVT) { 216239462Sdim // The result of EXTRACT_VECTOR_ELT may be larger than the element type of 217239462Sdim // the input vector. If so, extend the elements of the input vector to the 218239462Sdim // same bitwidth as the result before expanding. 219239462Sdim assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!"); 220239462Sdim EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts); 221239462Sdim OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0)); 222239462Sdim } 223239462Sdim 224218893Sdim SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 225207618Srdivacky EVT::getVectorVT(*DAG.getContext(), 226207618Srdivacky NewVT, 2*OldElts), 227207618Srdivacky OldVec); 228193323Sed 229193323Sed // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. 230193323Sed SDValue Idx = N->getOperand(1); 231193323Sed 232193323Sed Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 233193323Sed Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 234193323Sed 235193323Sed Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 236288943Sdim DAG.getConstant(1, dl, Idx.getValueType())); 237193323Sed Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx); 238193323Sed 239288943Sdim if (DAG.getDataLayout().isBigEndian()) 240193323Sed std::swap(Lo, Hi); 241193323Sed} 242193323Sed 243193323Sedvoid DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo, 244193323Sed SDValue &Hi) { 245193323Sed assert(ISD::isNormalLoad(N) && "This routine only for normal loads!"); 246261991Sdim SDLoc dl(N); 247193323Sed 248193323Sed LoadSDNode *LD = cast<LoadSDNode>(N); 249360784Sdim assert(!LD->isAtomic() && "Atomics can not be split"); 250276479Sdim EVT ValueVT = LD->getValueType(0); 251276479Sdim EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 252193323Sed SDValue Chain = LD->getChain(); 253193323Sed SDValue Ptr = LD->getBasePtr(); 254193323Sed unsigned Alignment = LD->getAlignment(); 255280031Sdim AAMDNodes AAInfo = LD->getAAInfo(); 256193323Sed 257193323Sed assert(NVT.isByteSized() && "Expanded type not byte sized!"); 258193323Sed 259309124Sdim Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), Alignment, 260309124Sdim LD->getMemOperand()->getFlags(), AAInfo); 261193323Sed 262193323Sed // Increment the pointer to the other half. 263193323Sed unsigned IncrementSize = NVT.getSizeInBits() / 8; 264360784Sdim Ptr = DAG.getMemBasePlusOffset(Ptr, IncrementSize, dl); 265218893Sdim Hi = DAG.getLoad(NVT, dl, Chain, Ptr, 266218893Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 267309124Sdim MinAlign(Alignment, IncrementSize), 268309124Sdim LD->getMemOperand()->getFlags(), AAInfo); 269193323Sed 270193323Sed // Build a factor node to remember that this load is independent of the 271193323Sed // other one. 272193323Sed Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 273193323Sed Hi.getValue(1)); 274193323Sed 275193323Sed // Handle endianness of the load. 276288943Sdim if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 277193323Sed std::swap(Lo, Hi); 278193323Sed 279193323Sed // Modified the chain - switch anything that used the old chain to use 280193323Sed // the new one. 281193323Sed ReplaceValueWith(SDValue(N, 1), Chain); 282193323Sed} 283193323Sed 284193323Sedvoid DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 285210299Sed EVT OVT = N->getValueType(0); 286210299Sed EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 287193323Sed SDValue Chain = N->getOperand(0); 288193323Sed SDValue Ptr = N->getOperand(1); 289261991Sdim SDLoc dl(N); 290210299Sed const unsigned Align = N->getConstantOperandVal(3); 291193323Sed 292210299Sed Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align); 293210299Sed Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0); 294341825Sdim Chain = Hi.getValue(1); 295193323Sed 296193323Sed // Handle endianness of the load. 297288943Sdim if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout())) 298193323Sed std::swap(Lo, Hi); 299193323Sed 300193323Sed // Modified the chain - switch anything that used the old chain to use 301193323Sed // the new one. 302341825Sdim ReplaceValueWith(SDValue(N, 1), Chain); 303193323Sed} 304193323Sed 305193323Sed 306193323Sed//===--------------------------------------------------------------------===// 307193323Sed// Generic Operand Expansion. 308193323Sed//===--------------------------------------------------------------------===// 309193323Sed 310261991Sdimvoid DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements, 311261991Sdim SmallVectorImpl<SDValue> &Ops, 312261991Sdim EVT EltVT) { 313261991Sdim assert(Op.getValueType().isInteger()); 314261991Sdim SDLoc DL(Op); 315261991Sdim SDValue Parts[2]; 316261991Sdim 317261991Sdim if (NumElements > 1) { 318261991Sdim NumElements >>= 1; 319261991Sdim SplitInteger(Op, Parts[0], Parts[1]); 320288943Sdim if (DAG.getDataLayout().isBigEndian()) 321344779Sdim std::swap(Parts[0], Parts[1]); 322261991Sdim IntegerToVector(Parts[0], NumElements, Ops, EltVT); 323261991Sdim IntegerToVector(Parts[1], NumElements, Ops, EltVT); 324261991Sdim } else { 325261991Sdim Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op)); 326261991Sdim } 327261991Sdim} 328261991Sdim 329218893SdimSDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) { 330261991Sdim SDLoc dl(N); 331314564Sdim if (N->getValueType(0).isVector() && 332314564Sdim N->getOperand(0).getValueType().isInteger()) { 333193323Sed // An illegal expanding type is being converted to a legal vector type. 334193323Sed // Make a two element vector out of the expanded parts and convert that 335193323Sed // instead, but only if the new vector type is legal (otherwise there 336193323Sed // is no point, and it might create expansion loops). For example, on 337218893Sdim // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32. 338261991Sdim // 339261991Sdim // FIXME: I'm not sure why we are first trying to split the input into 340261991Sdim // a 2 element vector, so I'm leaving it here to maintain the current 341261991Sdim // behavior. 342261991Sdim unsigned NumElts = 2; 343198090Srdivacky EVT OVT = N->getOperand(0).getValueType(); 344207618Srdivacky EVT NVT = EVT::getVectorVT(*DAG.getContext(), 345207618Srdivacky TLI.getTypeToTransformTo(*DAG.getContext(), OVT), 346261991Sdim NumElts); 347261991Sdim if (!isTypeLegal(NVT)) { 348261991Sdim // If we can't find a legal type by splitting the integer in half, 349261991Sdim // then we can use the node's value type. 350261991Sdim NumElts = N->getValueType(0).getVectorNumElements(); 351261991Sdim NVT = N->getValueType(0); 352261991Sdim } 353193323Sed 354261991Sdim SmallVector<SDValue, 8> Ops; 355261991Sdim IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType()); 356193323Sed 357321369Sdim SDValue Vec = 358321369Sdim DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts)); 359261991Sdim return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec); 360193323Sed } 361193323Sed 362193323Sed // Otherwise, store to a temporary and load out again as the new type. 363193323Sed return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 364193323Sed} 365193323Sed 366193323SedSDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { 367193323Sed // The vector type is legal but the element type needs expansion. 368198090Srdivacky EVT VecVT = N->getValueType(0); 369193323Sed unsigned NumElts = VecVT.getVectorNumElements(); 370198090Srdivacky EVT OldVT = N->getOperand(0).getValueType(); 371198090Srdivacky EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); 372261991Sdim SDLoc dl(N); 373193323Sed 374193323Sed assert(OldVT == VecVT.getVectorElementType() && 375193323Sed "BUILD_VECTOR operand type doesn't match vector element type!"); 376193323Sed 377193323Sed // Build a vector of twice the length out of the expanded elements. 378193323Sed // For example <3 x i64> -> <6 x i32>. 379341825Sdim SmallVector<SDValue, 16> NewElts; 380193323Sed NewElts.reserve(NumElts*2); 381193323Sed 382193323Sed for (unsigned i = 0; i < NumElts; ++i) { 383193323Sed SDValue Lo, Hi; 384193323Sed GetExpandedOp(N->getOperand(i), Lo, Hi); 385288943Sdim if (DAG.getDataLayout().isBigEndian()) 386193323Sed std::swap(Lo, Hi); 387193323Sed NewElts.push_back(Lo); 388193323Sed NewElts.push_back(Hi); 389193323Sed } 390193323Sed 391321369Sdim EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size()); 392321369Sdim SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts); 393193323Sed 394193323Sed // Convert the new vector to the old vector type. 395218893Sdim return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 396193323Sed} 397193323Sed 398193323SedSDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) { 399193323Sed SDValue Lo, Hi; 400193323Sed GetExpandedOp(N->getOperand(0), Lo, Hi); 401193323Sed return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo; 402193323Sed} 403193323Sed 404193323SedSDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) { 405193323Sed // The vector type is legal but the element type needs expansion. 406198090Srdivacky EVT VecVT = N->getValueType(0); 407193323Sed unsigned NumElts = VecVT.getVectorNumElements(); 408261991Sdim SDLoc dl(N); 409193323Sed 410193323Sed SDValue Val = N->getOperand(1); 411198090Srdivacky EVT OldEVT = Val.getValueType(); 412198090Srdivacky EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT); 413193323Sed 414193323Sed assert(OldEVT == VecVT.getVectorElementType() && 415193323Sed "Inserted element type doesn't match vector element type!"); 416193323Sed 417193323Sed // Bitconvert to a vector of twice the length with elements of the expanded 418193323Sed // type, insert the expanded vector elements, and then convert back. 419198090Srdivacky EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2); 420218893Sdim SDValue NewVec = DAG.getNode(ISD::BITCAST, dl, 421193323Sed NewVecVT, N->getOperand(0)); 422193323Sed 423193323Sed SDValue Lo, Hi; 424193323Sed GetExpandedOp(Val, Lo, Hi); 425288943Sdim if (DAG.getDataLayout().isBigEndian()) 426193323Sed std::swap(Lo, Hi); 427193323Sed 428193323Sed SDValue Idx = N->getOperand(2); 429193323Sed Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx); 430193323Sed NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx); 431193323Sed Idx = DAG.getNode(ISD::ADD, dl, 432261991Sdim Idx.getValueType(), Idx, 433288943Sdim DAG.getConstant(1, dl, Idx.getValueType())); 434193323Sed NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx); 435193323Sed 436193323Sed // Convert the new vector to the old vector type. 437218893Sdim return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec); 438193323Sed} 439193323Sed 440193323SedSDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) { 441261991Sdim SDLoc dl(N); 442198090Srdivacky EVT VT = N->getValueType(0); 443193323Sed assert(VT.getVectorElementType() == N->getOperand(0).getValueType() && 444193323Sed "SCALAR_TO_VECTOR operand type doesn't match vector element type!"); 445193323Sed unsigned NumElts = VT.getVectorNumElements(); 446193323Sed SmallVector<SDValue, 16> Ops(NumElts); 447193323Sed Ops[0] = N->getOperand(0); 448193323Sed SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType()); 449193323Sed for (unsigned i = 1; i < NumElts; ++i) 450193323Sed Ops[i] = UndefVal; 451321369Sdim return DAG.getBuildVector(VT, dl, Ops); 452193323Sed} 453193323Sed 454193323SedSDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { 455193323Sed assert(ISD::isNormalStore(N) && "This routine only for normal stores!"); 456193323Sed assert(OpNo == 1 && "Can only expand the stored value so far"); 457261991Sdim SDLoc dl(N); 458193323Sed 459193323Sed StoreSDNode *St = cast<StoreSDNode>(N); 460360784Sdim assert(!St->isAtomic() && "Atomics can not be split"); 461276479Sdim EVT ValueVT = St->getValue().getValueType(); 462276479Sdim EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT); 463193323Sed SDValue Chain = St->getChain(); 464193323Sed SDValue Ptr = St->getBasePtr(); 465193323Sed unsigned Alignment = St->getAlignment(); 466280031Sdim AAMDNodes AAInfo = St->getAAInfo(); 467193323Sed 468193323Sed assert(NVT.isByteSized() && "Expanded type not byte sized!"); 469193323Sed unsigned IncrementSize = NVT.getSizeInBits() / 8; 470193323Sed 471193323Sed SDValue Lo, Hi; 472193323Sed GetExpandedOp(St->getValue(), Lo, Hi); 473193323Sed 474288943Sdim if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 475193323Sed std::swap(Lo, Hi); 476193323Sed 477309124Sdim Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), Alignment, 478309124Sdim St->getMemOperand()->getFlags(), AAInfo); 479193323Sed 480327952Sdim Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 481218893Sdim Hi = DAG.getStore(Chain, dl, Hi, Ptr, 482218893Sdim St->getPointerInfo().getWithOffset(IncrementSize), 483309124Sdim MinAlign(Alignment, IncrementSize), 484309124Sdim St->getMemOperand()->getFlags(), AAInfo); 485193323Sed 486193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 487193323Sed} 488193323Sed 489193323Sed 490193323Sed//===--------------------------------------------------------------------===// 491193323Sed// Generic Result Splitting. 492193323Sed//===--------------------------------------------------------------------===// 493193323Sed 494193323Sed// Be careful to make no assumptions about which of Lo/Hi is stored first in 495193323Sed// memory (for vectors it is always Lo first followed by Hi in the following 496193323Sed// bytes; for integers and floats it is Lo first if and only if the machine is 497193323Sed// little-endian). 498193323Sed 499226633Sdimvoid DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 500193323Sed SDValue &Lo, SDValue &Hi) { 501226633Sdim SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 502226633Sdim GetSplitOp(Op, Lo, Hi); 503193323Sed} 504193323Sed 505321369Sdimvoid DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) { 506226633Sdim SDValue LL, LH, RL, RH, CL, CH; 507261991Sdim SDLoc dl(N); 508193323Sed GetSplitOp(N->getOperand(1), LL, LH); 509193323Sed GetSplitOp(N->getOperand(2), RL, RH); 510193323Sed 511193323Sed SDValue Cond = N->getOperand(0); 512226633Sdim CL = CH = Cond; 513226633Sdim if (Cond.getValueType().isVector()) { 514321369Sdim if (SDValue Res = WidenVSELECTAndMask(N)) 515321369Sdim std::tie(CL, CH) = DAG.SplitVector(Res->getOperand(0), dl); 516261991Sdim // Check if there are already splitted versions of the vector available and 517261991Sdim // use those instead of splitting the mask operand again. 518321369Sdim else if (getTypeAction(Cond.getValueType()) == 519321369Sdim TargetLowering::TypeSplitVector) 520261991Sdim GetSplitVector(Cond, CL, CH); 521360784Sdim // It seems to improve code to generate two narrow SETCCs as opposed to 522360784Sdim // splitting a wide result vector. 523360784Sdim else if (Cond.getOpcode() == ISD::SETCC) { 524360784Sdim // If the condition is a vXi1 vector, and the LHS of the setcc is a legal 525360784Sdim // type and the setcc result type is the same vXi1, then leave the setcc 526360784Sdim // alone. 527360784Sdim EVT CondLHSVT = Cond.getOperand(0).getValueType(); 528360784Sdim if (Cond.getValueType().getVectorElementType() == MVT::i1 && 529360784Sdim isTypeLegal(CondLHSVT) && 530360784Sdim getSetCCResultType(CondLHSVT) == Cond.getValueType()) 531360784Sdim std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 532360784Sdim else 533360784Sdim SplitVecRes_SETCC(Cond.getNode(), CL, CH); 534360784Sdim } else 535276479Sdim std::tie(CL, CH) = DAG.SplitVector(Cond, dl); 536226633Sdim } 537226633Sdim 538226633Sdim Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL); 539226633Sdim Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH); 540193323Sed} 541193323Sed 542193323Sedvoid DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, 543193323Sed SDValue &Hi) { 544193323Sed SDValue LL, LH, RL, RH; 545261991Sdim SDLoc dl(N); 546193323Sed GetSplitOp(N->getOperand(2), LL, LH); 547193323Sed GetSplitOp(N->getOperand(3), RL, RH); 548193323Sed 549193323Sed Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0), 550193323Sed N->getOperand(1), LL, RL, N->getOperand(4)); 551193323Sed Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0), 552193323Sed N->getOperand(1), LH, RH, N->getOperand(4)); 553193323Sed} 554193323Sed 555193323Sedvoid DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { 556198090Srdivacky EVT LoVT, HiVT; 557276479Sdim std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 558193323Sed Lo = DAG.getUNDEF(LoVT); 559193323Sed Hi = DAG.getUNDEF(HiVT); 560193323Sed} 561