LegalizeIntegerTypes.cpp revision 327952
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/IR/DerivedTypes.h" 23#include "llvm/Support/ErrorHandling.h" 24#include "llvm/Support/KnownBits.h" 25#include "llvm/Support/raw_ostream.h" 26using namespace llvm; 27 28#define DEBUG_TYPE "legalize-types" 29 30//===----------------------------------------------------------------------===// 31// Integer Result Promotion 32//===----------------------------------------------------------------------===// 33 34/// PromoteIntegerResult - This method is called when a result of a node is 35/// found to be in need of promotion to a larger type. At this point, the node 36/// may also have invalid operands or may have other results that need 37/// expansion, we just know that (at least) one result needs promotion. 38void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 39 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 40 SDValue Res = SDValue(); 41 42 // See if the target wants to custom expand this node. 43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 44 DEBUG(dbgs() << "Node has been custom expanded, done\n"); 45 return; 46 } 47 48 switch (N->getOpcode()) { 49 default: 50#ifndef NDEBUG 51 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 52 N->dump(&DAG); dbgs() << "\n"; 53#endif 54 llvm_unreachable("Do not know how to promote this operator!"); 55 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 56 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 57 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 58 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 59 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 60 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 61 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 62 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 63 case ISD::CTLZ_ZERO_UNDEF: 64 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 65 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 66 case ISD::CTTZ_ZERO_UNDEF: 67 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 68 case ISD::EXTRACT_VECTOR_ELT: 69 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 70 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 71 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 72 break; 73 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 74 break; 75 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 76 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 77 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 78 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 79 case ISD::SMIN: 80 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 81 case ISD::UMIN: 82 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break; 83 84 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 85 case ISD::SIGN_EXTEND_INREG: 86 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 87 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 88 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 89 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 90 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 91 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 92 93 case ISD::EXTRACT_SUBVECTOR: 94 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 95 case ISD::VECTOR_SHUFFLE: 96 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 97 case ISD::INSERT_VECTOR_ELT: 98 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 99 case ISD::BUILD_VECTOR: 100 Res = PromoteIntRes_BUILD_VECTOR(N); break; 101 case ISD::SCALAR_TO_VECTOR: 102 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 103 case ISD::CONCAT_VECTORS: 104 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 105 106 case ISD::ANY_EXTEND_VECTOR_INREG: 107 case ISD::SIGN_EXTEND_VECTOR_INREG: 108 case ISD::ZERO_EXTEND_VECTOR_INREG: 109 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 110 111 case ISD::SIGN_EXTEND: 112 case ISD::ZERO_EXTEND: 113 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 114 115 case ISD::FP_TO_SINT: 116 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 117 118 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 119 120 case ISD::AND: 121 case ISD::OR: 122 case ISD::XOR: 123 case ISD::ADD: 124 case ISD::SUB: 125 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 126 127 case ISD::SDIV: 128 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 129 130 case ISD::UDIV: 131 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 132 133 case ISD::SADDO: 134 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 135 case ISD::UADDO: 136 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 137 case ISD::SMULO: 138 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 139 140 case ISD::ADDCARRY: 141 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 142 143 case ISD::ATOMIC_LOAD: 144 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 145 146 case ISD::ATOMIC_LOAD_ADD: 147 case ISD::ATOMIC_LOAD_SUB: 148 case ISD::ATOMIC_LOAD_AND: 149 case ISD::ATOMIC_LOAD_OR: 150 case ISD::ATOMIC_LOAD_XOR: 151 case ISD::ATOMIC_LOAD_NAND: 152 case ISD::ATOMIC_LOAD_MIN: 153 case ISD::ATOMIC_LOAD_MAX: 154 case ISD::ATOMIC_LOAD_UMIN: 155 case ISD::ATOMIC_LOAD_UMAX: 156 case ISD::ATOMIC_SWAP: 157 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 158 159 case ISD::ATOMIC_CMP_SWAP: 160 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 161 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 162 break; 163 } 164 165 // If the result is null then the sub-method took care of registering it. 166 if (Res.getNode()) 167 SetPromotedInteger(SDValue(N, ResNo), Res); 168} 169 170SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 171 unsigned ResNo) { 172 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 173 return GetPromotedInteger(Op); 174} 175 176SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 177 // Sign-extend the new bits, and continue the assertion. 178 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 179 return DAG.getNode(ISD::AssertSext, SDLoc(N), 180 Op.getValueType(), Op, N->getOperand(1)); 181} 182 183SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 184 // Zero the new bits, and continue the assertion. 185 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 186 return DAG.getNode(ISD::AssertZext, SDLoc(N), 187 Op.getValueType(), Op, N->getOperand(1)); 188} 189 190SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 191 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 192 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 193 N->getMemoryVT(), ResVT, 194 N->getChain(), N->getBasePtr(), 195 N->getMemOperand()); 196 // Legalize the chain result - switch anything that used the old chain to 197 // use the new one. 198 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 199 return Res; 200} 201 202SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 203 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 204 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 205 N->getMemoryVT(), 206 N->getChain(), N->getBasePtr(), 207 Op2, N->getMemOperand()); 208 // Legalize the chain result - switch anything that used the old chain to 209 // use the new one. 210 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 211 return Res; 212} 213 214SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 215 unsigned ResNo) { 216 if (ResNo == 1) { 217 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 218 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 219 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 220 221 // Only use the result of getSetCCResultType if it is legal, 222 // otherwise just use the promoted result type (NVT). 223 if (!TLI.isTypeLegal(SVT)) 224 SVT = NVT; 225 226 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 227 SDValue Res = DAG.getAtomicCmpSwap( 228 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 229 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 230 N->getMemOperand()); 231 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 232 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 233 return Res.getValue(1); 234 } 235 236 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 237 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 238 SDVTList VTs = 239 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 240 SDValue Res = DAG.getAtomicCmpSwap( 241 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 242 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 243 // Update the use to N with the newly created Res. 244 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 245 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 246 return Res; 247} 248 249SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 250 SDValue InOp = N->getOperand(0); 251 EVT InVT = InOp.getValueType(); 252 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 253 EVT OutVT = N->getValueType(0); 254 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 255 SDLoc dl(N); 256 257 switch (getTypeAction(InVT)) { 258 case TargetLowering::TypeLegal: 259 break; 260 case TargetLowering::TypePromoteInteger: 261 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 262 // The input promotes to the same size. Convert the promoted value. 263 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 264 break; 265 case TargetLowering::TypeSoftenFloat: 266 // Promote the integer operand by hand. 267 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 268 case TargetLowering::TypePromoteFloat: { 269 // Convert the promoted float by hand. 270 SDValue PromotedOp = GetPromotedFloat(InOp); 271 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp); 272 break; 273 } 274 case TargetLowering::TypeExpandInteger: 275 case TargetLowering::TypeExpandFloat: 276 break; 277 case TargetLowering::TypeScalarizeVector: 278 // Convert the element to an integer and promote it by hand. 279 if (!NOutVT.isVector()) 280 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 281 BitConvertToInteger(GetScalarizedVector(InOp))); 282 break; 283 case TargetLowering::TypeSplitVector: { 284 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 285 // pieces of the input into integers and reassemble in the final type. 286 SDValue Lo, Hi; 287 GetSplitVector(N->getOperand(0), Lo, Hi); 288 Lo = BitConvertToInteger(Lo); 289 Hi = BitConvertToInteger(Hi); 290 291 if (DAG.getDataLayout().isBigEndian()) 292 std::swap(Lo, Hi); 293 294 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 295 EVT::getIntegerVT(*DAG.getContext(), 296 NOutVT.getSizeInBits()), 297 JoinIntegers(Lo, Hi)); 298 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 299 } 300 case TargetLowering::TypeWidenVector: 301 // The input is widened to the same size. Convert to the widened value. 302 // Make sure that the outgoing value is not a vector, because this would 303 // make us bitcast between two vectors which are legalized in different ways. 304 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) 305 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 306 } 307 308 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 309 CreateStackStoreLoad(InOp, OutVT)); 310} 311 312SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 313 SDValue Op = GetPromotedInteger(N->getOperand(0)); 314 EVT OVT = N->getValueType(0); 315 EVT NVT = Op.getValueType(); 316 SDLoc dl(N); 317 318 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 319 return DAG.getNode( 320 ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 321 DAG.getConstant(DiffBits, dl, 322 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 323} 324 325SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 326 SDValue Op = GetPromotedInteger(N->getOperand(0)); 327 EVT OVT = N->getValueType(0); 328 EVT NVT = Op.getValueType(); 329 SDLoc dl(N); 330 331 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 332 return DAG.getNode( 333 ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 334 DAG.getConstant(DiffBits, dl, 335 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 336} 337 338SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 339 // The pair element type may be legal, or may not promote to the same type as 340 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 341 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 342 TLI.getTypeToTransformTo(*DAG.getContext(), 343 N->getValueType(0)), JoinIntegers(N->getOperand(0), 344 N->getOperand(1))); 345} 346 347SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 348 EVT VT = N->getValueType(0); 349 // FIXME there is no actual debug info here 350 SDLoc dl(N); 351 // Zero extend things like i1, sign extend everything else. It shouldn't 352 // matter in theory which one we pick, but this tends to give better code? 353 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 354 SDValue Result = DAG.getNode(Opc, dl, 355 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 356 SDValue(N, 0)); 357 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 358 return Result; 359} 360 361SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 362 // Zero extend to the promoted type and do the count there. 363 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 364 SDLoc dl(N); 365 EVT OVT = N->getValueType(0); 366 EVT NVT = Op.getValueType(); 367 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 368 // Subtract off the extra leading bits in the bigger type. 369 return DAG.getNode( 370 ISD::SUB, dl, NVT, Op, 371 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 372 NVT)); 373} 374 375SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 376 // Zero extend to the promoted type and do the count there. 377 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 378 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op); 379} 380 381SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 382 SDValue Op = GetPromotedInteger(N->getOperand(0)); 383 EVT OVT = N->getValueType(0); 384 EVT NVT = Op.getValueType(); 385 SDLoc dl(N); 386 if (N->getOpcode() == ISD::CTTZ) { 387 // The count is the same in the promoted type except if the original 388 // value was zero. This can be handled by setting the bit just off 389 // the top of the original type. 390 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 391 OVT.getScalarSizeInBits()); 392 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 393 } 394 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 395} 396 397SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 398 SDLoc dl(N); 399 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 400 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 401 N->getOperand(1)); 402} 403 404SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 405 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 406 unsigned NewOpc = N->getOpcode(); 407 SDLoc dl(N); 408 409 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 410 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 411 // and SINT conversions are Custom, there is no way to tell which is 412 // preferable. We choose SINT because that's the right thing on PPC.) 413 if (N->getOpcode() == ISD::FP_TO_UINT && 414 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 415 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 416 NewOpc = ISD::FP_TO_SINT; 417 418 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 419 420 // Assert that the converted value fits in the original type. If it doesn't 421 // (eg: because the value being converted is too big), then the result of the 422 // original operation was undefined anyway, so the assert is still correct. 423 // 424 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 425 // before legalization: fp-to-uint16, 65534. -> 0xfffe 426 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 427 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 428 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 429 DAG.getValueType(N->getValueType(0).getScalarType())); 430} 431 432SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 433 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 434 SDLoc dl(N); 435 436 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 437} 438 439SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 440 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 441 SDLoc dl(N); 442 443 if (getTypeAction(N->getOperand(0).getValueType()) 444 == TargetLowering::TypePromoteInteger) { 445 SDValue Res = GetPromotedInteger(N->getOperand(0)); 446 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 447 448 // If the result and operand types are the same after promotion, simplify 449 // to an in-register extension. 450 if (NVT == Res.getValueType()) { 451 // The high bits are not guaranteed to be anything. Insert an extend. 452 if (N->getOpcode() == ISD::SIGN_EXTEND) 453 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 454 DAG.getValueType(N->getOperand(0).getValueType())); 455 if (N->getOpcode() == ISD::ZERO_EXTEND) 456 return DAG.getZeroExtendInReg(Res, dl, 457 N->getOperand(0).getValueType().getScalarType()); 458 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 459 return Res; 460 } 461 } 462 463 // Otherwise, just extend the original operand all the way to the larger type. 464 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 465} 466 467SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 468 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 469 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 470 ISD::LoadExtType ExtType = 471 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 472 SDLoc dl(N); 473 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 474 N->getMemoryVT(), N->getMemOperand()); 475 476 // Legalize the chain result - switch anything that used the old chain to 477 // use the new one. 478 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 479 return Res; 480} 481 482SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 483 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 484 SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0()); 485 486 SDLoc dl(N); 487 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 488 N->getMask(), ExtSrc0, N->getMemoryVT(), 489 N->getMemOperand(), ISD::SEXTLOAD); 490 // Legalize the chain result - switch anything that used the old chain to 491 // use the new one. 492 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 493 return Res; 494} 495 496SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 497 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 498 SDValue ExtSrc0 = GetPromotedInteger(N->getValue()); 499 assert(NVT == ExtSrc0.getValueType() && 500 "Gather result type and the passThru agrument type should be the same"); 501 502 SDLoc dl(N); 503 SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(), 504 N->getIndex()}; 505 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 506 N->getMemoryVT(), dl, Ops, 507 N->getMemOperand()); 508 // Legalize the chain result - switch anything that used the old chain to 509 // use the new one. 510 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 511 return Res; 512} 513 514/// Promote the overflow flag of an overflowing arithmetic node. 515SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 516 // Simply change the return type of the boolean result. 517 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 518 EVT ValueVTs[] = { N->getValueType(0), NVT }; 519 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 520 unsigned NumOps = N->getNumOperands(); 521 assert(NumOps <= 3 && "Too many operands"); 522 if (NumOps == 3) 523 Ops[2] = N->getOperand(2); 524 525 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 526 DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps)); 527 528 // Modified the sum result - switch anything that used the old sum to use 529 // the new one. 530 ReplaceValueWith(SDValue(N, 0), Res); 531 532 return SDValue(Res.getNode(), 1); 533} 534 535SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 536 if (ResNo == 1) 537 return PromoteIntRes_Overflow(N); 538 539 // The operation overflowed iff the result in the larger type is not the 540 // sign extension of its truncation to the original type. 541 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 542 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 543 EVT OVT = N->getOperand(0).getValueType(); 544 EVT NVT = LHS.getValueType(); 545 SDLoc dl(N); 546 547 // Do the arithmetic in the larger type. 548 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 549 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 550 551 // Calculate the overflow flag: sign extend the arithmetic result from 552 // the original type. 553 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 554 DAG.getValueType(OVT)); 555 // Overflowed if and only if this is not equal to Res. 556 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 557 558 // Use the calculated overflow everywhere. 559 ReplaceValueWith(SDValue(N, 1), Ofl); 560 561 return Res; 562} 563 564SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 565 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 566 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 567 return DAG.getSelect(SDLoc(N), 568 LHS.getValueType(), N->getOperand(0), LHS, RHS); 569} 570 571SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 572 SDValue Mask = N->getOperand(0); 573 574 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 575 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 576 return DAG.getNode(ISD::VSELECT, SDLoc(N), 577 LHS.getValueType(), Mask, LHS, RHS); 578} 579 580SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 581 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 582 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 583 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 584 LHS.getValueType(), N->getOperand(0), 585 N->getOperand(1), LHS, RHS, N->getOperand(4)); 586} 587 588SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 589 EVT SVT = getSetCCResultType(N->getOperand(0).getValueType()); 590 591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 592 593 // Only use the result of getSetCCResultType if it is legal, 594 // otherwise just use the promoted result type (NVT). 595 if (!TLI.isTypeLegal(SVT)) 596 SVT = NVT; 597 598 SDLoc dl(N); 599 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && 600 "Vector compare must return a vector result!"); 601 602 SDValue LHS = N->getOperand(0); 603 SDValue RHS = N->getOperand(1); 604 if (LHS.getValueType() != RHS.getValueType()) { 605 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger && 606 !LHS.getValueType().isVector()) 607 LHS = GetPromotedInteger(LHS); 608 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger && 609 !RHS.getValueType().isVector()) 610 RHS = GetPromotedInteger(RHS); 611 } 612 613 // Get the SETCC result using the canonical SETCC type. 614 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS, 615 N->getOperand(2)); 616 617 // Convert to the expected type. 618 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 619} 620 621SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 622 SDValue LHS = N->getOperand(0); 623 SDValue RHS = N->getOperand(1); 624 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 625 LHS = GetPromotedInteger(LHS); 626 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 627 RHS = ZExtPromotedInteger(RHS); 628 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); 629} 630 631SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 632 SDValue Op = GetPromotedInteger(N->getOperand(0)); 633 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 634 Op.getValueType(), Op, N->getOperand(1)); 635} 636 637SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 638 // The input may have strange things in the top bits of the registers, but 639 // these operations don't care. They may have weird bits going out, but 640 // that too is okay if they are integer operations. 641 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 642 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 643 return DAG.getNode(N->getOpcode(), SDLoc(N), 644 LHS.getValueType(), LHS, RHS); 645} 646 647SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 648 // Sign extend the input. 649 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 650 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 651 return DAG.getNode(N->getOpcode(), SDLoc(N), 652 LHS.getValueType(), LHS, RHS); 653} 654 655SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 656 // Zero extend the input. 657 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 658 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 659 return DAG.getNode(N->getOpcode(), SDLoc(N), 660 LHS.getValueType(), LHS, RHS); 661} 662 663SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 664 SDValue LHS = N->getOperand(0); 665 SDValue RHS = N->getOperand(1); 666 // The input value must be properly sign extended. 667 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 668 LHS = SExtPromotedInteger(LHS); 669 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 670 RHS = ZExtPromotedInteger(RHS); 671 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); 672} 673 674SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 675 SDValue LHS = N->getOperand(0); 676 SDValue RHS = N->getOperand(1); 677 // The input value must be properly zero extended. 678 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 679 LHS = ZExtPromotedInteger(LHS); 680 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 681 RHS = ZExtPromotedInteger(RHS); 682 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); 683} 684 685SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 686 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 687 SDValue Res; 688 SDValue InOp = N->getOperand(0); 689 SDLoc dl(N); 690 691 switch (getTypeAction(InOp.getValueType())) { 692 default: llvm_unreachable("Unknown type action!"); 693 case TargetLowering::TypeLegal: 694 case TargetLowering::TypeExpandInteger: 695 Res = InOp; 696 break; 697 case TargetLowering::TypePromoteInteger: 698 Res = GetPromotedInteger(InOp); 699 break; 700 case TargetLowering::TypeSplitVector: { 701 EVT InVT = InOp.getValueType(); 702 assert(InVT.isVector() && "Cannot split scalar types"); 703 unsigned NumElts = InVT.getVectorNumElements(); 704 assert(NumElts == NVT.getVectorNumElements() && 705 "Dst and Src must have the same number of elements"); 706 assert(isPowerOf2_32(NumElts) && 707 "Promoted vector type must be a power of two"); 708 709 SDValue EOp1, EOp2; 710 GetSplitVector(InOp, EOp1, EOp2); 711 712 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 713 NumElts/2); 714 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 715 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 716 717 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 718 } 719 case TargetLowering::TypeWidenVector: { 720 SDValue WideInOp = GetWidenedVector(InOp); 721 722 // Truncate widened InOp. 723 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 724 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 725 N->getValueType(0).getScalarType(), NumElem); 726 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 727 728 // Zero extend so that the elements are of same type as those of NVT 729 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 730 NumElem); 731 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 732 733 // Extract the low NVT subvector. 734 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 735 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy); 736 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 737 } 738 } 739 740 // Truncate to NVT instead of VT 741 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 742} 743 744SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 745 if (ResNo == 1) 746 return PromoteIntRes_Overflow(N); 747 748 // The operation overflowed iff the result in the larger type is not the 749 // zero extension of its truncation to the original type. 750 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 751 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 752 EVT OVT = N->getOperand(0).getValueType(); 753 EVT NVT = LHS.getValueType(); 754 SDLoc dl(N); 755 756 // Do the arithmetic in the larger type. 757 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 758 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 759 760 // Calculate the overflow flag: zero extend the arithmetic result from 761 // the original type. 762 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 763 // Overflowed if and only if this is not equal to Res. 764 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 765 766 // Use the calculated overflow everywhere. 767 ReplaceValueWith(SDValue(N, 1), Ofl); 768 769 return Res; 770} 771 772SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 773 if (ResNo == 1) 774 return PromoteIntRes_Overflow(N); 775 776 // We need to sign-extend the operands so the carry value computed by the 777 // wide operation will be equivalent to the carry value computed by the 778 // narrow operation. 779 // An ADDCARRY can generate carry only if any of the operands has its 780 // most significant bit set. Sign extension propagates the most significant 781 // bit into the higher bits which means the extra bit that the narrow 782 // addition would need (i.e. the carry) will be propagated through the higher 783 // bits of the wide addition. 784 // A SUBCARRY can generate borrow only if LHS < RHS and this property will be 785 // preserved by sign extension. 786 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 787 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 788 789 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; 790 791 // Do the arithmetic in the wide type. 792 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), 793 LHS, RHS, N->getOperand(2)); 794 795 // Update the users of the original carry/borrow value. 796 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 797 798 return SDValue(Res.getNode(), 0); 799} 800 801SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 802 // Promote the overflow bit trivially. 803 if (ResNo == 1) 804 return PromoteIntRes_Overflow(N); 805 806 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 807 SDLoc DL(N); 808 EVT SmallVT = LHS.getValueType(); 809 810 // To determine if the result overflowed in a larger type, we extend the 811 // input to the larger type, do the multiply (checking if it overflows), 812 // then also check the high bits of the result to see if overflow happened 813 // there. 814 if (N->getOpcode() == ISD::SMULO) { 815 LHS = SExtPromotedInteger(LHS); 816 RHS = SExtPromotedInteger(RHS); 817 } else { 818 LHS = ZExtPromotedInteger(LHS); 819 RHS = ZExtPromotedInteger(RHS); 820 } 821 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 822 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 823 824 // Overflow occurred if it occurred in the larger type, or if the high part 825 // of the result does not zero/sign-extend the low part. Check this second 826 // possibility first. 827 SDValue Overflow; 828 if (N->getOpcode() == ISD::UMULO) { 829 // Unsigned overflow occurred if the high part is non-zero. 830 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 831 DAG.getIntPtrConstant(SmallVT.getSizeInBits(), 832 DL)); 833 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 834 DAG.getConstant(0, DL, Hi.getValueType()), 835 ISD::SETNE); 836 } else { 837 // Signed overflow occurred if the high part does not sign extend the low. 838 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 839 Mul, DAG.getValueType(SmallVT)); 840 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 841 } 842 843 // The only other way for overflow to occur is if the multiplication in the 844 // larger type itself overflowed. 845 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 846 SDValue(Mul.getNode(), 1)); 847 848 // Use the calculated overflow everywhere. 849 ReplaceValueWith(SDValue(N, 1), Overflow); 850 return Mul; 851} 852 853SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 854 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 855 N->getValueType(0))); 856} 857 858SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 859 SDValue Chain = N->getOperand(0); // Get the chain. 860 SDValue Ptr = N->getOperand(1); // Get the pointer. 861 EVT VT = N->getValueType(0); 862 SDLoc dl(N); 863 864 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 865 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 866 // The argument is passed as NumRegs registers of type RegVT. 867 868 SmallVector<SDValue, 8> Parts(NumRegs); 869 for (unsigned i = 0; i < NumRegs; ++i) { 870 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 871 N->getConstantOperandVal(3)); 872 Chain = Parts[i].getValue(1); 873 } 874 875 // Handle endianness of the load. 876 if (DAG.getDataLayout().isBigEndian()) 877 std::reverse(Parts.begin(), Parts.end()); 878 879 // Assemble the parts in the promoted type. 880 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 881 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 882 for (unsigned i = 1; i < NumRegs; ++i) { 883 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 884 // Shift it to the right position and "or" it in. 885 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 886 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 887 TLI.getPointerTy(DAG.getDataLayout()))); 888 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 889 } 890 891 // Modified the chain result - switch anything that used the old chain to 892 // use the new one. 893 ReplaceValueWith(SDValue(N, 1), Chain); 894 895 return Res; 896} 897 898//===----------------------------------------------------------------------===// 899// Integer Operand Promotion 900//===----------------------------------------------------------------------===// 901 902/// PromoteIntegerOperand - This method is called when the specified operand of 903/// the specified node is found to need promotion. At this point, all of the 904/// result types of the node are known to be legal, but other operands of the 905/// node may need promotion or expansion as well as the specified one. 906bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 907 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 908 SDValue Res = SDValue(); 909 910 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 911 DEBUG(dbgs() << "Node has been custom lowered, done\n"); 912 return false; 913 } 914 915 switch (N->getOpcode()) { 916 default: 917 #ifndef NDEBUG 918 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 919 N->dump(&DAG); dbgs() << "\n"; 920 #endif 921 llvm_unreachable("Do not know how to promote this operator's operand!"); 922 923 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 924 case ISD::ATOMIC_STORE: 925 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 926 break; 927 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 928 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 929 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 930 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 931 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 932 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 933 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 934 case ISD::INSERT_VECTOR_ELT: 935 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 936 case ISD::SCALAR_TO_VECTOR: 937 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 938 case ISD::VSELECT: 939 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 940 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 941 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 942 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 943 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 944 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 945 OpNo); break; 946 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 947 OpNo); break; 948 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 949 OpNo); break; 950 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 951 OpNo); break; 952 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 953 OpNo); break; 954 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 955 case ISD::FP16_TO_FP: 956 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 957 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 958 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 959 960 case ISD::SHL: 961 case ISD::SRA: 962 case ISD::SRL: 963 case ISD::ROTL: 964 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 965 966 case ISD::ADDCARRY: 967 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 968 } 969 970 // If the result is null, the sub-method took care of registering results etc. 971 if (!Res.getNode()) return false; 972 973 // If the result is N, the sub-method updated N in place. Tell the legalizer 974 // core about this. 975 if (Res.getNode() == N) 976 return true; 977 978 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 979 "Invalid operand expansion"); 980 981 ReplaceValueWith(SDValue(N, 0), Res); 982 return false; 983} 984 985/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 986/// shared among BR_CC, SELECT_CC, and SETCC handlers. 987void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 988 ISD::CondCode CCCode) { 989 // We have to insert explicit sign or zero extends. Note that we could 990 // insert sign extends for ALL conditions, but zero extend is cheaper on 991 // many machines (an AND instead of two shifts), so prefer it. 992 switch (CCCode) { 993 default: llvm_unreachable("Unknown integer comparison!"); 994 case ISD::SETEQ: 995 case ISD::SETNE: { 996 SDValue OpL = GetPromotedInteger(NewLHS); 997 SDValue OpR = GetPromotedInteger(NewRHS); 998 999 // We would prefer to promote the comparison operand with sign extension. 1000 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 1001 // than the width of NewLHS/NewRH, we can avoid inserting real truncate 1002 // instruction, which is redudant eventually. 1003 unsigned OpLEffectiveBits = 1004 OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; 1005 unsigned OpREffectiveBits = 1006 OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; 1007 if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() && 1008 OpREffectiveBits <= NewRHS.getValueSizeInBits()) { 1009 NewLHS = OpL; 1010 NewRHS = OpR; 1011 } else { 1012 NewLHS = ZExtPromotedInteger(NewLHS); 1013 NewRHS = ZExtPromotedInteger(NewRHS); 1014 } 1015 break; 1016 } 1017 case ISD::SETUGE: 1018 case ISD::SETUGT: 1019 case ISD::SETULE: 1020 case ISD::SETULT: 1021 // ALL of these operations will work if we either sign or zero extend 1022 // the operands (including the unsigned comparisons!). Zero extend is 1023 // usually a simpler/cheaper operation, so prefer it. 1024 NewLHS = ZExtPromotedInteger(NewLHS); 1025 NewRHS = ZExtPromotedInteger(NewRHS); 1026 break; 1027 case ISD::SETGE: 1028 case ISD::SETGT: 1029 case ISD::SETLT: 1030 case ISD::SETLE: 1031 NewLHS = SExtPromotedInteger(NewLHS); 1032 NewRHS = SExtPromotedInteger(NewRHS); 1033 break; 1034 } 1035} 1036 1037SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1038 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1039 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1040} 1041 1042SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1043 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1044 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1045 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1046} 1047 1048SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1049 // This should only occur in unusual situations like bitcasting to an 1050 // x86_fp80, so just turn it into a store+load 1051 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1052} 1053 1054SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1055 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1056 1057 SDValue LHS = N->getOperand(2); 1058 SDValue RHS = N->getOperand(3); 1059 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1060 1061 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1062 // legal types. 1063 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1064 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1065 0); 1066} 1067 1068SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1069 assert(OpNo == 1 && "only know how to promote condition"); 1070 1071 // Promote all the way up to the canonical SetCC type. 1072 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1073 1074 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1075 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1076 N->getOperand(2)), 0); 1077} 1078 1079SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1080 // Since the result type is legal, the operands must promote to it. 1081 EVT OVT = N->getOperand(0).getValueType(); 1082 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1083 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1084 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1085 SDLoc dl(N); 1086 1087 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1088 DAG.getConstant(OVT.getSizeInBits(), dl, 1089 TLI.getPointerTy(DAG.getDataLayout()))); 1090 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1091} 1092 1093SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1094 // The vector type is legal but the element type is not. This implies 1095 // that the vector is a power-of-two in length and that the element 1096 // type does not have a strange size (eg: it is not i1). 1097 EVT VecVT = N->getValueType(0); 1098 unsigned NumElts = VecVT.getVectorNumElements(); 1099 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1100 "Legal vector of one illegal element?"); 1101 1102 // Promote the inserted value. The type does not need to match the 1103 // vector element type. Check that any extra bits introduced will be 1104 // truncated away. 1105 assert(N->getOperand(0).getValueSizeInBits() >= 1106 N->getValueType(0).getScalarSizeInBits() && 1107 "Type of inserted value narrower than vector element type!"); 1108 1109 SmallVector<SDValue, 16> NewOps; 1110 for (unsigned i = 0; i < NumElts; ++i) 1111 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1112 1113 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1114} 1115 1116SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1117 unsigned OpNo) { 1118 if (OpNo == 1) { 1119 // Promote the inserted value. This is valid because the type does not 1120 // have to match the vector element type. 1121 1122 // Check that any extra bits introduced will be truncated away. 1123 assert(N->getOperand(1).getValueSizeInBits() >= 1124 N->getValueType(0).getScalarSizeInBits() && 1125 "Type of inserted value narrower than vector element type!"); 1126 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1127 GetPromotedInteger(N->getOperand(1)), 1128 N->getOperand(2)), 1129 0); 1130 } 1131 1132 assert(OpNo == 2 && "Different operand and result vector types?"); 1133 1134 // Promote the index. 1135 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1136 TLI.getVectorIdxTy(DAG.getDataLayout())); 1137 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1138 N->getOperand(1), Idx), 0); 1139} 1140 1141SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1142 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1143 // the operand in place. 1144 return SDValue(DAG.UpdateNodeOperands(N, 1145 GetPromotedInteger(N->getOperand(0))), 0); 1146} 1147 1148SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1149 assert(OpNo == 0 && "Only know how to promote the condition!"); 1150 SDValue Cond = N->getOperand(0); 1151 EVT OpTy = N->getOperand(1).getValueType(); 1152 1153 if (N->getOpcode() == ISD::VSELECT) 1154 if (SDValue Res = WidenVSELECTAndMask(N)) 1155 return Res; 1156 1157 // Promote all the way up to the canonical SetCC type. 1158 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1159 Cond = PromoteTargetBoolean(Cond, OpVT); 1160 1161 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1162 N->getOperand(2)), 0); 1163} 1164 1165SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1166 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1167 1168 SDValue LHS = N->getOperand(0); 1169 SDValue RHS = N->getOperand(1); 1170 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1171 1172 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1173 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1174 N->getOperand(3), N->getOperand(4)), 0); 1175} 1176 1177SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1178 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1179 1180 SDValue LHS = N->getOperand(0); 1181 SDValue RHS = N->getOperand(1); 1182 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1183 1184 // The CC (#2) is always legal. 1185 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1186} 1187 1188SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1189 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1190 ZExtPromotedInteger(N->getOperand(1))), 0); 1191} 1192 1193SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1194 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1195 SDLoc dl(N); 1196 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1197 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1198 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1199} 1200 1201SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1202 return SDValue(DAG.UpdateNodeOperands(N, 1203 SExtPromotedInteger(N->getOperand(0))), 0); 1204} 1205 1206SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1207 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1208 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1209 SDLoc dl(N); 1210 1211 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1212 1213 // Truncate the value and store the result. 1214 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1215 N->getMemoryVT(), N->getMemOperand()); 1216} 1217 1218SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 1219 unsigned OpNo) { 1220 1221 SDValue DataOp = N->getValue(); 1222 EVT DataVT = DataOp.getValueType(); 1223 SDValue Mask = N->getMask(); 1224 SDLoc dl(N); 1225 1226 bool TruncateStore = false; 1227 if (OpNo == 2) { 1228 // Mask comes before the data operand. If the data operand is legal, we just 1229 // promote the mask. 1230 // When the data operand has illegal type, we should legalize the data 1231 // operand first. The mask will be promoted/splitted/widened according to 1232 // the data operand type. 1233 if (TLI.isTypeLegal(DataVT)) { 1234 Mask = PromoteTargetBoolean(Mask, DataVT); 1235 // Update in place. 1236 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1237 NewOps[2] = Mask; 1238 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1239 } 1240 1241 if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) 1242 return PromoteIntOp_MSTORE(N, 3); 1243 if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector) 1244 return WidenVecOp_MSTORE(N, 3); 1245 assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector); 1246 return SplitVecOp_MSTORE(N, 3); 1247 } else { // Data operand 1248 assert(OpNo == 3 && "Unexpected operand for promotion"); 1249 DataOp = GetPromotedInteger(DataOp); 1250 TruncateStore = true; 1251 } 1252 1253 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask, 1254 N->getMemoryVT(), N->getMemOperand(), 1255 TruncateStore, N->isCompressingStore()); 1256} 1257 1258SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 1259 unsigned OpNo) { 1260 assert(OpNo == 2 && "Only know how to promote the mask!"); 1261 EVT DataVT = N->getValueType(0); 1262 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1263 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1264 NewOps[OpNo] = Mask; 1265 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1266} 1267 1268SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 1269 unsigned OpNo) { 1270 1271 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1272 if (OpNo == 2) { 1273 // The Mask 1274 EVT DataVT = N->getValueType(0); 1275 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1276 } else if (OpNo == 4) { 1277 // Need to sign extend the index since the bits will likely be used. 1278 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1279 } else 1280 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1281 1282 SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1283 // updated in place. 1284 if (Res.getNode() == N) 1285 return Res; 1286 1287 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 1288 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1289 return SDValue(); 1290} 1291 1292SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 1293 unsigned OpNo) { 1294 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1295 if (OpNo == 2) { 1296 // The Mask 1297 EVT DataVT = N->getValue().getValueType(); 1298 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1299 } else if (OpNo == 4) { 1300 // Need to sign extend the index since the bits will likely be used. 1301 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); 1302 } else 1303 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1304 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1305} 1306 1307SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1308 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1309 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1310} 1311 1312SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1313 return SDValue(DAG.UpdateNodeOperands(N, 1314 ZExtPromotedInteger(N->getOperand(0))), 0); 1315} 1316 1317SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1318 SDLoc dl(N); 1319 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1320 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1321 return DAG.getZeroExtendInReg(Op, dl, 1322 N->getOperand(0).getValueType().getScalarType()); 1323} 1324 1325SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 1326 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1327 1328 SDValue LHS = N->getOperand(0); 1329 SDValue RHS = N->getOperand(1); 1330 SDValue Carry = N->getOperand(2); 1331 SDLoc DL(N); 1332 1333 auto VT = getSetCCResultType(LHS.getValueType()); 1334 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT); 1335 switch (BoolType) { 1336 case TargetLoweringBase::UndefinedBooleanContent: 1337 Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT); 1338 break; 1339 case TargetLoweringBase::ZeroOrOneBooleanContent: 1340 Carry = DAG.getZExtOrTrunc(Carry, DL, VT); 1341 break; 1342 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 1343 Carry = DAG.getSExtOrTrunc(Carry, DL, VT); 1344 break; 1345 } 1346 1347 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 1348} 1349 1350//===----------------------------------------------------------------------===// 1351// Integer Result Expansion 1352//===----------------------------------------------------------------------===// 1353 1354/// ExpandIntegerResult - This method is called when the specified result of the 1355/// specified node is found to need expansion. At this point, the node may also 1356/// have invalid operands or may have other results that need promotion, we just 1357/// know that (at least) one result needs expansion. 1358void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1359 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 1360 SDValue Lo, Hi; 1361 Lo = Hi = SDValue(); 1362 1363 // See if the target wants to custom expand this node. 1364 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1365 return; 1366 1367 switch (N->getOpcode()) { 1368 default: 1369#ifndef NDEBUG 1370 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1371 N->dump(&DAG); dbgs() << "\n"; 1372#endif 1373 llvm_unreachable("Do not know how to expand the result of this operator!"); 1374 1375 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1376 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1377 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1378 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1379 1380 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1381 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1382 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1383 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1384 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1385 1386 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1387 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1388 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1389 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 1390 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1391 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1392 case ISD::CTLZ_ZERO_UNDEF: 1393 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1394 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1395 case ISD::CTTZ_ZERO_UNDEF: 1396 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1397 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; 1398 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1399 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1400 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1401 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1402 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 1403 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1404 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1405 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1406 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1407 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1408 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1409 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1410 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1411 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1412 1413 case ISD::ATOMIC_LOAD_ADD: 1414 case ISD::ATOMIC_LOAD_SUB: 1415 case ISD::ATOMIC_LOAD_AND: 1416 case ISD::ATOMIC_LOAD_OR: 1417 case ISD::ATOMIC_LOAD_XOR: 1418 case ISD::ATOMIC_LOAD_NAND: 1419 case ISD::ATOMIC_LOAD_MIN: 1420 case ISD::ATOMIC_LOAD_MAX: 1421 case ISD::ATOMIC_LOAD_UMIN: 1422 case ISD::ATOMIC_LOAD_UMAX: 1423 case ISD::ATOMIC_SWAP: 1424 case ISD::ATOMIC_CMP_SWAP: { 1425 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1426 SplitInteger(Tmp.first, Lo, Hi); 1427 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1428 break; 1429 } 1430 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 1431 AtomicSDNode *AN = cast<AtomicSDNode>(N); 1432 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 1433 SDValue Tmp = DAG.getAtomicCmpSwap( 1434 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 1435 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 1436 AN->getMemOperand()); 1437 1438 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 1439 // success simply by comparing the loaded value against the ingoing 1440 // comparison. 1441 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 1442 N->getOperand(2), ISD::SETEQ); 1443 1444 SplitInteger(Tmp, Lo, Hi); 1445 ReplaceValueWith(SDValue(N, 1), Success); 1446 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 1447 break; 1448 } 1449 1450 case ISD::AND: 1451 case ISD::OR: 1452 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1453 1454 case ISD::UMAX: 1455 case ISD::SMAX: 1456 case ISD::UMIN: 1457 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 1458 1459 case ISD::ADD: 1460 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1461 1462 case ISD::ADDC: 1463 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1464 1465 case ISD::ADDE: 1466 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1467 1468 case ISD::ADDCARRY: 1469 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 1470 1471 case ISD::SHL: 1472 case ISD::SRA: 1473 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1474 1475 case ISD::SADDO: 1476 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1477 case ISD::UADDO: 1478 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1479 case ISD::UMULO: 1480 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1481 } 1482 1483 // If Lo/Hi is null, the sub-method took care of registering results etc. 1484 if (Lo.getNode()) 1485 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1486} 1487 1488/// Lower an atomic node to the appropriate builtin call. 1489std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1490 unsigned Opc = Node->getOpcode(); 1491 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1492 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); 1493 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); 1494 1495 return ExpandChainLibCall(LC, Node, false); 1496} 1497 1498/// N is a shift by a value that needs to be expanded, 1499/// and the shift amount is a constant 'Amt'. Expand the operation. 1500void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 1501 SDValue &Lo, SDValue &Hi) { 1502 SDLoc DL(N); 1503 // Expand the incoming operand to be shifted, so that we have its parts 1504 SDValue InL, InH; 1505 GetExpandedInteger(N->getOperand(0), InL, InH); 1506 1507 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 1508 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 1509 if (!Amt) { 1510 Lo = InL; 1511 Hi = InH; 1512 return; 1513 } 1514 1515 EVT NVT = InL.getValueType(); 1516 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1517 unsigned NVTBits = NVT.getSizeInBits(); 1518 EVT ShTy = N->getOperand(1).getValueType(); 1519 1520 if (N->getOpcode() == ISD::SHL) { 1521 if (Amt.ugt(VTBits)) { 1522 Lo = Hi = DAG.getConstant(0, DL, NVT); 1523 } else if (Amt.ugt(NVTBits)) { 1524 Lo = DAG.getConstant(0, DL, NVT); 1525 Hi = DAG.getNode(ISD::SHL, DL, 1526 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1527 } else if (Amt == NVTBits) { 1528 Lo = DAG.getConstant(0, DL, NVT); 1529 Hi = InL; 1530 } else { 1531 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 1532 Hi = DAG.getNode(ISD::OR, DL, NVT, 1533 DAG.getNode(ISD::SHL, DL, NVT, InH, 1534 DAG.getConstant(Amt, DL, ShTy)), 1535 DAG.getNode(ISD::SRL, DL, NVT, InL, 1536 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1537 } 1538 return; 1539 } 1540 1541 if (N->getOpcode() == ISD::SRL) { 1542 if (Amt.ugt(VTBits)) { 1543 Lo = Hi = DAG.getConstant(0, DL, NVT); 1544 } else if (Amt.ugt(NVTBits)) { 1545 Lo = DAG.getNode(ISD::SRL, DL, 1546 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1547 Hi = DAG.getConstant(0, DL, NVT); 1548 } else if (Amt == NVTBits) { 1549 Lo = InH; 1550 Hi = DAG.getConstant(0, DL, NVT); 1551 } else { 1552 Lo = DAG.getNode(ISD::OR, DL, NVT, 1553 DAG.getNode(ISD::SRL, DL, NVT, InL, 1554 DAG.getConstant(Amt, DL, ShTy)), 1555 DAG.getNode(ISD::SHL, DL, NVT, InH, 1556 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1557 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 1558 } 1559 return; 1560 } 1561 1562 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1563 if (Amt.ugt(VTBits)) { 1564 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1565 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1566 } else if (Amt.ugt(NVTBits)) { 1567 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1568 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1569 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1570 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1571 } else if (Amt == NVTBits) { 1572 Lo = InH; 1573 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1574 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1575 } else { 1576 Lo = DAG.getNode(ISD::OR, DL, NVT, 1577 DAG.getNode(ISD::SRL, DL, NVT, InL, 1578 DAG.getConstant(Amt, DL, ShTy)), 1579 DAG.getNode(ISD::SHL, DL, NVT, InH, 1580 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1581 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 1582 } 1583} 1584 1585/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1586/// this shift based on knowledge of the high bit of the shift amount. If we 1587/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1588/// shift amount. 1589bool DAGTypeLegalizer:: 1590ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1591 SDValue Amt = N->getOperand(1); 1592 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1593 EVT ShTy = Amt.getValueType(); 1594 unsigned ShBits = ShTy.getScalarSizeInBits(); 1595 unsigned NVTBits = NVT.getScalarSizeInBits(); 1596 assert(isPowerOf2_32(NVTBits) && 1597 "Expanded integer type size not a power of two!"); 1598 SDLoc dl(N); 1599 1600 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1601 KnownBits Known; 1602 DAG.computeKnownBits(N->getOperand(1), Known); 1603 1604 // If we don't know anything about the high bits, exit. 1605 if (((Known.Zero|Known.One) & HighBitMask) == 0) 1606 return false; 1607 1608 // Get the incoming operand to be shifted. 1609 SDValue InL, InH; 1610 GetExpandedInteger(N->getOperand(0), InL, InH); 1611 1612 // If we know that any of the high bits of the shift amount are one, then we 1613 // can do this as a couple of simple shifts. 1614 if (Known.One.intersects(HighBitMask)) { 1615 // Mask out the high bit, which we know is set. 1616 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1617 DAG.getConstant(~HighBitMask, dl, ShTy)); 1618 1619 switch (N->getOpcode()) { 1620 default: llvm_unreachable("Unknown shift"); 1621 case ISD::SHL: 1622 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 1623 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1624 return true; 1625 case ISD::SRL: 1626 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 1627 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1628 return true; 1629 case ISD::SRA: 1630 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1631 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1632 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1633 return true; 1634 } 1635 } 1636 1637 // If we know that all of the high bits of the shift amount are zero, then we 1638 // can do this as a couple of simple shifts. 1639 if (HighBitMask.isSubsetOf(Known.Zero)) { 1640 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 1641 // shift if x is zero. We can use XOR here because x is known to be smaller 1642 // than 32. 1643 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 1644 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1645 1646 unsigned Op1, Op2; 1647 switch (N->getOpcode()) { 1648 default: llvm_unreachable("Unknown shift"); 1649 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1650 case ISD::SRL: 1651 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1652 } 1653 1654 // When shifting right the arithmetic for Lo and Hi is swapped. 1655 if (N->getOpcode() != ISD::SHL) 1656 std::swap(InL, InH); 1657 1658 // Use a little trick to get the bits that move from Lo to Hi. First 1659 // shift by one bit. 1660 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 1661 // Then compute the remaining shift with amount-1. 1662 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 1663 1664 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 1665 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 1666 1667 if (N->getOpcode() != ISD::SHL) 1668 std::swap(Hi, Lo); 1669 return true; 1670 } 1671 1672 return false; 1673} 1674 1675/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1676/// of any size. 1677bool DAGTypeLegalizer:: 1678ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1679 SDValue Amt = N->getOperand(1); 1680 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1681 EVT ShTy = Amt.getValueType(); 1682 unsigned NVTBits = NVT.getSizeInBits(); 1683 assert(isPowerOf2_32(NVTBits) && 1684 "Expanded integer type size not a power of two!"); 1685 SDLoc dl(N); 1686 1687 // Get the incoming operand to be shifted. 1688 SDValue InL, InH; 1689 GetExpandedInteger(N->getOperand(0), InL, InH); 1690 1691 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 1692 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1693 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1694 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1695 Amt, NVBitsNode, ISD::SETULT); 1696 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1697 Amt, DAG.getConstant(0, dl, ShTy), 1698 ISD::SETEQ); 1699 1700 SDValue LoS, HiS, LoL, HiL; 1701 switch (N->getOpcode()) { 1702 default: llvm_unreachable("Unknown shift"); 1703 case ISD::SHL: 1704 // Short: ShAmt < NVTBits 1705 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1706 HiS = DAG.getNode(ISD::OR, dl, NVT, 1707 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1708 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1709 1710 // Long: ShAmt >= NVTBits 1711 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 1712 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1713 1714 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 1715 Hi = DAG.getSelect(dl, NVT, isZero, InH, 1716 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 1717 return true; 1718 case ISD::SRL: 1719 // Short: ShAmt < NVTBits 1720 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1721 LoS = DAG.getNode(ISD::OR, dl, NVT, 1722 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1723 // FIXME: If Amt is zero, the following shift generates an undefined result 1724 // on some architectures. 1725 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1726 1727 // Long: ShAmt >= NVTBits 1728 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 1729 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1730 1731 Lo = DAG.getSelect(dl, NVT, isZero, InL, 1732 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 1733 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1734 return true; 1735 case ISD::SRA: 1736 // Short: ShAmt < NVTBits 1737 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1738 LoS = DAG.getNode(ISD::OR, dl, NVT, 1739 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1740 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1741 1742 // Long: ShAmt >= NVTBits 1743 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1744 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1745 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1746 1747 Lo = DAG.getSelect(dl, NVT, isZero, InL, 1748 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 1749 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1750 return true; 1751 } 1752} 1753 1754static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 1755 1756 switch (Op) { 1757 default: llvm_unreachable("invalid min/max opcode"); 1758 case ISD::SMAX: 1759 return std::make_pair(ISD::SETGT, ISD::UMAX); 1760 case ISD::UMAX: 1761 return std::make_pair(ISD::SETUGT, ISD::UMAX); 1762 case ISD::SMIN: 1763 return std::make_pair(ISD::SETLT, ISD::UMIN); 1764 case ISD::UMIN: 1765 return std::make_pair(ISD::SETULT, ISD::UMIN); 1766 } 1767} 1768 1769void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 1770 SDValue &Lo, SDValue &Hi) { 1771 SDLoc DL(N); 1772 ISD::NodeType LoOpc; 1773 ISD::CondCode CondC; 1774 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 1775 1776 // Expand the subcomponents. 1777 SDValue LHSL, LHSH, RHSL, RHSH; 1778 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1779 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1780 1781 // Value types 1782 EVT NVT = LHSL.getValueType(); 1783 EVT CCT = getSetCCResultType(NVT); 1784 1785 // Hi part is always the same op 1786 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 1787 1788 // We need to know whether to select Lo part that corresponds to 'winning' 1789 // Hi part or if Hi parts are equal. 1790 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 1791 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 1792 1793 // Lo part corresponding to the 'winning' Hi part 1794 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 1795 1796 // Recursed Lo part if Hi parts are equal, this uses unsigned version 1797 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 1798 1799 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 1800} 1801 1802void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1803 SDValue &Lo, SDValue &Hi) { 1804 SDLoc dl(N); 1805 // Expand the subcomponents. 1806 SDValue LHSL, LHSH, RHSL, RHSH; 1807 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1808 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1809 1810 EVT NVT = LHSL.getValueType(); 1811 SDValue LoOps[2] = { LHSL, RHSL }; 1812 SDValue HiOps[3] = { LHSH, RHSH }; 1813 1814 bool HasOpCarry = TLI.isOperationLegalOrCustom( 1815 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 1816 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1817 if (HasOpCarry) { 1818 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 1819 if (N->getOpcode() == ISD::ADD) { 1820 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 1821 HiOps[2] = Lo.getValue(1); 1822 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 1823 } else { 1824 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 1825 HiOps[2] = Lo.getValue(1); 1826 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 1827 } 1828 return; 1829 } 1830 1831 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1832 // them. TODO: Teach operation legalization how to expand unsupported 1833 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1834 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1835 // generate a value of this type in the expanded code sequence. 1836 bool hasCarry = 1837 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1838 ISD::ADDC : ISD::SUBC, 1839 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1840 1841 if (hasCarry) { 1842 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1843 if (N->getOpcode() == ISD::ADD) { 1844 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1845 HiOps[2] = Lo.getValue(1); 1846 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1847 } else { 1848 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1849 HiOps[2] = Lo.getValue(1); 1850 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1851 } 1852 return; 1853 } 1854 1855 bool hasOVF = 1856 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1857 ISD::UADDO : ISD::USUBO, 1858 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1859 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 1860 1861 if (hasOVF) { 1862 EVT OvfVT = getSetCCResultType(NVT); 1863 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 1864 int RevOpc; 1865 if (N->getOpcode() == ISD::ADD) { 1866 RevOpc = ISD::SUB; 1867 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 1868 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1869 } else { 1870 RevOpc = ISD::ADD; 1871 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 1872 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1873 } 1874 SDValue OVF = Lo.getValue(1); 1875 1876 switch (BoolType) { 1877 case TargetLoweringBase::UndefinedBooleanContent: 1878 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 1879 LLVM_FALLTHROUGH; 1880 case TargetLoweringBase::ZeroOrOneBooleanContent: 1881 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 1882 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 1883 break; 1884 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 1885 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 1886 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 1887 } 1888 return; 1889 } 1890 1891 if (N->getOpcode() == ISD::ADD) { 1892 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 1893 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1894 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 1895 ISD::SETULT); 1896 1897 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { 1898 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); 1899 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 1900 return; 1901 } 1902 1903 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 1904 DAG.getConstant(1, dl, NVT), 1905 DAG.getConstant(0, dl, NVT)); 1906 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 1907 ISD::SETULT); 1908 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 1909 DAG.getConstant(1, dl, NVT), Carry1); 1910 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1911 } else { 1912 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 1913 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1914 SDValue Cmp = 1915 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 1916 LoOps[0], LoOps[1], ISD::SETULT); 1917 1918 SDValue Borrow; 1919 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 1920 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 1921 else 1922 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 1923 DAG.getConstant(0, dl, NVT)); 1924 1925 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1926 } 1927} 1928 1929void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1930 SDValue &Lo, SDValue &Hi) { 1931 // Expand the subcomponents. 1932 SDValue LHSL, LHSH, RHSL, RHSH; 1933 SDLoc dl(N); 1934 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1935 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1936 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1937 SDValue LoOps[2] = { LHSL, RHSL }; 1938 SDValue HiOps[3] = { LHSH, RHSH }; 1939 1940 if (N->getOpcode() == ISD::ADDC) { 1941 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1942 HiOps[2] = Lo.getValue(1); 1943 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1944 } else { 1945 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1946 HiOps[2] = Lo.getValue(1); 1947 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1948 } 1949 1950 // Legalized the flag result - switch anything that used the old flag to 1951 // use the new one. 1952 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1953} 1954 1955void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1956 SDValue &Lo, SDValue &Hi) { 1957 // Expand the subcomponents. 1958 SDValue LHSL, LHSH, RHSL, RHSH; 1959 SDLoc dl(N); 1960 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1961 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1962 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1963 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1964 SDValue HiOps[3] = { LHSH, RHSH }; 1965 1966 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 1967 HiOps[2] = Lo.getValue(1); 1968 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 1969 1970 // Legalized the flag result - switch anything that used the old flag to 1971 // use the new one. 1972 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1973} 1974 1975void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 1976 SDValue &Lo, SDValue &Hi) { 1977 SDValue LHS = N->getOperand(0); 1978 SDValue RHS = N->getOperand(1); 1979 SDLoc dl(N); 1980 1981 SDValue Ovf; 1982 1983 bool HasOpCarry = TLI.isOperationLegalOrCustom( 1984 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 1985 TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 1986 1987 if (HasOpCarry) { 1988 // Expand the subcomponents. 1989 SDValue LHSL, LHSH, RHSL, RHSH; 1990 GetExpandedInteger(LHS, LHSL, LHSH); 1991 GetExpandedInteger(RHS, RHSL, RHSH); 1992 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 1993 SDValue LoOps[2] = { LHSL, RHSL }; 1994 SDValue HiOps[3] = { LHSH, RHSH }; 1995 1996 unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY; 1997 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 1998 HiOps[2] = Lo.getValue(1); 1999 Hi = DAG.getNode(Opc, dl, VTList, HiOps); 2000 2001 Ovf = Hi.getValue(1); 2002 } else { 2003 // Expand the result by simply replacing it with the equivalent 2004 // non-overflow-checking operation. 2005 auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 2006 SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS); 2007 SplitInteger(Sum, Lo, Hi); 2008 2009 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2010 // overflows iff a - b > a. 2011 auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT; 2012 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 2013 } 2014 2015 // Legalized the flag result - switch anything that used the old flag to 2016 // use the new one. 2017 ReplaceValueWith(SDValue(N, 1), Ovf); 2018} 2019 2020void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 2021 SDValue &Lo, SDValue &Hi) { 2022 // Expand the subcomponents. 2023 SDValue LHSL, LHSH, RHSL, RHSH; 2024 SDLoc dl(N); 2025 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2026 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 2027 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 2028 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2029 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 2030 2031 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2032 HiOps[2] = Lo.getValue(1); 2033 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2034 2035 // Legalized the flag result - switch anything that used the old flag to 2036 // use the new one. 2037 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2038} 2039 2040void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 2041 SDValue &Lo, SDValue &Hi) { 2042 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2043 SDLoc dl(N); 2044 SDValue Op = N->getOperand(0); 2045 if (Op.getValueType().bitsLE(NVT)) { 2046 // The low part is any extension of the input (which degenerates to a copy). 2047 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 2048 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 2049 } else { 2050 // For example, extension of an i48 to an i64. The operand type necessarily 2051 // promotes to the result type, so will end up being expanded too. 2052 assert(getTypeAction(Op.getValueType()) == 2053 TargetLowering::TypePromoteInteger && 2054 "Only know how to promote this result!"); 2055 SDValue Res = GetPromotedInteger(Op); 2056 assert(Res.getValueType() == N->getValueType(0) && 2057 "Operand over promoted?"); 2058 // Split the promoted operand. This will simplify when it is expanded. 2059 SplitInteger(Res, Lo, Hi); 2060 } 2061} 2062 2063void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 2064 SDValue &Lo, SDValue &Hi) { 2065 SDLoc dl(N); 2066 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2067 EVT NVT = Lo.getValueType(); 2068 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2069 unsigned NVTBits = NVT.getSizeInBits(); 2070 unsigned EVTBits = EVT.getSizeInBits(); 2071 2072 if (NVTBits < EVTBits) { 2073 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 2074 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2075 EVTBits - NVTBits))); 2076 } else { 2077 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 2078 // The high part replicates the sign bit of Lo, make it explicit. 2079 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2080 DAG.getConstant(NVTBits - 1, dl, 2081 TLI.getPointerTy(DAG.getDataLayout()))); 2082 } 2083} 2084 2085void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 2086 SDValue &Lo, SDValue &Hi) { 2087 SDLoc dl(N); 2088 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2089 EVT NVT = Lo.getValueType(); 2090 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2091 unsigned NVTBits = NVT.getSizeInBits(); 2092 unsigned EVTBits = EVT.getSizeInBits(); 2093 2094 if (NVTBits < EVTBits) { 2095 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 2096 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2097 EVTBits - NVTBits))); 2098 } else { 2099 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 2100 // The high part must be zero, make it explicit. 2101 Hi = DAG.getConstant(0, dl, NVT); 2102 } 2103} 2104 2105void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 2106 SDValue &Lo, SDValue &Hi) { 2107 SDLoc dl(N); 2108 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2109 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 2110 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 2111} 2112 2113void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 2114 SDValue &Lo, SDValue &Hi) { 2115 SDLoc dl(N); 2116 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2117 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 2118 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 2119} 2120 2121void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 2122 SDValue &Lo, SDValue &Hi) { 2123 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2124 unsigned NBitWidth = NVT.getSizeInBits(); 2125 auto Constant = cast<ConstantSDNode>(N); 2126 const APInt &Cst = Constant->getAPIntValue(); 2127 bool IsTarget = Constant->isTargetOpcode(); 2128 bool IsOpaque = Constant->isOpaque(); 2129 SDLoc dl(N); 2130 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 2131 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 2132 IsOpaque); 2133} 2134 2135void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 2136 SDValue &Lo, SDValue &Hi) { 2137 SDLoc dl(N); 2138 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 2139 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2140 EVT NVT = Lo.getValueType(); 2141 2142 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 2143 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2144 2145 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 2146 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 2147 2148 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 2149 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 2150 DAG.getConstant(NVT.getSizeInBits(), dl, 2151 NVT))); 2152 Hi = DAG.getConstant(0, dl, NVT); 2153} 2154 2155void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 2156 SDValue &Lo, SDValue &Hi) { 2157 SDLoc dl(N); 2158 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 2159 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2160 EVT NVT = Lo.getValueType(); 2161 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 2162 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 2163 Hi = DAG.getConstant(0, dl, NVT); 2164} 2165 2166void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 2167 SDValue &Lo, SDValue &Hi) { 2168 SDLoc dl(N); 2169 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 2170 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2171 EVT NVT = Lo.getValueType(); 2172 2173 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 2174 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2175 2176 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 2177 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 2178 2179 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 2180 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 2181 DAG.getConstant(NVT.getSizeInBits(), dl, 2182 NVT))); 2183 Hi = DAG.getConstant(0, dl, NVT); 2184} 2185 2186void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, 2187 SDValue &Hi) { 2188 SDLoc dl(N); 2189 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2190 unsigned NBitWidth = NVT.getSizeInBits(); 2191 2192 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2193 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT); 2194 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS 2195 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2196 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); 2197} 2198 2199void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 2200 SDValue &Hi) { 2201 SDLoc dl(N); 2202 EVT VT = N->getValueType(0); 2203 2204 SDValue Op = N->getOperand(0); 2205 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2206 Op = GetPromotedFloat(Op); 2207 2208 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 2209 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 2210 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first, 2211 Lo, Hi); 2212} 2213 2214void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 2215 SDValue &Hi) { 2216 SDLoc dl(N); 2217 EVT VT = N->getValueType(0); 2218 2219 SDValue Op = N->getOperand(0); 2220 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2221 Op = GetPromotedFloat(Op); 2222 2223 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 2224 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 2225 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first, 2226 Lo, Hi); 2227} 2228 2229void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 2230 SDValue &Lo, SDValue &Hi) { 2231 if (ISD::isNormalLoad(N)) { 2232 ExpandRes_NormalLoad(N, Lo, Hi); 2233 return; 2234 } 2235 2236 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 2237 2238 EVT VT = N->getValueType(0); 2239 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2240 SDValue Ch = N->getChain(); 2241 SDValue Ptr = N->getBasePtr(); 2242 ISD::LoadExtType ExtType = N->getExtensionType(); 2243 unsigned Alignment = N->getAlignment(); 2244 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 2245 AAMDNodes AAInfo = N->getAAInfo(); 2246 SDLoc dl(N); 2247 2248 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2249 2250 if (N->getMemoryVT().bitsLE(NVT)) { 2251 EVT MemVT = N->getMemoryVT(); 2252 2253 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 2254 Alignment, MMOFlags, AAInfo); 2255 2256 // Remember the chain. 2257 Ch = Lo.getValue(1); 2258 2259 if (ExtType == ISD::SEXTLOAD) { 2260 // The high part is obtained by SRA'ing all but one of the bits of the 2261 // lo part. 2262 unsigned LoSize = Lo.getValueSizeInBits(); 2263 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2264 DAG.getConstant(LoSize - 1, dl, 2265 TLI.getPointerTy(DAG.getDataLayout()))); 2266 } else if (ExtType == ISD::ZEXTLOAD) { 2267 // The high part is just a zero. 2268 Hi = DAG.getConstant(0, dl, NVT); 2269 } else { 2270 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 2271 // The high part is undefined. 2272 Hi = DAG.getUNDEF(NVT); 2273 } 2274 } else if (DAG.getDataLayout().isLittleEndian()) { 2275 // Little-endian - low bits are at low addresses. 2276 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 2277 AAInfo); 2278 2279 unsigned ExcessBits = 2280 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2281 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2282 2283 // Increment the pointer to the other half. 2284 unsigned IncrementSize = NVT.getSizeInBits()/8; 2285 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2286 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 2287 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 2288 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 2289 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 2290 2291 // Build a factor node to remember that this load is independent of the 2292 // other one. 2293 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2294 Hi.getValue(1)); 2295 } else { 2296 // Big-endian - high bits are at low addresses. Favor aligned loads at 2297 // the cost of some bit-fiddling. 2298 EVT MemVT = N->getMemoryVT(); 2299 unsigned EBytes = MemVT.getStoreSize(); 2300 unsigned IncrementSize = NVT.getSizeInBits()/8; 2301 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2302 2303 // Load both the high bits and maybe some of the low bits. 2304 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 2305 EVT::getIntegerVT(*DAG.getContext(), 2306 MemVT.getSizeInBits() - ExcessBits), 2307 Alignment, MMOFlags, AAInfo); 2308 2309 // Increment the pointer to the other half. 2310 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2311 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 2312 // Load the rest of the low bits. 2313 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 2314 N->getPointerInfo().getWithOffset(IncrementSize), 2315 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2316 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 2317 2318 // Build a factor node to remember that this load is independent of the 2319 // other one. 2320 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2321 Hi.getValue(1)); 2322 2323 if (ExcessBits < NVT.getSizeInBits()) { 2324 // Transfer low bits from the bottom of Hi to the top of Lo. 2325 Lo = DAG.getNode( 2326 ISD::OR, dl, NVT, Lo, 2327 DAG.getNode(ISD::SHL, dl, NVT, Hi, 2328 DAG.getConstant(ExcessBits, dl, 2329 TLI.getPointerTy(DAG.getDataLayout())))); 2330 // Move high bits to the right position in Hi. 2331 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 2332 Hi, 2333 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 2334 TLI.getPointerTy(DAG.getDataLayout()))); 2335 } 2336 } 2337 2338 // Legalize the chain result - switch anything that used the old chain to 2339 // use the new one. 2340 ReplaceValueWith(SDValue(N, 1), Ch); 2341} 2342 2343void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 2344 SDValue &Lo, SDValue &Hi) { 2345 SDLoc dl(N); 2346 SDValue LL, LH, RL, RH; 2347 GetExpandedInteger(N->getOperand(0), LL, LH); 2348 GetExpandedInteger(N->getOperand(1), RL, RH); 2349 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 2350 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 2351} 2352 2353void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 2354 SDValue &Lo, SDValue &Hi) { 2355 EVT VT = N->getValueType(0); 2356 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2357 SDLoc dl(N); 2358 2359 SDValue LL, LH, RL, RH; 2360 GetExpandedInteger(N->getOperand(0), LL, LH); 2361 GetExpandedInteger(N->getOperand(1), RL, RH); 2362 2363 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 2364 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 2365 LL, LH, RL, RH)) 2366 return; 2367 2368 // If nothing else, we can make a libcall. 2369 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2370 if (VT == MVT::i16) 2371 LC = RTLIB::MUL_I16; 2372 else if (VT == MVT::i32) 2373 LC = RTLIB::MUL_I32; 2374 else if (VT == MVT::i64) 2375 LC = RTLIB::MUL_I64; 2376 else if (VT == MVT::i128) 2377 LC = RTLIB::MUL_I128; 2378 2379 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 2380 // We'll expand the multiplication by brute force because we have no other 2381 // options. This is a trivially-generalized version of the code from 2382 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 2383 // 4.3.1). 2384 unsigned Bits = NVT.getSizeInBits(); 2385 unsigned HalfBits = Bits >> 1; 2386 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 2387 NVT); 2388 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 2389 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 2390 2391 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 2392 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 2393 2394 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2395 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) { 2396 // The type from TLI is too small to fit the shift amount we want. 2397 // Override it with i32. The shift will have to be legalized. 2398 ShiftAmtTy = MVT::i32; 2399 } 2400 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); 2401 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 2402 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 2403 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 2404 2405 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 2406 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 2407 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 2408 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 2409 2410 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 2411 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 2412 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 2413 2414 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 2415 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 2416 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 2417 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 2418 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 2419 2420 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 2421 DAG.getNode(ISD::ADD, dl, NVT, 2422 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 2423 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 2424 return; 2425 } 2426 2427 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2428 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first, 2429 Lo, Hi); 2430} 2431 2432void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 2433 SDValue &Hi) { 2434 SDLoc DL(N); 2435 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2436 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 2437 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 2438 Lo = R.getValue(0); 2439 Hi = R.getValue(1); 2440 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 2441} 2442 2443void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 2444 SDValue &Lo, SDValue &Hi) { 2445 SDValue LHS = Node->getOperand(0); 2446 SDValue RHS = Node->getOperand(1); 2447 SDLoc dl(Node); 2448 2449 // Expand the result by simply replacing it with the equivalent 2450 // non-overflow-checking operation. 2451 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 2452 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2453 LHS, RHS); 2454 SplitInteger(Sum, Lo, Hi); 2455 2456 // Compute the overflow. 2457 // 2458 // LHSSign -> LHS >= 0 2459 // RHSSign -> RHS >= 0 2460 // SumSign -> Sum >= 0 2461 // 2462 // Add: 2463 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 2464 // Sub: 2465 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 2466 // 2467 EVT OType = Node->getValueType(1); 2468 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); 2469 2470 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 2471 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 2472 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 2473 Node->getOpcode() == ISD::SADDO ? 2474 ISD::SETEQ : ISD::SETNE); 2475 2476 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 2477 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 2478 2479 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 2480 2481 // Use the calculated overflow everywhere. 2482 ReplaceValueWith(SDValue(Node, 1), Cmp); 2483} 2484 2485void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 2486 SDValue &Lo, SDValue &Hi) { 2487 EVT VT = N->getValueType(0); 2488 SDLoc dl(N); 2489 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2490 2491 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 2492 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2493 SplitInteger(Res.getValue(0), Lo, Hi); 2494 return; 2495 } 2496 2497 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2498 if (VT == MVT::i16) 2499 LC = RTLIB::SDIV_I16; 2500 else if (VT == MVT::i32) 2501 LC = RTLIB::SDIV_I32; 2502 else if (VT == MVT::i64) 2503 LC = RTLIB::SDIV_I64; 2504 else if (VT == MVT::i128) 2505 LC = RTLIB::SDIV_I128; 2506 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 2507 2508 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); 2509} 2510 2511void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 2512 SDValue &Lo, SDValue &Hi) { 2513 EVT VT = N->getValueType(0); 2514 SDLoc dl(N); 2515 2516 // If we can emit an efficient shift operation, do so now. Check to see if 2517 // the RHS is a constant. 2518 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2519 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 2520 2521 // If we can determine that the high bit of the shift is zero or one, even if 2522 // the low bits are variable, emit this shift in an optimized form. 2523 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 2524 return; 2525 2526 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 2527 unsigned PartsOpc; 2528 if (N->getOpcode() == ISD::SHL) { 2529 PartsOpc = ISD::SHL_PARTS; 2530 } else if (N->getOpcode() == ISD::SRL) { 2531 PartsOpc = ISD::SRL_PARTS; 2532 } else { 2533 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2534 PartsOpc = ISD::SRA_PARTS; 2535 } 2536 2537 // Next check to see if the target supports this SHL_PARTS operation or if it 2538 // will custom expand it. 2539 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2540 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 2541 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 2542 Action == TargetLowering::Custom) { 2543 // Expand the subcomponents. 2544 SDValue LHSL, LHSH; 2545 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2546 EVT VT = LHSL.getValueType(); 2547 2548 // If the shift amount operand is coming from a vector legalization it may 2549 // have an illegal type. Fix that first by casting the operand, otherwise 2550 // the new SHL_PARTS operation would need further legalization. 2551 SDValue ShiftOp = N->getOperand(1); 2552 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2553 assert(ShiftTy.getScalarSizeInBits() >= 2554 Log2_32_Ceil(VT.getScalarSizeInBits()) && 2555 "ShiftAmountTy is too small to cover the range of this type!"); 2556 if (ShiftOp.getValueType() != ShiftTy) 2557 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 2558 2559 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 2560 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 2561 Hi = Lo.getValue(1); 2562 return; 2563 } 2564 2565 // Otherwise, emit a libcall. 2566 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2567 bool isSigned; 2568 if (N->getOpcode() == ISD::SHL) { 2569 isSigned = false; /*sign irrelevant*/ 2570 if (VT == MVT::i16) 2571 LC = RTLIB::SHL_I16; 2572 else if (VT == MVT::i32) 2573 LC = RTLIB::SHL_I32; 2574 else if (VT == MVT::i64) 2575 LC = RTLIB::SHL_I64; 2576 else if (VT == MVT::i128) 2577 LC = RTLIB::SHL_I128; 2578 } else if (N->getOpcode() == ISD::SRL) { 2579 isSigned = false; 2580 if (VT == MVT::i16) 2581 LC = RTLIB::SRL_I16; 2582 else if (VT == MVT::i32) 2583 LC = RTLIB::SRL_I32; 2584 else if (VT == MVT::i64) 2585 LC = RTLIB::SRL_I64; 2586 else if (VT == MVT::i128) 2587 LC = RTLIB::SRL_I128; 2588 } else { 2589 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2590 isSigned = true; 2591 if (VT == MVT::i16) 2592 LC = RTLIB::SRA_I16; 2593 else if (VT == MVT::i32) 2594 LC = RTLIB::SRA_I32; 2595 else if (VT == MVT::i64) 2596 LC = RTLIB::SRA_I64; 2597 else if (VT == MVT::i128) 2598 LC = RTLIB::SRA_I128; 2599 } 2600 2601 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2602 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2603 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi); 2604 return; 2605 } 2606 2607 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2608 llvm_unreachable("Unsupported shift!"); 2609} 2610 2611void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2612 SDValue &Lo, SDValue &Hi) { 2613 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2614 SDLoc dl(N); 2615 SDValue Op = N->getOperand(0); 2616 if (Op.getValueType().bitsLE(NVT)) { 2617 // The low part is sign extension of the input (degenerates to a copy). 2618 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2619 // The high part is obtained by SRA'ing all but one of the bits of low part. 2620 unsigned LoSize = NVT.getSizeInBits(); 2621 Hi = DAG.getNode( 2622 ISD::SRA, dl, NVT, Lo, 2623 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 2624 } else { 2625 // For example, extension of an i48 to an i64. The operand type necessarily 2626 // promotes to the result type, so will end up being expanded too. 2627 assert(getTypeAction(Op.getValueType()) == 2628 TargetLowering::TypePromoteInteger && 2629 "Only know how to promote this result!"); 2630 SDValue Res = GetPromotedInteger(Op); 2631 assert(Res.getValueType() == N->getValueType(0) && 2632 "Operand over promoted?"); 2633 // Split the promoted operand. This will simplify when it is expanded. 2634 SplitInteger(Res, Lo, Hi); 2635 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 2636 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2637 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2638 ExcessBits))); 2639 } 2640} 2641 2642void DAGTypeLegalizer:: 2643ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2644 SDLoc dl(N); 2645 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2646 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2647 2648 if (EVT.bitsLE(Lo.getValueType())) { 2649 // sext_inreg the low part if needed. 2650 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2651 N->getOperand(1)); 2652 2653 // The high part gets the sign extension from the lo-part. This handles 2654 // things like sextinreg V:i64 from i8. 2655 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2656 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 2657 TLI.getPointerTy(DAG.getDataLayout()))); 2658 } else { 2659 // For example, extension of an i48 to an i64. Leave the low part alone, 2660 // sext_inreg the high part. 2661 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 2662 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2663 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2664 ExcessBits))); 2665 } 2666} 2667 2668void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2669 SDValue &Lo, SDValue &Hi) { 2670 EVT VT = N->getValueType(0); 2671 SDLoc dl(N); 2672 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2673 2674 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 2675 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2676 SplitInteger(Res.getValue(1), Lo, Hi); 2677 return; 2678 } 2679 2680 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2681 if (VT == MVT::i16) 2682 LC = RTLIB::SREM_I16; 2683 else if (VT == MVT::i32) 2684 LC = RTLIB::SREM_I32; 2685 else if (VT == MVT::i64) 2686 LC = RTLIB::SREM_I64; 2687 else if (VT == MVT::i128) 2688 LC = RTLIB::SREM_I128; 2689 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2690 2691 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); 2692} 2693 2694void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2695 SDValue &Lo, SDValue &Hi) { 2696 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2697 SDLoc dl(N); 2698 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2699 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 2700 N->getOperand(0), 2701 DAG.getConstant(NVT.getSizeInBits(), dl, 2702 TLI.getPointerTy(DAG.getDataLayout()))); 2703 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2704} 2705 2706void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2707 SDValue &Lo, SDValue &Hi) { 2708 EVT VT = N->getValueType(0); 2709 SDLoc dl(N); 2710 2711 // A divide for UMULO should be faster than a function call. 2712 if (N->getOpcode() == ISD::UMULO) { 2713 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2714 2715 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS); 2716 SplitInteger(MUL, Lo, Hi); 2717 2718 // A divide for UMULO will be faster than a function call. Select to 2719 // make sure we aren't using 0. 2720 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT), 2721 RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ); 2722 SDValue NotZero = DAG.getSelect(dl, VT, isZero, 2723 DAG.getConstant(1, dl, VT), RHS); 2724 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero); 2725 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS, 2726 ISD::SETNE); 2727 Overflow = DAG.getSelect(dl, N->getValueType(1), isZero, 2728 DAG.getConstant(0, dl, N->getValueType(1)), 2729 Overflow); 2730 ReplaceValueWith(SDValue(N, 1), Overflow); 2731 return; 2732 } 2733 2734 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2735 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 2736 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2737 2738 // Replace this with a libcall that will check overflow. 2739 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2740 if (VT == MVT::i32) 2741 LC = RTLIB::MULO_I32; 2742 else if (VT == MVT::i64) 2743 LC = RTLIB::MULO_I64; 2744 else if (VT == MVT::i128) 2745 LC = RTLIB::MULO_I128; 2746 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2747 2748 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2749 // Temporary for the overflow value, default it to zero. 2750 SDValue Chain = 2751 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 2752 MachinePointerInfo()); 2753 2754 TargetLowering::ArgListTy Args; 2755 TargetLowering::ArgListEntry Entry; 2756 for (const SDValue &Op : N->op_values()) { 2757 EVT ArgVT = Op.getValueType(); 2758 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2759 Entry.Node = Op; 2760 Entry.Ty = ArgTy; 2761 Entry.IsSExt = true; 2762 Entry.IsZExt = false; 2763 Args.push_back(Entry); 2764 } 2765 2766 // Also pass the address of the overflow check. 2767 Entry.Node = Temp; 2768 Entry.Ty = PtrTy->getPointerTo(); 2769 Entry.IsSExt = true; 2770 Entry.IsZExt = false; 2771 Args.push_back(Entry); 2772 2773 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2774 2775 TargetLowering::CallLoweringInfo CLI(DAG); 2776 CLI.setDebugLoc(dl) 2777 .setChain(Chain) 2778 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 2779 .setSExtResult(); 2780 2781 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2782 2783 SplitInteger(CallInfo.first, Lo, Hi); 2784 SDValue Temp2 = 2785 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 2786 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2787 DAG.getConstant(0, dl, PtrVT), 2788 ISD::SETNE); 2789 // Use the overflow from the libcall everywhere. 2790 ReplaceValueWith(SDValue(N, 1), Ofl); 2791} 2792 2793void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2794 SDValue &Lo, SDValue &Hi) { 2795 EVT VT = N->getValueType(0); 2796 SDLoc dl(N); 2797 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2798 2799 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 2800 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2801 SplitInteger(Res.getValue(0), Lo, Hi); 2802 return; 2803 } 2804 2805 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2806 if (VT == MVT::i16) 2807 LC = RTLIB::UDIV_I16; 2808 else if (VT == MVT::i32) 2809 LC = RTLIB::UDIV_I32; 2810 else if (VT == MVT::i64) 2811 LC = RTLIB::UDIV_I64; 2812 else if (VT == MVT::i128) 2813 LC = RTLIB::UDIV_I128; 2814 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2815 2816 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); 2817} 2818 2819void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2820 SDValue &Lo, SDValue &Hi) { 2821 EVT VT = N->getValueType(0); 2822 SDLoc dl(N); 2823 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2824 2825 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 2826 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2827 SplitInteger(Res.getValue(1), Lo, Hi); 2828 return; 2829 } 2830 2831 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2832 if (VT == MVT::i16) 2833 LC = RTLIB::UREM_I16; 2834 else if (VT == MVT::i32) 2835 LC = RTLIB::UREM_I32; 2836 else if (VT == MVT::i64) 2837 LC = RTLIB::UREM_I64; 2838 else if (VT == MVT::i128) 2839 LC = RTLIB::UREM_I128; 2840 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2841 2842 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); 2843} 2844 2845void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2846 SDValue &Lo, SDValue &Hi) { 2847 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2848 SDLoc dl(N); 2849 SDValue Op = N->getOperand(0); 2850 if (Op.getValueType().bitsLE(NVT)) { 2851 // The low part is zero extension of the input (degenerates to a copy). 2852 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2853 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 2854 } else { 2855 // For example, extension of an i48 to an i64. The operand type necessarily 2856 // promotes to the result type, so will end up being expanded too. 2857 assert(getTypeAction(Op.getValueType()) == 2858 TargetLowering::TypePromoteInteger && 2859 "Only know how to promote this result!"); 2860 SDValue Res = GetPromotedInteger(Op); 2861 assert(Res.getValueType() == N->getValueType(0) && 2862 "Operand over promoted?"); 2863 // Split the promoted operand. This will simplify when it is expanded. 2864 SplitInteger(Res, Lo, Hi); 2865 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 2866 Hi = DAG.getZeroExtendInReg(Hi, dl, 2867 EVT::getIntegerVT(*DAG.getContext(), 2868 ExcessBits)); 2869 } 2870} 2871 2872void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 2873 SDValue &Lo, SDValue &Hi) { 2874 SDLoc dl(N); 2875 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 2876 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 2877 SDValue Zero = DAG.getConstant(0, dl, VT); 2878 SDValue Swap = DAG.getAtomicCmpSwap( 2879 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 2880 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 2881 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 2882 2883 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2884 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 2885} 2886 2887//===----------------------------------------------------------------------===// 2888// Integer Operand Expansion 2889//===----------------------------------------------------------------------===// 2890 2891/// ExpandIntegerOperand - This method is called when the specified operand of 2892/// the specified node is found to need expansion. At this point, all of the 2893/// result types of the node are known to be legal, but other operands of the 2894/// node may need promotion or expansion as well as the specified one. 2895bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2896 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2897 SDValue Res = SDValue(); 2898 2899 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2900 return false; 2901 2902 switch (N->getOpcode()) { 2903 default: 2904 #ifndef NDEBUG 2905 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2906 N->dump(&DAG); dbgs() << "\n"; 2907 #endif 2908 llvm_unreachable("Do not know how to expand this operator's operand!"); 2909 2910 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2911 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2912 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2913 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2914 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2915 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2916 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2917 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2918 case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break; 2919 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 2920 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2921 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2922 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2923 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2924 2925 case ISD::SHL: 2926 case ISD::SRA: 2927 case ISD::SRL: 2928 case ISD::ROTL: 2929 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2930 case ISD::RETURNADDR: 2931 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2932 2933 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 2934 } 2935 2936 // If the result is null, the sub-method took care of registering results etc. 2937 if (!Res.getNode()) return false; 2938 2939 // If the result is N, the sub-method updated N in place. Tell the legalizer 2940 // core about this. 2941 if (Res.getNode() == N) 2942 return true; 2943 2944 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2945 "Invalid operand expansion"); 2946 2947 ReplaceValueWith(SDValue(N, 0), Res); 2948 return false; 2949} 2950 2951/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2952/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2953void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2954 SDValue &NewRHS, 2955 ISD::CondCode &CCCode, 2956 const SDLoc &dl) { 2957 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2958 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2959 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2960 2961 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2962 if (RHSLo == RHSHi) { 2963 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2964 if (RHSCST->isAllOnesValue()) { 2965 // Equality comparison to -1. 2966 NewLHS = DAG.getNode(ISD::AND, dl, 2967 LHSLo.getValueType(), LHSLo, LHSHi); 2968 NewRHS = RHSLo; 2969 return; 2970 } 2971 } 2972 } 2973 2974 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2975 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2976 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2977 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 2978 return; 2979 } 2980 2981 // If this is a comparison of the sign bit, just look at the top part. 2982 // X > -1, x < 0 2983 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2984 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2985 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2986 NewLHS = LHSHi; 2987 NewRHS = RHSHi; 2988 return; 2989 } 2990 2991 // FIXME: This generated code sucks. 2992 ISD::CondCode LowCC; 2993 switch (CCCode) { 2994 default: llvm_unreachable("Unknown integer setcc!"); 2995 case ISD::SETLT: 2996 case ISD::SETULT: LowCC = ISD::SETULT; break; 2997 case ISD::SETGT: 2998 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2999 case ISD::SETLE: 3000 case ISD::SETULE: LowCC = ISD::SETULE; break; 3001 case ISD::SETGE: 3002 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 3003 } 3004 3005 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 3006 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 3007 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 3008 3009 // NOTE: on targets without efficient SELECT of bools, we can always use 3010 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 3011 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 3012 nullptr); 3013 SDValue LoCmp, HiCmp; 3014 if (TLI.isTypeLegal(LHSLo.getValueType()) && 3015 TLI.isTypeLegal(RHSLo.getValueType())) 3016 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 3017 RHSLo, LowCC, false, DagCombineInfo, dl); 3018 if (!LoCmp.getNode()) 3019 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 3020 RHSLo, LowCC); 3021 if (TLI.isTypeLegal(LHSHi.getValueType()) && 3022 TLI.isTypeLegal(RHSHi.getValueType())) 3023 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 3024 RHSHi, CCCode, false, DagCombineInfo, dl); 3025 if (!HiCmp.getNode()) 3026 HiCmp = 3027 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 3028 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 3029 3030 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 3031 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 3032 3033 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 3034 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 3035 3036 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) || 3037 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) || 3038 (LoCmpC && LoCmpC->isNullValue())))) { 3039 // For LE / GE, if high part is known false, ignore the low part. 3040 // For LT / GT: if low part is known false, return the high part. 3041 // if high part is known true, ignore the low part. 3042 NewLHS = HiCmp; 3043 NewRHS = SDValue(); 3044 return; 3045 } 3046 3047 if (LHSHi == RHSHi) { 3048 // Comparing the low bits is enough. 3049 NewLHS = LoCmp; 3050 NewRHS = SDValue(); 3051 return; 3052 } 3053 3054 // Lower with SETCCE or SETCCCARRY if the target supports it. 3055 EVT HiVT = LHSHi.getValueType(); 3056 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 3057 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 3058 3059 // FIXME: Make all targets support this, then remove the other lowering. 3060 if (HasSETCCCARRY || 3061 TLI.getOperationAction(ISD::SETCCE, ExpandVT) == TargetLowering::Custom) { 3062 // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip 3063 // operands and condition code. 3064 bool FlipOperands = false; 3065 switch (CCCode) { 3066 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 3067 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 3068 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 3069 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 3070 default: break; 3071 } 3072 if (FlipOperands) { 3073 std::swap(LHSLo, RHSLo); 3074 std::swap(LHSHi, RHSHi); 3075 } 3076 // Perform a wide subtraction, feeding the carry from the low part into 3077 // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially 3078 // looking at the high part of the result of LHS - RHS. It is negative 3079 // iff LHS < RHS. It is zero or positive iff LHS >= RHS. 3080 EVT LoVT = LHSLo.getValueType(); 3081 SDVTList VTList = DAG.getVTList( 3082 LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT) : MVT::Glue); 3083 SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO : ISD::SUBC, dl, 3084 VTList, LHSLo, RHSLo); 3085 SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY : ISD::SETCCE, dl, 3086 getSetCCResultType(HiVT), LHSHi, RHSHi, 3087 LowCmp.getValue(1), DAG.getCondCode(CCCode)); 3088 NewLHS = Res; 3089 NewRHS = SDValue(); 3090 return; 3091 } 3092 3093 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 3094 false, DagCombineInfo, dl); 3095 if (!NewLHS.getNode()) 3096 NewLHS = 3097 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 3098 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 3099 NewRHS = SDValue(); 3100} 3101 3102SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 3103 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 3104 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 3105 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3106 3107 // If ExpandSetCCOperands returned a scalar, we need to compare the result 3108 // against zero to select between true and false values. 3109 if (!NewRHS.getNode()) { 3110 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 3111 CCCode = ISD::SETNE; 3112 } 3113 3114 // Update N to have the operands specified. 3115 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 3116 DAG.getCondCode(CCCode), NewLHS, NewRHS, 3117 N->getOperand(4)), 0); 3118} 3119 3120SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 3121 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 3122 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 3123 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3124 3125 // If ExpandSetCCOperands returned a scalar, we need to compare the result 3126 // against zero to select between true and false values. 3127 if (!NewRHS.getNode()) { 3128 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 3129 CCCode = ISD::SETNE; 3130 } 3131 3132 // Update N to have the operands specified. 3133 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 3134 N->getOperand(2), N->getOperand(3), 3135 DAG.getCondCode(CCCode)), 0); 3136} 3137 3138SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 3139 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 3140 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3141 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3142 3143 // If ExpandSetCCOperands returned a scalar, use it. 3144 if (!NewRHS.getNode()) { 3145 assert(NewLHS.getValueType() == N->getValueType(0) && 3146 "Unexpected setcc expansion!"); 3147 return NewLHS; 3148 } 3149 3150 // Otherwise, update N to have the operands specified. 3151 return SDValue( 3152 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 3153} 3154 3155SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) { 3156 SDValue LHS = N->getOperand(0); 3157 SDValue RHS = N->getOperand(1); 3158 SDValue Carry = N->getOperand(2); 3159 SDValue Cond = N->getOperand(3); 3160 SDLoc dl = SDLoc(N); 3161 3162 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 3163 GetExpandedInteger(LHS, LHSLo, LHSHi); 3164 GetExpandedInteger(RHS, RHSLo, RHSHi); 3165 3166 // Expand to a SUBE for the low part and a smaller SETCCE for the high. 3167 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue); 3168 SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry); 3169 return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi, 3170 LowCmp.getValue(1), Cond); 3171} 3172 3173SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 3174 SDValue LHS = N->getOperand(0); 3175 SDValue RHS = N->getOperand(1); 3176 SDValue Carry = N->getOperand(2); 3177 SDValue Cond = N->getOperand(3); 3178 SDLoc dl = SDLoc(N); 3179 3180 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 3181 GetExpandedInteger(LHS, LHSLo, LHSHi); 3182 GetExpandedInteger(RHS, RHSLo, RHSHi); 3183 3184 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 3185 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 3186 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 3187 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 3188 LowCmp.getValue(1), Cond); 3189} 3190 3191SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 3192 // The value being shifted is legal, but the shift amount is too big. 3193 // It follows that either the result of the shift is undefined, or the 3194 // upper half of the shift amount is zero. Just use the lower half. 3195 SDValue Lo, Hi; 3196 GetExpandedInteger(N->getOperand(1), Lo, Hi); 3197 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 3198} 3199 3200SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 3201 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 3202 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 3203 // constant to valid type. 3204 SDValue Lo, Hi; 3205 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3206 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 3207} 3208 3209SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 3210 SDValue Op = N->getOperand(0); 3211 EVT DstVT = N->getValueType(0); 3212 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 3213 assert(LC != RTLIB::UNKNOWN_LIBCALL && 3214 "Don't know how to expand this SINT_TO_FP!"); 3215 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first; 3216} 3217 3218SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 3219 if (ISD::isNormalStore(N)) 3220 return ExpandOp_NormalStore(N, OpNo); 3221 3222 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 3223 assert(OpNo == 1 && "Can only expand the stored value so far"); 3224 3225 EVT VT = N->getOperand(1).getValueType(); 3226 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3227 SDValue Ch = N->getChain(); 3228 SDValue Ptr = N->getBasePtr(); 3229 unsigned Alignment = N->getAlignment(); 3230 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3231 AAMDNodes AAInfo = N->getAAInfo(); 3232 SDLoc dl(N); 3233 SDValue Lo, Hi; 3234 3235 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3236 3237 if (N->getMemoryVT().bitsLE(NVT)) { 3238 GetExpandedInteger(N->getValue(), Lo, Hi); 3239 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 3240 N->getMemoryVT(), Alignment, MMOFlags, AAInfo); 3241 } 3242 3243 if (DAG.getDataLayout().isLittleEndian()) { 3244 // Little-endian - low bits are at low addresses. 3245 GetExpandedInteger(N->getValue(), Lo, Hi); 3246 3247 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 3248 AAInfo); 3249 3250 unsigned ExcessBits = 3251 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3252 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3253 3254 // Increment the pointer to the other half. 3255 unsigned IncrementSize = NVT.getSizeInBits()/8; 3256 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 3257 Hi = DAG.getTruncStore( 3258 Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3259 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 3260 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 3261 } 3262 3263 // Big-endian - high bits are at low addresses. Favor aligned stores at 3264 // the cost of some bit-fiddling. 3265 GetExpandedInteger(N->getValue(), Lo, Hi); 3266 3267 EVT ExtVT = N->getMemoryVT(); 3268 unsigned EBytes = ExtVT.getStoreSize(); 3269 unsigned IncrementSize = NVT.getSizeInBits()/8; 3270 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3271 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 3272 ExtVT.getSizeInBits() - ExcessBits); 3273 3274 if (ExcessBits < NVT.getSizeInBits()) { 3275 // Transfer high bits from the top of Lo to the bottom of Hi. 3276 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 3277 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3278 TLI.getPointerTy(DAG.getDataLayout()))); 3279 Hi = DAG.getNode( 3280 ISD::OR, dl, NVT, Hi, 3281 DAG.getNode(ISD::SRL, dl, NVT, Lo, 3282 DAG.getConstant(ExcessBits, dl, 3283 TLI.getPointerTy(DAG.getDataLayout())))); 3284 } 3285 3286 // Store both the high bits and maybe some of the low bits. 3287 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment, 3288 MMOFlags, AAInfo); 3289 3290 // Increment the pointer to the other half. 3291 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 3292 // Store the lowest ExcessBits bits in the second half. 3293 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 3294 N->getPointerInfo().getWithOffset(IncrementSize), 3295 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3296 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 3297 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 3298} 3299 3300SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 3301 SDValue InL, InH; 3302 GetExpandedInteger(N->getOperand(0), InL, InH); 3303 // Just truncate the low part of the source. 3304 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 3305} 3306 3307SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 3308 SDValue Op = N->getOperand(0); 3309 EVT SrcVT = Op.getValueType(); 3310 EVT DstVT = N->getValueType(0); 3311 SDLoc dl(N); 3312 3313 // The following optimization is valid only if every value in SrcVT (when 3314 // treated as signed) is representable in DstVT. Check that the mantissa 3315 // size of DstVT is >= than the number of bits in SrcVT -1. 3316 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT); 3317 if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 && 3318 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 3319 // Do a signed conversion then adjust the result. 3320 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 3321 SignedConv = TLI.LowerOperation(SignedConv, DAG); 3322 3323 // The result of the signed conversion needs adjusting if the 'sign bit' of 3324 // the incoming integer was set. To handle this, we dynamically test to see 3325 // if it is set, and, if so, add a fudge factor. 3326 3327 const uint64_t F32TwoE32 = 0x4F800000ULL; 3328 const uint64_t F32TwoE64 = 0x5F800000ULL; 3329 const uint64_t F32TwoE128 = 0x7F800000ULL; 3330 3331 APInt FF(32, 0); 3332 if (SrcVT == MVT::i32) 3333 FF = APInt(32, F32TwoE32); 3334 else if (SrcVT == MVT::i64) 3335 FF = APInt(32, F32TwoE64); 3336 else if (SrcVT == MVT::i128) 3337 FF = APInt(32, F32TwoE128); 3338 else 3339 llvm_unreachable("Unsupported UINT_TO_FP!"); 3340 3341 // Check whether the sign bit is set. 3342 SDValue Lo, Hi; 3343 GetExpandedInteger(Op, Lo, Hi); 3344 SDValue SignSet = DAG.getSetCC(dl, 3345 getSetCCResultType(Hi.getValueType()), 3346 Hi, 3347 DAG.getConstant(0, dl, Hi.getValueType()), 3348 ISD::SETLT); 3349 3350 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 3351 SDValue FudgePtr = 3352 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)), 3353 TLI.getPointerTy(DAG.getDataLayout())); 3354 3355 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 3356 SDValue Zero = DAG.getIntPtrConstant(0, dl); 3357 SDValue Four = DAG.getIntPtrConstant(4, dl); 3358 if (DAG.getDataLayout().isBigEndian()) 3359 std::swap(Zero, Four); 3360 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, 3361 Zero, Four); 3362 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 3363 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(), 3364 FudgePtr, Offset); 3365 Alignment = std::min(Alignment, 4u); 3366 3367 // Load the value out, extending it from f32 to the destination float type. 3368 // FIXME: Avoid the extend by constructing the right constant pool? 3369 SDValue Fudge = DAG.getExtLoad( 3370 ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr, 3371 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, 3372 Alignment); 3373 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 3374 } 3375 3376 // Otherwise, use a libcall. 3377 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 3378 assert(LC != RTLIB::UNKNOWN_LIBCALL && 3379 "Don't know how to expand this UINT_TO_FP!"); 3380 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first; 3381} 3382 3383SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 3384 SDLoc dl(N); 3385 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 3386 cast<AtomicSDNode>(N)->getMemoryVT(), 3387 N->getOperand(0), 3388 N->getOperand(1), N->getOperand(2), 3389 cast<AtomicSDNode>(N)->getMemOperand()); 3390 return Swap.getValue(1); 3391} 3392 3393 3394SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 3395 SDValue InOp0 = N->getOperand(0); 3396 EVT InVT = InOp0.getValueType(); 3397 3398 EVT OutVT = N->getValueType(0); 3399 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3400 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3401 unsigned OutNumElems = OutVT.getVectorNumElements(); 3402 EVT NOutVTElem = NOutVT.getVectorElementType(); 3403 3404 SDLoc dl(N); 3405 SDValue BaseIdx = N->getOperand(1); 3406 3407 SmallVector<SDValue, 8> Ops; 3408 Ops.reserve(OutNumElems); 3409 for (unsigned i = 0; i != OutNumElems; ++i) { 3410 3411 // Extract the element from the original vector. 3412 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 3413 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 3414 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3415 InVT.getVectorElementType(), N->getOperand(0), Index); 3416 3417 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 3418 // Insert the converted element to the new vector. 3419 Ops.push_back(Op); 3420 } 3421 3422 return DAG.getBuildVector(NOutVT, dl, Ops); 3423} 3424 3425 3426SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 3427 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 3428 EVT VT = N->getValueType(0); 3429 SDLoc dl(N); 3430 3431 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 3432 3433 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3434 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 3435 EVT OutVT = V0.getValueType(); 3436 3437 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 3438} 3439 3440 3441SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 3442 EVT OutVT = N->getValueType(0); 3443 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3444 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3445 unsigned NumElems = N->getNumOperands(); 3446 EVT NOutVTElem = NOutVT.getVectorElementType(); 3447 3448 SDLoc dl(N); 3449 3450 SmallVector<SDValue, 8> Ops; 3451 Ops.reserve(NumElems); 3452 for (unsigned i = 0; i != NumElems; ++i) { 3453 SDValue Op; 3454 // BUILD_VECTOR integer operand types are allowed to be larger than the 3455 // result's element type. This may still be true after the promotion. For 3456 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 3457 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 3458 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 3459 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 3460 else 3461 Op = N->getOperand(i); 3462 Ops.push_back(Op); 3463 } 3464 3465 return DAG.getBuildVector(NOutVT, dl, Ops); 3466} 3467 3468SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 3469 3470 SDLoc dl(N); 3471 3472 assert(!N->getOperand(0).getValueType().isVector() && 3473 "Input must be a scalar"); 3474 3475 EVT OutVT = N->getValueType(0); 3476 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3477 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3478 EVT NOutVTElem = NOutVT.getVectorElementType(); 3479 3480 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 3481 3482 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 3483} 3484 3485SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 3486 SDLoc dl(N); 3487 3488 EVT OutVT = N->getValueType(0); 3489 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3490 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3491 3492 EVT OutElemTy = NOutVT.getVectorElementType(); 3493 3494 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 3495 unsigned NumOutElem = NOutVT.getVectorNumElements(); 3496 unsigned NumOperands = N->getNumOperands(); 3497 assert(NumElem * NumOperands == NumOutElem && 3498 "Unexpected number of elements"); 3499 3500 // If the input type is legal and we can promote it to a legal type with the 3501 // same element size, go ahead do that to create a new concat. 3502 if (getTypeAction(N->getOperand(0).getValueType()) == 3503 TargetLowering::TypeLegal) { 3504 EVT InPromotedTy = EVT::getVectorVT(*DAG.getContext(), OutElemTy, NumElem); 3505 if (TLI.isTypeLegal(InPromotedTy)) { 3506 SmallVector<SDValue, 8> Ops(NumOperands); 3507 for (unsigned i = 0; i < NumOperands; ++i) { 3508 Ops[i] = DAG.getNode(ISD::ANY_EXTEND, dl, InPromotedTy, 3509 N->getOperand(i)); 3510 } 3511 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NOutVT, Ops); 3512 } 3513 } 3514 3515 // Take the elements from the first vector. 3516 SmallVector<SDValue, 8> Ops(NumOutElem); 3517 for (unsigned i = 0; i < NumOperands; ++i) { 3518 SDValue Op = N->getOperand(i); 3519 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) 3520 Op = GetPromotedInteger(Op); 3521 EVT SclrTy = Op.getValueType().getVectorElementType(); 3522 assert(NumElem == Op.getValueType().getVectorNumElements() && 3523 "Unexpected number of elements"); 3524 3525 for (unsigned j = 0; j < NumElem; ++j) { 3526 SDValue Ext = DAG.getNode( 3527 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, 3528 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3529 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); 3530 } 3531 } 3532 3533 return DAG.getBuildVector(NOutVT, dl, Ops); 3534} 3535 3536SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 3537 EVT VT = N->getValueType(0); 3538 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3539 assert(NVT.isVector() && "This type must be promoted to a vector type"); 3540 3541 SDLoc dl(N); 3542 3543 // For operands whose TypeAction is to promote, extend the promoted node 3544 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 3545 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 3546 // type.. 3547 if (getTypeAction(N->getOperand(0).getValueType()) 3548 == TargetLowering::TypePromoteInteger) { 3549 SDValue Promoted; 3550 3551 switch(N->getOpcode()) { 3552 case ISD::SIGN_EXTEND_VECTOR_INREG: 3553 Promoted = SExtPromotedInteger(N->getOperand(0)); 3554 break; 3555 case ISD::ZERO_EXTEND_VECTOR_INREG: 3556 Promoted = ZExtPromotedInteger(N->getOperand(0)); 3557 break; 3558 case ISD::ANY_EXTEND_VECTOR_INREG: 3559 Promoted = GetPromotedInteger(N->getOperand(0)); 3560 break; 3561 default: 3562 llvm_unreachable("Node has unexpected Opcode"); 3563 } 3564 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 3565 } 3566 3567 // Directly extend to the appropriate transform-to type. 3568 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 3569} 3570 3571SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 3572 EVT OutVT = N->getValueType(0); 3573 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3574 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3575 3576 EVT NOutVTElem = NOutVT.getVectorElementType(); 3577 3578 SDLoc dl(N); 3579 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3580 3581 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 3582 NOutVTElem, N->getOperand(1)); 3583 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 3584 V0, ConvElem, N->getOperand(2)); 3585} 3586 3587SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3588 SDLoc dl(N); 3589 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3590 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 3591 TLI.getVectorIdxTy(DAG.getDataLayout())); 3592 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3593 V0->getValueType(0).getScalarType(), V0, V1); 3594 3595 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 3596 // element types. If this is the case then we need to expand the outgoing 3597 // value and not truncate it. 3598 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 3599} 3600 3601SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 3602 SDLoc dl(N); 3603 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3604 MVT InVT = V0.getValueType().getSimpleVT(); 3605 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 3606 N->getValueType(0).getVectorNumElements()); 3607 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 3608 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 3609} 3610 3611SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 3612 SDLoc dl(N); 3613 unsigned NumElems = N->getNumOperands(); 3614 3615 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 3616 3617 SmallVector<SDValue, 8> NewOps; 3618 NewOps.reserve(NumElems); 3619 3620 // For each incoming vector 3621 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 3622 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 3623 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 3624 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 3625 3626 for (unsigned i=0; i<NumElem; ++i) { 3627 // Extract element from incoming vector 3628 SDValue Ex = DAG.getNode( 3629 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 3630 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3631 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 3632 NewOps.push_back(Tr); 3633 } 3634 } 3635 3636 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 3637} 3638