LegalizeIntegerTypes.cpp revision 321369
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/IR/DerivedTypes.h" 23#include "llvm/Support/ErrorHandling.h" 24#include "llvm/Support/KnownBits.h" 25#include "llvm/Support/raw_ostream.h" 26using namespace llvm; 27 28#define DEBUG_TYPE "legalize-types" 29 30//===----------------------------------------------------------------------===// 31// Integer Result Promotion 32//===----------------------------------------------------------------------===// 33 34/// PromoteIntegerResult - This method is called when a result of a node is 35/// found to be in need of promotion to a larger type. At this point, the node 36/// may also have invalid operands or may have other results that need 37/// expansion, we just know that (at least) one result needs promotion. 38void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 39 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 40 SDValue Res = SDValue(); 41 42 // See if the target wants to custom expand this node. 43 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 44 return; 45 46 switch (N->getOpcode()) { 47 default: 48#ifndef NDEBUG 49 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 50 N->dump(&DAG); dbgs() << "\n"; 51#endif 52 llvm_unreachable("Do not know how to promote this operator!"); 53 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 54 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 55 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 56 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 57 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; 58 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 59 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 60 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 61 case ISD::CTLZ_ZERO_UNDEF: 62 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 63 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 64 case ISD::CTTZ_ZERO_UNDEF: 65 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 66 case ISD::EXTRACT_VECTOR_ELT: 67 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 68 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; 69 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); 70 break; 71 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); 72 break; 73 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 74 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 75 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 76 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 77 case ISD::SMIN: 78 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; 79 case ISD::UMIN: 80 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break; 81 82 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 83 case ISD::SIGN_EXTEND_INREG: 84 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 85 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 86 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 87 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 88 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 89 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 90 91 case ISD::EXTRACT_SUBVECTOR: 92 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 93 case ISD::VECTOR_SHUFFLE: 94 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 95 case ISD::INSERT_VECTOR_ELT: 96 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 97 case ISD::BUILD_VECTOR: 98 Res = PromoteIntRes_BUILD_VECTOR(N); break; 99 case ISD::SCALAR_TO_VECTOR: 100 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 101 case ISD::CONCAT_VECTORS: 102 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 103 104 case ISD::ANY_EXTEND_VECTOR_INREG: 105 case ISD::SIGN_EXTEND_VECTOR_INREG: 106 case ISD::ZERO_EXTEND_VECTOR_INREG: 107 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; 108 109 case ISD::SIGN_EXTEND: 110 case ISD::ZERO_EXTEND: 111 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 112 113 case ISD::FP_TO_SINT: 114 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 115 116 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; 117 118 case ISD::AND: 119 case ISD::OR: 120 case ISD::XOR: 121 case ISD::ADD: 122 case ISD::SUB: 123 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 124 125 case ISD::SDIV: 126 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; 127 128 case ISD::UDIV: 129 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; 130 131 case ISD::SADDO: 132 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 133 case ISD::UADDO: 134 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 135 case ISD::SMULO: 136 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 137 138 case ISD::ADDCARRY: 139 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; 140 141 case ISD::ATOMIC_LOAD: 142 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 143 144 case ISD::ATOMIC_LOAD_ADD: 145 case ISD::ATOMIC_LOAD_SUB: 146 case ISD::ATOMIC_LOAD_AND: 147 case ISD::ATOMIC_LOAD_OR: 148 case ISD::ATOMIC_LOAD_XOR: 149 case ISD::ATOMIC_LOAD_NAND: 150 case ISD::ATOMIC_LOAD_MIN: 151 case ISD::ATOMIC_LOAD_MAX: 152 case ISD::ATOMIC_LOAD_UMIN: 153 case ISD::ATOMIC_LOAD_UMAX: 154 case ISD::ATOMIC_SWAP: 155 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 156 157 case ISD::ATOMIC_CMP_SWAP: 158 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 159 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); 160 break; 161 } 162 163 // If the result is null then the sub-method took care of registering it. 164 if (Res.getNode()) 165 SetPromotedInteger(SDValue(N, ResNo), Res); 166} 167 168SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 169 unsigned ResNo) { 170 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 171 return GetPromotedInteger(Op); 172} 173 174SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 175 // Sign-extend the new bits, and continue the assertion. 176 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 177 return DAG.getNode(ISD::AssertSext, SDLoc(N), 178 Op.getValueType(), Op, N->getOperand(1)); 179} 180 181SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 182 // Zero the new bits, and continue the assertion. 183 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 184 return DAG.getNode(ISD::AssertZext, SDLoc(N), 185 Op.getValueType(), Op, N->getOperand(1)); 186} 187 188SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 189 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 190 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 191 N->getMemoryVT(), ResVT, 192 N->getChain(), N->getBasePtr(), 193 N->getMemOperand()); 194 // Legalize the chain result - switch anything that used the old chain to 195 // use the new one. 196 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 197 return Res; 198} 199 200SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 201 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 202 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), 203 N->getMemoryVT(), 204 N->getChain(), N->getBasePtr(), 205 Op2, N->getMemOperand()); 206 // Legalize the chain result - switch anything that used the old chain to 207 // use the new one. 208 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 209 return Res; 210} 211 212SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, 213 unsigned ResNo) { 214 if (ResNo == 1) { 215 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 216 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); 217 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 218 219 // Only use the result of getSetCCResultType if it is legal, 220 // otherwise just use the promoted result type (NVT). 221 if (!TLI.isTypeLegal(SVT)) 222 SVT = NVT; 223 224 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); 225 SDValue Res = DAG.getAtomicCmpSwap( 226 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, 227 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), 228 N->getMemOperand()); 229 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 230 ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); 231 return Res.getValue(1); 232 } 233 234 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 235 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 236 SDVTList VTs = 237 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); 238 SDValue Res = DAG.getAtomicCmpSwap( 239 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), 240 N->getBasePtr(), Op2, Op3, N->getMemOperand()); 241 // Update the use to N with the newly created Res. 242 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) 243 ReplaceValueWith(SDValue(N, i), Res.getValue(i)); 244 return Res; 245} 246 247SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 248 SDValue InOp = N->getOperand(0); 249 EVT InVT = InOp.getValueType(); 250 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 251 EVT OutVT = N->getValueType(0); 252 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 253 SDLoc dl(N); 254 255 switch (getTypeAction(InVT)) { 256 case TargetLowering::TypeLegal: 257 break; 258 case TargetLowering::TypePromoteInteger: 259 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 260 // The input promotes to the same size. Convert the promoted value. 261 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 262 break; 263 case TargetLowering::TypeSoftenFloat: 264 // Promote the integer operand by hand. 265 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 266 case TargetLowering::TypePromoteFloat: { 267 // Convert the promoted float by hand. 268 SDValue PromotedOp = GetPromotedFloat(InOp); 269 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp); 270 break; 271 } 272 case TargetLowering::TypeExpandInteger: 273 case TargetLowering::TypeExpandFloat: 274 break; 275 case TargetLowering::TypeScalarizeVector: 276 // Convert the element to an integer and promote it by hand. 277 if (!NOutVT.isVector()) 278 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 279 BitConvertToInteger(GetScalarizedVector(InOp))); 280 break; 281 case TargetLowering::TypeSplitVector: { 282 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 283 // pieces of the input into integers and reassemble in the final type. 284 SDValue Lo, Hi; 285 GetSplitVector(N->getOperand(0), Lo, Hi); 286 Lo = BitConvertToInteger(Lo); 287 Hi = BitConvertToInteger(Hi); 288 289 if (DAG.getDataLayout().isBigEndian()) 290 std::swap(Lo, Hi); 291 292 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 293 EVT::getIntegerVT(*DAG.getContext(), 294 NOutVT.getSizeInBits()), 295 JoinIntegers(Lo, Hi)); 296 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 297 } 298 case TargetLowering::TypeWidenVector: 299 // The input is widened to the same size. Convert to the widened value. 300 // Make sure that the outgoing value is not a vector, because this would 301 // make us bitcast between two vectors which are legalized in different ways. 302 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) 303 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 304 } 305 306 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 307 CreateStackStoreLoad(InOp, OutVT)); 308} 309 310SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 311 SDValue Op = GetPromotedInteger(N->getOperand(0)); 312 EVT OVT = N->getValueType(0); 313 EVT NVT = Op.getValueType(); 314 SDLoc dl(N); 315 316 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 317 return DAG.getNode( 318 ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 319 DAG.getConstant(DiffBits, dl, 320 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 321} 322 323SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { 324 SDValue Op = GetPromotedInteger(N->getOperand(0)); 325 EVT OVT = N->getValueType(0); 326 EVT NVT = Op.getValueType(); 327 SDLoc dl(N); 328 329 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); 330 return DAG.getNode( 331 ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), 332 DAG.getConstant(DiffBits, dl, 333 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); 334} 335 336SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 337 // The pair element type may be legal, or may not promote to the same type as 338 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 339 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), 340 TLI.getTypeToTransformTo(*DAG.getContext(), 341 N->getValueType(0)), JoinIntegers(N->getOperand(0), 342 N->getOperand(1))); 343} 344 345SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 346 EVT VT = N->getValueType(0); 347 // FIXME there is no actual debug info here 348 SDLoc dl(N); 349 // Zero extend things like i1, sign extend everything else. It shouldn't 350 // matter in theory which one we pick, but this tends to give better code? 351 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 352 SDValue Result = DAG.getNode(Opc, dl, 353 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 354 SDValue(N, 0)); 355 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 356 return Result; 357} 358 359SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 360 // Zero extend to the promoted type and do the count there. 361 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 362 SDLoc dl(N); 363 EVT OVT = N->getValueType(0); 364 EVT NVT = Op.getValueType(); 365 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 366 // Subtract off the extra leading bits in the bigger type. 367 return DAG.getNode( 368 ISD::SUB, dl, NVT, Op, 369 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, 370 NVT)); 371} 372 373SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 374 // Zero extend to the promoted type and do the count there. 375 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 376 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op); 377} 378 379SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 380 SDValue Op = GetPromotedInteger(N->getOperand(0)); 381 EVT OVT = N->getValueType(0); 382 EVT NVT = Op.getValueType(); 383 SDLoc dl(N); 384 if (N->getOpcode() == ISD::CTTZ) { 385 // The count is the same in the promoted type except if the original 386 // value was zero. This can be handled by setting the bit just off 387 // the top of the original type. 388 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), 389 OVT.getScalarSizeInBits()); 390 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); 391 } 392 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 393} 394 395SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 396 SDLoc dl(N); 397 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 398 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 399 N->getOperand(1)); 400} 401 402SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 403 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 404 unsigned NewOpc = N->getOpcode(); 405 SDLoc dl(N); 406 407 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 408 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 409 // and SINT conversions are Custom, there is no way to tell which is 410 // preferable. We choose SINT because that's the right thing on PPC.) 411 if (N->getOpcode() == ISD::FP_TO_UINT && 412 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 413 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 414 NewOpc = ISD::FP_TO_SINT; 415 416 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 417 418 // Assert that the converted value fits in the original type. If it doesn't 419 // (eg: because the value being converted is too big), then the result of the 420 // original operation was undefined anyway, so the assert is still correct. 421 // 422 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: 423 // before legalization: fp-to-uint16, 65534. -> 0xfffe 424 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe 425 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 426 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 427 DAG.getValueType(N->getValueType(0).getScalarType())); 428} 429 430SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { 431 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 432 SDLoc dl(N); 433 434 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 435} 436 437SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 438 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 439 SDLoc dl(N); 440 441 if (getTypeAction(N->getOperand(0).getValueType()) 442 == TargetLowering::TypePromoteInteger) { 443 SDValue Res = GetPromotedInteger(N->getOperand(0)); 444 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 445 446 // If the result and operand types are the same after promotion, simplify 447 // to an in-register extension. 448 if (NVT == Res.getValueType()) { 449 // The high bits are not guaranteed to be anything. Insert an extend. 450 if (N->getOpcode() == ISD::SIGN_EXTEND) 451 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 452 DAG.getValueType(N->getOperand(0).getValueType())); 453 if (N->getOpcode() == ISD::ZERO_EXTEND) 454 return DAG.getZeroExtendInReg(Res, dl, 455 N->getOperand(0).getValueType().getScalarType()); 456 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 457 return Res; 458 } 459 } 460 461 // Otherwise, just extend the original operand all the way to the larger type. 462 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 463} 464 465SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 466 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 468 ISD::LoadExtType ExtType = 469 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 470 SDLoc dl(N); 471 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 472 N->getMemoryVT(), N->getMemOperand()); 473 474 // Legalize the chain result - switch anything that used the old chain to 475 // use the new one. 476 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 477 return Res; 478} 479 480SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { 481 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 482 SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0()); 483 484 SDLoc dl(N); 485 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), 486 N->getMask(), ExtSrc0, N->getMemoryVT(), 487 N->getMemOperand(), ISD::SEXTLOAD); 488 // Legalize the chain result - switch anything that used the old chain to 489 // use the new one. 490 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 491 return Res; 492} 493 494SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { 495 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 496 SDValue ExtSrc0 = GetPromotedInteger(N->getValue()); 497 assert(NVT == ExtSrc0.getValueType() && 498 "Gather result type and the passThru agrument type should be the same"); 499 500 SDLoc dl(N); 501 SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(), 502 N->getIndex()}; 503 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), 504 N->getMemoryVT(), dl, Ops, 505 N->getMemOperand()); 506 // Legalize the chain result - switch anything that used the old chain to 507 // use the new one. 508 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 509 return Res; 510} 511 512/// Promote the overflow flag of an overflowing arithmetic node. 513SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 514 // Simply change the return type of the boolean result. 515 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 516 EVT ValueVTs[] = { N->getValueType(0), NVT }; 517 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; 518 unsigned NumOps = N->getNumOperands(); 519 assert(NumOps <= 3 && "Too many operands"); 520 if (NumOps == 3) 521 Ops[2] = N->getOperand(2); 522 523 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 524 DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps)); 525 526 // Modified the sum result - switch anything that used the old sum to use 527 // the new one. 528 ReplaceValueWith(SDValue(N, 0), Res); 529 530 return SDValue(Res.getNode(), 1); 531} 532 533SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 534 if (ResNo == 1) 535 return PromoteIntRes_Overflow(N); 536 537 // The operation overflowed iff the result in the larger type is not the 538 // sign extension of its truncation to the original type. 539 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 540 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 541 EVT OVT = N->getOperand(0).getValueType(); 542 EVT NVT = LHS.getValueType(); 543 SDLoc dl(N); 544 545 // Do the arithmetic in the larger type. 546 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 547 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 548 549 // Calculate the overflow flag: sign extend the arithmetic result from 550 // the original type. 551 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 552 DAG.getValueType(OVT)); 553 // Overflowed if and only if this is not equal to Res. 554 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 555 556 // Use the calculated overflow everywhere. 557 ReplaceValueWith(SDValue(N, 1), Ofl); 558 559 return Res; 560} 561 562SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 563 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 564 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 565 return DAG.getSelect(SDLoc(N), 566 LHS.getValueType(), N->getOperand(0), LHS, RHS); 567} 568 569SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 570 SDValue Mask = N->getOperand(0); 571 EVT OpTy = N->getOperand(1).getValueType(); 572 573 // Promote all the way up to the canonical SetCC type. 574 Mask = PromoteTargetBoolean(Mask, OpTy); 575 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 576 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 577 return DAG.getNode(ISD::VSELECT, SDLoc(N), 578 LHS.getValueType(), Mask, LHS, RHS); 579} 580 581SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 582 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 583 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 584 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 585 LHS.getValueType(), N->getOperand(0), 586 N->getOperand(1), LHS, RHS, N->getOperand(4)); 587} 588 589SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 590 EVT SVT = getSetCCResultType(N->getOperand(0).getValueType()); 591 592 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 593 594 // Only use the result of getSetCCResultType if it is legal, 595 // otherwise just use the promoted result type (NVT). 596 if (!TLI.isTypeLegal(SVT)) 597 SVT = NVT; 598 599 SDLoc dl(N); 600 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && 601 "Vector compare must return a vector result!"); 602 603 SDValue LHS = N->getOperand(0); 604 SDValue RHS = N->getOperand(1); 605 if (LHS.getValueType() != RHS.getValueType()) { 606 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger && 607 !LHS.getValueType().isVector()) 608 LHS = GetPromotedInteger(LHS); 609 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger && 610 !RHS.getValueType().isVector()) 611 RHS = GetPromotedInteger(RHS); 612 } 613 614 // Get the SETCC result using the canonical SETCC type. 615 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS, 616 N->getOperand(2)); 617 618 // Convert to the expected type. 619 return DAG.getSExtOrTrunc(SetCC, dl, NVT); 620} 621 622SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 623 SDValue LHS = N->getOperand(0); 624 SDValue RHS = N->getOperand(1); 625 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 626 LHS = GetPromotedInteger(LHS); 627 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 628 RHS = ZExtPromotedInteger(RHS); 629 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); 630} 631 632SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 633 SDValue Op = GetPromotedInteger(N->getOperand(0)); 634 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), 635 Op.getValueType(), Op, N->getOperand(1)); 636} 637 638SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 639 // The input may have strange things in the top bits of the registers, but 640 // these operations don't care. They may have weird bits going out, but 641 // that too is okay if they are integer operations. 642 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 643 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 644 return DAG.getNode(N->getOpcode(), SDLoc(N), 645 LHS.getValueType(), LHS, RHS); 646} 647 648SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { 649 // Sign extend the input. 650 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 651 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 652 return DAG.getNode(N->getOpcode(), SDLoc(N), 653 LHS.getValueType(), LHS, RHS); 654} 655 656SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { 657 // Zero extend the input. 658 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 659 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 660 return DAG.getNode(N->getOpcode(), SDLoc(N), 661 LHS.getValueType(), LHS, RHS); 662} 663 664SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 665 SDValue LHS = N->getOperand(0); 666 SDValue RHS = N->getOperand(1); 667 // The input value must be properly sign extended. 668 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 669 LHS = SExtPromotedInteger(LHS); 670 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 671 RHS = ZExtPromotedInteger(RHS); 672 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); 673} 674 675SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 676 SDValue LHS = N->getOperand(0); 677 SDValue RHS = N->getOperand(1); 678 // The input value must be properly zero extended. 679 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) 680 LHS = ZExtPromotedInteger(LHS); 681 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) 682 RHS = ZExtPromotedInteger(RHS); 683 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); 684} 685 686SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 688 SDValue Res; 689 SDValue InOp = N->getOperand(0); 690 SDLoc dl(N); 691 692 switch (getTypeAction(InOp.getValueType())) { 693 default: llvm_unreachable("Unknown type action!"); 694 case TargetLowering::TypeLegal: 695 case TargetLowering::TypeExpandInteger: 696 Res = InOp; 697 break; 698 case TargetLowering::TypePromoteInteger: 699 Res = GetPromotedInteger(InOp); 700 break; 701 case TargetLowering::TypeSplitVector: { 702 EVT InVT = InOp.getValueType(); 703 assert(InVT.isVector() && "Cannot split scalar types"); 704 unsigned NumElts = InVT.getVectorNumElements(); 705 assert(NumElts == NVT.getVectorNumElements() && 706 "Dst and Src must have the same number of elements"); 707 assert(isPowerOf2_32(NumElts) && 708 "Promoted vector type must be a power of two"); 709 710 SDValue EOp1, EOp2; 711 GetSplitVector(InOp, EOp1, EOp2); 712 713 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 714 NumElts/2); 715 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 716 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 717 718 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 719 } 720 case TargetLowering::TypeWidenVector: { 721 SDValue WideInOp = GetWidenedVector(InOp); 722 723 // Truncate widened InOp. 724 unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); 725 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), 726 N->getValueType(0).getScalarType(), NumElem); 727 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); 728 729 // Zero extend so that the elements are of same type as those of NVT 730 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), 731 NumElem); 732 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); 733 734 // Extract the low NVT subvector. 735 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 736 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy); 737 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); 738 } 739 } 740 741 // Truncate to NVT instead of VT 742 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 743} 744 745SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 746 if (ResNo == 1) 747 return PromoteIntRes_Overflow(N); 748 749 // The operation overflowed iff the result in the larger type is not the 750 // zero extension of its truncation to the original type. 751 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 752 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 753 EVT OVT = N->getOperand(0).getValueType(); 754 EVT NVT = LHS.getValueType(); 755 SDLoc dl(N); 756 757 // Do the arithmetic in the larger type. 758 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 759 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 760 761 // Calculate the overflow flag: zero extend the arithmetic result from 762 // the original type. 763 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 764 // Overflowed if and only if this is not equal to Res. 765 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 766 767 // Use the calculated overflow everywhere. 768 ReplaceValueWith(SDValue(N, 1), Ofl); 769 770 return Res; 771} 772 773SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { 774 if (ResNo == 1) 775 return PromoteIntRes_Overflow(N); 776 llvm_unreachable("Not implemented"); 777} 778 779SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 780 // Promote the overflow bit trivially. 781 if (ResNo == 1) 782 return PromoteIntRes_Overflow(N); 783 784 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 785 SDLoc DL(N); 786 EVT SmallVT = LHS.getValueType(); 787 788 // To determine if the result overflowed in a larger type, we extend the 789 // input to the larger type, do the multiply (checking if it overflows), 790 // then also check the high bits of the result to see if overflow happened 791 // there. 792 if (N->getOpcode() == ISD::SMULO) { 793 LHS = SExtPromotedInteger(LHS); 794 RHS = SExtPromotedInteger(RHS); 795 } else { 796 LHS = ZExtPromotedInteger(LHS); 797 RHS = ZExtPromotedInteger(RHS); 798 } 799 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 800 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 801 802 // Overflow occurred if it occurred in the larger type, or if the high part 803 // of the result does not zero/sign-extend the low part. Check this second 804 // possibility first. 805 SDValue Overflow; 806 if (N->getOpcode() == ISD::UMULO) { 807 // Unsigned overflow occurred if the high part is non-zero. 808 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 809 DAG.getIntPtrConstant(SmallVT.getSizeInBits(), 810 DL)); 811 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 812 DAG.getConstant(0, DL, Hi.getValueType()), 813 ISD::SETNE); 814 } else { 815 // Signed overflow occurred if the high part does not sign extend the low. 816 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 817 Mul, DAG.getValueType(SmallVT)); 818 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 819 } 820 821 // The only other way for overflow to occur is if the multiplication in the 822 // larger type itself overflowed. 823 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 824 SDValue(Mul.getNode(), 1)); 825 826 // Use the calculated overflow everywhere. 827 ReplaceValueWith(SDValue(N, 1), Overflow); 828 return Mul; 829} 830 831SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 832 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 833 N->getValueType(0))); 834} 835 836SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 837 SDValue Chain = N->getOperand(0); // Get the chain. 838 SDValue Ptr = N->getOperand(1); // Get the pointer. 839 EVT VT = N->getValueType(0); 840 SDLoc dl(N); 841 842 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 843 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 844 // The argument is passed as NumRegs registers of type RegVT. 845 846 SmallVector<SDValue, 8> Parts(NumRegs); 847 for (unsigned i = 0; i < NumRegs; ++i) { 848 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 849 N->getConstantOperandVal(3)); 850 Chain = Parts[i].getValue(1); 851 } 852 853 // Handle endianness of the load. 854 if (DAG.getDataLayout().isBigEndian()) 855 std::reverse(Parts.begin(), Parts.end()); 856 857 // Assemble the parts in the promoted type. 858 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 859 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 860 for (unsigned i = 1; i < NumRegs; ++i) { 861 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 862 // Shift it to the right position and "or" it in. 863 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 864 DAG.getConstant(i * RegVT.getSizeInBits(), dl, 865 TLI.getPointerTy(DAG.getDataLayout()))); 866 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 867 } 868 869 // Modified the chain result - switch anything that used the old chain to 870 // use the new one. 871 ReplaceValueWith(SDValue(N, 1), Chain); 872 873 return Res; 874} 875 876//===----------------------------------------------------------------------===// 877// Integer Operand Promotion 878//===----------------------------------------------------------------------===// 879 880/// PromoteIntegerOperand - This method is called when the specified operand of 881/// the specified node is found to need promotion. At this point, all of the 882/// result types of the node are known to be legal, but other operands of the 883/// node may need promotion or expansion as well as the specified one. 884bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 885 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 886 SDValue Res = SDValue(); 887 888 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 889 return false; 890 891 switch (N->getOpcode()) { 892 default: 893 #ifndef NDEBUG 894 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 895 N->dump(&DAG); dbgs() << "\n"; 896 #endif 897 llvm_unreachable("Do not know how to promote this operator's operand!"); 898 899 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 900 case ISD::ATOMIC_STORE: 901 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 902 break; 903 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 904 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 905 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 906 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 907 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 908 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 909 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 910 case ISD::INSERT_VECTOR_ELT: 911 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 912 case ISD::SCALAR_TO_VECTOR: 913 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 914 case ISD::VSELECT: 915 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 916 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 917 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 918 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 919 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 920 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 921 OpNo); break; 922 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), 923 OpNo); break; 924 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), 925 OpNo); break; 926 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), 927 OpNo); break; 928 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), 929 OpNo); break; 930 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 931 case ISD::FP16_TO_FP: 932 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 933 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 934 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; 935 936 case ISD::SHL: 937 case ISD::SRA: 938 case ISD::SRL: 939 case ISD::ROTL: 940 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 941 942 case ISD::ADDCARRY: 943 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; 944 } 945 946 // If the result is null, the sub-method took care of registering results etc. 947 if (!Res.getNode()) return false; 948 949 // If the result is N, the sub-method updated N in place. Tell the legalizer 950 // core about this. 951 if (Res.getNode() == N) 952 return true; 953 954 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 955 "Invalid operand expansion"); 956 957 ReplaceValueWith(SDValue(N, 0), Res); 958 return false; 959} 960 961/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 962/// shared among BR_CC, SELECT_CC, and SETCC handlers. 963void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 964 ISD::CondCode CCCode) { 965 // We have to insert explicit sign or zero extends. Note that we could 966 // insert sign extends for ALL conditions, but zero extend is cheaper on 967 // many machines (an AND instead of two shifts), so prefer it. 968 switch (CCCode) { 969 default: llvm_unreachable("Unknown integer comparison!"); 970 case ISD::SETEQ: 971 case ISD::SETNE: { 972 SDValue OpL = GetPromotedInteger(NewLHS); 973 SDValue OpR = GetPromotedInteger(NewRHS); 974 975 // We would prefer to promote the comparison operand with sign extension. 976 // If the width of OpL/OpR excluding the duplicated sign bits is no greater 977 // than the width of NewLHS/NewRH, we can avoid inserting real truncate 978 // instruction, which is redudant eventually. 979 unsigned OpLEffectiveBits = 980 OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; 981 unsigned OpREffectiveBits = 982 OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; 983 if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() && 984 OpREffectiveBits <= NewRHS.getValueSizeInBits()) { 985 NewLHS = OpL; 986 NewRHS = OpR; 987 } else { 988 NewLHS = ZExtPromotedInteger(NewLHS); 989 NewRHS = ZExtPromotedInteger(NewRHS); 990 } 991 break; 992 } 993 case ISD::SETUGE: 994 case ISD::SETUGT: 995 case ISD::SETULE: 996 case ISD::SETULT: 997 // ALL of these operations will work if we either sign or zero extend 998 // the operands (including the unsigned comparisons!). Zero extend is 999 // usually a simpler/cheaper operation, so prefer it. 1000 NewLHS = ZExtPromotedInteger(NewLHS); 1001 NewRHS = ZExtPromotedInteger(NewRHS); 1002 break; 1003 case ISD::SETGE: 1004 case ISD::SETGT: 1005 case ISD::SETLT: 1006 case ISD::SETLE: 1007 NewLHS = SExtPromotedInteger(NewLHS); 1008 NewRHS = SExtPromotedInteger(NewRHS); 1009 break; 1010 } 1011} 1012 1013SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 1014 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1015 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); 1016} 1017 1018SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 1019 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 1020 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), 1021 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); 1022} 1023 1024SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 1025 // This should only occur in unusual situations like bitcasting to an 1026 // x86_fp80, so just turn it into a store+load 1027 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 1028} 1029 1030SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 1031 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1032 1033 SDValue LHS = N->getOperand(2); 1034 SDValue RHS = N->getOperand(3); 1035 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 1036 1037 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 1038 // legal types. 1039 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1040 N->getOperand(1), LHS, RHS, N->getOperand(4)), 1041 0); 1042} 1043 1044SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 1045 assert(OpNo == 1 && "only know how to promote condition"); 1046 1047 // Promote all the way up to the canonical SetCC type. 1048 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); 1049 1050 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 1051 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 1052 N->getOperand(2)), 0); 1053} 1054 1055SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 1056 // Since the result type is legal, the operands must promote to it. 1057 EVT OVT = N->getOperand(0).getValueType(); 1058 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 1059 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 1060 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 1061 SDLoc dl(N); 1062 1063 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 1064 DAG.getConstant(OVT.getSizeInBits(), dl, 1065 TLI.getPointerTy(DAG.getDataLayout()))); 1066 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 1067} 1068 1069SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 1070 // The vector type is legal but the element type is not. This implies 1071 // that the vector is a power-of-two in length and that the element 1072 // type does not have a strange size (eg: it is not i1). 1073 EVT VecVT = N->getValueType(0); 1074 unsigned NumElts = VecVT.getVectorNumElements(); 1075 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && 1076 "Legal vector of one illegal element?"); 1077 1078 // Promote the inserted value. The type does not need to match the 1079 // vector element type. Check that any extra bits introduced will be 1080 // truncated away. 1081 assert(N->getOperand(0).getValueSizeInBits() >= 1082 N->getValueType(0).getScalarSizeInBits() && 1083 "Type of inserted value narrower than vector element type!"); 1084 1085 SmallVector<SDValue, 16> NewOps; 1086 for (unsigned i = 0; i < NumElts; ++i) 1087 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 1088 1089 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1090} 1091 1092SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 1093 unsigned OpNo) { 1094 if (OpNo == 1) { 1095 // Promote the inserted value. This is valid because the type does not 1096 // have to match the vector element type. 1097 1098 // Check that any extra bits introduced will be truncated away. 1099 assert(N->getOperand(1).getValueSizeInBits() >= 1100 N->getValueType(0).getScalarSizeInBits() && 1101 "Type of inserted value narrower than vector element type!"); 1102 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1103 GetPromotedInteger(N->getOperand(1)), 1104 N->getOperand(2)), 1105 0); 1106 } 1107 1108 assert(OpNo == 2 && "Different operand and result vector types?"); 1109 1110 // Promote the index. 1111 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), 1112 TLI.getVectorIdxTy(DAG.getDataLayout())); 1113 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1114 N->getOperand(1), Idx), 0); 1115} 1116 1117SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 1118 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 1119 // the operand in place. 1120 return SDValue(DAG.UpdateNodeOperands(N, 1121 GetPromotedInteger(N->getOperand(0))), 0); 1122} 1123 1124SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 1125 assert(OpNo == 0 && "Only know how to promote the condition!"); 1126 SDValue Cond = N->getOperand(0); 1127 EVT OpTy = N->getOperand(1).getValueType(); 1128 1129 if (N->getOpcode() == ISD::VSELECT) 1130 if (SDValue Res = WidenVSELECTAndMask(N)) 1131 return Res; 1132 1133 // Promote all the way up to the canonical SetCC type. 1134 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; 1135 Cond = PromoteTargetBoolean(Cond, OpVT); 1136 1137 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 1138 N->getOperand(2)), 0); 1139} 1140 1141SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 1142 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1143 1144 SDValue LHS = N->getOperand(0); 1145 SDValue RHS = N->getOperand(1); 1146 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1147 1148 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1149 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1150 N->getOperand(3), N->getOperand(4)), 0); 1151} 1152 1153SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1154 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1155 1156 SDValue LHS = N->getOperand(0); 1157 SDValue RHS = N->getOperand(1); 1158 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1159 1160 // The CC (#2) is always legal. 1161 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1162} 1163 1164SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1165 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1166 ZExtPromotedInteger(N->getOperand(1))), 0); 1167} 1168 1169SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1170 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1171 SDLoc dl(N); 1172 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1173 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1174 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1175} 1176 1177SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1178 return SDValue(DAG.UpdateNodeOperands(N, 1179 SExtPromotedInteger(N->getOperand(0))), 0); 1180} 1181 1182SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1183 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1184 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1185 SDLoc dl(N); 1186 1187 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1188 1189 // Truncate the value and store the result. 1190 return DAG.getTruncStore(Ch, dl, Val, Ptr, 1191 N->getMemoryVT(), N->getMemOperand()); 1192} 1193 1194SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, 1195 unsigned OpNo) { 1196 1197 SDValue DataOp = N->getValue(); 1198 EVT DataVT = DataOp.getValueType(); 1199 SDValue Mask = N->getMask(); 1200 SDLoc dl(N); 1201 1202 bool TruncateStore = false; 1203 if (OpNo == 2) { 1204 // Mask comes before the data operand. If the data operand is legal, we just 1205 // promote the mask. 1206 // When the data operand has illegal type, we should legalize the data 1207 // operand first. The mask will be promoted/splitted/widened according to 1208 // the data operand type. 1209 if (TLI.isTypeLegal(DataVT)) 1210 Mask = PromoteTargetBoolean(Mask, DataVT); 1211 else { 1212 if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) 1213 return PromoteIntOp_MSTORE(N, 3); 1214 1215 else if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector) 1216 return WidenVecOp_MSTORE(N, 3); 1217 1218 else { 1219 assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector); 1220 return SplitVecOp_MSTORE(N, 3); 1221 } 1222 } 1223 } else { // Data operand 1224 assert(OpNo == 3 && "Unexpected operand for promotion"); 1225 DataOp = GetPromotedInteger(DataOp); 1226 Mask = PromoteTargetBoolean(Mask, DataOp.getValueType()); 1227 TruncateStore = true; 1228 } 1229 1230 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask, 1231 N->getMemoryVT(), N->getMemOperand(), 1232 TruncateStore, N->isCompressingStore()); 1233} 1234 1235SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, 1236 unsigned OpNo) { 1237 assert(OpNo == 2 && "Only know how to promote the mask!"); 1238 EVT DataVT = N->getValueType(0); 1239 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1240 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 1241 NewOps[OpNo] = Mask; 1242 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1243} 1244 1245SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, 1246 unsigned OpNo) { 1247 1248 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1249 if (OpNo == 2) { 1250 // The Mask 1251 EVT DataVT = N->getValueType(0); 1252 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1253 } else 1254 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1255 1256 SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1257 // updated in place. 1258 if (Res.getNode() == N) 1259 return Res; 1260 1261 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 1262 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 1263 return SDValue(); 1264} 1265 1266SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, 1267 unsigned OpNo) { 1268 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); 1269 if (OpNo == 2) { 1270 // The Mask 1271 EVT DataVT = N->getValue().getValueType(); 1272 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); 1273 } else 1274 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); 1275 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); 1276} 1277 1278SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1279 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1280 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); 1281} 1282 1283SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1284 return SDValue(DAG.UpdateNodeOperands(N, 1285 ZExtPromotedInteger(N->getOperand(0))), 0); 1286} 1287 1288SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1289 SDLoc dl(N); 1290 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1291 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1292 return DAG.getZeroExtendInReg(Op, dl, 1293 N->getOperand(0).getValueType().getScalarType()); 1294} 1295 1296SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { 1297 assert(OpNo == 2 && "Don't know how to promote this operand!"); 1298 1299 SDValue LHS = N->getOperand(0); 1300 SDValue RHS = N->getOperand(1); 1301 SDValue Carry = N->getOperand(2); 1302 SDLoc DL(N); 1303 1304 auto VT = getSetCCResultType(LHS.getValueType()); 1305 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT); 1306 switch (BoolType) { 1307 case TargetLoweringBase::UndefinedBooleanContent: 1308 Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT); 1309 break; 1310 case TargetLoweringBase::ZeroOrOneBooleanContent: 1311 Carry = DAG.getZExtOrTrunc(Carry, DL, VT); 1312 break; 1313 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 1314 Carry = DAG.getSExtOrTrunc(Carry, DL, VT); 1315 break; 1316 } 1317 1318 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); 1319} 1320 1321//===----------------------------------------------------------------------===// 1322// Integer Result Expansion 1323//===----------------------------------------------------------------------===// 1324 1325/// ExpandIntegerResult - This method is called when the specified result of the 1326/// specified node is found to need expansion. At this point, the node may also 1327/// have invalid operands or may have other results that need promotion, we just 1328/// know that (at least) one result needs expansion. 1329void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1330 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 1331 SDValue Lo, Hi; 1332 Lo = Hi = SDValue(); 1333 1334 // See if the target wants to custom expand this node. 1335 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1336 return; 1337 1338 switch (N->getOpcode()) { 1339 default: 1340#ifndef NDEBUG 1341 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1342 N->dump(&DAG); dbgs() << "\n"; 1343#endif 1344 llvm_unreachable("Do not know how to expand the result of this operator!"); 1345 1346 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1347 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1348 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1349 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1350 1351 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1352 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1353 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1354 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1355 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1356 1357 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1358 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1359 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1360 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; 1361 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1362 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1363 case ISD::CTLZ_ZERO_UNDEF: 1364 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1365 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1366 case ISD::CTTZ_ZERO_UNDEF: 1367 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1368 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; 1369 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1370 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1371 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1372 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1373 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; 1374 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1375 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1376 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1377 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1378 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1379 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1380 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1381 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1382 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1383 1384 case ISD::ATOMIC_LOAD_ADD: 1385 case ISD::ATOMIC_LOAD_SUB: 1386 case ISD::ATOMIC_LOAD_AND: 1387 case ISD::ATOMIC_LOAD_OR: 1388 case ISD::ATOMIC_LOAD_XOR: 1389 case ISD::ATOMIC_LOAD_NAND: 1390 case ISD::ATOMIC_LOAD_MIN: 1391 case ISD::ATOMIC_LOAD_MAX: 1392 case ISD::ATOMIC_LOAD_UMIN: 1393 case ISD::ATOMIC_LOAD_UMAX: 1394 case ISD::ATOMIC_SWAP: 1395 case ISD::ATOMIC_CMP_SWAP: { 1396 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1397 SplitInteger(Tmp.first, Lo, Hi); 1398 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1399 break; 1400 } 1401 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { 1402 AtomicSDNode *AN = cast<AtomicSDNode>(N); 1403 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); 1404 SDValue Tmp = DAG.getAtomicCmpSwap( 1405 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, 1406 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), 1407 AN->getMemOperand()); 1408 1409 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine 1410 // success simply by comparing the loaded value against the ingoing 1411 // comparison. 1412 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, 1413 N->getOperand(2), ISD::SETEQ); 1414 1415 SplitInteger(Tmp, Lo, Hi); 1416 ReplaceValueWith(SDValue(N, 1), Success); 1417 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); 1418 break; 1419 } 1420 1421 case ISD::AND: 1422 case ISD::OR: 1423 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1424 1425 case ISD::UMAX: 1426 case ISD::SMAX: 1427 case ISD::UMIN: 1428 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; 1429 1430 case ISD::ADD: 1431 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1432 1433 case ISD::ADDC: 1434 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1435 1436 case ISD::ADDE: 1437 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1438 1439 case ISD::ADDCARRY: 1440 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; 1441 1442 case ISD::SHL: 1443 case ISD::SRA: 1444 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1445 1446 case ISD::SADDO: 1447 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1448 case ISD::UADDO: 1449 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1450 case ISD::UMULO: 1451 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1452 } 1453 1454 // If Lo/Hi is null, the sub-method took care of registering results etc. 1455 if (Lo.getNode()) 1456 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1457} 1458 1459/// Lower an atomic node to the appropriate builtin call. 1460std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1461 unsigned Opc = Node->getOpcode(); 1462 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1463 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); 1464 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); 1465 1466 return ExpandChainLibCall(LC, Node, false); 1467} 1468 1469/// N is a shift by a value that needs to be expanded, 1470/// and the shift amount is a constant 'Amt'. Expand the operation. 1471void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, 1472 SDValue &Lo, SDValue &Hi) { 1473 SDLoc DL(N); 1474 // Expand the incoming operand to be shifted, so that we have its parts 1475 SDValue InL, InH; 1476 GetExpandedInteger(N->getOperand(0), InL, InH); 1477 1478 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization 1479 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. 1480 if (!Amt) { 1481 Lo = InL; 1482 Hi = InH; 1483 return; 1484 } 1485 1486 EVT NVT = InL.getValueType(); 1487 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1488 unsigned NVTBits = NVT.getSizeInBits(); 1489 EVT ShTy = N->getOperand(1).getValueType(); 1490 1491 if (N->getOpcode() == ISD::SHL) { 1492 if (Amt.ugt(VTBits)) { 1493 Lo = Hi = DAG.getConstant(0, DL, NVT); 1494 } else if (Amt.ugt(NVTBits)) { 1495 Lo = DAG.getConstant(0, DL, NVT); 1496 Hi = DAG.getNode(ISD::SHL, DL, 1497 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1498 } else if (Amt == NVTBits) { 1499 Lo = DAG.getConstant(0, DL, NVT); 1500 Hi = InL; 1501 } else { 1502 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); 1503 Hi = DAG.getNode(ISD::OR, DL, NVT, 1504 DAG.getNode(ISD::SHL, DL, NVT, InH, 1505 DAG.getConstant(Amt, DL, ShTy)), 1506 DAG.getNode(ISD::SRL, DL, NVT, InL, 1507 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1508 } 1509 return; 1510 } 1511 1512 if (N->getOpcode() == ISD::SRL) { 1513 if (Amt.ugt(VTBits)) { 1514 Lo = Hi = DAG.getConstant(0, DL, NVT); 1515 } else if (Amt.ugt(NVTBits)) { 1516 Lo = DAG.getNode(ISD::SRL, DL, 1517 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1518 Hi = DAG.getConstant(0, DL, NVT); 1519 } else if (Amt == NVTBits) { 1520 Lo = InH; 1521 Hi = DAG.getConstant(0, DL, NVT); 1522 } else { 1523 Lo = DAG.getNode(ISD::OR, DL, NVT, 1524 DAG.getNode(ISD::SRL, DL, NVT, InL, 1525 DAG.getConstant(Amt, DL, ShTy)), 1526 DAG.getNode(ISD::SHL, DL, NVT, InH, 1527 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1528 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 1529 } 1530 return; 1531 } 1532 1533 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1534 if (Amt.ugt(VTBits)) { 1535 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1536 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1537 } else if (Amt.ugt(NVTBits)) { 1538 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1539 DAG.getConstant(Amt - NVTBits, DL, ShTy)); 1540 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1541 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1542 } else if (Amt == NVTBits) { 1543 Lo = InH; 1544 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1545 DAG.getConstant(NVTBits - 1, DL, ShTy)); 1546 } else { 1547 Lo = DAG.getNode(ISD::OR, DL, NVT, 1548 DAG.getNode(ISD::SRL, DL, NVT, InL, 1549 DAG.getConstant(Amt, DL, ShTy)), 1550 DAG.getNode(ISD::SHL, DL, NVT, InH, 1551 DAG.getConstant(-Amt + NVTBits, DL, ShTy))); 1552 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); 1553 } 1554} 1555 1556/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1557/// this shift based on knowledge of the high bit of the shift amount. If we 1558/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1559/// shift amount. 1560bool DAGTypeLegalizer:: 1561ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1562 SDValue Amt = N->getOperand(1); 1563 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1564 EVT ShTy = Amt.getValueType(); 1565 unsigned ShBits = ShTy.getScalarSizeInBits(); 1566 unsigned NVTBits = NVT.getScalarSizeInBits(); 1567 assert(isPowerOf2_32(NVTBits) && 1568 "Expanded integer type size not a power of two!"); 1569 SDLoc dl(N); 1570 1571 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1572 KnownBits Known; 1573 DAG.computeKnownBits(N->getOperand(1), Known); 1574 1575 // If we don't know anything about the high bits, exit. 1576 if (((Known.Zero|Known.One) & HighBitMask) == 0) 1577 return false; 1578 1579 // Get the incoming operand to be shifted. 1580 SDValue InL, InH; 1581 GetExpandedInteger(N->getOperand(0), InL, InH); 1582 1583 // If we know that any of the high bits of the shift amount are one, then we 1584 // can do this as a couple of simple shifts. 1585 if (Known.One.intersects(HighBitMask)) { 1586 // Mask out the high bit, which we know is set. 1587 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1588 DAG.getConstant(~HighBitMask, dl, ShTy)); 1589 1590 switch (N->getOpcode()) { 1591 default: llvm_unreachable("Unknown shift"); 1592 case ISD::SHL: 1593 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. 1594 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1595 return true; 1596 case ISD::SRL: 1597 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. 1598 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1599 return true; 1600 case ISD::SRA: 1601 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1602 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1603 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1604 return true; 1605 } 1606 } 1607 1608 // If we know that all of the high bits of the shift amount are zero, then we 1609 // can do this as a couple of simple shifts. 1610 if (HighBitMask.isSubsetOf(Known.Zero)) { 1611 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 1612 // shift if x is zero. We can use XOR here because x is known to be smaller 1613 // than 32. 1614 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 1615 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1616 1617 unsigned Op1, Op2; 1618 switch (N->getOpcode()) { 1619 default: llvm_unreachable("Unknown shift"); 1620 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1621 case ISD::SRL: 1622 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1623 } 1624 1625 // When shifting right the arithmetic for Lo and Hi is swapped. 1626 if (N->getOpcode() != ISD::SHL) 1627 std::swap(InL, InH); 1628 1629 // Use a little trick to get the bits that move from Lo to Hi. First 1630 // shift by one bit. 1631 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); 1632 // Then compute the remaining shift with amount-1. 1633 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 1634 1635 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 1636 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 1637 1638 if (N->getOpcode() != ISD::SHL) 1639 std::swap(Hi, Lo); 1640 return true; 1641 } 1642 1643 return false; 1644} 1645 1646/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1647/// of any size. 1648bool DAGTypeLegalizer:: 1649ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1650 SDValue Amt = N->getOperand(1); 1651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1652 EVT ShTy = Amt.getValueType(); 1653 unsigned NVTBits = NVT.getSizeInBits(); 1654 assert(isPowerOf2_32(NVTBits) && 1655 "Expanded integer type size not a power of two!"); 1656 SDLoc dl(N); 1657 1658 // Get the incoming operand to be shifted. 1659 SDValue InL, InH; 1660 GetExpandedInteger(N->getOperand(0), InL, InH); 1661 1662 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); 1663 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1664 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1665 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1666 Amt, NVBitsNode, ISD::SETULT); 1667 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), 1668 Amt, DAG.getConstant(0, dl, ShTy), 1669 ISD::SETEQ); 1670 1671 SDValue LoS, HiS, LoL, HiL; 1672 switch (N->getOpcode()) { 1673 default: llvm_unreachable("Unknown shift"); 1674 case ISD::SHL: 1675 // Short: ShAmt < NVTBits 1676 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1677 HiS = DAG.getNode(ISD::OR, dl, NVT, 1678 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1679 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1680 1681 // Long: ShAmt >= NVTBits 1682 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. 1683 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1684 1685 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); 1686 Hi = DAG.getSelect(dl, NVT, isZero, InH, 1687 DAG.getSelect(dl, NVT, isShort, HiS, HiL)); 1688 return true; 1689 case ISD::SRL: 1690 // Short: ShAmt < NVTBits 1691 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1692 LoS = DAG.getNode(ISD::OR, dl, NVT, 1693 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1694 // FIXME: If Amt is zero, the following shift generates an undefined result 1695 // on some architectures. 1696 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1697 1698 // Long: ShAmt >= NVTBits 1699 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. 1700 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1701 1702 Lo = DAG.getSelect(dl, NVT, isZero, InL, 1703 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 1704 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1705 return true; 1706 case ISD::SRA: 1707 // Short: ShAmt < NVTBits 1708 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1709 LoS = DAG.getNode(ISD::OR, dl, NVT, 1710 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1711 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1712 1713 // Long: ShAmt >= NVTBits 1714 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1715 DAG.getConstant(NVTBits - 1, dl, ShTy)); 1716 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1717 1718 Lo = DAG.getSelect(dl, NVT, isZero, InL, 1719 DAG.getSelect(dl, NVT, isShort, LoS, LoL)); 1720 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); 1721 return true; 1722 } 1723} 1724 1725static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { 1726 1727 switch (Op) { 1728 default: llvm_unreachable("invalid min/max opcode"); 1729 case ISD::SMAX: 1730 return std::make_pair(ISD::SETGT, ISD::UMAX); 1731 case ISD::UMAX: 1732 return std::make_pair(ISD::SETUGT, ISD::UMAX); 1733 case ISD::SMIN: 1734 return std::make_pair(ISD::SETLT, ISD::UMIN); 1735 case ISD::UMIN: 1736 return std::make_pair(ISD::SETULT, ISD::UMIN); 1737 } 1738} 1739 1740void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, 1741 SDValue &Lo, SDValue &Hi) { 1742 SDLoc DL(N); 1743 ISD::NodeType LoOpc; 1744 ISD::CondCode CondC; 1745 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); 1746 1747 // Expand the subcomponents. 1748 SDValue LHSL, LHSH, RHSL, RHSH; 1749 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1750 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1751 1752 // Value types 1753 EVT NVT = LHSL.getValueType(); 1754 EVT CCT = getSetCCResultType(NVT); 1755 1756 // Hi part is always the same op 1757 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); 1758 1759 // We need to know whether to select Lo part that corresponds to 'winning' 1760 // Hi part or if Hi parts are equal. 1761 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); 1762 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); 1763 1764 // Lo part corresponding to the 'winning' Hi part 1765 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); 1766 1767 // Recursed Lo part if Hi parts are equal, this uses unsigned version 1768 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); 1769 1770 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); 1771} 1772 1773void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1774 SDValue &Lo, SDValue &Hi) { 1775 SDLoc dl(N); 1776 // Expand the subcomponents. 1777 SDValue LHSL, LHSH, RHSL, RHSH; 1778 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1779 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1780 1781 EVT NVT = LHSL.getValueType(); 1782 SDValue LoOps[2] = { LHSL, RHSL }; 1783 SDValue HiOps[3] = { LHSH, RHSH }; 1784 1785 bool HasOpCarry = TLI.isOperationLegalOrCustom( 1786 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 1787 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1788 if (HasOpCarry) { 1789 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); 1790 if (N->getOpcode() == ISD::ADD) { 1791 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 1792 HiOps[2] = Lo.getValue(1); 1793 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); 1794 } else { 1795 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 1796 HiOps[2] = Lo.getValue(1); 1797 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); 1798 } 1799 return; 1800 } 1801 1802 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1803 // them. TODO: Teach operation legalization how to expand unsupported 1804 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1805 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1806 // generate a value of this type in the expanded code sequence. 1807 bool hasCarry = 1808 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1809 ISD::ADDC : ISD::SUBC, 1810 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1811 1812 if (hasCarry) { 1813 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1814 if (N->getOpcode() == ISD::ADD) { 1815 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1816 HiOps[2] = Lo.getValue(1); 1817 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1818 } else { 1819 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1820 HiOps[2] = Lo.getValue(1); 1821 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1822 } 1823 return; 1824 } 1825 1826 bool hasOVF = 1827 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1828 ISD::UADDO : ISD::USUBO, 1829 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1830 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); 1831 1832 if (hasOVF) { 1833 EVT OvfVT = getSetCCResultType(NVT); 1834 SDVTList VTList = DAG.getVTList(NVT, OvfVT); 1835 int RevOpc; 1836 if (N->getOpcode() == ISD::ADD) { 1837 RevOpc = ISD::SUB; 1838 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); 1839 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1840 } else { 1841 RevOpc = ISD::ADD; 1842 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); 1843 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1844 } 1845 SDValue OVF = Lo.getValue(1); 1846 1847 switch (BoolType) { 1848 case TargetLoweringBase::UndefinedBooleanContent: 1849 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); 1850 LLVM_FALLTHROUGH; 1851 case TargetLoweringBase::ZeroOrOneBooleanContent: 1852 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); 1853 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); 1854 break; 1855 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: 1856 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); 1857 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); 1858 } 1859 return; 1860 } 1861 1862 if (N->getOpcode() == ISD::ADD) { 1863 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); 1864 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); 1865 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], 1866 ISD::SETULT); 1867 1868 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { 1869 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); 1870 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); 1871 return; 1872 } 1873 1874 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, 1875 DAG.getConstant(1, dl, NVT), 1876 DAG.getConstant(0, dl, NVT)); 1877 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], 1878 ISD::SETULT); 1879 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, 1880 DAG.getConstant(1, dl, NVT), Carry1); 1881 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1882 } else { 1883 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); 1884 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); 1885 SDValue Cmp = 1886 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), 1887 LoOps[0], LoOps[1], ISD::SETULT); 1888 1889 SDValue Borrow; 1890 if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) 1891 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); 1892 else 1893 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), 1894 DAG.getConstant(0, dl, NVT)); 1895 1896 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1897 } 1898} 1899 1900void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1901 SDValue &Lo, SDValue &Hi) { 1902 // Expand the subcomponents. 1903 SDValue LHSL, LHSH, RHSL, RHSH; 1904 SDLoc dl(N); 1905 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1906 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1907 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1908 SDValue LoOps[2] = { LHSL, RHSL }; 1909 SDValue HiOps[3] = { LHSH, RHSH }; 1910 1911 if (N->getOpcode() == ISD::ADDC) { 1912 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); 1913 HiOps[2] = Lo.getValue(1); 1914 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); 1915 } else { 1916 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); 1917 HiOps[2] = Lo.getValue(1); 1918 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); 1919 } 1920 1921 // Legalized the flag result - switch anything that used the old flag to 1922 // use the new one. 1923 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1924} 1925 1926void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1927 SDValue &Lo, SDValue &Hi) { 1928 // Expand the subcomponents. 1929 SDValue LHSL, LHSH, RHSL, RHSH; 1930 SDLoc dl(N); 1931 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1932 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1933 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1934 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1935 SDValue HiOps[3] = { LHSH, RHSH }; 1936 1937 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 1938 HiOps[2] = Lo.getValue(1); 1939 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 1940 1941 // Legalized the flag result - switch anything that used the old flag to 1942 // use the new one. 1943 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1944} 1945 1946void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 1947 SDValue &Lo, SDValue &Hi) { 1948 SDValue LHS = N->getOperand(0); 1949 SDValue RHS = N->getOperand(1); 1950 SDLoc dl(N); 1951 1952 SDValue Ovf; 1953 1954 bool HasOpCarry = TLI.isOperationLegalOrCustom( 1955 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, 1956 TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); 1957 1958 if (HasOpCarry) { 1959 // Expand the subcomponents. 1960 SDValue LHSL, LHSH, RHSL, RHSH; 1961 GetExpandedInteger(LHS, LHSL, LHSH); 1962 GetExpandedInteger(RHS, RHSL, RHSH); 1963 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 1964 SDValue LoOps[2] = { LHSL, RHSL }; 1965 SDValue HiOps[3] = { LHSH, RHSH }; 1966 1967 unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY; 1968 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 1969 HiOps[2] = Lo.getValue(1); 1970 Hi = DAG.getNode(Opc, dl, VTList, HiOps); 1971 1972 Ovf = Hi.getValue(1); 1973 } else { 1974 // Expand the result by simply replacing it with the equivalent 1975 // non-overflow-checking operation. 1976 auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 1977 SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS); 1978 SplitInteger(Sum, Lo, Hi); 1979 1980 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 1981 // overflows iff a - b > a. 1982 auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT; 1983 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); 1984 } 1985 1986 // Legalized the flag result - switch anything that used the old flag to 1987 // use the new one. 1988 ReplaceValueWith(SDValue(N, 1), Ovf); 1989} 1990 1991void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, 1992 SDValue &Lo, SDValue &Hi) { 1993 // Expand the subcomponents. 1994 SDValue LHSL, LHSH, RHSL, RHSH; 1995 SDLoc dl(N); 1996 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1997 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1998 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); 1999 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 2000 SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; 2001 2002 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); 2003 HiOps[2] = Lo.getValue(1); 2004 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); 2005 2006 // Legalized the flag result - switch anything that used the old flag to 2007 // use the new one. 2008 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 2009} 2010 2011void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 2012 SDValue &Lo, SDValue &Hi) { 2013 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2014 SDLoc dl(N); 2015 SDValue Op = N->getOperand(0); 2016 if (Op.getValueType().bitsLE(NVT)) { 2017 // The low part is any extension of the input (which degenerates to a copy). 2018 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 2019 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 2020 } else { 2021 // For example, extension of an i48 to an i64. The operand type necessarily 2022 // promotes to the result type, so will end up being expanded too. 2023 assert(getTypeAction(Op.getValueType()) == 2024 TargetLowering::TypePromoteInteger && 2025 "Only know how to promote this result!"); 2026 SDValue Res = GetPromotedInteger(Op); 2027 assert(Res.getValueType() == N->getValueType(0) && 2028 "Operand over promoted?"); 2029 // Split the promoted operand. This will simplify when it is expanded. 2030 SplitInteger(Res, Lo, Hi); 2031 } 2032} 2033 2034void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 2035 SDValue &Lo, SDValue &Hi) { 2036 SDLoc dl(N); 2037 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2038 EVT NVT = Lo.getValueType(); 2039 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2040 unsigned NVTBits = NVT.getSizeInBits(); 2041 unsigned EVTBits = EVT.getSizeInBits(); 2042 2043 if (NVTBits < EVTBits) { 2044 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 2045 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2046 EVTBits - NVTBits))); 2047 } else { 2048 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 2049 // The high part replicates the sign bit of Lo, make it explicit. 2050 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2051 DAG.getConstant(NVTBits - 1, dl, 2052 TLI.getPointerTy(DAG.getDataLayout()))); 2053 } 2054} 2055 2056void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 2057 SDValue &Lo, SDValue &Hi) { 2058 SDLoc dl(N); 2059 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2060 EVT NVT = Lo.getValueType(); 2061 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2062 unsigned NVTBits = NVT.getSizeInBits(); 2063 unsigned EVTBits = EVT.getSizeInBits(); 2064 2065 if (NVTBits < EVTBits) { 2066 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 2067 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2068 EVTBits - NVTBits))); 2069 } else { 2070 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 2071 // The high part must be zero, make it explicit. 2072 Hi = DAG.getConstant(0, dl, NVT); 2073 } 2074} 2075 2076void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, 2077 SDValue &Lo, SDValue &Hi) { 2078 SDLoc dl(N); 2079 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2080 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); 2081 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); 2082} 2083 2084void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 2085 SDValue &Lo, SDValue &Hi) { 2086 SDLoc dl(N); 2087 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 2088 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 2089 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 2090} 2091 2092void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 2093 SDValue &Lo, SDValue &Hi) { 2094 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2095 unsigned NBitWidth = NVT.getSizeInBits(); 2096 auto Constant = cast<ConstantSDNode>(N); 2097 const APInt &Cst = Constant->getAPIntValue(); 2098 bool IsTarget = Constant->isTargetOpcode(); 2099 bool IsOpaque = Constant->isOpaque(); 2100 SDLoc dl(N); 2101 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); 2102 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, 2103 IsOpaque); 2104} 2105 2106void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 2107 SDValue &Lo, SDValue &Hi) { 2108 SDLoc dl(N); 2109 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 2110 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2111 EVT NVT = Lo.getValueType(); 2112 2113 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, 2114 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2115 2116 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 2117 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 2118 2119 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, 2120 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 2121 DAG.getConstant(NVT.getSizeInBits(), dl, 2122 NVT))); 2123 Hi = DAG.getConstant(0, dl, NVT); 2124} 2125 2126void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 2127 SDValue &Lo, SDValue &Hi) { 2128 SDLoc dl(N); 2129 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 2130 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2131 EVT NVT = Lo.getValueType(); 2132 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 2133 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 2134 Hi = DAG.getConstant(0, dl, NVT); 2135} 2136 2137void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 2138 SDValue &Lo, SDValue &Hi) { 2139 SDLoc dl(N); 2140 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 2141 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2142 EVT NVT = Lo.getValueType(); 2143 2144 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, 2145 DAG.getConstant(0, dl, NVT), ISD::SETNE); 2146 2147 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 2148 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 2149 2150 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, 2151 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 2152 DAG.getConstant(NVT.getSizeInBits(), dl, 2153 NVT))); 2154 Hi = DAG.getConstant(0, dl, NVT); 2155} 2156 2157void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, 2158 SDValue &Hi) { 2159 SDLoc dl(N); 2160 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2161 unsigned NBitWidth = NVT.getSizeInBits(); 2162 2163 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2164 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT); 2165 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS 2166 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2167 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); 2168} 2169 2170void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 2171 SDValue &Hi) { 2172 SDLoc dl(N); 2173 EVT VT = N->getValueType(0); 2174 2175 SDValue Op = N->getOperand(0); 2176 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2177 Op = GetPromotedFloat(Op); 2178 2179 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 2180 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 2181 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first, 2182 Lo, Hi); 2183} 2184 2185void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 2186 SDValue &Hi) { 2187 SDLoc dl(N); 2188 EVT VT = N->getValueType(0); 2189 2190 SDValue Op = N->getOperand(0); 2191 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) 2192 Op = GetPromotedFloat(Op); 2193 2194 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 2195 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 2196 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first, 2197 Lo, Hi); 2198} 2199 2200void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 2201 SDValue &Lo, SDValue &Hi) { 2202 if (ISD::isNormalLoad(N)) { 2203 ExpandRes_NormalLoad(N, Lo, Hi); 2204 return; 2205 } 2206 2207 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 2208 2209 EVT VT = N->getValueType(0); 2210 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2211 SDValue Ch = N->getChain(); 2212 SDValue Ptr = N->getBasePtr(); 2213 ISD::LoadExtType ExtType = N->getExtensionType(); 2214 unsigned Alignment = N->getAlignment(); 2215 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 2216 AAMDNodes AAInfo = N->getAAInfo(); 2217 SDLoc dl(N); 2218 2219 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2220 2221 if (N->getMemoryVT().bitsLE(NVT)) { 2222 EVT MemVT = N->getMemoryVT(); 2223 2224 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, 2225 Alignment, MMOFlags, AAInfo); 2226 2227 // Remember the chain. 2228 Ch = Lo.getValue(1); 2229 2230 if (ExtType == ISD::SEXTLOAD) { 2231 // The high part is obtained by SRA'ing all but one of the bits of the 2232 // lo part. 2233 unsigned LoSize = Lo.getValueSizeInBits(); 2234 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2235 DAG.getConstant(LoSize - 1, dl, 2236 TLI.getPointerTy(DAG.getDataLayout()))); 2237 } else if (ExtType == ISD::ZEXTLOAD) { 2238 // The high part is just a zero. 2239 Hi = DAG.getConstant(0, dl, NVT); 2240 } else { 2241 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 2242 // The high part is undefined. 2243 Hi = DAG.getUNDEF(NVT); 2244 } 2245 } else if (DAG.getDataLayout().isLittleEndian()) { 2246 // Little-endian - low bits are at low addresses. 2247 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 2248 AAInfo); 2249 2250 unsigned ExcessBits = 2251 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2252 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2253 2254 // Increment the pointer to the other half. 2255 unsigned IncrementSize = NVT.getSizeInBits()/8; 2256 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2257 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 2258 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 2259 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 2260 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 2261 2262 // Build a factor node to remember that this load is independent of the 2263 // other one. 2264 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2265 Hi.getValue(1)); 2266 } else { 2267 // Big-endian - high bits are at low addresses. Favor aligned loads at 2268 // the cost of some bit-fiddling. 2269 EVT MemVT = N->getMemoryVT(); 2270 unsigned EBytes = MemVT.getStoreSize(); 2271 unsigned IncrementSize = NVT.getSizeInBits()/8; 2272 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2273 2274 // Load both the high bits and maybe some of the low bits. 2275 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 2276 EVT::getIntegerVT(*DAG.getContext(), 2277 MemVT.getSizeInBits() - ExcessBits), 2278 Alignment, MMOFlags, AAInfo); 2279 2280 // Increment the pointer to the other half. 2281 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2282 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 2283 // Load the rest of the low bits. 2284 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 2285 N->getPointerInfo().getWithOffset(IncrementSize), 2286 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2287 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 2288 2289 // Build a factor node to remember that this load is independent of the 2290 // other one. 2291 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2292 Hi.getValue(1)); 2293 2294 if (ExcessBits < NVT.getSizeInBits()) { 2295 // Transfer low bits from the bottom of Hi to the top of Lo. 2296 Lo = DAG.getNode( 2297 ISD::OR, dl, NVT, Lo, 2298 DAG.getNode(ISD::SHL, dl, NVT, Hi, 2299 DAG.getConstant(ExcessBits, dl, 2300 TLI.getPointerTy(DAG.getDataLayout())))); 2301 // Move high bits to the right position in Hi. 2302 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, 2303 Hi, 2304 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 2305 TLI.getPointerTy(DAG.getDataLayout()))); 2306 } 2307 } 2308 2309 // Legalize the chain result - switch anything that used the old chain to 2310 // use the new one. 2311 ReplaceValueWith(SDValue(N, 1), Ch); 2312} 2313 2314void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 2315 SDValue &Lo, SDValue &Hi) { 2316 SDLoc dl(N); 2317 SDValue LL, LH, RL, RH; 2318 GetExpandedInteger(N->getOperand(0), LL, LH); 2319 GetExpandedInteger(N->getOperand(1), RL, RH); 2320 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 2321 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 2322} 2323 2324void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 2325 SDValue &Lo, SDValue &Hi) { 2326 EVT VT = N->getValueType(0); 2327 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2328 SDLoc dl(N); 2329 2330 SDValue LL, LH, RL, RH; 2331 GetExpandedInteger(N->getOperand(0), LL, LH); 2332 GetExpandedInteger(N->getOperand(1), RL, RH); 2333 2334 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, 2335 TargetLowering::MulExpansionKind::OnlyLegalOrCustom, 2336 LL, LH, RL, RH)) 2337 return; 2338 2339 // If nothing else, we can make a libcall. 2340 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2341 if (VT == MVT::i16) 2342 LC = RTLIB::MUL_I16; 2343 else if (VT == MVT::i32) 2344 LC = RTLIB::MUL_I32; 2345 else if (VT == MVT::i64) 2346 LC = RTLIB::MUL_I64; 2347 else if (VT == MVT::i128) 2348 LC = RTLIB::MUL_I128; 2349 2350 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { 2351 // We'll expand the multiplication by brute force because we have no other 2352 // options. This is a trivially-generalized version of the code from 2353 // Hacker's Delight (itself derived from Knuth's Algorithm M from section 2354 // 4.3.1). 2355 unsigned Bits = NVT.getSizeInBits(); 2356 unsigned HalfBits = Bits >> 1; 2357 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, 2358 NVT); 2359 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); 2360 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); 2361 2362 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); 2363 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); 2364 2365 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); 2366 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) { 2367 // The type from TLI is too small to fit the shift amount we want. 2368 // Override it with i32. The shift will have to be legalized. 2369 ShiftAmtTy = MVT::i32; 2370 } 2371 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); 2372 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); 2373 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); 2374 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); 2375 2376 SDValue U = DAG.getNode(ISD::ADD, dl, NVT, 2377 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); 2378 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); 2379 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); 2380 2381 SDValue V = DAG.getNode(ISD::ADD, dl, NVT, 2382 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); 2383 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); 2384 2385 SDValue W = DAG.getNode(ISD::ADD, dl, NVT, 2386 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), 2387 DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); 2388 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, 2389 DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); 2390 2391 Hi = DAG.getNode(ISD::ADD, dl, NVT, W, 2392 DAG.getNode(ISD::ADD, dl, NVT, 2393 DAG.getNode(ISD::MUL, dl, NVT, RH, LL), 2394 DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); 2395 return; 2396 } 2397 2398 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2399 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first, 2400 Lo, Hi); 2401} 2402 2403void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, 2404 SDValue &Hi) { 2405 SDLoc DL(N); 2406 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2407 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); 2408 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); 2409 Lo = R.getValue(0); 2410 Hi = R.getValue(1); 2411 ReplaceValueWith(SDValue(N, 1), R.getValue(2)); 2412} 2413 2414void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 2415 SDValue &Lo, SDValue &Hi) { 2416 SDValue LHS = Node->getOperand(0); 2417 SDValue RHS = Node->getOperand(1); 2418 SDLoc dl(Node); 2419 2420 // Expand the result by simply replacing it with the equivalent 2421 // non-overflow-checking operation. 2422 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 2423 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2424 LHS, RHS); 2425 SplitInteger(Sum, Lo, Hi); 2426 2427 // Compute the overflow. 2428 // 2429 // LHSSign -> LHS >= 0 2430 // RHSSign -> RHS >= 0 2431 // SumSign -> Sum >= 0 2432 // 2433 // Add: 2434 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 2435 // Sub: 2436 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 2437 // 2438 EVT OType = Node->getValueType(1); 2439 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); 2440 2441 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 2442 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 2443 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 2444 Node->getOpcode() == ISD::SADDO ? 2445 ISD::SETEQ : ISD::SETNE); 2446 2447 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 2448 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 2449 2450 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 2451 2452 // Use the calculated overflow everywhere. 2453 ReplaceValueWith(SDValue(Node, 1), Cmp); 2454} 2455 2456void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 2457 SDValue &Lo, SDValue &Hi) { 2458 EVT VT = N->getValueType(0); 2459 SDLoc dl(N); 2460 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2461 2462 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 2463 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2464 SplitInteger(Res.getValue(0), Lo, Hi); 2465 return; 2466 } 2467 2468 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2469 if (VT == MVT::i16) 2470 LC = RTLIB::SDIV_I16; 2471 else if (VT == MVT::i32) 2472 LC = RTLIB::SDIV_I32; 2473 else if (VT == MVT::i64) 2474 LC = RTLIB::SDIV_I64; 2475 else if (VT == MVT::i128) 2476 LC = RTLIB::SDIV_I128; 2477 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 2478 2479 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); 2480} 2481 2482void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 2483 SDValue &Lo, SDValue &Hi) { 2484 EVT VT = N->getValueType(0); 2485 SDLoc dl(N); 2486 2487 // If we can emit an efficient shift operation, do so now. Check to see if 2488 // the RHS is a constant. 2489 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2490 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); 2491 2492 // If we can determine that the high bit of the shift is zero or one, even if 2493 // the low bits are variable, emit this shift in an optimized form. 2494 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 2495 return; 2496 2497 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 2498 unsigned PartsOpc; 2499 if (N->getOpcode() == ISD::SHL) { 2500 PartsOpc = ISD::SHL_PARTS; 2501 } else if (N->getOpcode() == ISD::SRL) { 2502 PartsOpc = ISD::SRL_PARTS; 2503 } else { 2504 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2505 PartsOpc = ISD::SRA_PARTS; 2506 } 2507 2508 // Next check to see if the target supports this SHL_PARTS operation or if it 2509 // will custom expand it. 2510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2511 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 2512 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 2513 Action == TargetLowering::Custom) { 2514 // Expand the subcomponents. 2515 SDValue LHSL, LHSH; 2516 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2517 EVT VT = LHSL.getValueType(); 2518 2519 // If the shift amount operand is coming from a vector legalization it may 2520 // have an illegal type. Fix that first by casting the operand, otherwise 2521 // the new SHL_PARTS operation would need further legalization. 2522 SDValue ShiftOp = N->getOperand(1); 2523 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); 2524 assert(ShiftTy.getScalarSizeInBits() >= 2525 Log2_32_Ceil(VT.getScalarSizeInBits()) && 2526 "ShiftAmountTy is too small to cover the range of this type!"); 2527 if (ShiftOp.getValueType() != ShiftTy) 2528 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); 2529 2530 SDValue Ops[] = { LHSL, LHSH, ShiftOp }; 2531 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); 2532 Hi = Lo.getValue(1); 2533 return; 2534 } 2535 2536 // Otherwise, emit a libcall. 2537 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2538 bool isSigned; 2539 if (N->getOpcode() == ISD::SHL) { 2540 isSigned = false; /*sign irrelevant*/ 2541 if (VT == MVT::i16) 2542 LC = RTLIB::SHL_I16; 2543 else if (VT == MVT::i32) 2544 LC = RTLIB::SHL_I32; 2545 else if (VT == MVT::i64) 2546 LC = RTLIB::SHL_I64; 2547 else if (VT == MVT::i128) 2548 LC = RTLIB::SHL_I128; 2549 } else if (N->getOpcode() == ISD::SRL) { 2550 isSigned = false; 2551 if (VT == MVT::i16) 2552 LC = RTLIB::SRL_I16; 2553 else if (VT == MVT::i32) 2554 LC = RTLIB::SRL_I32; 2555 else if (VT == MVT::i64) 2556 LC = RTLIB::SRL_I64; 2557 else if (VT == MVT::i128) 2558 LC = RTLIB::SRL_I128; 2559 } else { 2560 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2561 isSigned = true; 2562 if (VT == MVT::i16) 2563 LC = RTLIB::SRA_I16; 2564 else if (VT == MVT::i32) 2565 LC = RTLIB::SRA_I32; 2566 else if (VT == MVT::i64) 2567 LC = RTLIB::SRA_I64; 2568 else if (VT == MVT::i128) 2569 LC = RTLIB::SRA_I128; 2570 } 2571 2572 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2573 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2574 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi); 2575 return; 2576 } 2577 2578 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2579 llvm_unreachable("Unsupported shift!"); 2580} 2581 2582void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2583 SDValue &Lo, SDValue &Hi) { 2584 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2585 SDLoc dl(N); 2586 SDValue Op = N->getOperand(0); 2587 if (Op.getValueType().bitsLE(NVT)) { 2588 // The low part is sign extension of the input (degenerates to a copy). 2589 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2590 // The high part is obtained by SRA'ing all but one of the bits of low part. 2591 unsigned LoSize = NVT.getSizeInBits(); 2592 Hi = DAG.getNode( 2593 ISD::SRA, dl, NVT, Lo, 2594 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); 2595 } else { 2596 // For example, extension of an i48 to an i64. The operand type necessarily 2597 // promotes to the result type, so will end up being expanded too. 2598 assert(getTypeAction(Op.getValueType()) == 2599 TargetLowering::TypePromoteInteger && 2600 "Only know how to promote this result!"); 2601 SDValue Res = GetPromotedInteger(Op); 2602 assert(Res.getValueType() == N->getValueType(0) && 2603 "Operand over promoted?"); 2604 // Split the promoted operand. This will simplify when it is expanded. 2605 SplitInteger(Res, Lo, Hi); 2606 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 2607 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2608 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2609 ExcessBits))); 2610 } 2611} 2612 2613void DAGTypeLegalizer:: 2614ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2615 SDLoc dl(N); 2616 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2617 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2618 2619 if (EVT.bitsLE(Lo.getValueType())) { 2620 // sext_inreg the low part if needed. 2621 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2622 N->getOperand(1)); 2623 2624 // The high part gets the sign extension from the lo-part. This handles 2625 // things like sextinreg V:i64 from i8. 2626 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2627 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, 2628 TLI.getPointerTy(DAG.getDataLayout()))); 2629 } else { 2630 // For example, extension of an i48 to an i64. Leave the low part alone, 2631 // sext_inreg the high part. 2632 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); 2633 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2634 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2635 ExcessBits))); 2636 } 2637} 2638 2639void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2640 SDValue &Lo, SDValue &Hi) { 2641 EVT VT = N->getValueType(0); 2642 SDLoc dl(N); 2643 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2644 2645 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { 2646 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2647 SplitInteger(Res.getValue(1), Lo, Hi); 2648 return; 2649 } 2650 2651 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2652 if (VT == MVT::i16) 2653 LC = RTLIB::SREM_I16; 2654 else if (VT == MVT::i32) 2655 LC = RTLIB::SREM_I32; 2656 else if (VT == MVT::i64) 2657 LC = RTLIB::SREM_I64; 2658 else if (VT == MVT::i128) 2659 LC = RTLIB::SREM_I128; 2660 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2661 2662 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); 2663} 2664 2665void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2666 SDValue &Lo, SDValue &Hi) { 2667 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2668 SDLoc dl(N); 2669 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2670 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), 2671 N->getOperand(0), 2672 DAG.getConstant(NVT.getSizeInBits(), dl, 2673 TLI.getPointerTy(DAG.getDataLayout()))); 2674 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2675} 2676 2677void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2678 SDValue &Lo, SDValue &Hi) { 2679 EVT VT = N->getValueType(0); 2680 SDLoc dl(N); 2681 2682 // A divide for UMULO should be faster than a function call. 2683 if (N->getOpcode() == ISD::UMULO) { 2684 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2685 2686 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS); 2687 SplitInteger(MUL, Lo, Hi); 2688 2689 // A divide for UMULO will be faster than a function call. Select to 2690 // make sure we aren't using 0. 2691 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT), 2692 RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ); 2693 SDValue NotZero = DAG.getSelect(dl, VT, isZero, 2694 DAG.getConstant(1, dl, VT), RHS); 2695 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero); 2696 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS, 2697 ISD::SETNE); 2698 Overflow = DAG.getSelect(dl, N->getValueType(1), isZero, 2699 DAG.getConstant(0, dl, N->getValueType(1)), 2700 Overflow); 2701 ReplaceValueWith(SDValue(N, 1), Overflow); 2702 return; 2703 } 2704 2705 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2706 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 2707 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2708 2709 // Replace this with a libcall that will check overflow. 2710 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2711 if (VT == MVT::i32) 2712 LC = RTLIB::MULO_I32; 2713 else if (VT == MVT::i64) 2714 LC = RTLIB::MULO_I64; 2715 else if (VT == MVT::i128) 2716 LC = RTLIB::MULO_I128; 2717 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2718 2719 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2720 // Temporary for the overflow value, default it to zero. 2721 SDValue Chain = 2722 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, 2723 MachinePointerInfo()); 2724 2725 TargetLowering::ArgListTy Args; 2726 TargetLowering::ArgListEntry Entry; 2727 for (const SDValue &Op : N->op_values()) { 2728 EVT ArgVT = Op.getValueType(); 2729 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2730 Entry.Node = Op; 2731 Entry.Ty = ArgTy; 2732 Entry.IsSExt = true; 2733 Entry.IsZExt = false; 2734 Args.push_back(Entry); 2735 } 2736 2737 // Also pass the address of the overflow check. 2738 Entry.Node = Temp; 2739 Entry.Ty = PtrTy->getPointerTo(); 2740 Entry.IsSExt = true; 2741 Entry.IsZExt = false; 2742 Args.push_back(Entry); 2743 2744 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2745 2746 TargetLowering::CallLoweringInfo CLI(DAG); 2747 CLI.setDebugLoc(dl) 2748 .setChain(Chain) 2749 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) 2750 .setSExtResult(); 2751 2752 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2753 2754 SplitInteger(CallInfo.first, Lo, Hi); 2755 SDValue Temp2 = 2756 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); 2757 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2758 DAG.getConstant(0, dl, PtrVT), 2759 ISD::SETNE); 2760 // Use the overflow from the libcall everywhere. 2761 ReplaceValueWith(SDValue(N, 1), Ofl); 2762} 2763 2764void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2765 SDValue &Lo, SDValue &Hi) { 2766 EVT VT = N->getValueType(0); 2767 SDLoc dl(N); 2768 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2769 2770 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 2771 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2772 SplitInteger(Res.getValue(0), Lo, Hi); 2773 return; 2774 } 2775 2776 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2777 if (VT == MVT::i16) 2778 LC = RTLIB::UDIV_I16; 2779 else if (VT == MVT::i32) 2780 LC = RTLIB::UDIV_I32; 2781 else if (VT == MVT::i64) 2782 LC = RTLIB::UDIV_I64; 2783 else if (VT == MVT::i128) 2784 LC = RTLIB::UDIV_I128; 2785 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2786 2787 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); 2788} 2789 2790void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2791 SDValue &Lo, SDValue &Hi) { 2792 EVT VT = N->getValueType(0); 2793 SDLoc dl(N); 2794 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2795 2796 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { 2797 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); 2798 SplitInteger(Res.getValue(1), Lo, Hi); 2799 return; 2800 } 2801 2802 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2803 if (VT == MVT::i16) 2804 LC = RTLIB::UREM_I16; 2805 else if (VT == MVT::i32) 2806 LC = RTLIB::UREM_I32; 2807 else if (VT == MVT::i64) 2808 LC = RTLIB::UREM_I64; 2809 else if (VT == MVT::i128) 2810 LC = RTLIB::UREM_I128; 2811 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2812 2813 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); 2814} 2815 2816void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2817 SDValue &Lo, SDValue &Hi) { 2818 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2819 SDLoc dl(N); 2820 SDValue Op = N->getOperand(0); 2821 if (Op.getValueType().bitsLE(NVT)) { 2822 // The low part is zero extension of the input (degenerates to a copy). 2823 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2824 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. 2825 } else { 2826 // For example, extension of an i48 to an i64. The operand type necessarily 2827 // promotes to the result type, so will end up being expanded too. 2828 assert(getTypeAction(Op.getValueType()) == 2829 TargetLowering::TypePromoteInteger && 2830 "Only know how to promote this result!"); 2831 SDValue Res = GetPromotedInteger(Op); 2832 assert(Res.getValueType() == N->getValueType(0) && 2833 "Operand over promoted?"); 2834 // Split the promoted operand. This will simplify when it is expanded. 2835 SplitInteger(Res, Lo, Hi); 2836 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); 2837 Hi = DAG.getZeroExtendInReg(Hi, dl, 2838 EVT::getIntegerVT(*DAG.getContext(), 2839 ExcessBits)); 2840 } 2841} 2842 2843void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 2844 SDValue &Lo, SDValue &Hi) { 2845 SDLoc dl(N); 2846 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 2847 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); 2848 SDValue Zero = DAG.getConstant(0, dl, VT); 2849 SDValue Swap = DAG.getAtomicCmpSwap( 2850 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, 2851 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), 2852 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); 2853 2854 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2855 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); 2856} 2857 2858//===----------------------------------------------------------------------===// 2859// Integer Operand Expansion 2860//===----------------------------------------------------------------------===// 2861 2862/// ExpandIntegerOperand - This method is called when the specified operand of 2863/// the specified node is found to need expansion. At this point, all of the 2864/// result types of the node are known to be legal, but other operands of the 2865/// node may need promotion or expansion as well as the specified one. 2866bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2867 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2868 SDValue Res = SDValue(); 2869 2870 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2871 return false; 2872 2873 switch (N->getOpcode()) { 2874 default: 2875 #ifndef NDEBUG 2876 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2877 N->dump(&DAG); dbgs() << "\n"; 2878 #endif 2879 llvm_unreachable("Do not know how to expand this operator's operand!"); 2880 2881 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2882 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2883 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2884 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2885 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2886 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2887 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2888 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2889 case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break; 2890 case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; 2891 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2892 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2893 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2894 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2895 2896 case ISD::SHL: 2897 case ISD::SRA: 2898 case ISD::SRL: 2899 case ISD::ROTL: 2900 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2901 case ISD::RETURNADDR: 2902 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2903 2904 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 2905 } 2906 2907 // If the result is null, the sub-method took care of registering results etc. 2908 if (!Res.getNode()) return false; 2909 2910 // If the result is N, the sub-method updated N in place. Tell the legalizer 2911 // core about this. 2912 if (Res.getNode() == N) 2913 return true; 2914 2915 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2916 "Invalid operand expansion"); 2917 2918 ReplaceValueWith(SDValue(N, 0), Res); 2919 return false; 2920} 2921 2922/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2923/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2924void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2925 SDValue &NewRHS, 2926 ISD::CondCode &CCCode, 2927 const SDLoc &dl) { 2928 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2929 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2930 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2931 2932 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2933 if (RHSLo == RHSHi) { 2934 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2935 if (RHSCST->isAllOnesValue()) { 2936 // Equality comparison to -1. 2937 NewLHS = DAG.getNode(ISD::AND, dl, 2938 LHSLo.getValueType(), LHSLo, LHSHi); 2939 NewRHS = RHSLo; 2940 return; 2941 } 2942 } 2943 } 2944 2945 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2946 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2947 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2948 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); 2949 return; 2950 } 2951 2952 // If this is a comparison of the sign bit, just look at the top part. 2953 // X > -1, x < 0 2954 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2955 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2956 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2957 NewLHS = LHSHi; 2958 NewRHS = RHSHi; 2959 return; 2960 } 2961 2962 // FIXME: This generated code sucks. 2963 ISD::CondCode LowCC; 2964 switch (CCCode) { 2965 default: llvm_unreachable("Unknown integer setcc!"); 2966 case ISD::SETLT: 2967 case ISD::SETULT: LowCC = ISD::SETULT; break; 2968 case ISD::SETGT: 2969 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2970 case ISD::SETLE: 2971 case ISD::SETULE: LowCC = ISD::SETULE; break; 2972 case ISD::SETGE: 2973 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2974 } 2975 2976 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison 2977 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands 2978 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; 2979 2980 // NOTE: on targets without efficient SELECT of bools, we can always use 2981 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2982 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, 2983 nullptr); 2984 SDValue LoCmp, HiCmp; 2985 if (TLI.isTypeLegal(LHSLo.getValueType()) && 2986 TLI.isTypeLegal(RHSLo.getValueType())) 2987 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, 2988 RHSLo, LowCC, false, DagCombineInfo, dl); 2989 if (!LoCmp.getNode()) 2990 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 2991 RHSLo, LowCC); 2992 if (TLI.isTypeLegal(LHSHi.getValueType()) && 2993 TLI.isTypeLegal(RHSHi.getValueType())) 2994 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, 2995 RHSHi, CCCode, false, DagCombineInfo, dl); 2996 if (!HiCmp.getNode()) 2997 HiCmp = 2998 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), 2999 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 3000 3001 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); 3002 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); 3003 3004 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 3005 CCCode == ISD::SETUGE || CCCode == ISD::SETULE); 3006 3007 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) || 3008 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) || 3009 (LoCmpC && LoCmpC->isNullValue())))) { 3010 // For LE / GE, if high part is known false, ignore the low part. 3011 // For LT / GT: if low part is known false, return the high part. 3012 // if high part is known true, ignore the low part. 3013 NewLHS = HiCmp; 3014 NewRHS = SDValue(); 3015 return; 3016 } 3017 3018 if (LHSHi == RHSHi) { 3019 // Comparing the low bits is enough. 3020 NewLHS = LoCmp; 3021 NewRHS = SDValue(); 3022 return; 3023 } 3024 3025 // Lower with SETCCE or SETCCCARRY if the target supports it. 3026 EVT HiVT = LHSHi.getValueType(); 3027 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); 3028 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); 3029 3030 // FIXME: Make all targets support this, then remove the other lowering. 3031 if (HasSETCCCARRY || 3032 TLI.getOperationAction(ISD::SETCCE, ExpandVT) == TargetLowering::Custom) { 3033 // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip 3034 // operands and condition code. 3035 bool FlipOperands = false; 3036 switch (CCCode) { 3037 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; 3038 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; 3039 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; 3040 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; 3041 default: break; 3042 } 3043 if (FlipOperands) { 3044 std::swap(LHSLo, RHSLo); 3045 std::swap(LHSHi, RHSHi); 3046 } 3047 // Perform a wide subtraction, feeding the carry from the low part into 3048 // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially 3049 // looking at the high part of the result of LHS - RHS. It is negative 3050 // iff LHS < RHS. It is zero or positive iff LHS >= RHS. 3051 EVT LoVT = LHSLo.getValueType(); 3052 SDVTList VTList = DAG.getVTList( 3053 LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT) : MVT::Glue); 3054 SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO : ISD::SUBC, dl, 3055 VTList, LHSLo, RHSLo); 3056 SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY : ISD::SETCCE, dl, 3057 getSetCCResultType(HiVT), LHSHi, RHSHi, 3058 LowCmp.getValue(1), DAG.getCondCode(CCCode)); 3059 NewLHS = Res; 3060 NewRHS = SDValue(); 3061 return; 3062 } 3063 3064 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, 3065 false, DagCombineInfo, dl); 3066 if (!NewLHS.getNode()) 3067 NewLHS = 3068 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); 3069 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); 3070 NewRHS = SDValue(); 3071} 3072 3073SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 3074 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 3075 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 3076 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3077 3078 // If ExpandSetCCOperands returned a scalar, we need to compare the result 3079 // against zero to select between true and false values. 3080 if (!NewRHS.getNode()) { 3081 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 3082 CCCode = ISD::SETNE; 3083 } 3084 3085 // Update N to have the operands specified. 3086 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 3087 DAG.getCondCode(CCCode), NewLHS, NewRHS, 3088 N->getOperand(4)), 0); 3089} 3090 3091SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 3092 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 3093 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 3094 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3095 3096 // If ExpandSetCCOperands returned a scalar, we need to compare the result 3097 // against zero to select between true and false values. 3098 if (!NewRHS.getNode()) { 3099 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 3100 CCCode = ISD::SETNE; 3101 } 3102 3103 // Update N to have the operands specified. 3104 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 3105 N->getOperand(2), N->getOperand(3), 3106 DAG.getCondCode(CCCode)), 0); 3107} 3108 3109SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 3110 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 3111 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3112 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); 3113 3114 // If ExpandSetCCOperands returned a scalar, use it. 3115 if (!NewRHS.getNode()) { 3116 assert(NewLHS.getValueType() == N->getValueType(0) && 3117 "Unexpected setcc expansion!"); 3118 return NewLHS; 3119 } 3120 3121 // Otherwise, update N to have the operands specified. 3122 return SDValue( 3123 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); 3124} 3125 3126SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) { 3127 SDValue LHS = N->getOperand(0); 3128 SDValue RHS = N->getOperand(1); 3129 SDValue Carry = N->getOperand(2); 3130 SDValue Cond = N->getOperand(3); 3131 SDLoc dl = SDLoc(N); 3132 3133 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 3134 GetExpandedInteger(LHS, LHSLo, LHSHi); 3135 GetExpandedInteger(RHS, RHSLo, RHSHi); 3136 3137 // Expand to a SUBE for the low part and a smaller SETCCE for the high. 3138 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue); 3139 SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry); 3140 return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi, 3141 LowCmp.getValue(1), Cond); 3142} 3143 3144SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { 3145 SDValue LHS = N->getOperand(0); 3146 SDValue RHS = N->getOperand(1); 3147 SDValue Carry = N->getOperand(2); 3148 SDValue Cond = N->getOperand(3); 3149 SDLoc dl = SDLoc(N); 3150 3151 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 3152 GetExpandedInteger(LHS, LHSLo, LHSHi); 3153 GetExpandedInteger(RHS, RHSLo, RHSHi); 3154 3155 // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. 3156 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); 3157 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); 3158 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, 3159 LowCmp.getValue(1), Cond); 3160} 3161 3162SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 3163 // The value being shifted is legal, but the shift amount is too big. 3164 // It follows that either the result of the shift is undefined, or the 3165 // upper half of the shift amount is zero. Just use the lower half. 3166 SDValue Lo, Hi; 3167 GetExpandedInteger(N->getOperand(1), Lo, Hi); 3168 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 3169} 3170 3171SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 3172 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 3173 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 3174 // constant to valid type. 3175 SDValue Lo, Hi; 3176 GetExpandedInteger(N->getOperand(0), Lo, Hi); 3177 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 3178} 3179 3180SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 3181 SDValue Op = N->getOperand(0); 3182 EVT DstVT = N->getValueType(0); 3183 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 3184 assert(LC != RTLIB::UNKNOWN_LIBCALL && 3185 "Don't know how to expand this SINT_TO_FP!"); 3186 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first; 3187} 3188 3189SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 3190 if (ISD::isNormalStore(N)) 3191 return ExpandOp_NormalStore(N, OpNo); 3192 3193 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 3194 assert(OpNo == 1 && "Can only expand the stored value so far"); 3195 3196 EVT VT = N->getOperand(1).getValueType(); 3197 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3198 SDValue Ch = N->getChain(); 3199 SDValue Ptr = N->getBasePtr(); 3200 unsigned Alignment = N->getAlignment(); 3201 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 3202 AAMDNodes AAInfo = N->getAAInfo(); 3203 SDLoc dl(N); 3204 SDValue Lo, Hi; 3205 3206 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 3207 3208 if (N->getMemoryVT().bitsLE(NVT)) { 3209 GetExpandedInteger(N->getValue(), Lo, Hi); 3210 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 3211 N->getMemoryVT(), Alignment, MMOFlags, AAInfo); 3212 } 3213 3214 if (DAG.getDataLayout().isLittleEndian()) { 3215 // Little-endian - low bits are at low addresses. 3216 GetExpandedInteger(N->getValue(), Lo, Hi); 3217 3218 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 3219 AAInfo); 3220 3221 unsigned ExcessBits = 3222 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 3223 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 3224 3225 // Increment the pointer to the other half. 3226 unsigned IncrementSize = NVT.getSizeInBits()/8; 3227 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 3228 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 3229 Hi = DAG.getTruncStore( 3230 Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 3231 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 3232 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 3233 } 3234 3235 // Big-endian - high bits are at low addresses. Favor aligned stores at 3236 // the cost of some bit-fiddling. 3237 GetExpandedInteger(N->getValue(), Lo, Hi); 3238 3239 EVT ExtVT = N->getMemoryVT(); 3240 unsigned EBytes = ExtVT.getStoreSize(); 3241 unsigned IncrementSize = NVT.getSizeInBits()/8; 3242 unsigned ExcessBits = (EBytes - IncrementSize)*8; 3243 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 3244 ExtVT.getSizeInBits() - ExcessBits); 3245 3246 if (ExcessBits < NVT.getSizeInBits()) { 3247 // Transfer high bits from the top of Lo to the bottom of Hi. 3248 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 3249 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, 3250 TLI.getPointerTy(DAG.getDataLayout()))); 3251 Hi = DAG.getNode( 3252 ISD::OR, dl, NVT, Hi, 3253 DAG.getNode(ISD::SRL, dl, NVT, Lo, 3254 DAG.getConstant(ExcessBits, dl, 3255 TLI.getPointerTy(DAG.getDataLayout())))); 3256 } 3257 3258 // Store both the high bits and maybe some of the low bits. 3259 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment, 3260 MMOFlags, AAInfo); 3261 3262 // Increment the pointer to the other half. 3263 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 3264 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 3265 // Store the lowest ExcessBits bits in the second half. 3266 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 3267 N->getPointerInfo().getWithOffset(IncrementSize), 3268 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 3269 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); 3270 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 3271} 3272 3273SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 3274 SDValue InL, InH; 3275 GetExpandedInteger(N->getOperand(0), InL, InH); 3276 // Just truncate the low part of the source. 3277 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); 3278} 3279 3280SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 3281 SDValue Op = N->getOperand(0); 3282 EVT SrcVT = Op.getValueType(); 3283 EVT DstVT = N->getValueType(0); 3284 SDLoc dl(N); 3285 3286 // The following optimization is valid only if every value in SrcVT (when 3287 // treated as signed) is representable in DstVT. Check that the mantissa 3288 // size of DstVT is >= than the number of bits in SrcVT -1. 3289 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT); 3290 if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 && 3291 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 3292 // Do a signed conversion then adjust the result. 3293 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 3294 SignedConv = TLI.LowerOperation(SignedConv, DAG); 3295 3296 // The result of the signed conversion needs adjusting if the 'sign bit' of 3297 // the incoming integer was set. To handle this, we dynamically test to see 3298 // if it is set, and, if so, add a fudge factor. 3299 3300 const uint64_t F32TwoE32 = 0x4F800000ULL; 3301 const uint64_t F32TwoE64 = 0x5F800000ULL; 3302 const uint64_t F32TwoE128 = 0x7F800000ULL; 3303 3304 APInt FF(32, 0); 3305 if (SrcVT == MVT::i32) 3306 FF = APInt(32, F32TwoE32); 3307 else if (SrcVT == MVT::i64) 3308 FF = APInt(32, F32TwoE64); 3309 else if (SrcVT == MVT::i128) 3310 FF = APInt(32, F32TwoE128); 3311 else 3312 llvm_unreachable("Unsupported UINT_TO_FP!"); 3313 3314 // Check whether the sign bit is set. 3315 SDValue Lo, Hi; 3316 GetExpandedInteger(Op, Lo, Hi); 3317 SDValue SignSet = DAG.getSetCC(dl, 3318 getSetCCResultType(Hi.getValueType()), 3319 Hi, 3320 DAG.getConstant(0, dl, Hi.getValueType()), 3321 ISD::SETLT); 3322 3323 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 3324 SDValue FudgePtr = 3325 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)), 3326 TLI.getPointerTy(DAG.getDataLayout())); 3327 3328 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 3329 SDValue Zero = DAG.getIntPtrConstant(0, dl); 3330 SDValue Four = DAG.getIntPtrConstant(4, dl); 3331 if (DAG.getDataLayout().isBigEndian()) 3332 std::swap(Zero, Four); 3333 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, 3334 Zero, Four); 3335 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 3336 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(), 3337 FudgePtr, Offset); 3338 Alignment = std::min(Alignment, 4u); 3339 3340 // Load the value out, extending it from f32 to the destination float type. 3341 // FIXME: Avoid the extend by constructing the right constant pool? 3342 SDValue Fudge = DAG.getExtLoad( 3343 ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr, 3344 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, 3345 Alignment); 3346 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 3347 } 3348 3349 // Otherwise, use a libcall. 3350 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 3351 assert(LC != RTLIB::UNKNOWN_LIBCALL && 3352 "Don't know how to expand this UINT_TO_FP!"); 3353 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first; 3354} 3355 3356SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 3357 SDLoc dl(N); 3358 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 3359 cast<AtomicSDNode>(N)->getMemoryVT(), 3360 N->getOperand(0), 3361 N->getOperand(1), N->getOperand(2), 3362 cast<AtomicSDNode>(N)->getMemOperand()); 3363 return Swap.getValue(1); 3364} 3365 3366 3367SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 3368 SDValue InOp0 = N->getOperand(0); 3369 EVT InVT = InOp0.getValueType(); 3370 3371 EVT OutVT = N->getValueType(0); 3372 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3373 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3374 unsigned OutNumElems = OutVT.getVectorNumElements(); 3375 EVT NOutVTElem = NOutVT.getVectorElementType(); 3376 3377 SDLoc dl(N); 3378 SDValue BaseIdx = N->getOperand(1); 3379 3380 SmallVector<SDValue, 8> Ops; 3381 Ops.reserve(OutNumElems); 3382 for (unsigned i = 0; i != OutNumElems; ++i) { 3383 3384 // Extract the element from the original vector. 3385 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 3386 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); 3387 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3388 InVT.getVectorElementType(), N->getOperand(0), Index); 3389 3390 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 3391 // Insert the converted element to the new vector. 3392 Ops.push_back(Op); 3393 } 3394 3395 return DAG.getBuildVector(NOutVT, dl, Ops); 3396} 3397 3398 3399SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 3400 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 3401 EVT VT = N->getValueType(0); 3402 SDLoc dl(N); 3403 3404 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); 3405 3406 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3407 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 3408 EVT OutVT = V0.getValueType(); 3409 3410 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); 3411} 3412 3413 3414SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 3415 EVT OutVT = N->getValueType(0); 3416 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3417 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3418 unsigned NumElems = N->getNumOperands(); 3419 EVT NOutVTElem = NOutVT.getVectorElementType(); 3420 3421 SDLoc dl(N); 3422 3423 SmallVector<SDValue, 8> Ops; 3424 Ops.reserve(NumElems); 3425 for (unsigned i = 0; i != NumElems; ++i) { 3426 SDValue Op; 3427 // BUILD_VECTOR integer operand types are allowed to be larger than the 3428 // result's element type. This may still be true after the promotion. For 3429 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to 3430 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. 3431 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) 3432 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 3433 else 3434 Op = N->getOperand(i); 3435 Ops.push_back(Op); 3436 } 3437 3438 return DAG.getBuildVector(NOutVT, dl, Ops); 3439} 3440 3441SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 3442 3443 SDLoc dl(N); 3444 3445 assert(!N->getOperand(0).getValueType().isVector() && 3446 "Input must be a scalar"); 3447 3448 EVT OutVT = N->getValueType(0); 3449 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3450 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3451 EVT NOutVTElem = NOutVT.getVectorElementType(); 3452 3453 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 3454 3455 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 3456} 3457 3458SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 3459 SDLoc dl(N); 3460 3461 EVT OutVT = N->getValueType(0); 3462 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3463 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3464 3465 EVT InElemTy = OutVT.getVectorElementType(); 3466 EVT OutElemTy = NOutVT.getVectorElementType(); 3467 3468 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 3469 unsigned NumOutElem = NOutVT.getVectorNumElements(); 3470 unsigned NumOperands = N->getNumOperands(); 3471 assert(NumElem * NumOperands == NumOutElem && 3472 "Unexpected number of elements"); 3473 3474 // Take the elements from the first vector. 3475 SmallVector<SDValue, 8> Ops(NumOutElem); 3476 for (unsigned i = 0; i < NumOperands; ++i) { 3477 SDValue Op = N->getOperand(i); 3478 for (unsigned j = 0; j < NumElem; ++j) { 3479 SDValue Ext = DAG.getNode( 3480 ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op, 3481 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3482 Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); 3483 } 3484 } 3485 3486 return DAG.getBuildVector(NOutVT, dl, Ops); 3487} 3488 3489SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { 3490 EVT VT = N->getValueType(0); 3491 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3492 assert(NVT.isVector() && "This type must be promoted to a vector type"); 3493 3494 SDLoc dl(N); 3495 3496 // For operands whose TypeAction is to promote, extend the promoted node 3497 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion 3498 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to 3499 // type.. 3500 if (getTypeAction(N->getOperand(0).getValueType()) 3501 == TargetLowering::TypePromoteInteger) { 3502 SDValue Promoted; 3503 3504 switch(N->getOpcode()) { 3505 case ISD::SIGN_EXTEND_VECTOR_INREG: 3506 Promoted = SExtPromotedInteger(N->getOperand(0)); 3507 break; 3508 case ISD::ZERO_EXTEND_VECTOR_INREG: 3509 Promoted = ZExtPromotedInteger(N->getOperand(0)); 3510 break; 3511 case ISD::ANY_EXTEND_VECTOR_INREG: 3512 Promoted = GetPromotedInteger(N->getOperand(0)); 3513 break; 3514 default: 3515 llvm_unreachable("Node has unexpected Opcode"); 3516 } 3517 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); 3518 } 3519 3520 // Directly extend to the appropriate transform-to type. 3521 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 3522} 3523 3524SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 3525 EVT OutVT = N->getValueType(0); 3526 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 3527 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 3528 3529 EVT NOutVTElem = NOutVT.getVectorElementType(); 3530 3531 SDLoc dl(N); 3532 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3533 3534 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 3535 NOutVTElem, N->getOperand(1)); 3536 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 3537 V0, ConvElem, N->getOperand(2)); 3538} 3539 3540SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3541 SDLoc dl(N); 3542 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3543 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, 3544 TLI.getVectorIdxTy(DAG.getDataLayout())); 3545 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3546 V0->getValueType(0).getScalarType(), V0, V1); 3547 3548 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 3549 // element types. If this is the case then we need to expand the outgoing 3550 // value and not truncate it. 3551 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 3552} 3553 3554SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { 3555 SDLoc dl(N); 3556 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3557 MVT InVT = V0.getValueType().getSimpleVT(); 3558 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), 3559 N->getValueType(0).getVectorNumElements()); 3560 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); 3561 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); 3562} 3563 3564SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 3565 SDLoc dl(N); 3566 unsigned NumElems = N->getNumOperands(); 3567 3568 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 3569 3570 SmallVector<SDValue, 8> NewOps; 3571 NewOps.reserve(NumElems); 3572 3573 // For each incoming vector 3574 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 3575 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 3576 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 3577 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 3578 3579 for (unsigned i=0; i<NumElem; ++i) { 3580 // Extract element from incoming vector 3581 SDValue Ex = DAG.getNode( 3582 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, 3583 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3584 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 3585 NewOps.push_back(Tr); 3586 } 3587 } 3588 3589 return DAG.getBuildVector(N->getValueType(0), dl, NewOps); 3590} 3591