1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements integer type expansion and promotion for LegalizeTypes. 11// Promotion is the act of changing a computation in an illegal type into a 12// computation in a larger type. For example, implementing i8 arithmetic in an 13// i32 register (often needed on powerpc). 14// Expansion is the act of changing a computation in an illegal type into a 15// computation in two identical registers of a smaller type. For example, 16// implementing i64 arithmetic in two i32 registers (often needed on 32-bit 17// targets). 18// 19//===----------------------------------------------------------------------===// 20 21#include "LegalizeTypes.h" 22#include "llvm/DerivedTypes.h" 23#include "llvm/Support/ErrorHandling.h" 24#include "llvm/Support/raw_ostream.h" 25using namespace llvm; 26 27//===----------------------------------------------------------------------===// 28// Integer Result Promotion 29//===----------------------------------------------------------------------===// 30 31/// PromoteIntegerResult - This method is called when a result of a node is 32/// found to be in need of promotion to a larger type. At this point, the node 33/// may also have invalid operands or may have other results that need 34/// expansion, we just know that (at least) one result needs promotion. 35void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { 36 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); 37 SDValue Res = SDValue(); 38 39 // See if the target wants to custom expand this node. 40 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 41 return; 42 43 switch (N->getOpcode()) { 44 default: 45#ifndef NDEBUG 46 dbgs() << "PromoteIntegerResult #" << ResNo << ": "; 47 N->dump(&DAG); dbgs() << "\n"; 48#endif 49 llvm_unreachable("Do not know how to promote this operator!"); 50 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; 51 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; 52 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; 53 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; 54 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; 55 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; 56 case ISD::Constant: Res = PromoteIntRes_Constant(N); break; 57 case ISD::CONVERT_RNDSAT: 58 Res = PromoteIntRes_CONVERT_RNDSAT(N); break; 59 case ISD::CTLZ_ZERO_UNDEF: 60 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; 61 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; 62 case ISD::CTTZ_ZERO_UNDEF: 63 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; 64 case ISD::EXTRACT_VECTOR_ELT: 65 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; 66 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; 67 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; 68 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; 69 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; 70 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; 71 case ISD::SHL: Res = PromoteIntRes_SHL(N); break; 72 case ISD::SIGN_EXTEND_INREG: 73 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; 74 case ISD::SRA: Res = PromoteIntRes_SRA(N); break; 75 case ISD::SRL: Res = PromoteIntRes_SRL(N); break; 76 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; 77 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; 78 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; 79 80 case ISD::EXTRACT_SUBVECTOR: 81 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; 82 case ISD::VECTOR_SHUFFLE: 83 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; 84 case ISD::INSERT_VECTOR_ELT: 85 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; 86 case ISD::BUILD_VECTOR: 87 Res = PromoteIntRes_BUILD_VECTOR(N); break; 88 case ISD::SCALAR_TO_VECTOR: 89 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; 90 case ISD::CONCAT_VECTORS: 91 Res = PromoteIntRes_CONCAT_VECTORS(N); break; 92 93 case ISD::SIGN_EXTEND: 94 case ISD::ZERO_EXTEND: 95 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; 96 97 case ISD::FP_TO_SINT: 98 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; 99 100 case ISD::FP32_TO_FP16:Res = PromoteIntRes_FP32_TO_FP16(N); break; 101 102 case ISD::AND: 103 case ISD::OR: 104 case ISD::XOR: 105 case ISD::ADD: 106 case ISD::SUB: 107 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; 108 109 case ISD::SDIV: 110 case ISD::SREM: Res = PromoteIntRes_SDIV(N); break; 111 112 case ISD::UDIV: 113 case ISD::UREM: Res = PromoteIntRes_UDIV(N); break; 114 115 case ISD::SADDO: 116 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; 117 case ISD::UADDO: 118 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; 119 case ISD::SMULO: 120 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; 121 122 case ISD::ATOMIC_LOAD: 123 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; 124 125 case ISD::ATOMIC_LOAD_ADD: 126 case ISD::ATOMIC_LOAD_SUB: 127 case ISD::ATOMIC_LOAD_AND: 128 case ISD::ATOMIC_LOAD_OR: 129 case ISD::ATOMIC_LOAD_XOR: 130 case ISD::ATOMIC_LOAD_NAND: 131 case ISD::ATOMIC_LOAD_MIN: 132 case ISD::ATOMIC_LOAD_MAX: 133 case ISD::ATOMIC_LOAD_UMIN: 134 case ISD::ATOMIC_LOAD_UMAX: 135 case ISD::ATOMIC_SWAP: 136 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; 137 138 case ISD::ATOMIC_CMP_SWAP: 139 Res = PromoteIntRes_Atomic2(cast<AtomicSDNode>(N)); break; 140 } 141 142 // If the result is null then the sub-method took care of registering it. 143 if (Res.getNode()) 144 SetPromotedInteger(SDValue(N, ResNo), Res); 145} 146 147SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, 148 unsigned ResNo) { 149 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 150 return GetPromotedInteger(Op); 151} 152 153SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { 154 // Sign-extend the new bits, and continue the assertion. 155 SDValue Op = SExtPromotedInteger(N->getOperand(0)); 156 return DAG.getNode(ISD::AssertSext, N->getDebugLoc(), 157 Op.getValueType(), Op, N->getOperand(1)); 158} 159 160SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { 161 // Zero the new bits, and continue the assertion. 162 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 163 return DAG.getNode(ISD::AssertZext, N->getDebugLoc(), 164 Op.getValueType(), Op, N->getOperand(1)); 165} 166 167SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { 168 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 169 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 170 N->getMemoryVT(), ResVT, 171 N->getChain(), N->getBasePtr(), 172 N->getMemOperand(), N->getOrdering(), 173 N->getSynchScope()); 174 // Legalized the chain result - switch anything that used the old chain to 175 // use the new one. 176 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 177 return Res; 178} 179 180SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { 181 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 182 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 183 N->getMemoryVT(), 184 N->getChain(), N->getBasePtr(), 185 Op2, N->getMemOperand(), N->getOrdering(), 186 N->getSynchScope()); 187 // Legalized the chain result - switch anything that used the old chain to 188 // use the new one. 189 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 190 return Res; 191} 192 193SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { 194 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 195 SDValue Op3 = GetPromotedInteger(N->getOperand(3)); 196 SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), 197 N->getMemoryVT(), N->getChain(), N->getBasePtr(), 198 Op2, Op3, N->getMemOperand(), N->getOrdering(), 199 N->getSynchScope()); 200 // Legalized the chain result - switch anything that used the old chain to 201 // use the new one. 202 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 203 return Res; 204} 205 206SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { 207 SDValue InOp = N->getOperand(0); 208 EVT InVT = InOp.getValueType(); 209 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); 210 EVT OutVT = N->getValueType(0); 211 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 212 DebugLoc dl = N->getDebugLoc(); 213 214 switch (getTypeAction(InVT)) { 215 case TargetLowering::TypeLegal: 216 break; 217 case TargetLowering::TypePromoteInteger: 218 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) 219 // The input promotes to the same size. Convert the promoted value. 220 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); 221 break; 222 case TargetLowering::TypeSoftenFloat: 223 // Promote the integer operand by hand. 224 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); 225 case TargetLowering::TypeExpandInteger: 226 case TargetLowering::TypeExpandFloat: 227 break; 228 case TargetLowering::TypeScalarizeVector: 229 // Convert the element to an integer and promote it by hand. 230 if (!NOutVT.isVector()) 231 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 232 BitConvertToInteger(GetScalarizedVector(InOp))); 233 break; 234 case TargetLowering::TypeSplitVector: { 235 // For example, i32 = BITCAST v2i16 on alpha. Convert the split 236 // pieces of the input into integers and reassemble in the final type. 237 SDValue Lo, Hi; 238 GetSplitVector(N->getOperand(0), Lo, Hi); 239 Lo = BitConvertToInteger(Lo); 240 Hi = BitConvertToInteger(Hi); 241 242 if (TLI.isBigEndian()) 243 std::swap(Lo, Hi); 244 245 InOp = DAG.getNode(ISD::ANY_EXTEND, dl, 246 EVT::getIntegerVT(*DAG.getContext(), 247 NOutVT.getSizeInBits()), 248 JoinIntegers(Lo, Hi)); 249 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); 250 } 251 case TargetLowering::TypeWidenVector: 252 // The input is widened to the same size. Convert to the widened value. 253 // Make sure that the outgoing value is not a vector, because this would 254 // make us bitcast between two vectors which are legalized in different ways. 255 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) 256 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); 257 } 258 259 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, 260 CreateStackStoreLoad(InOp, OutVT)); 261} 262 263SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { 264 SDValue Op = GetPromotedInteger(N->getOperand(0)); 265 EVT OVT = N->getValueType(0); 266 EVT NVT = Op.getValueType(); 267 DebugLoc dl = N->getDebugLoc(); 268 269 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 270 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), 271 DAG.getConstant(DiffBits, TLI.getPointerTy())); 272} 273 274SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { 275 // The pair element type may be legal, or may not promote to the same type as 276 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. 277 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), 278 TLI.getTypeToTransformTo(*DAG.getContext(), 279 N->getValueType(0)), JoinIntegers(N->getOperand(0), 280 N->getOperand(1))); 281} 282 283SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { 284 EVT VT = N->getValueType(0); 285 // FIXME there is no actual debug info here 286 DebugLoc dl = N->getDebugLoc(); 287 // Zero extend things like i1, sign extend everything else. It shouldn't 288 // matter in theory which one we pick, but this tends to give better code? 289 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 290 SDValue Result = DAG.getNode(Opc, dl, 291 TLI.getTypeToTransformTo(*DAG.getContext(), VT), 292 SDValue(N, 0)); 293 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); 294 return Result; 295} 296 297SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) { 298 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 299 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 300 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 301 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && 302 "can only promote integers"); 303 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 304 return DAG.getConvertRndSat(OutVT, N->getDebugLoc(), N->getOperand(0), 305 N->getOperand(1), N->getOperand(2), 306 N->getOperand(3), N->getOperand(4), CvtCode); 307} 308 309SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { 310 // Zero extend to the promoted type and do the count there. 311 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 312 DebugLoc dl = N->getDebugLoc(); 313 EVT OVT = N->getValueType(0); 314 EVT NVT = Op.getValueType(); 315 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); 316 // Subtract off the extra leading bits in the bigger type. 317 return DAG.getNode(ISD::SUB, dl, NVT, Op, 318 DAG.getConstant(NVT.getSizeInBits() - 319 OVT.getSizeInBits(), NVT)); 320} 321 322SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { 323 // Zero extend to the promoted type and do the count there. 324 SDValue Op = ZExtPromotedInteger(N->getOperand(0)); 325 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), Op.getValueType(), Op); 326} 327 328SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { 329 SDValue Op = GetPromotedInteger(N->getOperand(0)); 330 EVT OVT = N->getValueType(0); 331 EVT NVT = Op.getValueType(); 332 DebugLoc dl = N->getDebugLoc(); 333 if (N->getOpcode() == ISD::CTTZ) { 334 // The count is the same in the promoted type except if the original 335 // value was zero. This can be handled by setting the bit just off 336 // the top of the original type. 337 APInt TopBit(NVT.getSizeInBits(), 0); 338 TopBit.setBit(OVT.getSizeInBits()); 339 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); 340 } 341 return DAG.getNode(N->getOpcode(), dl, NVT, Op); 342} 343 344SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { 345 DebugLoc dl = N->getDebugLoc(); 346 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 347 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), 348 N->getOperand(1)); 349} 350 351SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { 352 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 353 unsigned NewOpc = N->getOpcode(); 354 DebugLoc dl = N->getDebugLoc(); 355 356 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is 357 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT 358 // and SINT conversions are Custom, there is no way to tell which is 359 // preferable. We choose SINT because that's the right thing on PPC.) 360 if (N->getOpcode() == ISD::FP_TO_UINT && 361 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && 362 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) 363 NewOpc = ISD::FP_TO_SINT; 364 365 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); 366 367 // Assert that the converted value fits in the original type. If it doesn't 368 // (eg: because the value being converted is too big), then the result of the 369 // original operation was undefined anyway, so the assert is still correct. 370 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? 371 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, 372 DAG.getValueType(N->getValueType(0).getScalarType())); 373} 374 375SDValue DAGTypeLegalizer::PromoteIntRes_FP32_TO_FP16(SDNode *N) { 376 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 377 DebugLoc dl = N->getDebugLoc(); 378 379 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 380 381 return DAG.getNode(ISD::AssertZext, dl, 382 NVT, Res, DAG.getValueType(N->getValueType(0))); 383} 384 385SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { 386 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 387 DebugLoc dl = N->getDebugLoc(); 388 389 if (getTypeAction(N->getOperand(0).getValueType()) 390 == TargetLowering::TypePromoteInteger) { 391 SDValue Res = GetPromotedInteger(N->getOperand(0)); 392 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); 393 394 // If the result and operand types are the same after promotion, simplify 395 // to an in-register extension. 396 if (NVT == Res.getValueType()) { 397 // The high bits are not guaranteed to be anything. Insert an extend. 398 if (N->getOpcode() == ISD::SIGN_EXTEND) 399 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 400 DAG.getValueType(N->getOperand(0).getValueType())); 401 if (N->getOpcode() == ISD::ZERO_EXTEND) 402 return DAG.getZeroExtendInReg(Res, dl, 403 N->getOperand(0).getValueType().getScalarType()); 404 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); 405 return Res; 406 } 407 } 408 409 // Otherwise, just extend the original operand all the way to the larger type. 410 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 411} 412 413SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { 414 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 415 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 416 ISD::LoadExtType ExtType = 417 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); 418 DebugLoc dl = N->getDebugLoc(); 419 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), 420 N->getPointerInfo(), 421 N->getMemoryVT(), N->isVolatile(), 422 N->isNonTemporal(), N->getAlignment()); 423 424 // Legalized the chain result - switch anything that used the old chain to 425 // use the new one. 426 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 427 return Res; 428} 429 430/// Promote the overflow flag of an overflowing arithmetic node. 431SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { 432 // Simply change the return type of the boolean result. 433 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); 434 EVT ValueVTs[] = { N->getValueType(0), NVT }; 435 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) }; 436 SDValue Res = DAG.getNode(N->getOpcode(), N->getDebugLoc(), 437 DAG.getVTList(ValueVTs, 2), Ops, 2); 438 439 // Modified the sum result - switch anything that used the old sum to use 440 // the new one. 441 ReplaceValueWith(SDValue(N, 0), Res); 442 443 return SDValue(Res.getNode(), 1); 444} 445 446SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { 447 if (ResNo == 1) 448 return PromoteIntRes_Overflow(N); 449 450 // The operation overflowed iff the result in the larger type is not the 451 // sign extension of its truncation to the original type. 452 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 453 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 454 EVT OVT = N->getOperand(0).getValueType(); 455 EVT NVT = LHS.getValueType(); 456 DebugLoc dl = N->getDebugLoc(); 457 458 // Do the arithmetic in the larger type. 459 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; 460 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 461 462 // Calculate the overflow flag: sign extend the arithmetic result from 463 // the original type. 464 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, 465 DAG.getValueType(OVT)); 466 // Overflowed if and only if this is not equal to Res. 467 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 468 469 // Use the calculated overflow everywhere. 470 ReplaceValueWith(SDValue(N, 1), Ofl); 471 472 return Res; 473} 474 475SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) { 476 // Sign extend the input. 477 SDValue LHS = SExtPromotedInteger(N->getOperand(0)); 478 SDValue RHS = SExtPromotedInteger(N->getOperand(1)); 479 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 480 LHS.getValueType(), LHS, RHS); 481} 482 483SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { 484 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 485 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 486 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 487 LHS.getValueType(), N->getOperand(0),LHS,RHS); 488} 489 490SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { 491 SDValue Mask = N->getOperand(0); 492 EVT OpTy = N->getOperand(1).getValueType(); 493 494 // Promote all the way up to the canonical SetCC type. 495 Mask = PromoteTargetBoolean(Mask, TLI.getSetCCResultType(OpTy)); 496 SDValue LHS = GetPromotedInteger(N->getOperand(1)); 497 SDValue RHS = GetPromotedInteger(N->getOperand(2)); 498 return DAG.getNode(ISD::VSELECT, N->getDebugLoc(), 499 LHS.getValueType(), Mask, LHS, RHS); 500} 501 502SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { 503 SDValue LHS = GetPromotedInteger(N->getOperand(2)); 504 SDValue RHS = GetPromotedInteger(N->getOperand(3)); 505 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 506 LHS.getValueType(), N->getOperand(0), 507 N->getOperand(1), LHS, RHS, N->getOperand(4)); 508} 509 510SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { 511 EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType()); 512 513 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 514 515 // Only use the result of getSetCCResultType if it is legal, 516 // otherwise just use the promoted result type (NVT). 517 if (!TLI.isTypeLegal(SVT)) 518 SVT = NVT; 519 520 DebugLoc dl = N->getDebugLoc(); 521 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && 522 "Vector compare must return a vector result!"); 523 524 // Get the SETCC result using the canonical SETCC type. 525 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), 526 N->getOperand(1), N->getOperand(2)); 527 528 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); 529 // Convert to the expected type. 530 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); 531} 532 533SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { 534 return DAG.getNode(ISD::SHL, N->getDebugLoc(), 535 TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), 536 GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); 537} 538 539SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { 540 SDValue Op = GetPromotedInteger(N->getOperand(0)); 541 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), 542 Op.getValueType(), Op, N->getOperand(1)); 543} 544 545SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { 546 // The input may have strange things in the top bits of the registers, but 547 // these operations don't care. They may have weird bits going out, but 548 // that too is okay if they are integer operations. 549 SDValue LHS = GetPromotedInteger(N->getOperand(0)); 550 SDValue RHS = GetPromotedInteger(N->getOperand(1)); 551 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 552 LHS.getValueType(), LHS, RHS); 553} 554 555SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { 556 // The input value must be properly sign extended. 557 SDValue Res = SExtPromotedInteger(N->getOperand(0)); 558 return DAG.getNode(ISD::SRA, N->getDebugLoc(), 559 Res.getValueType(), Res, N->getOperand(1)); 560} 561 562SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { 563 // The input value must be properly zero extended. 564 EVT VT = N->getValueType(0); 565 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 566 SDValue Res = ZExtPromotedInteger(N->getOperand(0)); 567 return DAG.getNode(ISD::SRL, N->getDebugLoc(), NVT, Res, N->getOperand(1)); 568} 569 570SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { 571 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 572 SDValue Res; 573 SDValue InOp = N->getOperand(0); 574 DebugLoc dl = N->getDebugLoc(); 575 576 switch (getTypeAction(InOp.getValueType())) { 577 default: llvm_unreachable("Unknown type action!"); 578 case TargetLowering::TypeLegal: 579 case TargetLowering::TypeExpandInteger: 580 Res = InOp; 581 break; 582 case TargetLowering::TypePromoteInteger: 583 Res = GetPromotedInteger(InOp); 584 break; 585 case TargetLowering::TypeSplitVector: 586 EVT InVT = InOp.getValueType(); 587 assert(InVT.isVector() && "Cannot split scalar types"); 588 unsigned NumElts = InVT.getVectorNumElements(); 589 assert(NumElts == NVT.getVectorNumElements() && 590 "Dst and Src must have the same number of elements"); 591 assert(isPowerOf2_32(NumElts) && 592 "Promoted vector type must be a power of two"); 593 594 SDValue EOp1, EOp2; 595 GetSplitVector(InOp, EOp1, EOp2); 596 597 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), 598 NumElts/2); 599 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); 600 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); 601 602 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); 603 } 604 605 // Truncate to NVT instead of VT 606 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 607} 608 609SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { 610 if (ResNo == 1) 611 return PromoteIntRes_Overflow(N); 612 613 // The operation overflowed iff the result in the larger type is not the 614 // zero extension of its truncation to the original type. 615 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 616 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 617 EVT OVT = N->getOperand(0).getValueType(); 618 EVT NVT = LHS.getValueType(); 619 DebugLoc dl = N->getDebugLoc(); 620 621 // Do the arithmetic in the larger type. 622 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; 623 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); 624 625 // Calculate the overflow flag: zero extend the arithmetic result from 626 // the original type. 627 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); 628 // Overflowed if and only if this is not equal to Res. 629 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); 630 631 // Use the calculated overflow everywhere. 632 ReplaceValueWith(SDValue(N, 1), Ofl); 633 634 return Res; 635} 636 637SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { 638 // Promote the overflow bit trivially. 639 if (ResNo == 1) 640 return PromoteIntRes_Overflow(N); 641 642 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 643 DebugLoc DL = N->getDebugLoc(); 644 EVT SmallVT = LHS.getValueType(); 645 646 // To determine if the result overflowed in a larger type, we extend the 647 // input to the larger type, do the multiply (checking if it overflows), 648 // then also check the high bits of the result to see if overflow happened 649 // there. 650 if (N->getOpcode() == ISD::SMULO) { 651 LHS = SExtPromotedInteger(LHS); 652 RHS = SExtPromotedInteger(RHS); 653 } else { 654 LHS = ZExtPromotedInteger(LHS); 655 RHS = ZExtPromotedInteger(RHS); 656 } 657 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); 658 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); 659 660 // Overflow occurred if it occurred in the larger type, or if the high part 661 // of the result does not zero/sign-extend the low part. Check this second 662 // possibility first. 663 SDValue Overflow; 664 if (N->getOpcode() == ISD::UMULO) { 665 // Unsigned overflow occurred if the high part is non-zero. 666 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, 667 DAG.getIntPtrConstant(SmallVT.getSizeInBits())); 668 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, 669 DAG.getConstant(0, Hi.getValueType()), ISD::SETNE); 670 } else { 671 // Signed overflow occurred if the high part does not sign extend the low. 672 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), 673 Mul, DAG.getValueType(SmallVT)); 674 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); 675 } 676 677 // The only other way for overflow to occur is if the multiplication in the 678 // larger type itself overflowed. 679 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, 680 SDValue(Mul.getNode(), 1)); 681 682 // Use the calculated overflow everywhere. 683 ReplaceValueWith(SDValue(N, 1), Overflow); 684 return Mul; 685} 686 687SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { 688 // Zero extend the input. 689 SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); 690 SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); 691 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 692 LHS.getValueType(), LHS, RHS); 693} 694 695SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { 696 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 697 N->getValueType(0))); 698} 699 700SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { 701 SDValue Chain = N->getOperand(0); // Get the chain. 702 SDValue Ptr = N->getOperand(1); // Get the pointer. 703 EVT VT = N->getValueType(0); 704 DebugLoc dl = N->getDebugLoc(); 705 706 EVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); 707 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); 708 // The argument is passed as NumRegs registers of type RegVT. 709 710 SmallVector<SDValue, 8> Parts(NumRegs); 711 for (unsigned i = 0; i < NumRegs; ++i) { 712 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), 713 N->getConstantOperandVal(3)); 714 Chain = Parts[i].getValue(1); 715 } 716 717 // Handle endianness of the load. 718 if (TLI.isBigEndian()) 719 std::reverse(Parts.begin(), Parts.end()); 720 721 // Assemble the parts in the promoted type. 722 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 723 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); 724 for (unsigned i = 1; i < NumRegs; ++i) { 725 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); 726 // Shift it to the right position and "or" it in. 727 Part = DAG.getNode(ISD::SHL, dl, NVT, Part, 728 DAG.getConstant(i * RegVT.getSizeInBits(), 729 TLI.getPointerTy())); 730 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); 731 } 732 733 // Modified the chain result - switch anything that used the old chain to 734 // use the new one. 735 ReplaceValueWith(SDValue(N, 1), Chain); 736 737 return Res; 738} 739 740//===----------------------------------------------------------------------===// 741// Integer Operand Promotion 742//===----------------------------------------------------------------------===// 743 744/// PromoteIntegerOperand - This method is called when the specified operand of 745/// the specified node is found to need promotion. At this point, all of the 746/// result types of the node are known to be legal, but other operands of the 747/// node may need promotion or expansion as well as the specified one. 748bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { 749 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); 750 SDValue Res = SDValue(); 751 752 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 753 return false; 754 755 switch (N->getOpcode()) { 756 default: 757 #ifndef NDEBUG 758 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; 759 N->dump(&DAG); dbgs() << "\n"; 760 #endif 761 llvm_unreachable("Do not know how to promote this operator's operand!"); 762 763 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; 764 case ISD::ATOMIC_STORE: 765 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); 766 break; 767 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; 768 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; 769 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; 770 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; 771 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; 772 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; 773 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; 774 case ISD::CONVERT_RNDSAT: 775 Res = PromoteIntOp_CONVERT_RNDSAT(N); break; 776 case ISD::INSERT_VECTOR_ELT: 777 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; 778 case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; 779 case ISD::SCALAR_TO_VECTOR: 780 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; 781 case ISD::VSELECT: 782 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; 783 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; 784 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; 785 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; 786 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; 787 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), 788 OpNo); break; 789 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; 790 case ISD::FP16_TO_FP32: 791 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; 792 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; 793 794 case ISD::SHL: 795 case ISD::SRA: 796 case ISD::SRL: 797 case ISD::ROTL: 798 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; 799 } 800 801 // If the result is null, the sub-method took care of registering results etc. 802 if (!Res.getNode()) return false; 803 804 // If the result is N, the sub-method updated N in place. Tell the legalizer 805 // core about this. 806 if (Res.getNode() == N) 807 return true; 808 809 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 810 "Invalid operand expansion"); 811 812 ReplaceValueWith(SDValue(N, 0), Res); 813 return false; 814} 815 816/// PromoteSetCCOperands - Promote the operands of a comparison. This code is 817/// shared among BR_CC, SELECT_CC, and SETCC handlers. 818void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, 819 ISD::CondCode CCCode) { 820 // We have to insert explicit sign or zero extends. Note that we could 821 // insert sign extends for ALL conditions, but zero extend is cheaper on 822 // many machines (an AND instead of two shifts), so prefer it. 823 switch (CCCode) { 824 default: llvm_unreachable("Unknown integer comparison!"); 825 case ISD::SETEQ: 826 case ISD::SETNE: 827 case ISD::SETUGE: 828 case ISD::SETUGT: 829 case ISD::SETULE: 830 case ISD::SETULT: 831 // ALL of these operations will work if we either sign or zero extend 832 // the operands (including the unsigned comparisons!). Zero extend is 833 // usually a simpler/cheaper operation, so prefer it. 834 NewLHS = ZExtPromotedInteger(NewLHS); 835 NewRHS = ZExtPromotedInteger(NewRHS); 836 break; 837 case ISD::SETGE: 838 case ISD::SETGT: 839 case ISD::SETLT: 840 case ISD::SETLE: 841 NewLHS = SExtPromotedInteger(NewLHS); 842 NewRHS = SExtPromotedInteger(NewRHS); 843 break; 844 } 845} 846 847SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { 848 SDValue Op = GetPromotedInteger(N->getOperand(0)); 849 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op); 850} 851 852SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { 853 SDValue Op2 = GetPromotedInteger(N->getOperand(2)); 854 return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), 855 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), 856 N->getOrdering(), N->getSynchScope()); 857} 858 859SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { 860 // This should only occur in unusual situations like bitcasting to an 861 // x86_fp80, so just turn it into a store+load 862 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); 863} 864 865SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { 866 assert(OpNo == 2 && "Don't know how to promote this operand!"); 867 868 SDValue LHS = N->getOperand(2); 869 SDValue RHS = N->getOperand(3); 870 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); 871 872 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always 873 // legal types. 874 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 875 N->getOperand(1), LHS, RHS, N->getOperand(4)), 876 0); 877} 878 879SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { 880 assert(OpNo == 1 && "only know how to promote condition"); 881 882 // Promote all the way up to the canonical SetCC type. 883 EVT SVT = TLI.getSetCCResultType(MVT::Other); 884 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), SVT); 885 886 // The chain (Op#0) and basic block destination (Op#2) are always legal types. 887 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, 888 N->getOperand(2)), 0); 889} 890 891SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { 892 // Since the result type is legal, the operands must promote to it. 893 EVT OVT = N->getOperand(0).getValueType(); 894 SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); 895 SDValue Hi = GetPromotedInteger(N->getOperand(1)); 896 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); 897 DebugLoc dl = N->getDebugLoc(); 898 899 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, 900 DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy())); 901 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); 902} 903 904SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { 905 // The vector type is legal but the element type is not. This implies 906 // that the vector is a power-of-two in length and that the element 907 // type does not have a strange size (eg: it is not i1). 908 EVT VecVT = N->getValueType(0); 909 unsigned NumElts = VecVT.getVectorNumElements(); 910 assert(!(NumElts & 1) && "Legal vector of one illegal element?"); 911 912 // Promote the inserted value. The type does not need to match the 913 // vector element type. Check that any extra bits introduced will be 914 // truncated away. 915 assert(N->getOperand(0).getValueType().getSizeInBits() >= 916 N->getValueType(0).getVectorElementType().getSizeInBits() && 917 "Type of inserted value narrower than vector element type!"); 918 919 SmallVector<SDValue, 16> NewOps; 920 for (unsigned i = 0; i < NumElts; ++i) 921 NewOps.push_back(GetPromotedInteger(N->getOperand(i))); 922 923 return SDValue(DAG.UpdateNodeOperands(N, &NewOps[0], NumElts), 0); 924} 925 926SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { 927 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 928 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || 929 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || 930 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) && 931 "can only promote integer arguments"); 932 SDValue InOp = GetPromotedInteger(N->getOperand(0)); 933 return DAG.getConvertRndSat(N->getValueType(0), N->getDebugLoc(), InOp, 934 N->getOperand(1), N->getOperand(2), 935 N->getOperand(3), N->getOperand(4), CvtCode); 936} 937 938SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, 939 unsigned OpNo) { 940 if (OpNo == 1) { 941 // Promote the inserted value. This is valid because the type does not 942 // have to match the vector element type. 943 944 // Check that any extra bits introduced will be truncated away. 945 assert(N->getOperand(1).getValueType().getSizeInBits() >= 946 N->getValueType(0).getVectorElementType().getSizeInBits() && 947 "Type of inserted value narrower than vector element type!"); 948 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 949 GetPromotedInteger(N->getOperand(1)), 950 N->getOperand(2)), 951 0); 952 } 953 954 assert(OpNo == 2 && "Different operand and result vector types?"); 955 956 // Promote the index. 957 SDValue Idx = ZExtPromotedInteger(N->getOperand(2)); 958 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 959 N->getOperand(1), Idx), 0); 960} 961 962SDValue DAGTypeLegalizer::PromoteIntOp_MEMBARRIER(SDNode *N) { 963 SDValue NewOps[6]; 964 DebugLoc dl = N->getDebugLoc(); 965 NewOps[0] = N->getOperand(0); 966 for (unsigned i = 1; i < array_lengthof(NewOps); ++i) { 967 SDValue Flag = GetPromotedInteger(N->getOperand(i)); 968 NewOps[i] = DAG.getZeroExtendInReg(Flag, dl, MVT::i1); 969 } 970 return SDValue(DAG.UpdateNodeOperands(N, NewOps, array_lengthof(NewOps)), 0); 971} 972 973SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { 974 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote 975 // the operand in place. 976 return SDValue(DAG.UpdateNodeOperands(N, 977 GetPromotedInteger(N->getOperand(0))), 0); 978} 979 980SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { 981 assert(OpNo == 0 && "Only know how to promote the condition!"); 982 SDValue Cond = N->getOperand(0); 983 EVT OpTy = N->getOperand(1).getValueType(); 984 985 // Promote all the way up to the canonical SetCC type. 986 EVT SVT = TLI.getSetCCResultType(N->getOpcode() == ISD::SELECT ? 987 OpTy.getScalarType() : OpTy); 988 Cond = PromoteTargetBoolean(Cond, SVT); 989 990 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), 991 N->getOperand(2)), 0); 992} 993 994SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { 995 assert(OpNo == 0 && "Don't know how to promote this operand!"); 996 997 SDValue LHS = N->getOperand(0); 998 SDValue RHS = N->getOperand(1); 999 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); 1000 1001 // The CC (#4) and the possible return values (#2 and #3) have legal types. 1002 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), 1003 N->getOperand(3), N->getOperand(4)), 0); 1004} 1005 1006SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { 1007 assert(OpNo == 0 && "Don't know how to promote this operand!"); 1008 1009 SDValue LHS = N->getOperand(0); 1010 SDValue RHS = N->getOperand(1); 1011 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); 1012 1013 // The CC (#2) is always legal. 1014 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); 1015} 1016 1017SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { 1018 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1019 ZExtPromotedInteger(N->getOperand(1))), 0); 1020} 1021 1022SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { 1023 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1024 DebugLoc dl = N->getDebugLoc(); 1025 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1026 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), 1027 Op, DAG.getValueType(N->getOperand(0).getValueType())); 1028} 1029 1030SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { 1031 return SDValue(DAG.UpdateNodeOperands(N, 1032 SExtPromotedInteger(N->getOperand(0))), 0); 1033} 1034 1035SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ 1036 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1037 SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); 1038 unsigned Alignment = N->getAlignment(); 1039 bool isVolatile = N->isVolatile(); 1040 bool isNonTemporal = N->isNonTemporal(); 1041 DebugLoc dl = N->getDebugLoc(); 1042 1043 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. 1044 1045 // Truncate the value and store the result. 1046 return DAG.getTruncStore(Ch, dl, Val, Ptr, N->getPointerInfo(), 1047 N->getMemoryVT(), 1048 isVolatile, isNonTemporal, Alignment); 1049} 1050 1051SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { 1052 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1053 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), Op); 1054} 1055 1056SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { 1057 return SDValue(DAG.UpdateNodeOperands(N, 1058 ZExtPromotedInteger(N->getOperand(0))), 0); 1059} 1060 1061SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { 1062 DebugLoc dl = N->getDebugLoc(); 1063 SDValue Op = GetPromotedInteger(N->getOperand(0)); 1064 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); 1065 return DAG.getZeroExtendInReg(Op, dl, 1066 N->getOperand(0).getValueType().getScalarType()); 1067} 1068 1069 1070//===----------------------------------------------------------------------===// 1071// Integer Result Expansion 1072//===----------------------------------------------------------------------===// 1073 1074/// ExpandIntegerResult - This method is called when the specified result of the 1075/// specified node is found to need expansion. At this point, the node may also 1076/// have invalid operands or may have other results that need promotion, we just 1077/// know that (at least) one result needs expansion. 1078void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { 1079 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); 1080 SDValue Lo, Hi; 1081 Lo = Hi = SDValue(); 1082 1083 // See if the target wants to custom expand this node. 1084 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1085 return; 1086 1087 switch (N->getOpcode()) { 1088 default: 1089#ifndef NDEBUG 1090 dbgs() << "ExpandIntegerResult #" << ResNo << ": "; 1091 N->dump(&DAG); dbgs() << "\n"; 1092#endif 1093 llvm_unreachable("Do not know how to expand the result of this operator!"); 1094 1095 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1096 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 1097 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1098 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1099 1100 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1101 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1102 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1103 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1104 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1105 1106 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; 1107 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; 1108 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; 1109 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; 1110 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; 1111 case ISD::CTLZ_ZERO_UNDEF: 1112 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; 1113 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; 1114 case ISD::CTTZ_ZERO_UNDEF: 1115 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; 1116 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; 1117 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; 1118 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; 1119 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; 1120 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; 1121 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; 1122 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; 1123 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; 1124 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; 1125 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; 1126 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; 1127 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; 1128 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; 1129 1130 case ISD::ATOMIC_LOAD_ADD: 1131 case ISD::ATOMIC_LOAD_SUB: 1132 case ISD::ATOMIC_LOAD_AND: 1133 case ISD::ATOMIC_LOAD_OR: 1134 case ISD::ATOMIC_LOAD_XOR: 1135 case ISD::ATOMIC_LOAD_NAND: 1136 case ISD::ATOMIC_LOAD_MIN: 1137 case ISD::ATOMIC_LOAD_MAX: 1138 case ISD::ATOMIC_LOAD_UMIN: 1139 case ISD::ATOMIC_LOAD_UMAX: 1140 case ISD::ATOMIC_SWAP: { 1141 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); 1142 SplitInteger(Tmp.first, Lo, Hi); 1143 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1144 break; 1145 } 1146 1147 case ISD::AND: 1148 case ISD::OR: 1149 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; 1150 1151 case ISD::ADD: 1152 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; 1153 1154 case ISD::ADDC: 1155 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; 1156 1157 case ISD::ADDE: 1158 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; 1159 1160 case ISD::SHL: 1161 case ISD::SRA: 1162 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; 1163 1164 case ISD::SADDO: 1165 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; 1166 case ISD::UADDO: 1167 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; 1168 case ISD::UMULO: 1169 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; 1170 } 1171 1172 // If Lo/Hi is null, the sub-method took care of registering results etc. 1173 if (Lo.getNode()) 1174 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); 1175} 1176 1177/// Lower an atomic node to the appropriate builtin call. 1178std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { 1179 unsigned Opc = Node->getOpcode(); 1180 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 1181 RTLIB::Libcall LC; 1182 1183 switch (Opc) { 1184 default: 1185 llvm_unreachable("Unhandled atomic intrinsic Expand!"); 1186 case ISD::ATOMIC_SWAP: 1187 switch (VT.SimpleTy) { 1188 default: llvm_unreachable("Unexpected value type for atomic!"); 1189 case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 1190 case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 1191 case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 1192 case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 1193 } 1194 break; 1195 case ISD::ATOMIC_CMP_SWAP: 1196 switch (VT.SimpleTy) { 1197 default: llvm_unreachable("Unexpected value type for atomic!"); 1198 case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 1199 case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 1200 case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 1201 case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 1202 } 1203 break; 1204 case ISD::ATOMIC_LOAD_ADD: 1205 switch (VT.SimpleTy) { 1206 default: llvm_unreachable("Unexpected value type for atomic!"); 1207 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 1208 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 1209 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 1210 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 1211 } 1212 break; 1213 case ISD::ATOMIC_LOAD_SUB: 1214 switch (VT.SimpleTy) { 1215 default: llvm_unreachable("Unexpected value type for atomic!"); 1216 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 1217 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 1218 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 1219 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 1220 } 1221 break; 1222 case ISD::ATOMIC_LOAD_AND: 1223 switch (VT.SimpleTy) { 1224 default: llvm_unreachable("Unexpected value type for atomic!"); 1225 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 1226 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 1227 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 1228 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 1229 } 1230 break; 1231 case ISD::ATOMIC_LOAD_OR: 1232 switch (VT.SimpleTy) { 1233 default: llvm_unreachable("Unexpected value type for atomic!"); 1234 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 1235 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 1236 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 1237 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 1238 } 1239 break; 1240 case ISD::ATOMIC_LOAD_XOR: 1241 switch (VT.SimpleTy) { 1242 default: llvm_unreachable("Unexpected value type for atomic!"); 1243 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 1244 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 1245 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 1246 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 1247 } 1248 break; 1249 case ISD::ATOMIC_LOAD_NAND: 1250 switch (VT.SimpleTy) { 1251 default: llvm_unreachable("Unexpected value type for atomic!"); 1252 case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 1253 case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 1254 case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 1255 case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 1256 } 1257 break; 1258 } 1259 1260 return ExpandChainLibCall(LC, Node, false); 1261} 1262 1263/// ExpandShiftByConstant - N is a shift by a value that needs to be expanded, 1264/// and the shift amount is a constant 'Amt'. Expand the operation. 1265void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 1266 SDValue &Lo, SDValue &Hi) { 1267 DebugLoc DL = N->getDebugLoc(); 1268 // Expand the incoming operand to be shifted, so that we have its parts 1269 SDValue InL, InH; 1270 GetExpandedInteger(N->getOperand(0), InL, InH); 1271 1272 EVT NVT = InL.getValueType(); 1273 unsigned VTBits = N->getValueType(0).getSizeInBits(); 1274 unsigned NVTBits = NVT.getSizeInBits(); 1275 EVT ShTy = N->getOperand(1).getValueType(); 1276 1277 if (N->getOpcode() == ISD::SHL) { 1278 if (Amt > VTBits) { 1279 Lo = Hi = DAG.getConstant(0, NVT); 1280 } else if (Amt > NVTBits) { 1281 Lo = DAG.getConstant(0, NVT); 1282 Hi = DAG.getNode(ISD::SHL, DL, 1283 NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy)); 1284 } else if (Amt == NVTBits) { 1285 Lo = DAG.getConstant(0, NVT); 1286 Hi = InL; 1287 } else if (Amt == 1 && 1288 TLI.isOperationLegalOrCustom(ISD::ADDC, 1289 TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) { 1290 // Emit this X << 1 as X+X. 1291 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1292 SDValue LoOps[2] = { InL, InL }; 1293 Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps, 2); 1294 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) }; 1295 Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps, 3); 1296 } else { 1297 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy)); 1298 Hi = DAG.getNode(ISD::OR, DL, NVT, 1299 DAG.getNode(ISD::SHL, DL, NVT, InH, 1300 DAG.getConstant(Amt, ShTy)), 1301 DAG.getNode(ISD::SRL, DL, NVT, InL, 1302 DAG.getConstant(NVTBits-Amt, ShTy))); 1303 } 1304 return; 1305 } 1306 1307 if (N->getOpcode() == ISD::SRL) { 1308 if (Amt > VTBits) { 1309 Lo = DAG.getConstant(0, NVT); 1310 Hi = DAG.getConstant(0, NVT); 1311 } else if (Amt > NVTBits) { 1312 Lo = DAG.getNode(ISD::SRL, DL, 1313 NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy)); 1314 Hi = DAG.getConstant(0, NVT); 1315 } else if (Amt == NVTBits) { 1316 Lo = InH; 1317 Hi = DAG.getConstant(0, NVT); 1318 } else { 1319 Lo = DAG.getNode(ISD::OR, DL, NVT, 1320 DAG.getNode(ISD::SRL, DL, NVT, InL, 1321 DAG.getConstant(Amt, ShTy)), 1322 DAG.getNode(ISD::SHL, DL, NVT, InH, 1323 DAG.getConstant(NVTBits-Amt, ShTy))); 1324 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1325 } 1326 return; 1327 } 1328 1329 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 1330 if (Amt > VTBits) { 1331 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1332 DAG.getConstant(NVTBits-1, ShTy)); 1333 } else if (Amt > NVTBits) { 1334 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, 1335 DAG.getConstant(Amt-NVTBits, ShTy)); 1336 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1337 DAG.getConstant(NVTBits-1, ShTy)); 1338 } else if (Amt == NVTBits) { 1339 Lo = InH; 1340 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, 1341 DAG.getConstant(NVTBits-1, ShTy)); 1342 } else { 1343 Lo = DAG.getNode(ISD::OR, DL, NVT, 1344 DAG.getNode(ISD::SRL, DL, NVT, InL, 1345 DAG.getConstant(Amt, ShTy)), 1346 DAG.getNode(ISD::SHL, DL, NVT, InH, 1347 DAG.getConstant(NVTBits-Amt, ShTy))); 1348 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy)); 1349 } 1350} 1351 1352/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify 1353/// this shift based on knowledge of the high bit of the shift amount. If we 1354/// can tell this, we know that it is >= 32 or < 32, without knowing the actual 1355/// shift amount. 1356bool DAGTypeLegalizer:: 1357ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1358 SDValue Amt = N->getOperand(1); 1359 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1360 EVT ShTy = Amt.getValueType(); 1361 unsigned ShBits = ShTy.getScalarType().getSizeInBits(); 1362 unsigned NVTBits = NVT.getScalarType().getSizeInBits(); 1363 assert(isPowerOf2_32(NVTBits) && 1364 "Expanded integer type size not a power of two!"); 1365 DebugLoc dl = N->getDebugLoc(); 1366 1367 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); 1368 APInt KnownZero, KnownOne; 1369 DAG.ComputeMaskedBits(N->getOperand(1), KnownZero, KnownOne); 1370 1371 // If we don't know anything about the high bits, exit. 1372 if (((KnownZero|KnownOne) & HighBitMask) == 0) 1373 return false; 1374 1375 // Get the incoming operand to be shifted. 1376 SDValue InL, InH; 1377 GetExpandedInteger(N->getOperand(0), InL, InH); 1378 1379 // If we know that any of the high bits of the shift amount are one, then we 1380 // can do this as a couple of simple shifts. 1381 if (KnownOne.intersects(HighBitMask)) { 1382 // Mask out the high bit, which we know is set. 1383 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, 1384 DAG.getConstant(~HighBitMask, ShTy)); 1385 1386 switch (N->getOpcode()) { 1387 default: llvm_unreachable("Unknown shift"); 1388 case ISD::SHL: 1389 Lo = DAG.getConstant(0, NVT); // Low part is zero. 1390 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. 1391 return true; 1392 case ISD::SRL: 1393 Hi = DAG.getConstant(0, NVT); // Hi part is zero. 1394 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. 1395 return true; 1396 case ISD::SRA: 1397 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. 1398 DAG.getConstant(NVTBits-1, ShTy)); 1399 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. 1400 return true; 1401 } 1402 } 1403 1404 // If we know that all of the high bits of the shift amount are zero, then we 1405 // can do this as a couple of simple shifts. 1406 if ((KnownZero & HighBitMask) == HighBitMask) { 1407 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined 1408 // shift if x is zero. We can use XOR here because x is known to be smaller 1409 // than 32. 1410 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, 1411 DAG.getConstant(NVTBits-1, ShTy)); 1412 1413 unsigned Op1, Op2; 1414 switch (N->getOpcode()) { 1415 default: llvm_unreachable("Unknown shift"); 1416 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; 1417 case ISD::SRL: 1418 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; 1419 } 1420 1421 // When shifting right the arithmetic for Lo and Hi is swapped. 1422 if (N->getOpcode() != ISD::SHL) 1423 std::swap(InL, InH); 1424 1425 // Use a little trick to get the bits that move from Lo to Hi. First 1426 // shift by one bit. 1427 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, ShTy)); 1428 // Then compute the remaining shift with amount-1. 1429 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); 1430 1431 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); 1432 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); 1433 1434 if (N->getOpcode() != ISD::SHL) 1435 std::swap(Hi, Lo); 1436 return true; 1437 } 1438 1439 return false; 1440} 1441 1442/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift 1443/// of any size. 1444bool DAGTypeLegalizer:: 1445ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { 1446 SDValue Amt = N->getOperand(1); 1447 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1448 EVT ShTy = Amt.getValueType(); 1449 unsigned NVTBits = NVT.getSizeInBits(); 1450 assert(isPowerOf2_32(NVTBits) && 1451 "Expanded integer type size not a power of two!"); 1452 DebugLoc dl = N->getDebugLoc(); 1453 1454 // Get the incoming operand to be shifted. 1455 SDValue InL, InH; 1456 GetExpandedInteger(N->getOperand(0), InL, InH); 1457 1458 SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy); 1459 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); 1460 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); 1461 SDValue isShort = DAG.getSetCC(dl, TLI.getSetCCResultType(ShTy), 1462 Amt, NVBitsNode, ISD::SETULT); 1463 1464 SDValue LoS, HiS, LoL, HiL; 1465 switch (N->getOpcode()) { 1466 default: llvm_unreachable("Unknown shift"); 1467 case ISD::SHL: 1468 // Short: ShAmt < NVTBits 1469 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); 1470 HiS = DAG.getNode(ISD::OR, dl, NVT, 1471 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), 1472 // FIXME: If Amt is zero, the following shift generates an undefined result 1473 // on some architectures. 1474 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); 1475 1476 // Long: ShAmt >= NVTBits 1477 LoL = DAG.getConstant(0, NVT); // Lo part is zero. 1478 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. 1479 1480 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1481 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1482 return true; 1483 case ISD::SRL: 1484 // Short: ShAmt < NVTBits 1485 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); 1486 LoS = DAG.getNode(ISD::OR, dl, NVT, 1487 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1488 // FIXME: If Amt is zero, the following shift generates an undefined result 1489 // on some architectures. 1490 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1491 1492 // Long: ShAmt >= NVTBits 1493 HiL = DAG.getConstant(0, NVT); // Hi part is zero. 1494 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1495 1496 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1497 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1498 return true; 1499 case ISD::SRA: 1500 // Short: ShAmt < NVTBits 1501 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); 1502 LoS = DAG.getNode(ISD::OR, dl, NVT, 1503 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), 1504 // FIXME: If Amt is zero, the following shift generates an undefined result 1505 // on some architectures. 1506 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); 1507 1508 // Long: ShAmt >= NVTBits 1509 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. 1510 DAG.getConstant(NVTBits-1, ShTy)); 1511 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. 1512 1513 Lo = DAG.getNode(ISD::SELECT, dl, NVT, isShort, LoS, LoL); 1514 Hi = DAG.getNode(ISD::SELECT, dl, NVT, isShort, HiS, HiL); 1515 return true; 1516 } 1517} 1518 1519void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, 1520 SDValue &Lo, SDValue &Hi) { 1521 DebugLoc dl = N->getDebugLoc(); 1522 // Expand the subcomponents. 1523 SDValue LHSL, LHSH, RHSL, RHSH; 1524 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1525 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1526 1527 EVT NVT = LHSL.getValueType(); 1528 SDValue LoOps[2] = { LHSL, RHSL }; 1529 SDValue HiOps[3] = { LHSH, RHSH }; 1530 1531 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support 1532 // them. TODO: Teach operation legalization how to expand unsupported 1533 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate 1534 // a carry of type MVT::Glue, but there doesn't seem to be any way to 1535 // generate a value of this type in the expanded code sequence. 1536 bool hasCarry = 1537 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? 1538 ISD::ADDC : ISD::SUBC, 1539 TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); 1540 1541 if (hasCarry) { 1542 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); 1543 if (N->getOpcode() == ISD::ADD) { 1544 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1545 HiOps[2] = Lo.getValue(1); 1546 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1547 } else { 1548 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1549 HiOps[2] = Lo.getValue(1); 1550 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1551 } 1552 return; 1553 } 1554 1555 if (N->getOpcode() == ISD::ADD) { 1556 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); 1557 Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); 1558 SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[0], 1559 ISD::SETULT); 1560 SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, 1561 DAG.getConstant(1, NVT), 1562 DAG.getConstant(0, NVT)); 1563 SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, LoOps[1], 1564 ISD::SETULT); 1565 SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, 1566 DAG.getConstant(1, NVT), Carry1); 1567 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); 1568 } else { 1569 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); 1570 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); 1571 SDValue Cmp = 1572 DAG.getSetCC(dl, TLI.getSetCCResultType(LoOps[0].getValueType()), 1573 LoOps[0], LoOps[1], ISD::SETULT); 1574 SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, 1575 DAG.getConstant(1, NVT), 1576 DAG.getConstant(0, NVT)); 1577 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); 1578 } 1579} 1580 1581void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, 1582 SDValue &Lo, SDValue &Hi) { 1583 // Expand the subcomponents. 1584 SDValue LHSL, LHSH, RHSL, RHSH; 1585 DebugLoc dl = N->getDebugLoc(); 1586 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1587 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1588 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1589 SDValue LoOps[2] = { LHSL, RHSL }; 1590 SDValue HiOps[3] = { LHSH, RHSH }; 1591 1592 if (N->getOpcode() == ISD::ADDC) { 1593 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); 1594 HiOps[2] = Lo.getValue(1); 1595 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); 1596 } else { 1597 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); 1598 HiOps[2] = Lo.getValue(1); 1599 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); 1600 } 1601 1602 // Legalized the flag result - switch anything that used the old flag to 1603 // use the new one. 1604 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1605} 1606 1607void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, 1608 SDValue &Lo, SDValue &Hi) { 1609 // Expand the subcomponents. 1610 SDValue LHSL, LHSH, RHSL, RHSH; 1611 DebugLoc dl = N->getDebugLoc(); 1612 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 1613 GetExpandedInteger(N->getOperand(1), RHSL, RHSH); 1614 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); 1615 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; 1616 SDValue HiOps[3] = { LHSH, RHSH }; 1617 1618 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps, 3); 1619 HiOps[2] = Lo.getValue(1); 1620 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps, 3); 1621 1622 // Legalized the flag result - switch anything that used the old flag to 1623 // use the new one. 1624 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); 1625} 1626 1627void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo, 1628 SDValue &Lo, SDValue &Hi) { 1629 SDValue Res = DisintegrateMERGE_VALUES(N, ResNo); 1630 SplitInteger(Res, Lo, Hi); 1631} 1632 1633void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, 1634 SDValue &Lo, SDValue &Hi) { 1635 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1636 DebugLoc dl = N->getDebugLoc(); 1637 SDValue Op = N->getOperand(0); 1638 if (Op.getValueType().bitsLE(NVT)) { 1639 // The low part is any extension of the input (which degenerates to a copy). 1640 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); 1641 Hi = DAG.getUNDEF(NVT); // The high part is undefined. 1642 } else { 1643 // For example, extension of an i48 to an i64. The operand type necessarily 1644 // promotes to the result type, so will end up being expanded too. 1645 assert(getTypeAction(Op.getValueType()) == 1646 TargetLowering::TypePromoteInteger && 1647 "Only know how to promote this result!"); 1648 SDValue Res = GetPromotedInteger(Op); 1649 assert(Res.getValueType() == N->getValueType(0) && 1650 "Operand over promoted?"); 1651 // Split the promoted operand. This will simplify when it is expanded. 1652 SplitInteger(Res, Lo, Hi); 1653 } 1654} 1655 1656void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, 1657 SDValue &Lo, SDValue &Hi) { 1658 DebugLoc dl = N->getDebugLoc(); 1659 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1660 EVT NVT = Lo.getValueType(); 1661 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1662 unsigned NVTBits = NVT.getSizeInBits(); 1663 unsigned EVTBits = EVT.getSizeInBits(); 1664 1665 if (NVTBits < EVTBits) { 1666 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, 1667 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1668 EVTBits - NVTBits))); 1669 } else { 1670 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); 1671 // The high part replicates the sign bit of Lo, make it explicit. 1672 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1673 DAG.getConstant(NVTBits-1, TLI.getPointerTy())); 1674 } 1675} 1676 1677void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, 1678 SDValue &Lo, SDValue &Hi) { 1679 DebugLoc dl = N->getDebugLoc(); 1680 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1681 EVT NVT = Lo.getValueType(); 1682 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 1683 unsigned NVTBits = NVT.getSizeInBits(); 1684 unsigned EVTBits = EVT.getSizeInBits(); 1685 1686 if (NVTBits < EVTBits) { 1687 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, 1688 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 1689 EVTBits - NVTBits))); 1690 } else { 1691 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); 1692 // The high part must be zero, make it explicit. 1693 Hi = DAG.getConstant(0, NVT); 1694 } 1695} 1696 1697void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, 1698 SDValue &Lo, SDValue &Hi) { 1699 DebugLoc dl = N->getDebugLoc(); 1700 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. 1701 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); 1702 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); 1703} 1704 1705void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, 1706 SDValue &Lo, SDValue &Hi) { 1707 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1708 unsigned NBitWidth = NVT.getSizeInBits(); 1709 const APInt &Cst = cast<ConstantSDNode>(N)->getAPIntValue(); 1710 Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT); 1711 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT); 1712} 1713 1714void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, 1715 SDValue &Lo, SDValue &Hi) { 1716 DebugLoc dl = N->getDebugLoc(); 1717 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) 1718 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1719 EVT NVT = Lo.getValueType(); 1720 1721 SDValue HiNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Hi, 1722 DAG.getConstant(0, NVT), ISD::SETNE); 1723 1724 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); 1725 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); 1726 1727 Lo = DAG.getNode(ISD::SELECT, dl, NVT, HiNotZero, HiLZ, 1728 DAG.getNode(ISD::ADD, dl, NVT, LoLZ, 1729 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1730 Hi = DAG.getConstant(0, NVT); 1731} 1732 1733void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, 1734 SDValue &Lo, SDValue &Hi) { 1735 DebugLoc dl = N->getDebugLoc(); 1736 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) 1737 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1738 EVT NVT = Lo.getValueType(); 1739 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), 1740 DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); 1741 Hi = DAG.getConstant(0, NVT); 1742} 1743 1744void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, 1745 SDValue &Lo, SDValue &Hi) { 1746 DebugLoc dl = N->getDebugLoc(); 1747 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) 1748 GetExpandedInteger(N->getOperand(0), Lo, Hi); 1749 EVT NVT = Lo.getValueType(); 1750 1751 SDValue LoNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, 1752 DAG.getConstant(0, NVT), ISD::SETNE); 1753 1754 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); 1755 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); 1756 1757 Lo = DAG.getNode(ISD::SELECT, dl, NVT, LoNotZero, LoLZ, 1758 DAG.getNode(ISD::ADD, dl, NVT, HiLZ, 1759 DAG.getConstant(NVT.getSizeInBits(), NVT))); 1760 Hi = DAG.getConstant(0, NVT); 1761} 1762 1763void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, 1764 SDValue &Hi) { 1765 DebugLoc dl = N->getDebugLoc(); 1766 EVT VT = N->getValueType(0); 1767 SDValue Op = N->getOperand(0); 1768 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); 1769 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); 1770 SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*irrelevant*/, dl), Lo, Hi); 1771} 1772 1773void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, 1774 SDValue &Hi) { 1775 DebugLoc dl = N->getDebugLoc(); 1776 EVT VT = N->getValueType(0); 1777 SDValue Op = N->getOperand(0); 1778 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); 1779 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); 1780 SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*irrelevant*/, dl), Lo, Hi); 1781} 1782 1783void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, 1784 SDValue &Lo, SDValue &Hi) { 1785 if (ISD::isNormalLoad(N)) { 1786 ExpandRes_NormalLoad(N, Lo, Hi); 1787 return; 1788 } 1789 1790 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1791 1792 EVT VT = N->getValueType(0); 1793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1794 SDValue Ch = N->getChain(); 1795 SDValue Ptr = N->getBasePtr(); 1796 ISD::LoadExtType ExtType = N->getExtensionType(); 1797 unsigned Alignment = N->getAlignment(); 1798 bool isVolatile = N->isVolatile(); 1799 bool isNonTemporal = N->isNonTemporal(); 1800 bool isInvariant = N->isInvariant(); 1801 DebugLoc dl = N->getDebugLoc(); 1802 1803 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1804 1805 if (N->getMemoryVT().bitsLE(NVT)) { 1806 EVT MemVT = N->getMemoryVT(); 1807 1808 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1809 MemVT, isVolatile, isNonTemporal, Alignment); 1810 1811 // Remember the chain. 1812 Ch = Lo.getValue(1); 1813 1814 if (ExtType == ISD::SEXTLOAD) { 1815 // The high part is obtained by SRA'ing all but one of the bits of the 1816 // lo part. 1817 unsigned LoSize = Lo.getValueType().getSizeInBits(); 1818 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 1819 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 1820 } else if (ExtType == ISD::ZEXTLOAD) { 1821 // The high part is just a zero. 1822 Hi = DAG.getConstant(0, NVT); 1823 } else { 1824 assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); 1825 // The high part is undefined. 1826 Hi = DAG.getUNDEF(NVT); 1827 } 1828 } else if (TLI.isLittleEndian()) { 1829 // Little-endian - low bits are at low addresses. 1830 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), 1831 isVolatile, isNonTemporal, isInvariant, Alignment); 1832 1833 unsigned ExcessBits = 1834 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 1835 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 1836 1837 // Increment the pointer to the other half. 1838 unsigned IncrementSize = NVT.getSizeInBits()/8; 1839 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1840 DAG.getIntPtrConstant(IncrementSize)); 1841 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, 1842 N->getPointerInfo().getWithOffset(IncrementSize), NEVT, 1843 isVolatile, isNonTemporal, 1844 MinAlign(Alignment, IncrementSize)); 1845 1846 // Build a factor node to remember that this load is independent of the 1847 // other one. 1848 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1849 Hi.getValue(1)); 1850 } else { 1851 // Big-endian - high bits are at low addresses. Favor aligned loads at 1852 // the cost of some bit-fiddling. 1853 EVT MemVT = N->getMemoryVT(); 1854 unsigned EBytes = MemVT.getStoreSize(); 1855 unsigned IncrementSize = NVT.getSizeInBits()/8; 1856 unsigned ExcessBits = (EBytes - IncrementSize)*8; 1857 1858 // Load both the high bits and maybe some of the low bits. 1859 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), 1860 EVT::getIntegerVT(*DAG.getContext(), 1861 MemVT.getSizeInBits() - ExcessBits), 1862 isVolatile, isNonTemporal, Alignment); 1863 1864 // Increment the pointer to the other half. 1865 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1866 DAG.getIntPtrConstant(IncrementSize)); 1867 // Load the rest of the low bits. 1868 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, 1869 N->getPointerInfo().getWithOffset(IncrementSize), 1870 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 1871 isVolatile, isNonTemporal, 1872 MinAlign(Alignment, IncrementSize)); 1873 1874 // Build a factor node to remember that this load is independent of the 1875 // other one. 1876 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1877 Hi.getValue(1)); 1878 1879 if (ExcessBits < NVT.getSizeInBits()) { 1880 // Transfer low bits from the bottom of Hi to the top of Lo. 1881 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo, 1882 DAG.getNode(ISD::SHL, dl, NVT, Hi, 1883 DAG.getConstant(ExcessBits, 1884 TLI.getPointerTy()))); 1885 // Move high bits to the right position in Hi. 1886 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, 1887 NVT, Hi, 1888 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 1889 TLI.getPointerTy())); 1890 } 1891 } 1892 1893 // Legalized the chain result - switch anything that used the old chain to 1894 // use the new one. 1895 ReplaceValueWith(SDValue(N, 1), Ch); 1896} 1897 1898void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, 1899 SDValue &Lo, SDValue &Hi) { 1900 DebugLoc dl = N->getDebugLoc(); 1901 SDValue LL, LH, RL, RH; 1902 GetExpandedInteger(N->getOperand(0), LL, LH); 1903 GetExpandedInteger(N->getOperand(1), RL, RH); 1904 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); 1905 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); 1906} 1907 1908void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, 1909 SDValue &Lo, SDValue &Hi) { 1910 EVT VT = N->getValueType(0); 1911 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1912 DebugLoc dl = N->getDebugLoc(); 1913 1914 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); 1915 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); 1916 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); 1917 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); 1918 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { 1919 SDValue LL, LH, RL, RH; 1920 GetExpandedInteger(N->getOperand(0), LL, LH); 1921 GetExpandedInteger(N->getOperand(1), RL, RH); 1922 unsigned OuterBitSize = VT.getSizeInBits(); 1923 unsigned InnerBitSize = NVT.getSizeInBits(); 1924 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0)); 1925 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1)); 1926 1927 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); 1928 if (DAG.MaskedValueIsZero(N->getOperand(0), HighMask) && 1929 DAG.MaskedValueIsZero(N->getOperand(1), HighMask)) { 1930 // The inputs are both zero-extended. 1931 if (HasUMUL_LOHI) { 1932 // We can emit a umul_lohi. 1933 Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1934 Hi = SDValue(Lo.getNode(), 1); 1935 return; 1936 } 1937 if (HasMULHU) { 1938 // We can emit a mulhu+mul. 1939 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1940 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1941 return; 1942 } 1943 } 1944 if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { 1945 // The input values are both sign-extended. 1946 if (HasSMUL_LOHI) { 1947 // We can emit a smul_lohi. 1948 Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); 1949 Hi = SDValue(Lo.getNode(), 1); 1950 return; 1951 } 1952 if (HasMULHS) { 1953 // We can emit a mulhs+mul. 1954 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1955 Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); 1956 return; 1957 } 1958 } 1959 if (HasUMUL_LOHI) { 1960 // Lo,Hi = umul LHS, RHS. 1961 SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, 1962 DAG.getVTList(NVT, NVT), LL, RL); 1963 Lo = UMulLOHI; 1964 Hi = UMulLOHI.getValue(1); 1965 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1966 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1967 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1968 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1969 return; 1970 } 1971 if (HasMULHU) { 1972 Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); 1973 Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); 1974 RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); 1975 LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); 1976 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); 1977 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); 1978 return; 1979 } 1980 } 1981 1982 // If nothing else, we can make a libcall. 1983 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1984 if (VT == MVT::i16) 1985 LC = RTLIB::MUL_I16; 1986 else if (VT == MVT::i32) 1987 LC = RTLIB::MUL_I32; 1988 else if (VT == MVT::i64) 1989 LC = RTLIB::MUL_I64; 1990 else if (VT == MVT::i128) 1991 LC = RTLIB::MUL_I128; 1992 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); 1993 1994 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 1995 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*irrelevant*/, dl), Lo, Hi); 1996} 1997 1998void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, 1999 SDValue &Lo, SDValue &Hi) { 2000 SDValue LHS = Node->getOperand(0); 2001 SDValue RHS = Node->getOperand(1); 2002 DebugLoc dl = Node->getDebugLoc(); 2003 2004 // Expand the result by simply replacing it with the equivalent 2005 // non-overflow-checking operation. 2006 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 2007 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2008 LHS, RHS); 2009 SplitInteger(Sum, Lo, Hi); 2010 2011 // Compute the overflow. 2012 // 2013 // LHSSign -> LHS >= 0 2014 // RHSSign -> RHS >= 0 2015 // SumSign -> Sum >= 0 2016 // 2017 // Add: 2018 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 2019 // Sub: 2020 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 2021 // 2022 EVT OType = Node->getValueType(1); 2023 SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 2024 2025 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 2026 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 2027 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 2028 Node->getOpcode() == ISD::SADDO ? 2029 ISD::SETEQ : ISD::SETNE); 2030 2031 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 2032 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 2033 2034 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 2035 2036 // Use the calculated overflow everywhere. 2037 ReplaceValueWith(SDValue(Node, 1), Cmp); 2038} 2039 2040void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, 2041 SDValue &Lo, SDValue &Hi) { 2042 EVT VT = N->getValueType(0); 2043 DebugLoc dl = N->getDebugLoc(); 2044 2045 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2046 if (VT == MVT::i16) 2047 LC = RTLIB::SDIV_I16; 2048 else if (VT == MVT::i32) 2049 LC = RTLIB::SDIV_I32; 2050 else if (VT == MVT::i64) 2051 LC = RTLIB::SDIV_I64; 2052 else if (VT == MVT::i128) 2053 LC = RTLIB::SDIV_I128; 2054 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); 2055 2056 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2057 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 2058} 2059 2060void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, 2061 SDValue &Lo, SDValue &Hi) { 2062 EVT VT = N->getValueType(0); 2063 DebugLoc dl = N->getDebugLoc(); 2064 2065 // If we can emit an efficient shift operation, do so now. Check to see if 2066 // the RHS is a constant. 2067 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2068 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi); 2069 2070 // If we can determine that the high bit of the shift is zero or one, even if 2071 // the low bits are variable, emit this shift in an optimized form. 2072 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) 2073 return; 2074 2075 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. 2076 unsigned PartsOpc; 2077 if (N->getOpcode() == ISD::SHL) { 2078 PartsOpc = ISD::SHL_PARTS; 2079 } else if (N->getOpcode() == ISD::SRL) { 2080 PartsOpc = ISD::SRL_PARTS; 2081 } else { 2082 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2083 PartsOpc = ISD::SRA_PARTS; 2084 } 2085 2086 // Next check to see if the target supports this SHL_PARTS operation or if it 2087 // will custom expand it. 2088 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2089 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); 2090 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || 2091 Action == TargetLowering::Custom) { 2092 // Expand the subcomponents. 2093 SDValue LHSL, LHSH; 2094 GetExpandedInteger(N->getOperand(0), LHSL, LHSH); 2095 2096 SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; 2097 EVT VT = LHSL.getValueType(); 2098 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops, 3); 2099 Hi = Lo.getValue(1); 2100 return; 2101 } 2102 2103 // Otherwise, emit a libcall. 2104 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2105 bool isSigned; 2106 if (N->getOpcode() == ISD::SHL) { 2107 isSigned = false; /*sign irrelevant*/ 2108 if (VT == MVT::i16) 2109 LC = RTLIB::SHL_I16; 2110 else if (VT == MVT::i32) 2111 LC = RTLIB::SHL_I32; 2112 else if (VT == MVT::i64) 2113 LC = RTLIB::SHL_I64; 2114 else if (VT == MVT::i128) 2115 LC = RTLIB::SHL_I128; 2116 } else if (N->getOpcode() == ISD::SRL) { 2117 isSigned = false; 2118 if (VT == MVT::i16) 2119 LC = RTLIB::SRL_I16; 2120 else if (VT == MVT::i32) 2121 LC = RTLIB::SRL_I32; 2122 else if (VT == MVT::i64) 2123 LC = RTLIB::SRL_I64; 2124 else if (VT == MVT::i128) 2125 LC = RTLIB::SRL_I128; 2126 } else { 2127 assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); 2128 isSigned = true; 2129 if (VT == MVT::i16) 2130 LC = RTLIB::SRA_I16; 2131 else if (VT == MVT::i32) 2132 LC = RTLIB::SRA_I32; 2133 else if (VT == MVT::i64) 2134 LC = RTLIB::SRA_I64; 2135 else if (VT == MVT::i128) 2136 LC = RTLIB::SRA_I128; 2137 } 2138 2139 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { 2140 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2141 SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned, dl), Lo, Hi); 2142 return; 2143 } 2144 2145 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) 2146 llvm_unreachable("Unsupported shift!"); 2147} 2148 2149void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, 2150 SDValue &Lo, SDValue &Hi) { 2151 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2152 DebugLoc dl = N->getDebugLoc(); 2153 SDValue Op = N->getOperand(0); 2154 if (Op.getValueType().bitsLE(NVT)) { 2155 // The low part is sign extension of the input (degenerates to a copy). 2156 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); 2157 // The high part is obtained by SRA'ing all but one of the bits of low part. 2158 unsigned LoSize = NVT.getSizeInBits(); 2159 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, 2160 DAG.getConstant(LoSize-1, TLI.getPointerTy())); 2161 } else { 2162 // For example, extension of an i48 to an i64. The operand type necessarily 2163 // promotes to the result type, so will end up being expanded too. 2164 assert(getTypeAction(Op.getValueType()) == 2165 TargetLowering::TypePromoteInteger && 2166 "Only know how to promote this result!"); 2167 SDValue Res = GetPromotedInteger(Op); 2168 assert(Res.getValueType() == N->getValueType(0) && 2169 "Operand over promoted?"); 2170 // Split the promoted operand. This will simplify when it is expanded. 2171 SplitInteger(Res, Lo, Hi); 2172 unsigned ExcessBits = 2173 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2174 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2175 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2176 ExcessBits))); 2177 } 2178} 2179 2180void DAGTypeLegalizer:: 2181ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2182 DebugLoc dl = N->getDebugLoc(); 2183 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2184 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 2185 2186 if (EVT.bitsLE(Lo.getValueType())) { 2187 // sext_inreg the low part if needed. 2188 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, 2189 N->getOperand(1)); 2190 2191 // The high part gets the sign extension from the lo-part. This handles 2192 // things like sextinreg V:i64 from i8. 2193 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, 2194 DAG.getConstant(Hi.getValueType().getSizeInBits()-1, 2195 TLI.getPointerTy())); 2196 } else { 2197 // For example, extension of an i48 to an i64. Leave the low part alone, 2198 // sext_inreg the high part. 2199 unsigned ExcessBits = 2200 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits(); 2201 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, 2202 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 2203 ExcessBits))); 2204 } 2205} 2206 2207void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, 2208 SDValue &Lo, SDValue &Hi) { 2209 EVT VT = N->getValueType(0); 2210 DebugLoc dl = N->getDebugLoc(); 2211 2212 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2213 if (VT == MVT::i16) 2214 LC = RTLIB::SREM_I16; 2215 else if (VT == MVT::i32) 2216 LC = RTLIB::SREM_I32; 2217 else if (VT == MVT::i64) 2218 LC = RTLIB::SREM_I64; 2219 else if (VT == MVT::i128) 2220 LC = RTLIB::SREM_I128; 2221 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); 2222 2223 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2224 SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); 2225} 2226 2227void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, 2228 SDValue &Lo, SDValue &Hi) { 2229 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2230 DebugLoc dl = N->getDebugLoc(); 2231 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); 2232 Hi = DAG.getNode(ISD::SRL, dl, 2233 N->getOperand(0).getValueType(), N->getOperand(0), 2234 DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy())); 2235 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); 2236} 2237 2238void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, 2239 SDValue &Lo, SDValue &Hi) { 2240 SDValue LHS = N->getOperand(0); 2241 SDValue RHS = N->getOperand(1); 2242 DebugLoc dl = N->getDebugLoc(); 2243 2244 // Expand the result by simply replacing it with the equivalent 2245 // non-overflow-checking operation. 2246 SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ? 2247 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 2248 LHS, RHS); 2249 SplitInteger(Sum, Lo, Hi); 2250 2251 // Calculate the overflow: addition overflows iff a + b < a, and subtraction 2252 // overflows iff a - b > a. 2253 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, 2254 N->getOpcode () == ISD::UADDO ? 2255 ISD::SETULT : ISD::SETUGT); 2256 2257 // Use the calculated overflow everywhere. 2258 ReplaceValueWith(SDValue(N, 1), Ofl); 2259} 2260 2261void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, 2262 SDValue &Lo, SDValue &Hi) { 2263 EVT VT = N->getValueType(0); 2264 DebugLoc dl = N->getDebugLoc(); 2265 2266 // A divide for UMULO should be faster than a function call. 2267 if (N->getOpcode() == ISD::UMULO) { 2268 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); 2269 2270 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS); 2271 SplitInteger(MUL, Lo, Hi); 2272 2273 // A divide for UMULO will be faster than a function call. Select to 2274 // make sure we aren't using 0. 2275 SDValue isZero = DAG.getSetCC(dl, TLI.getSetCCResultType(VT), 2276 RHS, DAG.getConstant(0, VT), ISD::SETEQ); 2277 SDValue NotZero = DAG.getNode(ISD::SELECT, dl, VT, isZero, 2278 DAG.getConstant(1, VT), RHS); 2279 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero); 2280 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS, 2281 ISD::SETNE); 2282 Overflow = DAG.getNode(ISD::SELECT, dl, N->getValueType(1), isZero, 2283 DAG.getConstant(0, N->getValueType(1)), 2284 Overflow); 2285 ReplaceValueWith(SDValue(N, 1), Overflow); 2286 return; 2287 } 2288 2289 Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); 2290 EVT PtrVT = TLI.getPointerTy(); 2291 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); 2292 2293 // Replace this with a libcall that will check overflow. 2294 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2295 if (VT == MVT::i32) 2296 LC = RTLIB::MULO_I32; 2297 else if (VT == MVT::i64) 2298 LC = RTLIB::MULO_I64; 2299 else if (VT == MVT::i128) 2300 LC = RTLIB::MULO_I128; 2301 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); 2302 2303 SDValue Temp = DAG.CreateStackTemporary(PtrVT); 2304 // Temporary for the overflow value, default it to zero. 2305 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, 2306 DAG.getConstant(0, PtrVT), Temp, 2307 MachinePointerInfo(), false, false, 0); 2308 2309 TargetLowering::ArgListTy Args; 2310 TargetLowering::ArgListEntry Entry; 2311 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2312 EVT ArgVT = N->getOperand(i).getValueType(); 2313 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2314 Entry.Node = N->getOperand(i); 2315 Entry.Ty = ArgTy; 2316 Entry.isSExt = true; 2317 Entry.isZExt = false; 2318 Args.push_back(Entry); 2319 } 2320 2321 // Also pass the address of the overflow check. 2322 Entry.Node = Temp; 2323 Entry.Ty = PtrTy->getPointerTo(); 2324 Entry.isSExt = true; 2325 Entry.isZExt = false; 2326 Args.push_back(Entry); 2327 2328 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); 2329 TargetLowering:: 2330 CallLoweringInfo CLI(Chain, RetTy, true, false, false, false, 2331 0, TLI.getLibcallCallingConv(LC), 2332 /*isTailCall=*/false, 2333 /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2334 Func, Args, DAG, dl); 2335 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2336 2337 SplitInteger(CallInfo.first, Lo, Hi); 2338 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, 2339 MachinePointerInfo(), false, false, false, 0); 2340 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, 2341 DAG.getConstant(0, PtrVT), 2342 ISD::SETNE); 2343 // Use the overflow from the libcall everywhere. 2344 ReplaceValueWith(SDValue(N, 1), Ofl); 2345} 2346 2347void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, 2348 SDValue &Lo, SDValue &Hi) { 2349 EVT VT = N->getValueType(0); 2350 DebugLoc dl = N->getDebugLoc(); 2351 2352 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2353 if (VT == MVT::i16) 2354 LC = RTLIB::UDIV_I16; 2355 else if (VT == MVT::i32) 2356 LC = RTLIB::UDIV_I32; 2357 else if (VT == MVT::i64) 2358 LC = RTLIB::UDIV_I64; 2359 else if (VT == MVT::i128) 2360 LC = RTLIB::UDIV_I128; 2361 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); 2362 2363 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2364 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 2365} 2366 2367void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, 2368 SDValue &Lo, SDValue &Hi) { 2369 EVT VT = N->getValueType(0); 2370 DebugLoc dl = N->getDebugLoc(); 2371 2372 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2373 if (VT == MVT::i16) 2374 LC = RTLIB::UREM_I16; 2375 else if (VT == MVT::i32) 2376 LC = RTLIB::UREM_I32; 2377 else if (VT == MVT::i64) 2378 LC = RTLIB::UREM_I64; 2379 else if (VT == MVT::i128) 2380 LC = RTLIB::UREM_I128; 2381 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); 2382 2383 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; 2384 SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); 2385} 2386 2387void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, 2388 SDValue &Lo, SDValue &Hi) { 2389 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2390 DebugLoc dl = N->getDebugLoc(); 2391 SDValue Op = N->getOperand(0); 2392 if (Op.getValueType().bitsLE(NVT)) { 2393 // The low part is zero extension of the input (degenerates to a copy). 2394 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); 2395 Hi = DAG.getConstant(0, NVT); // The high part is just a zero. 2396 } else { 2397 // For example, extension of an i48 to an i64. The operand type necessarily 2398 // promotes to the result type, so will end up being expanded too. 2399 assert(getTypeAction(Op.getValueType()) == 2400 TargetLowering::TypePromoteInteger && 2401 "Only know how to promote this result!"); 2402 SDValue Res = GetPromotedInteger(Op); 2403 assert(Res.getValueType() == N->getValueType(0) && 2404 "Operand over promoted?"); 2405 // Split the promoted operand. This will simplify when it is expanded. 2406 SplitInteger(Res, Lo, Hi); 2407 unsigned ExcessBits = 2408 Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); 2409 Hi = DAG.getZeroExtendInReg(Hi, dl, 2410 EVT::getIntegerVT(*DAG.getContext(), 2411 ExcessBits)); 2412 } 2413} 2414 2415void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, 2416 SDValue &Lo, SDValue &Hi) { 2417 DebugLoc dl = N->getDebugLoc(); 2418 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); 2419 SDValue Zero = DAG.getConstant(0, VT); 2420 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT, 2421 N->getOperand(0), 2422 N->getOperand(1), Zero, Zero, 2423 cast<AtomicSDNode>(N)->getMemOperand(), 2424 cast<AtomicSDNode>(N)->getOrdering(), 2425 cast<AtomicSDNode>(N)->getSynchScope()); 2426 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); 2427 ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); 2428} 2429 2430//===----------------------------------------------------------------------===// 2431// Integer Operand Expansion 2432//===----------------------------------------------------------------------===// 2433 2434/// ExpandIntegerOperand - This method is called when the specified operand of 2435/// the specified node is found to need expansion. At this point, all of the 2436/// result types of the node are known to be legal, but other operands of the 2437/// node may need promotion or expansion as well as the specified one. 2438bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { 2439 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); 2440 SDValue Res = SDValue(); 2441 2442 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2443 return false; 2444 2445 switch (N->getOpcode()) { 2446 default: 2447 #ifndef NDEBUG 2448 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; 2449 N->dump(&DAG); dbgs() << "\n"; 2450 #endif 2451 llvm_unreachable("Do not know how to expand this operator's operand!"); 2452 2453 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2454 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; 2455 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2456 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2457 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; 2458 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; 2459 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; 2460 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; 2461 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; 2462 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; 2463 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; 2464 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; 2465 2466 case ISD::SHL: 2467 case ISD::SRA: 2468 case ISD::SRL: 2469 case ISD::ROTL: 2470 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; 2471 case ISD::RETURNADDR: 2472 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; 2473 2474 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; 2475 } 2476 2477 // If the result is null, the sub-method took care of registering results etc. 2478 if (!Res.getNode()) return false; 2479 2480 // If the result is N, the sub-method updated N in place. Tell the legalizer 2481 // core about this. 2482 if (Res.getNode() == N) 2483 return true; 2484 2485 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2486 "Invalid operand expansion"); 2487 2488 ReplaceValueWith(SDValue(N, 0), Res); 2489 return false; 2490} 2491 2492/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code 2493/// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2494void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, 2495 SDValue &NewRHS, 2496 ISD::CondCode &CCCode, 2497 DebugLoc dl) { 2498 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2499 GetExpandedInteger(NewLHS, LHSLo, LHSHi); 2500 GetExpandedInteger(NewRHS, RHSLo, RHSHi); 2501 2502 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { 2503 if (RHSLo == RHSHi) { 2504 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { 2505 if (RHSCST->isAllOnesValue()) { 2506 // Equality comparison to -1. 2507 NewLHS = DAG.getNode(ISD::AND, dl, 2508 LHSLo.getValueType(), LHSLo, LHSHi); 2509 NewRHS = RHSLo; 2510 return; 2511 } 2512 } 2513 } 2514 2515 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); 2516 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); 2517 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); 2518 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2519 return; 2520 } 2521 2522 // If this is a comparison of the sign bit, just look at the top part. 2523 // X > -1, x < 0 2524 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) 2525 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0 2526 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1 2527 NewLHS = LHSHi; 2528 NewRHS = RHSHi; 2529 return; 2530 } 2531 2532 // FIXME: This generated code sucks. 2533 ISD::CondCode LowCC; 2534 switch (CCCode) { 2535 default: llvm_unreachable("Unknown integer setcc!"); 2536 case ISD::SETLT: 2537 case ISD::SETULT: LowCC = ISD::SETULT; break; 2538 case ISD::SETGT: 2539 case ISD::SETUGT: LowCC = ISD::SETUGT; break; 2540 case ISD::SETLE: 2541 case ISD::SETULE: LowCC = ISD::SETULE; break; 2542 case ISD::SETGE: 2543 case ISD::SETUGE: LowCC = ISD::SETUGE; break; 2544 } 2545 2546 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison 2547 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands 2548 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; 2549 2550 // NOTE: on targets without efficient SELECT of bools, we can always use 2551 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) 2552 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, true, NULL); 2553 SDValue Tmp1, Tmp2; 2554 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), 2555 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); 2556 if (!Tmp1.getNode()) 2557 Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), 2558 LHSLo, RHSLo, LowCC); 2559 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2560 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); 2561 if (!Tmp2.getNode()) 2562 Tmp2 = DAG.getNode(ISD::SETCC, dl, 2563 TLI.getSetCCResultType(LHSHi.getValueType()), 2564 LHSHi, RHSHi, DAG.getCondCode(CCCode)); 2565 2566 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode()); 2567 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode()); 2568 if ((Tmp1C && Tmp1C->isNullValue()) || 2569 (Tmp2C && Tmp2C->isNullValue() && 2570 (CCCode == ISD::SETLE || CCCode == ISD::SETGE || 2571 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || 2572 (Tmp2C && Tmp2C->getAPIntValue() == 1 && 2573 (CCCode == ISD::SETLT || CCCode == ISD::SETGT || 2574 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { 2575 // low part is known false, returns high part. 2576 // For LE / GE, if high part is known false, ignore the low part. 2577 // For LT / GT, if high part is known true, ignore the low part. 2578 NewLHS = Tmp2; 2579 NewRHS = SDValue(); 2580 return; 2581 } 2582 2583 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), 2584 LHSHi, RHSHi, ISD::SETEQ, false, 2585 DagCombineInfo, dl); 2586 if (!NewLHS.getNode()) 2587 NewLHS = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), 2588 LHSHi, RHSHi, ISD::SETEQ); 2589 NewLHS = DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), 2590 NewLHS, Tmp1, Tmp2); 2591 NewRHS = SDValue(); 2592} 2593 2594SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { 2595 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2596 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2597 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2598 2599 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2600 // against zero to select between true and false values. 2601 if (NewRHS.getNode() == 0) { 2602 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2603 CCCode = ISD::SETNE; 2604 } 2605 2606 // Update N to have the operands specified. 2607 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2608 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2609 N->getOperand(4)), 0); 2610} 2611 2612SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { 2613 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2614 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2615 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2616 2617 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2618 // against zero to select between true and false values. 2619 if (NewRHS.getNode() == 0) { 2620 NewRHS = DAG.getConstant(0, NewLHS.getValueType()); 2621 CCCode = ISD::SETNE; 2622 } 2623 2624 // Update N to have the operands specified. 2625 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2626 N->getOperand(2), N->getOperand(3), 2627 DAG.getCondCode(CCCode)), 0); 2628} 2629 2630SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { 2631 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2632 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2633 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); 2634 2635 // If ExpandSetCCOperands returned a scalar, use it. 2636 if (NewRHS.getNode() == 0) { 2637 assert(NewLHS.getValueType() == N->getValueType(0) && 2638 "Unexpected setcc expansion!"); 2639 return NewLHS; 2640 } 2641 2642 // Otherwise, update N to have the operands specified. 2643 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2644 DAG.getCondCode(CCCode)), 0); 2645} 2646 2647SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { 2648 // The value being shifted is legal, but the shift amount is too big. 2649 // It follows that either the result of the shift is undefined, or the 2650 // upper half of the shift amount is zero. Just use the lower half. 2651 SDValue Lo, Hi; 2652 GetExpandedInteger(N->getOperand(1), Lo, Hi); 2653 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); 2654} 2655 2656SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { 2657 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This 2658 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this 2659 // constant to valid type. 2660 SDValue Lo, Hi; 2661 GetExpandedInteger(N->getOperand(0), Lo, Hi); 2662 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); 2663} 2664 2665SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { 2666 SDValue Op = N->getOperand(0); 2667 EVT DstVT = N->getValueType(0); 2668 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); 2669 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2670 "Don't know how to expand this SINT_TO_FP!"); 2671 return MakeLibCall(LC, DstVT, &Op, 1, true, N->getDebugLoc()); 2672} 2673 2674SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { 2675 if (ISD::isNormalStore(N)) 2676 return ExpandOp_NormalStore(N, OpNo); 2677 2678 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2679 assert(OpNo == 1 && "Can only expand the stored value so far"); 2680 2681 EVT VT = N->getOperand(1).getValueType(); 2682 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2683 SDValue Ch = N->getChain(); 2684 SDValue Ptr = N->getBasePtr(); 2685 unsigned Alignment = N->getAlignment(); 2686 bool isVolatile = N->isVolatile(); 2687 bool isNonTemporal = N->isNonTemporal(); 2688 DebugLoc dl = N->getDebugLoc(); 2689 SDValue Lo, Hi; 2690 2691 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2692 2693 if (N->getMemoryVT().bitsLE(NVT)) { 2694 GetExpandedInteger(N->getValue(), Lo, Hi); 2695 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2696 N->getMemoryVT(), isVolatile, isNonTemporal, 2697 Alignment); 2698 } 2699 2700 if (TLI.isLittleEndian()) { 2701 // Little-endian - low bits are at low addresses. 2702 GetExpandedInteger(N->getValue(), Lo, Hi); 2703 2704 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), 2705 isVolatile, isNonTemporal, Alignment); 2706 2707 unsigned ExcessBits = 2708 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); 2709 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); 2710 2711 // Increment the pointer to the other half. 2712 unsigned IncrementSize = NVT.getSizeInBits()/8; 2713 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2714 DAG.getIntPtrConstant(IncrementSize)); 2715 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 2716 N->getPointerInfo().getWithOffset(IncrementSize), 2717 NEVT, isVolatile, isNonTemporal, 2718 MinAlign(Alignment, IncrementSize)); 2719 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2720 } 2721 2722 // Big-endian - high bits are at low addresses. Favor aligned stores at 2723 // the cost of some bit-fiddling. 2724 GetExpandedInteger(N->getValue(), Lo, Hi); 2725 2726 EVT ExtVT = N->getMemoryVT(); 2727 unsigned EBytes = ExtVT.getStoreSize(); 2728 unsigned IncrementSize = NVT.getSizeInBits()/8; 2729 unsigned ExcessBits = (EBytes - IncrementSize)*8; 2730 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), 2731 ExtVT.getSizeInBits() - ExcessBits); 2732 2733 if (ExcessBits < NVT.getSizeInBits()) { 2734 // Transfer high bits from the top of Lo to the bottom of Hi. 2735 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, 2736 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, 2737 TLI.getPointerTy())); 2738 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, 2739 DAG.getNode(ISD::SRL, dl, NVT, Lo, 2740 DAG.getConstant(ExcessBits, 2741 TLI.getPointerTy()))); 2742 } 2743 2744 // Store both the high bits and maybe some of the low bits. 2745 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), 2746 HiVT, isVolatile, isNonTemporal, Alignment); 2747 2748 // Increment the pointer to the other half. 2749 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 2750 DAG.getIntPtrConstant(IncrementSize)); 2751 // Store the lowest ExcessBits bits in the second half. 2752 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, 2753 N->getPointerInfo().getWithOffset(IncrementSize), 2754 EVT::getIntegerVT(*DAG.getContext(), ExcessBits), 2755 isVolatile, isNonTemporal, 2756 MinAlign(Alignment, IncrementSize)); 2757 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 2758} 2759 2760SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { 2761 SDValue InL, InH; 2762 GetExpandedInteger(N->getOperand(0), InL, InH); 2763 // Just truncate the low part of the source. 2764 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL); 2765} 2766 2767static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { 2768 switch (VT.getSimpleVT().SimpleTy) { 2769 default: llvm_unreachable("Unknown FP format"); 2770 case MVT::f32: return &APFloat::IEEEsingle; 2771 case MVT::f64: return &APFloat::IEEEdouble; 2772 case MVT::f80: return &APFloat::x87DoubleExtended; 2773 case MVT::f128: return &APFloat::IEEEquad; 2774 case MVT::ppcf128: return &APFloat::PPCDoubleDouble; 2775 } 2776} 2777 2778SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { 2779 SDValue Op = N->getOperand(0); 2780 EVT SrcVT = Op.getValueType(); 2781 EVT DstVT = N->getValueType(0); 2782 DebugLoc dl = N->getDebugLoc(); 2783 2784 // The following optimization is valid only if every value in SrcVT (when 2785 // treated as signed) is representable in DstVT. Check that the mantissa 2786 // size of DstVT is >= than the number of bits in SrcVT -1. 2787 const fltSemantics *sem = EVTToAPFloatSemantics(DstVT); 2788 if (APFloat::semanticsPrecision(*sem) >= SrcVT.getSizeInBits()-1 && 2789 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ 2790 // Do a signed conversion then adjust the result. 2791 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); 2792 SignedConv = TLI.LowerOperation(SignedConv, DAG); 2793 2794 // The result of the signed conversion needs adjusting if the 'sign bit' of 2795 // the incoming integer was set. To handle this, we dynamically test to see 2796 // if it is set, and, if so, add a fudge factor. 2797 2798 const uint64_t F32TwoE32 = 0x4F800000ULL; 2799 const uint64_t F32TwoE64 = 0x5F800000ULL; 2800 const uint64_t F32TwoE128 = 0x7F800000ULL; 2801 2802 APInt FF(32, 0); 2803 if (SrcVT == MVT::i32) 2804 FF = APInt(32, F32TwoE32); 2805 else if (SrcVT == MVT::i64) 2806 FF = APInt(32, F32TwoE64); 2807 else if (SrcVT == MVT::i128) 2808 FF = APInt(32, F32TwoE128); 2809 else 2810 llvm_unreachable("Unsupported UINT_TO_FP!"); 2811 2812 // Check whether the sign bit is set. 2813 SDValue Lo, Hi; 2814 GetExpandedInteger(Op, Lo, Hi); 2815 SDValue SignSet = DAG.getSetCC(dl, 2816 TLI.getSetCCResultType(Hi.getValueType()), 2817 Hi, DAG.getConstant(0, Hi.getValueType()), 2818 ISD::SETLT); 2819 2820 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. 2821 SDValue FudgePtr = DAG.getConstantPool( 2822 ConstantInt::get(*DAG.getContext(), FF.zext(64)), 2823 TLI.getPointerTy()); 2824 2825 // Get a pointer to FF if the sign bit was set, or to 0 otherwise. 2826 SDValue Zero = DAG.getIntPtrConstant(0); 2827 SDValue Four = DAG.getIntPtrConstant(4); 2828 if (TLI.isBigEndian()) std::swap(Zero, Four); 2829 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, 2830 Zero, Four); 2831 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); 2832 FudgePtr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), FudgePtr, Offset); 2833 Alignment = std::min(Alignment, 4u); 2834 2835 // Load the value out, extending it from f32 to the destination float type. 2836 // FIXME: Avoid the extend by constructing the right constant pool? 2837 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), 2838 FudgePtr, 2839 MachinePointerInfo::getConstantPool(), 2840 MVT::f32, 2841 false, false, Alignment); 2842 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); 2843 } 2844 2845 // Otherwise, use a libcall. 2846 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); 2847 assert(LC != RTLIB::UNKNOWN_LIBCALL && 2848 "Don't know how to expand this UINT_TO_FP!"); 2849 return MakeLibCall(LC, DstVT, &Op, 1, true, dl); 2850} 2851 2852SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { 2853 DebugLoc dl = N->getDebugLoc(); 2854 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2855 cast<AtomicSDNode>(N)->getMemoryVT(), 2856 N->getOperand(0), 2857 N->getOperand(1), N->getOperand(2), 2858 cast<AtomicSDNode>(N)->getMemOperand(), 2859 cast<AtomicSDNode>(N)->getOrdering(), 2860 cast<AtomicSDNode>(N)->getSynchScope()); 2861 return Swap.getValue(1); 2862} 2863 2864 2865SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { 2866 SDValue InOp0 = N->getOperand(0); 2867 EVT InVT = InOp0.getValueType(); 2868 2869 EVT OutVT = N->getValueType(0); 2870 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2871 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2872 unsigned OutNumElems = OutVT.getVectorNumElements(); 2873 EVT NOutVTElem = NOutVT.getVectorElementType(); 2874 2875 DebugLoc dl = N->getDebugLoc(); 2876 SDValue BaseIdx = N->getOperand(1); 2877 2878 SmallVector<SDValue, 8> Ops; 2879 Ops.reserve(OutNumElems); 2880 for (unsigned i = 0; i != OutNumElems; ++i) { 2881 2882 // Extract the element from the original vector. 2883 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), 2884 BaseIdx, DAG.getIntPtrConstant(i)); 2885 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2886 InVT.getVectorElementType(), N->getOperand(0), Index); 2887 2888 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); 2889 // Insert the converted element to the new vector. 2890 Ops.push_back(Op); 2891 } 2892 2893 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2894} 2895 2896 2897SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { 2898 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); 2899 EVT VT = N->getValueType(0); 2900 DebugLoc dl = N->getDebugLoc(); 2901 2902 unsigned NumElts = VT.getVectorNumElements(); 2903 SmallVector<int, 8> NewMask; 2904 for (unsigned i = 0; i != NumElts; ++i) { 2905 NewMask.push_back(SV->getMaskElt(i)); 2906 } 2907 2908 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2909 SDValue V1 = GetPromotedInteger(N->getOperand(1)); 2910 EVT OutVT = V0.getValueType(); 2911 2912 return DAG.getVectorShuffle(OutVT, dl, V0, V1, &NewMask[0]); 2913} 2914 2915 2916SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { 2917 EVT OutVT = N->getValueType(0); 2918 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2919 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2920 unsigned NumElems = N->getNumOperands(); 2921 EVT NOutVTElem = NOutVT.getVectorElementType(); 2922 2923 DebugLoc dl = N->getDebugLoc(); 2924 2925 SmallVector<SDValue, 8> Ops; 2926 Ops.reserve(NumElems); 2927 for (unsigned i = 0; i != NumElems; ++i) { 2928 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); 2929 Ops.push_back(Op); 2930 } 2931 2932 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2933} 2934 2935SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { 2936 2937 DebugLoc dl = N->getDebugLoc(); 2938 2939 assert(!N->getOperand(0).getValueType().isVector() && 2940 "Input must be a scalar"); 2941 2942 EVT OutVT = N->getValueType(0); 2943 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2944 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2945 EVT NOutVTElem = NOutVT.getVectorElementType(); 2946 2947 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); 2948 2949 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); 2950} 2951 2952SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { 2953 DebugLoc dl = N->getDebugLoc(); 2954 2955 EVT OutVT = N->getValueType(0); 2956 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2957 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2958 2959 EVT InElemTy = OutVT.getVectorElementType(); 2960 EVT OutElemTy = NOutVT.getVectorElementType(); 2961 2962 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); 2963 unsigned NumOutElem = NOutVT.getVectorNumElements(); 2964 unsigned NumOperands = N->getNumOperands(); 2965 assert(NumElem * NumOperands == NumOutElem && 2966 "Unexpected number of elements"); 2967 2968 // Take the elements from the first vector. 2969 SmallVector<SDValue, 8> Ops(NumOutElem); 2970 for (unsigned i = 0; i < NumOperands; ++i) { 2971 SDValue Op = N->getOperand(i); 2972 for (unsigned j = 0; j < NumElem; ++j) { 2973 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 2974 InElemTy, Op, DAG.getIntPtrConstant(j)); 2975 Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); 2976 } 2977 } 2978 2979 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); 2980} 2981 2982SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { 2983 EVT OutVT = N->getValueType(0); 2984 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); 2985 assert(NOutVT.isVector() && "This type must be promoted to a vector type"); 2986 2987 EVT NOutVTElem = NOutVT.getVectorElementType(); 2988 2989 DebugLoc dl = N->getDebugLoc(); 2990 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 2991 2992 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, 2993 NOutVTElem, N->getOperand(1)); 2994 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, 2995 V0, ConvElem, N->getOperand(2)); 2996} 2997 2998SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2999 DebugLoc dl = N->getDebugLoc(); 3000 SDValue V0 = GetPromotedInteger(N->getOperand(0)); 3001 SDValue V1 = N->getOperand(1); 3002 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3003 V0->getValueType(0).getScalarType(), V0, V1); 3004 3005 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming 3006 // element types. If this is the case then we need to expand the outgoing 3007 // value and not truncate it. 3008 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); 3009} 3010 3011SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { 3012 DebugLoc dl = N->getDebugLoc(); 3013 unsigned NumElems = N->getNumOperands(); 3014 3015 EVT RetSclrTy = N->getValueType(0).getVectorElementType(); 3016 3017 SmallVector<SDValue, 8> NewOps; 3018 NewOps.reserve(NumElems); 3019 3020 // For each incoming vector 3021 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { 3022 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); 3023 EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); 3024 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); 3025 3026 for (unsigned i=0; i<NumElem; ++i) { 3027 // Extract element from incoming vector 3028 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, 3029 Incoming, DAG.getIntPtrConstant(i)); 3030 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); 3031 NewOps.push_back(Tr); 3032 } 3033 } 3034 3035 return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), 3036 &NewOps[0], NewOps.size()); 3037 } 3038