Lines Matching refs:ISD
94 if (Value->getOpcode() != ISD::CopyFromReg)
227 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
231 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
277 ISD::CondCode &CCCode,
287 ISD::CondCode &CCCode,
303 case ISD::SETEQ:
304 case ISD::SETOEQ:
309 case ISD::SETNE:
310 case ISD::SETUNE:
315 case ISD::SETGE:
316 case ISD::SETOGE:
321 case ISD::SETLT:
322 case ISD::SETOLT:
327 case ISD::SETLE:
328 case ISD::SETOLE:
333 case ISD::SETGT:
334 case ISD::SETOGT:
339 case ISD::SETO:
342 case ISD::SETUO:
347 case ISD::SETONE:
351 case ISD::SETUEQ:
363 case ISD::SETULT:
368 case ISD::SETULE:
373 case ISD::SETUGT:
378 case ISD::SETUGE:
417 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
419 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
497 // FIXME: ISD::SELECT, ISD::SELECT_CC
501 case ISD::XOR:
502 case ISD::AND:
503 case ISD::OR: {
510 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
574 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
575 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
577 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
643 case ISD::BITCAST: {
701 case ISD::AND: {
714 case ISD::OR: {
727 case ISD::XOR: {
739 case ISD::SHL: {
754 case ISD::SETCC: {
757 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
769 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
770 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
775 case ISD::SIGN_EXTEND_INREG: {
788 case ISD::ANY_EXTEND_VECTOR_INREG:
789 case ISD::SIGN_EXTEND_VECTOR_INREG:
790 case ISD::ZERO_EXTEND_VECTOR_INREG: {
803 case ISD::INSERT_VECTOR_ELT: {
813 case ISD::INSERT_SUBVECTOR: {
823 case ISD::VECTOR_SHUFFLE: {
847 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
913 if (Op.getOpcode() == ISD::Constant) {
943 case ISD::TargetConstant:
945 case ISD::SCALAR_TO_VECTOR: {
962 case ISD::BUILD_VECTOR:
967 case ISD::LOAD: {
972 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
981 case ISD::INSERT_VECTOR_ELT: {
1019 case ISD::INSERT_SUBVECTOR: {
1066 case ISD::EXTRACT_SUBVECTOR: {
1091 case ISD::CONCAT_VECTORS: {
1111 case ISD::VECTOR_SHUFFLE: {
1170 case ISD::AND: {
1199 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1247 case ISD::OR: {
1290 case ISD::XOR: {
1331 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1343 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1365 case ISD::SELECT:
1383 case ISD::SELECT_CC:
1401 case ISD::SETCC: {
1404 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1416 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1417 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1429 case ISD::SHL: {
1444 if (Op0.getOpcode() == ISD::SRL) {
1449 unsigned Opc = ISD::SHL;
1453 Opc = ISD::SRL;
1465 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1470 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1475 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1478 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1487 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1498 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1501 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1537 case ISD::SRL: {
1552 if (Op0.getOpcode() == ISD::SHL) {
1557 unsigned Opc = ISD::SRL;
1561 Opc = ISD::SHL;
1589 case ISD::SRA: {
1606 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1640 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1647 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1659 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1666 case ISD::FSHL:
1667 case ISD::FSHR: {
1671 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1713 case ISD::ROTL:
1714 case ISD::ROTR: {
1731 case ISD::BITREVERSE: {
1741 case ISD::BSWAP: {
1751 case ISD::SIGN_EXTEND_INREG: {
1773 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1808 case ISD::BUILD_PAIR: {
1830 case ISD::ZERO_EXTEND:
1831 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1836 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1848 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1863 case ISD::SIGN_EXTEND:
1864 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1869 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1881 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1905 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1911 case ISD::ANY_EXTEND:
1912 case ISD::ANY_EXTEND_VECTOR_INREG: {
1917 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1941 case ISD::TRUNCATE: {
1955 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
1963 case ISD::SRL:
1966 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1988 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
1990 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, ShAmt));
1999 case ISD::AssertZext: {
2012 case ISD::EXTRACT_VECTOR_ELT: {
2050 case ISD::BITCAST: {
2060 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2061 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2068 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2071 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2075 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2142 case ISD::ADD:
2143 case ISD::MUL:
2144 case ISD::SUB: {
2207 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2365 case ISD::SCALAR_TO_VECTOR: {
2373 case ISD::BITCAST: {
2459 case ISD::BUILD_VECTOR: {
2489 case ISD::CONCAT_VECTORS: {
2505 case ISD::INSERT_SUBVECTOR: {
2523 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2550 case ISD::EXTRACT_SUBVECTOR: {
2578 case ISD::INSERT_VECTOR_ELT: {
2613 case ISD::VSELECT: {
2641 case ISD::VECTOR_SHUFFLE: {
2714 case ISD::ANY_EXTEND_VECTOR_INREG:
2715 case ISD::SIGN_EXTEND_VECTOR_INREG:
2716 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2727 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2734 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2745 case ISD::OR:
2746 case ISD::XOR:
2747 case ISD::ADD:
2748 case ISD::SUB:
2749 case ISD::FADD:
2750 case ISD::FSUB:
2751 case ISD::FMUL:
2752 case ISD::FDIV:
2753 case ISD::FREM: {
2776 case ISD::SHL:
2777 case ISD::SRL:
2778 case ISD::SRA:
2779 case ISD::ROTL:
2780 case ISD::ROTR: {
2803 case ISD::MUL:
2804 case ISD::AND: {
2831 case ISD::TRUNCATE:
2832 case ISD::SIGN_EXTEND:
2833 case ISD::ZERO_EXTEND:
2838 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2846 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2877 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2878 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2879 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2880 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2911 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2912 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2913 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2914 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2929 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2930 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2931 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2932 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2941 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2942 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2943 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2944 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2955 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2956 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2957 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2958 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2989 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2990 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2991 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2992 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2998 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3080 ISD::CondCode Cond, const SDLoc &DL,
3085 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3089 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3090 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3112 Cond = ISD::getSetCCInverse(Cond, OpVT);
3131 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3149 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3157 if (N0->getOpcode() != ISD::ADD)
3172 ISD::CondCode NewCond;
3173 if (Cond == ISD::CondCode::SETULT) {
3174 NewCond = ISD::CondCode::SETEQ;
3175 } else if (Cond == ISD::CondCode::SETULE) {
3176 NewCond = ISD::CondCode::SETEQ;
3179 } else if (Cond == ISD::CondCode::SETUGT) {
3180 NewCond = ISD::CondCode::SETNE;
3183 } else if (Cond == ISD::CondCode::SETUGE) {
3184 NewCond = ISD::CondCode::SETNE;
3229 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3230 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3238 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3243 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3259 case ISD::SHL:
3260 NewShiftOpcode = ISD::SRL;
3262 case ISD::SRL:
3263 NewShiftOpcode = ISD::SHL;
3284 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3302 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3311 ISD::CondCode Cond, const SDLoc &DL,
3314 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
3316 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
3333 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3344 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3353 ISD::CondCode Cond, bool foldBooleans,
3366 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3379 DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N1, N0 } ) &&
3380 !DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N0, N1 } ))
3389 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3390 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3391 N0.getOperand(1).getOpcode() == ISD::Constant) {
3393 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3395 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3398 Cond = ISD::SETNE;
3402 Cond = ISD::SETEQ;
3412 if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
3415 if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
3423 if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
3425 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3426 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3427 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3432 if (C1 == 1 && !isOperationLegalOrCustom(ISD::CTPOP, CTVT) &&
3433 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3439 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3440 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3441 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3444 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3451 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3456 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3460 } else if (N0->getOpcode() == ISD::AND) {
3467 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3474 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3477 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3494 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3496 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3500 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3515 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3517 TopSetCC.getOpcode() == ISD::SETCC &&
3518 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3522 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3523 (!N1C->isNullValue() && Cond == ISD::SETNE);
3528 ISD::CondCode InvCond = ISD::getSetCCInverse(
3542 !ISD::isSignedIntSetCC(Cond) &&
3543 N0.getOpcode() == ISD::AND && C1 == 0 &&
3556 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3578 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3587 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3596 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3604 case ISD::SETUGT:
3605 case ISD::SETUGE:
3606 case ISD::SETEQ:
3608 case ISD::SETULT:
3609 case ISD::SETULE:
3610 case ISD::SETNE:
3612 case ISD::SETGT:
3613 case ISD::SETGE:
3616 case ISD::SETLT:
3617 case ISD::SETLE:
3627 case ISD::SETEQ:
3628 case ISD::SETNE:
3629 case ISD::SETUGT:
3630 case ISD::SETUGE:
3631 case ISD::SETULT:
3632 case ISD::SETULE: {
3635 (isOperationLegal(ISD::SETCC, newVT) &&
3649 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3650 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3659 return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
3667 ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
3680 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3682 if (N0.getOpcode() == ISD::SETCC &&
3687 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3689 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3691 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3692 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3698 if ((N0.getOpcode() == ISD::XOR ||
3699 (N0.getOpcode() == ISD::AND &&
3700 N0.getOperand(0).getOpcode() == ISD::XOR &&
3712 if (N0.getOpcode() == ISD::XOR) {
3715 assert(N0.getOpcode() == ISD::AND &&
3716 N0.getOperand(0).getOpcode() == ISD::XOR);
3718 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3724 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3728 if (Op0.getOpcode() == ISD::TRUNCATE)
3731 if ((Op0.getOpcode() == ISD::XOR) &&
3732 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3733 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3743 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3747 if (Op0.getOpcode() == ISD::AND &&
3752 Op0 = DAG.getNode(ISD::AND, dl, VT,
3753 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3756 Op0 = DAG.getNode(ISD::AND, dl, VT,
3757 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3762 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3764 if (Op0.getOpcode() == ISD::AssertZext &&
3768 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3776 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3777 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3796 if (ISD::isSignedIntSetCC(Cond)) {
3805 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3813 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3825 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3833 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3845 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3853 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3859 ISD::SETEQ);
3863 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3871 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3877 ISD::SETEQ);
3881 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
3894 if (Cond == ISD::SETUGT &&
3898 ISD::SETLT);
3901 if (Cond == ISD::SETULT &&
3906 return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
3919 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3921 N0.getOpcode() == ISD::AND) {
3925 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
3930 return DAG.getNode(ISD::TRUNCATE, dl, VT,
3931 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3934 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
3940 return DAG.getNode(ISD::TRUNCATE, dl, VT,
3941 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3952 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3953 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
3960 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
3967 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
3968 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
3969 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
3976 ISD::CondCode NewCond = Cond;
3980 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
3988 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4005 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4009 if (N0.getOpcode() == ISD::FNEG) {
4010 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4013 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4026 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4028 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4029 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4030 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4031 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4034 if (NewCond != ISD::SETCC_INVALID &&
4047 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4048 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4055 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4062 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4064 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4065 N0.getOpcode() == ISD::XOR) {
4090 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4098 if (N0.getOpcode() == ISD::XOR)
4112 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4136 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4137 N1.getOpcode() == ISD::XOR)
4146 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4147 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4153 if (N0.getOpcode() == ISD::UREM) {
4156 } else if (N0.getOpcode() == ISD::SREM) {
4168 case ISD::SETEQ: // X == Y -> ~(X^Y)
4169 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4174 case ISD::SETNE: // X != Y --> (X^Y)
4175 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4177 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4178 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4180 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4184 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4185 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4187 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4191 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4192 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4194 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4198 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4199 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4201 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4208 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4231 if (N->getOpcode() == ISD::ADD) {
4333 if (Op.getOpcode() == ISD::BasicBlock ||
4334 Op.getOpcode() == ISD::TargetBlockAddress) {
4352 // likely an ISD::ADD.
4366 ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
4367 : ISD::SIGN_EXTEND;
4368 int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue()
4381 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4385 else if (OpCode == ISD::ADD &&
4390 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4839 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
4886 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
4905 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
4909 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
4922 /// Given an ISD::SDIV node expressing a divide by constant,
4981 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5000 if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
5001 : isOperationLegalOrCustom(ISD::MULHS, VT))
5002 Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
5003 else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
5004 : isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) {
5006 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
5013 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5015 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5019 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5024 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5026 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5028 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5031 /// Given an ISD::UDIV node expressing a divide by constant,
5099 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5115 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5120 if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
5121 : isOperationLegalOrCustom(ISD::MULHU, VT))
5122 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5123 if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
5124 : isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) {
5126 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5140 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5148 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5152 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5156 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5160 SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
5192 /// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5199 ISD::CondCode Cond,
5215 SDValue CompTargetNode, ISD::CondCode Cond,
5223 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5234 if (!isOperationLegalOrCustom(ISD::MUL, VT))
5330 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5364 if (!isOperationLegalOrCustom(ISD::SUB, VT))
5365 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5368 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5372 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5379 if (!isOperationLegalOrCustom(ISD::ROTR, VT))
5384 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5391 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5406 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5409 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5412 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5414 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5419 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5420 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5426 /// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5433 ISD::CondCode Cond,
5450 SDValue CompTargetNode, ISD::CondCode Cond,
5463 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5474 if (!isOperationLegalOrCustom(ISD::MUL, VT))
5574 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5614 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5619 if (!isOperationLegalOrCustom(ISD::ADD, VT))
5623 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
5631 if (!isOperationLegalOrCustom(ISD::ROTR, VT))
5636 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5643 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5654 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
5655 !isOperationLegalOrCustom(ISD::AND, VT) ||
5657 !isOperationLegalOrCustom(ISD::VSELECT, VT))
5670 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5674 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5684 DAG.getNode(ISD::VSELECT, DL, VT, DivisorIsIntMin, MaskedIsZero, Fold);
5705 if (Op.getOpcode() == ISD::FNEG) {
5722 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
5723 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
5737 case ISD::ConstantFP: {
5741 isOperationLegal(ISD::ConstantFP, VT) ||
5759 case ISD::BUILD_VECTOR: {
5767 (isOperationLegal(ISD::ConstantFP, VT) &&
5768 isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
5791 case ISD::FADD: {
5796 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
5812 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
5821 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
5828 case ISD::FSUB: {
5843 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
5845 case ISD::FMUL:
5846 case ISD::FDIV: {
5869 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
5882 case ISD::FMA:
5883 case ISD::FMAD: {
5924 case ISD::FP_EXTEND:
5925 case ISD::FSIN:
5930 case ISD::FP_ROUND:
5933 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
5950 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
5951 Opcode == ISD::SMUL_LOHI);
5954 isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
5956 isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
5958 isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
5960 isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);
5978 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
5983 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
5984 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
5993 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
5994 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
5995 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
6008 if (Opcode != ISD::MUL) {
6017 if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
6039 isOperationLegalOrCustom(ISD::SRL, VT) &&
6040 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6041 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
6042 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
6043 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
6044 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
6055 if (Opcode == ISD::MUL) {
6056 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
6057 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
6058 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
6059 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
6066 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
6067 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6068 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
6069 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
6072 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6078 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6086 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
6087 isOperationLegalOrCustom(ISD::ADDE, VT));
6089 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
6092 Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
6096 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6097 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6099 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
6103 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
6106 Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
6109 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6111 if (Opcode == ISD::SMUL_LOHI) {
6112 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6113 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
6114 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
6116 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6117 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
6118 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
6121 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6122 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6123 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6145 return ISD::matchUnaryPredicate(
6155 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6156 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6157 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6158 !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
6166 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
6178 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6179 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
6180 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
6181 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
6188 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
6190 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
6193 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6194 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
6199 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
6200 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
6201 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
6203 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
6204 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
6205 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
6208 Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
6217 bool IsLeft = Node->getOpcode() == ISD::ROTL;
6229 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
6231 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6236 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6237 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6238 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6239 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
6240 !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6247 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
6248 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
6250 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6251 SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
6252 SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
6253 Result = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ShOpc, DL, VT, Op0, And0),
6291 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
6294 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
6296 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
6298 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
6299 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
6303 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
6304 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
6311 DAG.getNode(ISD::SHL, dl, DstVT, R,
6313 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
6315 DAG.getNode(ISD::SRL, dl, DstVT, R,
6317 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
6319 ISD::SETGT);
6321 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
6322 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
6325 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
6344 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
6345 ISD::FP_TO_SINT;
6347 !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))
6359 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6363 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6371 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
6375 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
6398 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
6400 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6404 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
6405 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
6407 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
6414 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6416 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
6417 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
6418 False = DAG.getNode(ISD::XOR, dl, DstVT, False,
6438 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
6439 !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
6440 !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
6441 !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||
6442 !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))
6459 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
6460 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
6461 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
6462 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
6467 DAG.getNode(ISD::STRICT_FSUB, dl, {DstVT, MVT::Other},
6469 Result = DAG.getNode(ISD::STRICT_FADD, dl, {DstVT, MVT::Other},
6474 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
6475 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
6483 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
6484 ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
6494 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
6498 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
6510 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
6524 ISD::CondCode Pred =
6525 Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
6554 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||
6555 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6556 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6557 (Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||
6558 !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6573 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
6574 DAG.getNode(ISD::AND, dl, VT,
6575 DAG.getNode(ISD::SRL, dl, VT, Op,
6579 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
6580 DAG.getNode(ISD::AND, dl, VT,
6581 DAG.getNode(ISD::SRL, dl, VT, Op,
6585 Op = DAG.getNode(ISD::AND, dl, VT,
6586 DAG.getNode(ISD::ADD, dl, VT, Op,
6587 DAG.getNode(ISD::SRL, dl, VT, Op,
6593 DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
6609 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
6610 isOperationLegalOrCustom(ISD::CTLZ, VT)) {
6611 Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);
6616 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
6619 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
6621 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
6622 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
6629 !isOperationLegalOrCustom(ISD::CTPOP, VT) ||
6630 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6631 !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
6645 Op = DAG.getNode(ISD::OR, dl, VT, Op,
6646 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
6649 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
6661 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
6662 isOperationLegalOrCustom(ISD::CTTZ, VT)) {
6663 Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
6668 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
6671 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
6673 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
6674 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
6681 (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
6682 !isOperationLegalOrCustom(ISD::CTLZ, VT)) ||
6683 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6684 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
6685 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
6693 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
6694 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
6696 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
6697 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
6699 DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
6700 DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
6704 Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
6716 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SRA, VT) ||
6717 !isOperationLegalOrCustom(ISD::ADD, VT) ||
6718 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
6722 DAG.getNode(ISD::SRA, dl, VT, Op,
6724 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
6725 Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
6737 ISD::LoadExtType ExtType = LD->getExtensionType();
6764 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
6775 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
6777 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
6778 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
6780 if (ExtType != ISD::NON_EXTLOAD) {
6781 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
6811 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
6848 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
6850 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
6851 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
6857 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
6858 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
6873 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
6887 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
6892 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
6904 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
6914 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
6916 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
6917 ISD::ANY_EXTEND, dl, VT, Result);
6964 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
6976 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
6999 ISD::LoadExtType HiExtType = LD->getExtensionType();
7002 if (HiExtType == ISD::NON_EXTLOAD)
7003 HiExtType = ISD::ZEXTLOAD;
7008 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7023 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
7033 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
7034 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
7036 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
7044 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
7058 if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
7067 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
7123 ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
7132 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7147 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
7164 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
7183 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
7188 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
7193 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
7197 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
7212 return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
7216 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
7238 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
7286 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
7289 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
7292 if (C->isNullValue() && CC == ISD::SETEQ) {
7297 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
7300 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
7301 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
7303 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
7320 if (Opcode == ISD::USUBSAT && isOperationLegalOrCustom(ISD::UMAX, VT)) {
7321 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
7322 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
7325 if (Opcode == ISD::UADDSAT && isOperationLegalOrCustom(ISD::UMIN, VT)) {
7327 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
7328 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
7333 case ISD::SADDSAT:
7334 OverflowOp = ISD::SADDO;
7336 case ISD::UADDSAT:
7337 OverflowOp = ISD::UADDO;
7339 case ISD::SSUBSAT:
7340 OverflowOp = ISD::SSUBO;
7342 case ISD::USUBSAT:
7343 OverflowOp = ISD::USUBO;
7359 if (Opcode == ISD::UADDSAT) {
7363 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
7367 } else if (Opcode == ISD::USUBSAT) {
7372 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
7383 SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT);
7391 assert((Node->getOpcode() == ISD::SMULFIX ||
7392 Node->getOpcode() == ISD::UMULFIX ||
7393 Node->getOpcode() == ISD::SMULFIXSAT ||
7394 Node->getOpcode() == ISD::UMULFIXSAT) &&
7402 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
7403 Node->getOpcode() == ISD::UMULFIXSAT);
7404 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
7405 Node->getOpcode() == ISD::SMULFIXSAT);
7412 if (isOperationLegalOrCustom(ISD::MUL, VT))
7413 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
7414 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
7416 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
7425 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
7428 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
7430 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
7448 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
7449 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
7455 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
7473 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
7489 ISD::SETUGT);
7501 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
7503 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
7508 ISD::SETLT);
7519 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
7525 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
7533 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
7534 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
7538 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
7539 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
7571 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
7574 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
7587 isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
7588 Quot = DAG.getNode(ISD::SDIVREM, dl,
7594 Quot = DAG.getNode(ISD::SDIV, dl, VT,
7596 Rem = DAG.getNode(ISD::SREM, dl, VT,
7600 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
7601 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
7602 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
7603 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
7604 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
7607 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
7610 Quot = DAG.getNode(ISD::UDIV, dl, VT,
7621 bool IsAdd = Node->getOpcode() == ISD::UADDO;
7624 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
7634 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
7640 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
7650 bool IsAdd = Node->getOpcode() == ISD::SADDO;
7652 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
7660 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
7663 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
7676 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
7678 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);
7681 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
7692 bool isSigned = Node->getOpcode() == ISD::SMULO;
7703 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
7705 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
7707 LHS, ISD::SETNE);
7720 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
7721 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
7723 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
7732 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
7733 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
7736 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
7737 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
7764 DAG.getNode(ISD::SRA, dl, VT, LHS,
7768 DAG.getNode(ISD::SRA, dl, VT, RHS,
7795 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
7812 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
7813 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
7816 DAG.getConstant(0, dl, VT), ISD::SETNE);
7822 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);
7835 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
7836 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
7837 case ISD::VECREDUCE_ADD: BaseOpcode = ISD::ADD; break;
7838 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
7839 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
7840 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
7841 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
7842 case ISD::VECREDUCE_SMAX: BaseOpcode = ISD::SMAX; break;
7843 case ISD::VECREDUCE_SMIN: BaseOpcode = ISD::SMIN; break;
7844 case ISD::VECREDUCE_UMAX: BaseOpcode = ISD::UMAX; break;
7845 case ISD::VECREDUCE_UMIN: BaseOpcode = ISD::UMIN; break;
7846 case ISD::VECREDUCE_FMAX:
7847 BaseOpcode = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM;
7849 case ISD::VECREDUCE_FMIN:
7850 BaseOpcode = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM;
7883 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
7891 bool isSigned = Node->getOpcode() == ISD::SREM;
7892 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
7893 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
7903 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
7904 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);