Lines Matching refs:SDValue
52 SDValue &Chain) const {
81 const SmallVectorImpl<SDValue> &OutVals) const {
93 SDValue Value = OutVals[I];
129 std::pair<SDValue, SDValue>
131 ArrayRef<SDValue> Ops,
134 SDValue InChain) const {
143 SDValue NewOp = Ops[i];
159 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
276 SDValue &NewLHS, SDValue &NewRHS,
278 const SDLoc &dl, const SDValue OldLHS,
279 const SDValue OldRHS) const {
280 SDValue Chain;
286 SDValue &NewLHS, SDValue &NewRHS,
288 const SDLoc &dl, const SDValue OldLHS,
289 const SDValue OldRHS,
290 SDValue &Chain,
389 SDValue Ops[2] = {NewLHS, NewRHS};
410 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
421 NewRHS = SDValue();
440 SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
486 bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
515 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
516 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
527 bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
540 bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
572 SDValue X = DAG.getNode(
577 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
584 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
599 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
624 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
625 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
629 return SDValue();
633 return SDValue();
644 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
653 if (SDValue V = SimplifyMultipleUseDemandedBits(
675 if (SDValue V = SimplifyMultipleUseDemandedBits(
694 if (SDValue V = SimplifyMultipleUseDemandedBits(
744 SDValue Op0 = Op.getOperand(0);
755 SDValue Op0 = Op.getOperand(0);
756 SDValue Op1 = Op.getOperand(1);
777 SDValue Op0 = Op.getOperand(0);
793 SDValue Src = Op.getOperand(0);
805 SDValue Vec = Op.getOperand(0);
815 SDValue Vec = Op.getOperand(0);
816 SDValue Sub = Op.getOperand(1);
848 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
853 return SDValue();
856 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
857 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
867 SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
868 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
883 SDValue Op, const APInt &OriginalDemandedBits,
950 SDValue Src = Op.getOperand(0);
982 SDValue Vec = Op.getOperand(0);
983 SDValue Scl = Op.getOperand(1);
1022 SDValue Src = Op.getOperand(0);
1023 SDValue Sub = Op.getOperand(1);
1052 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1054 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1059 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1068 SDValue Src = Op.getOperand(0);
1081 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1084 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1136 SDValue Op0 = Op.getOperand(0);
1137 SDValue Op1 = Op.getOperand(1);
1157 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1159 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1164 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1171 SDValue Op0 = Op.getOperand(0);
1172 SDValue Op1 = Op.getOperand(1);
1199 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1215 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1217 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1222 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1248 SDValue Op0 = Op.getOperand(0);
1249 SDValue Op1 = Op.getOperand(1);
1262 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1264 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1269 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1291 SDValue Op0 = Op.getOperand(0);
1292 SDValue Op1 = Op.getOperand(1);
1305 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1307 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1312 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1341 SDValue ANDC =
1352 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1402 SDValue Op0 = Op.getOperand(0);
1403 SDValue Op1 = Op.getOperand(1);
1430 SDValue Op0 = Op.getOperand(0);
1431 SDValue Op1 = Op.getOperand(1);
1455 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1466 SDValue InnerOp = Op0.getOperand(0);
1474 SDValue NarrowShl =
1496 SDValue NewSA =
1498 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1538 SDValue Op0 = Op.getOperand(0);
1539 SDValue Op1 = Op.getOperand(1);
1563 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1590 SDValue Op0 = Op.getOperand(0);
1591 SDValue Op1 = Op.getOperand(1);
1646 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1656 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1659 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1668 SDValue Op0 = Op.getOperand(0);
1669 SDValue Op1 = Op.getOperand(1);
1670 SDValue Op2 = Op.getOperand(2);
1715 SDValue Op0 = Op.getOperand(0);
1716 SDValue Op1 = Op.getOperand(1);
1732 SDValue Src = Op.getOperand(0);
1742 SDValue Src = Op.getOperand(0);
1752 SDValue Op0 = Op.getOperand(0);
1770 SDValue ShiftAmt =
1832 SDValue Src = Op.getOperand(0);
1865 SDValue Src = Op.getOperand(0);
1913 SDValue Src = Op.getOperand(0);
1936 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1942 SDValue Src = Op.getOperand(0);
1953 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1971 SDValue ShAmt = Src.getOperand(1);
1987 SDValue NewTrunc =
2013 SDValue Src = Op.getOperand(0);
2014 SDValue Idx = Op.getOperand(1);
2037 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2039 SDValue NewOp =
2051 SDValue Src = Op.getOperand(0);
2068 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2073 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2147 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2162 SDValue NewOp =
2171 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2173 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2180 SDValue NewOp =
2195 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2200 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2240 bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2261 static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2273 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2281 SDValue Elt = BV->getOperand(Index);
2287 return SDValue();
2298 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2299 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2308 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2351 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2352 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2354 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2357 SDValue NewOp = TLO.DAG.getNode(
2374 SDValue Src = Op.getOperand(0);
2464 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2465 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2479 SDValue SrcOp = Op.getOperand(i);
2494 SDValue SubOp = Op.getOperand(i);
2508 SDValue Src = Op.getOperand(0);
2509 SDValue Sub = Op.getOperand(1);
2536 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2538 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2543 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2552 SDValue Src = Op.getOperand(0);
2568 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2571 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2579 SDValue Vec = Op.getOperand(0);
2580 SDValue Scl = Op.getOperand(1);
2688 SDValue LegalShuffle =
2718 SDValue Src = Op.getOperand(0);
2754 SDValue Op0 = Op.getOperand(0);
2755 SDValue Op1 = Op.getOperand(1);
2781 SDValue Op0 = Op.getOperand(0);
2782 SDValue Op1 = Op.getOperand(1);
2805 SDValue Op0 = Op.getOperand(0);
2806 SDValue Op1 = Op.getOperand(1);
2872 void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2907 unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2927 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2939 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2951 SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
2952 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2961 return SDValue();
2964 SDValue
2965 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
2966 SDValue N1, MutableArrayRef<int> Mask,
2976 return SDValue();
2985 bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3079 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3091 return SDValue();
3093 SDValue X, Y;
3101 return SDValue();
3105 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3127 return SDValue();
3130 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3131 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3135 return SDValue();
3148 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3149 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3154 return SDValue();
3158 return SDValue();
3163 return SDValue();
3165 SDValue X = N0->getOperand(0);
3186 return SDValue();
3204 return SDValue();
3214 return SDValue();
3221 return SDValue();
3228 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3229 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3230 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3231 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3237 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3238 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3247 SDValue X, C, Y;
3253 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3285 return SDValue();
3288 SDValue Mask = N0.getOperand(1);
3294 return SDValue();
3301 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3302 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3303 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3310 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3323 SDValue X = N0.getOperand(0);
3324 SDValue Y = N0.getOperand(1);
3329 return SDValue();
3338 return SDValue();
3343 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3344 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3351 /// unable to simplify it, return a null SDValue.
3352 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3361 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3404 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3410 SDValue CTPOP = N0;
3419 SDValue CTOp = CTPOP.getOperand(0);
3424 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3425 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3426 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3436 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3437 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3440 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3441 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3442 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3443 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3454 SDValue PreExt;
3496 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3501 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3513 SDValue TopSetCC = N0->getOperand(0);
3579 SDValue Ptr = Lod->getBasePtr();
3583 SDValue NewLoad = DAG.getLoad(
3638 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3640 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3661 SDValue ZextOp;
3711 SDValue Val;
3727 SDValue Op0 = N0;
3734 SDValue XorLHS = Op0.getOperand(0);
3735 SDValue XorRHS = Op0.getOperand(1);
3784 if (SDValue V =
3884 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3903 SDValue ConstMinusOne =
3959 SDValue Shift =
3962 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
3988 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3990 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4013 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4132 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4138 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4141 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4154 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4157 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4165 SDValue Temp;
4215 return SDValue();
4223 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4232 SDValue N1 = N->getOperand(0);
4233 SDValue N2 = N->getOperand(1);
4250 SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4253 return SDValue();
4314 SDValue TargetLowering::LowerAsmOutputForConstraint(
4315 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
4317 return SDValue();
4322 void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4324 std::vector<SDValue> &Ops,
4760 SDValue Op, SelectionDAG *DAG) {
4785 std::vector<SDValue> ResultOps;
4816 SDValue Op,
4853 static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
4856 SDValue Op0 = N->getOperand(0);
4857 SDValue Op1 = N->getOperand(1);
4864 SmallVector<SDValue, 16> Shifts, Factors;
4887 return SDValue();
4889 SDValue Shift, Factor;
4898 SDValue Res = Op0;
4912 SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
4918 return SDValue(N, 0); // Lower SDIV as SDIV
4919 return SDValue();
4926 SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
4939 return SDValue();
4945 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
4977 SDValue N0 = N->getOperand(0);
4978 SDValue N1 = N->getOperand(1);
4982 return SDValue();
4984 SDValue MagicFactor, Factor, Shift, ShiftMask;
4999 SDValue Q;
5005 SDValue LoHi =
5007 Q = SDValue(LoHi.getNode(), 1);
5009 return SDValue(); // No mulhs or equivalent.
5023 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5024 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5035 SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5048 return SDValue();
5051 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5095 SDValue N0 = N->getOperand(0);
5096 SDValue N1 = N->getOperand(1);
5100 return SDValue();
5102 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5114 SDValue Q = N0;
5119 auto GetMULHU = [&](SDValue X, SDValue Y) {
5125 SDValue LoHi =
5127 return SDValue(LoHi.getNode(), 1);
5129 return SDValue(); // No mulhu or equivalent
5135 return SDValue();
5140 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5159 SDValue One = DAG.getConstant(1, dl, VT);
5160 SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
5169 turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5170 std::function<bool(SDValue)> Predicate,
5171 SDValue AlternativeReplacement = SDValue()) {
5172 SDValue Replacement;
5177 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5197 SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5198 SDValue CompTargetNode,
5203 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5210 return SDValue();
5213 SDValue
5214 TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5215 SDValue CompTargetNode, ISD::CondCode Cond,
5235 return SDValue();
5244 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5326 SDValue N = REMNode.getOperand(0);
5327 SDValue D = REMNode.getOperand(1);
5331 return SDValue();
5335 return SDValue();
5340 return SDValue();
5342 SDValue PVal, KVal, QVal;
5365 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5372 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5380 return SDValue();
5389 SDValue NewCC =
5405 SDValue TautologicalInvertedChannels =
5412 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5423 return SDValue(); // Don't know how to lower.
5431 SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5432 SDValue CompTargetNode,
5437 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5445 return SDValue();
5448 SDValue
5449 TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5450 SDValue CompTargetNode, ISD::CondCode Cond,
5475 return SDValue();
5480 return SDValue();
5488 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5570 SDValue N = REMNode.getOperand(0);
5571 SDValue D = REMNode.getOperand(1);
5575 return SDValue();
5579 return SDValue();
5584 return SDValue();
5586 SDValue PVal, AVal, KVal, QVal;
5614 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5620 return SDValue();
5632 return SDValue();
5641 SDValue Fold =
5658 return SDValue();
5662 SDValue IntMin = DAG.getConstant(
5664 SDValue IntMax = DAG.getConstant(
5666 SDValue Zero =
5670 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5674 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5676 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5683 SDValue Blended =
5690 verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5700 SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
5712 return SDValue();
5726 return SDValue();
5729 auto RemoveDeadNode = [&](SDValue N) {
5750 SDValue CFP = DAG.getConstantFP(V, DL, VT);
5761 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
5769 llvm::all_of(Op->op_values(), [&](SDValue N) {
5778 SmallVector<SDValue, 4> Ops;
5779 for (SDValue C : Op->op_values()) {
5798 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
5802 SDValue NegX =
5806 SDValue NegY =
5812 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
5821 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
5833 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
5847 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
5851 SDValue NegX =
5855 SDValue NegY =
5861 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
5875 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
5887 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
5889 SDValue NegZ =
5897 SDValue NegX =
5901 SDValue NegY =
5907 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
5916 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
5926 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
5931 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
5937 return SDValue();
5945 SDValue LHS, SDValue RHS,
5946 SmallVectorImpl<SDValue> &Result,
5948 MulExpansionKind Kind, SDValue LL,
5949 SDValue LH, SDValue RL, SDValue RH) const {
5975 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
5979 Hi = SDValue(Lo.getNode(), 1);
5990 SDValue Lo, Hi;
6009 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6036 SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
6065 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
6072 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6083 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6095 SDValue Carry = Next.getValue(1);
6112 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6127 bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
6129 SDValue LL, SDValue LH, SDValue RL,
6130 SDValue RH) const {
6131 SmallVector<SDValue, 2> Result;
6144 static bool isNonZeroModBitWidth(SDValue Z, unsigned BW) {
6151 bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
6161 SDValue X = Node->getOperand(0);
6162 SDValue Y = Node->getOperand(1);
6163 SDValue Z = Node->getOperand(2);
6167 SDLoc DL(SDValue(Node, 0));
6171 SDValue ShX, ShY;
6172 SDValue ShAmt, InvShAmt;
6177 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6185 SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
6192 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6197 SDValue One = DAG.getConstant(1, DL, ShVT);
6200 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
6203 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
6213 bool TargetLowering::expandROT(SDNode *Node, SDValue &Result,
6218 SDValue Op0 = Node->getOperand(0);
6219 SDValue Op1 = Node->getOperand(1);
6220 SDLoc DL(SDValue(Node, 0));
6223 SDValue Zero = DAG.getConstant(0, DL, ShVT);
6231 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6249 SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
6250 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6251 SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
6252 SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
6258 bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
6261 SDValue Src = Node->getOperand(OpNo);
6264 SDLoc dl(SDValue(Node, 0));
6284 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
6285 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
6286 SDValue Bias = DAG.getConstant(127, dl, IntVT);
6287 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
6288 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
6289 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
6291 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
6293 SDValue ExponentBits = DAG.getNode(
6296 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
6298 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
6303 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
6321 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
6329 bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
6330 SDValue &Chain,
6332 SDLoc dl(SDValue(Node, 0));
6334 SDValue Src = Node->getOperand(OpNo);
6367 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
6368 SDValue Sel;
6390 SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
6393 SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
6396 SDValue SInt;
6398 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
6404 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
6414 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6416 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
6426 bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
6427 SDValue &Chain,
6430 SDValue Src = Node->getOperand(OpNo);
6445 SDLoc dl(SDValue(Node, 0));
6452 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
6453 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
6455 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
6456 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
6457 SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
6459 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
6460 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
6461 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
6462 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
6463 SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
6464 SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
6466 SDValue HiSub =
6473 SDValue HiSub =
6480 SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
6487 SDValue Quiet0 = Node->getOperand(0);
6488 SDValue Quiet1 = Node->getOperand(1);
6526 SDValue Op1 = Node->getOperand(0);
6527 SDValue Op2 = Node->getOperand(1);
6528 SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);
6537 return SDValue();
6540 bool TargetLowering::expandCTPOP(SDNode *Node, SDValue &Result,
6545 SDValue Op = Node->getOperand(0);
6563 SDValue Mask55 =
6565 SDValue Mask33 =
6567 SDValue Mask0F =
6569 SDValue Mask01 =
6600 bool TargetLowering::expandCTLZ(SDNode *Node, SDValue &Result,
6605 SDValue Op = Node->getOperand(0);
6619 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
6620 SDValue Zero = DAG.getConstant(0, dl, VT);
6621 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
6644 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
6653 bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
6657 SDValue Op = Node->getOperand(0);
6671 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
6672 SDValue Zero = DAG.getConstant(0, dl, VT);
6673 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
6692 SDValue Tmp = DAG.getNode(
6708 bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
6713 SDValue Op = N->getOperand(0);
6721 SDValue Shift =
6724 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
6729 std::pair<SDValue, SDValue>
6733 SDValue Chain = LD->getChain();
6734 SDValue BasePTR = LD->getBasePtr();
6758 SDValue SrcEltBitMask = DAG.getConstant(
6763 SDValue Load =
6768 SmallVector<SDValue, 8> Vals;
6772 SDValue ShiftAmount =
6775 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
6776 SDValue Elt =
6778 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
6788 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
6795 SmallVector<SDValue, 8> Vals;
6796 SmallVector<SDValue, 8> LoadChains;
6799 SDValue ScalarLoad =
6811 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
6812 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
6817 SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
6821 SDValue Chain = ST->getChain();
6822 SDValue BasePtr = ST->getBasePtr();
6823 SDValue Value = ST->getValue();
6845 SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
6848 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
6850 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
6851 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
6854 SDValue ShiftAmount =
6856 SDValue ShiftedElt =
6871 SmallVector<SDValue, 8> Stores;
6873 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
6876 SDValue Ptr = DAG.getObjectPtrOffset(SL, BasePtr, Idx * Stride);
6879 SDValue Store = DAG.getTruncStore(
6890 std::pair<SDValue, SDValue>
6894 SDValue Chain = LD->getChain();
6895 SDValue Ptr = LD->getBasePtr();
6912 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
6914 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
6930 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
6932 SmallVector<SDValue, 8> Stores;
6933 SDValue StackPtr = StackBase;
6939 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
6940 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
6945 SDValue Load = DAG.getLoad(
6963 SDValue Load =
6976 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7006 SDValue Lo, Hi;
7030 SDValue ShiftAmount =
7033 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
7036 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
7042 SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
7046 SDValue Chain = ST->getChain();
7047 SDValue Ptr = ST->getBasePtr();
7048 SDValue Val = ST->getValue();
7061 SDValue Result = scalarizeVectorStore(ST, DAG);
7067 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
7083 SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
7087 SDValue Store = DAG.getTruncStore(
7093 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7094 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7095 SmallVector<SDValue, 8> Stores;
7101 SDValue Load = DAG.getLoad(
7122 SDValue Load = DAG.getExtLoad(
7132 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7144 SDValue ShiftAmount = DAG.getConstant(
7146 SDValue Lo = Val;
7147 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
7150 SDValue Store1, Store2;
7163 SDValue Result =
7168 SDValue
7169 TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
7173 SDValue Increment;
7181 SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
7191 SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
7200 static SDValue clampDynamicVectorIndex(SelectionDAG &DAG,
7201 SDValue Idx,
7220 SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
7221 SDValue VecPtr, EVT VecVT,
7222 SDValue Index) const {
7247 SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
7266 SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
7271 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
7284 SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
7288 return SDValue();
7294 SDValue Zext = Op.getOperand(0);
7300 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
7301 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
7306 return SDValue();
7309 SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
7311 SDValue LHS = Node->getOperand(0);
7312 SDValue RHS = Node->getOperand(1);
7321 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
7326 SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
7327 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
7352 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT),
7354 SDValue SumDiff = Result.getValue(0);
7355 SDValue Overflow = Result.getValue(1);
7356 SDValue Zero = DAG.getConstant(0, dl, VT);
7357 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
7362 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
7370 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
7371 SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
7381 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
7382 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
7383 SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT);
7389 SDValue
7398 SDValue LHS = Node->getOperand(0);
7399 SDValue RHS = Node->getOperand(1);
7415 SDValue Result =
7417 SDValue Product = Result.getValue(0);
7418 SDValue Overflow = Result.getValue(1);
7419 SDValue Zero = DAG.getConstant(0, dl, VT);
7423 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
7424 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
7425 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
7429 SDValue Result =
7431 SDValue Product = Result.getValue(0);
7432 SDValue Overflow = Result.getValue(1);
7435 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
7447 SDValue Lo, Hi;
7451 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
7458 return SDValue();
7473 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
7485 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
7497 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
7498 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
7501 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
7503 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
7506 SDValue Zero = DAG.getConstant(0, dl, VT);
7507 SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
7517 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
7522 SDValue HighMask =
7529 SDValue
7531 SDValue LHS, SDValue RHS,
7560 return SDValue();
7577 SDValue Quot;
7582 SDValue Rem;
7599 SDValue Zero = DAG.getConstant(0, dl, VT);
7600 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
7601 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
7602 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
7603 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
7604 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
7617 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
7619 SDValue LHS = Node->getOperand(0);
7620 SDValue RHS = Node->getOperand(1);
7626 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
7627 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
7629 Result = SDValue(NodeCarry.getNode(), 0);
7630 Overflow = SDValue(NodeCarry.getNode(), 1);
7641 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
7646 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
7648 SDValue LHS = Node->getOperand(0);
7649 SDValue RHS = Node->getOperand(1);
7662 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
7663 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
7668 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
7676 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
7677 SDValue ConditionRHS =
7685 bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
7686 SDValue &Overflow, SelectionDAG &DAG) const {
7690 SDValue LHS = Node->getOperand(0);
7691 SDValue RHS = Node->getOperand(1);
7702 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);
7717 SDValue BottomHalf;
7718 SDValue TopHalf;
7732 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
7734 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,
7757 SDValue HiLHS;
7758 SDValue HiRHS;
7780 SDValue Ret;
7789 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
7792 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
7809 SDValue ShiftAmt = DAG.getConstant(
7812 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
7829 SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
7854 SDValue Op = Node->getOperand(0);
7864 SDValue Lo, Hi;
7874 SmallVector<SDValue, 8> Ops;
7877 SDValue Res = Ops[0];
7887 bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
7894 SDValue Dividend = Node->getOperand(0);
7895 SDValue Divisor = Node->getOperand(1);
7902 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
7903 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);