Lines Matching refs:SDValue
234 SDValue visit(SDNode *N);
295 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
299 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
304 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
306 SDValue To[] = { Res0, Res1 };
318 bool SimplifyDemandedBits(SDValue Op) {
324 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
340 bool SimplifyDemandedVectorElts(SDValue Op) {
350 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
353 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
358 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
362 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
363 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
364 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
373 /// \returns EVE on success SDValue() on failure.
374 SDValue scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
375 SDValue EltNo,
378 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
379 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
380 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
381 SDValue PromoteIntBinOp(SDValue Op);
382 SDValue PromoteIntShiftOp(SDValue Op);
383 SDValue PromoteExtend(SDValue Op);
384 bool PromoteLoad(SDValue Op);
389 SDValue combine(SDNode *N);
394 // SDValue.getNode() == 0 - No change was made
395 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
398 SDValue visitTokenFactor(SDNode *N);
399 SDValue visitMERGE_VALUES(SDNode *N);
400 SDValue visitADD(SDNode *N);
401 SDValue visitADDLike(SDNode *N);
402 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
403 SDValue visitSUB(SDNode *N);
404 SDValue visitADDSAT(SDNode *N);
405 SDValue visitSUBSAT(SDNode *N);
406 SDValue visitADDC(SDNode *N);
407 SDValue visitADDO(SDNode *N);
408 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
409 SDValue visitSUBC(SDNode *N);
410 SDValue visitSUBO(SDNode *N);
411 SDValue visitADDE(SDNode *N);
412 SDValue visitADDCARRY(SDNode *N);
413 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
414 SDValue visitSUBE(SDNode *N);
415 SDValue visitSUBCARRY(SDNode *N);
416 SDValue visitMUL(SDNode *N);
417 SDValue visitMULFIX(SDNode *N);
418 SDValue useDivRem(SDNode *N);
419 SDValue visitSDIV(SDNode *N);
420 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
421 SDValue visitUDIV(SDNode *N);
422 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
423 SDValue visitREM(SDNode *N);
424 SDValue visitMULHU(SDNode *N);
425 SDValue visitMULHS(SDNode *N);
426 SDValue visitSMUL_LOHI(SDNode *N);
427 SDValue visitUMUL_LOHI(SDNode *N);
428 SDValue visitMULO(SDNode *N);
429 SDValue visitIMINMAX(SDNode *N);
430 SDValue visitAND(SDNode *N);
431 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
432 SDValue visitOR(SDNode *N);
433 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
434 SDValue visitXOR(SDNode *N);
435 SDValue SimplifyVBinOp(SDNode *N);
436 SDValue visitSHL(SDNode *N);
437 SDValue visitSRA(SDNode *N);
438 SDValue visitSRL(SDNode *N);
439 SDValue visitFunnelShift(SDNode *N);
440 SDValue visitRotate(SDNode *N);
441 SDValue visitABS(SDNode *N);
442 SDValue visitBSWAP(SDNode *N);
443 SDValue visitBITREVERSE(SDNode *N);
444 SDValue visitCTLZ(SDNode *N);
445 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
446 SDValue visitCTTZ(SDNode *N);
447 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
448 SDValue visitCTPOP(SDNode *N);
449 SDValue visitSELECT(SDNode *N);
450 SDValue visitVSELECT(SDNode *N);
451 SDValue visitSELECT_CC(SDNode *N);
452 SDValue visitSETCC(SDNode *N);
453 SDValue visitSETCCCARRY(SDNode *N);
454 SDValue visitSIGN_EXTEND(SDNode *N);
455 SDValue visitZERO_EXTEND(SDNode *N);
456 SDValue visitANY_EXTEND(SDNode *N);
457 SDValue visitAssertExt(SDNode *N);
458 SDValue visitAssertAlign(SDNode *N);
459 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
460 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
461 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
462 SDValue visitTRUNCATE(SDNode *N);
463 SDValue visitBITCAST(SDNode *N);
464 SDValue visitFREEZE(SDNode *N);
465 SDValue visitBUILD_PAIR(SDNode *N);
466 SDValue visitFADD(SDNode *N);
467 SDValue visitFSUB(SDNode *N);
468 SDValue visitFMUL(SDNode *N);
469 SDValue visitFMA(SDNode *N);
470 SDValue visitFDIV(SDNode *N);
471 SDValue visitFREM(SDNode *N);
472 SDValue visitFSQRT(SDNode *N);
473 SDValue visitFCOPYSIGN(SDNode *N);
474 SDValue visitFPOW(SDNode *N);
475 SDValue visitSINT_TO_FP(SDNode *N);
476 SDValue visitUINT_TO_FP(SDNode *N);
477 SDValue visitFP_TO_SINT(SDNode *N);
478 SDValue visitFP_TO_UINT(SDNode *N);
479 SDValue visitFP_ROUND(SDNode *N);
480 SDValue visitFP_EXTEND(SDNode *N);
481 SDValue visitFNEG(SDNode *N);
482 SDValue visitFABS(SDNode *N);
483 SDValue visitFCEIL(SDNode *N);
484 SDValue visitFTRUNC(SDNode *N);
485 SDValue visitFFLOOR(SDNode *N);
486 SDValue visitFMINNUM(SDNode *N);
487 SDValue visitFMAXNUM(SDNode *N);
488 SDValue visitFMINIMUM(SDNode *N);
489 SDValue visitFMAXIMUM(SDNode *N);
490 SDValue visitBRCOND(SDNode *N);
491 SDValue visitBR_CC(SDNode *N);
492 SDValue visitLOAD(SDNode *N);
494 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
495 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
497 SDValue visitSTORE(SDNode *N);
498 SDValue visitLIFETIME_END(SDNode *N);
499 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
500 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
501 SDValue visitBUILD_VECTOR(SDNode *N);
502 SDValue visitCONCAT_VECTORS(SDNode *N);
503 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
504 SDValue visitVECTOR_SHUFFLE(SDNode *N);
505 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
506 SDValue visitINSERT_SUBVECTOR(SDNode *N);
507 SDValue visitMLOAD(SDNode *N);
508 SDValue visitMSTORE(SDNode *N);
509 SDValue visitMGATHER(SDNode *N);
510 SDValue visitMSCATTER(SDNode *N);
511 SDValue visitFP_TO_FP16(SDNode *N);
512 SDValue visitFP16_TO_FP(SDNode *N);
513 SDValue visitVECREDUCE(SDNode *N);
515 SDValue visitFADDForFMACombine(SDNode *N);
516 SDValue visitFSUBForFMACombine(SDNode *N);
517 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
519 SDValue XformToShuffleWithZero(SDNode *N);
521 const SDLoc &DL, SDValue N0,
522 SDValue N1);
523 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
524 SDValue N1);
525 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
526 SDValue N1, SDNodeFlags Flags);
528 SDValue visitShiftByConstant(SDNode *N);
530 SDValue foldSelectOfConstants(SDNode *N);
531 SDValue foldVSelectOfConstants(SDNode *N);
532 SDValue foldBinOpIntoSelect(SDNode *BO);
533 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
534 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
535 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
536 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
537 SDValue N2, SDValue N3, ISD::CondCode CC,
539 SDValue convertSelectOfFPConstantsToLoadOffset(
540 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
542 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
543 SDValue N2, SDValue N3, ISD::CondCode CC);
544 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
546 SDValue unfoldMaskedMerge(SDNode *N);
547 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
548 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
550 SDValue rebuildSetCC(SDValue N);
552 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
553 SDValue &CC, bool MatchStrict = false) const;
554 bool isOneUseSetCC(SDValue N) const;
556 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
558 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
559 SDValue CombineExtLoad(SDNode *N);
560 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
561 SDValue combineRepeatedFPDivisors(SDNode *N);
562 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
563 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
564 SDValue BuildSDIV(SDNode *N);
565 SDValue BuildSDIVPow2(SDNode *N);
566 SDValue BuildUDIV(SDNode *N);
567 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
568 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
569 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
570 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
571 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
572 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
574 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
576 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
578 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
579 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
580 SDValue InnerPos, SDValue InnerNeg,
583 SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
584 SDValue InnerPos, SDValue InnerNeg,
587 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
588 SDValue MatchLoadCombine(SDNode *N);
589 SDValue MatchStoreCombine(StoreSDNode *N);
590 SDValue ReduceLoadWidth(SDNode *N);
591 SDValue ReduceLoadOpStoreWidth(SDNode *N);
592 SDValue splitMergedValStore(StoreSDNode *ST);
593 SDValue TransformFPLoadStorePair(SDNode *N);
594 SDValue convertBuildVecZextToZext(SDNode *N);
595 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
596 SDValue reduceBuildVecTruncToBitCast(SDNode *N);
597 SDValue reduceBuildVecToShuffle(SDNode *N);
598 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
599 ArrayRef<int> VectorMask, SDValue VecIn1,
600 SDValue VecIn2, unsigned LeftIdx,
602 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
606 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
607 SmallVectorImpl<SDValue> &Aliases);
614 SDValue FindBetterChain(SDNode *N, SDValue Chain);
643 StoreSource getStoreSource(SDValue StoreVal) {
659 SDValue &AddNode,
660 SDValue &ConstNode);
683 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
747 /// single-use) and if missed an empty SDValue is returned.
748 SDValue distributeTruncateThroughAnd(SDNode *N);
786 SDValue OrigLoad, SDValue ExtLoad,
826 SDValue TargetLowering::DAGCombinerInfo::
827 CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
831 SDValue TargetLowering::DAGCombinerInfo::
832 CombineTo(SDNode *N, SDValue Res, bool AddTo) {
836 SDValue TargetLowering::DAGCombinerInfo::
837 CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
860 for (const SDValue &Op : N->ops())
884 bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
885 SDValue &CC, bool MatchStrict) const {
920 bool DAGCombiner::isOneUseSetCC(SDValue N) const {
921 SDValue N0, N1, N2;
929 static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
940 static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
946 for (const SDValue &Op : N->op_values()) {
959 static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
975 SDValue N0,
976 SDValue N1) {
1031 SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
1032 SDValue N0, SDValue N1) {
1036 return SDValue();
1041 if (SDValue OpNode =
1044 return SDValue();
1049 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
1051 return SDValue();
1055 return SDValue();
1059 SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
1060 SDValue N1, SDNodeFlags Flags) {
1067 return SDValue();
1069 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1))
1071 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0))
1073 return SDValue();
1076 SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1105 return SDValue(N, 0);
1133 bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1152 bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1171 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1176 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1177 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1182 SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1200 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1204 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1215 return SDValue();
1219 SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1221 return SDValue();
1225 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1227 return SDValue();
1236 SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1240 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1242 return SDValue();
1253 SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1255 return SDValue();
1259 return SDValue();
1265 return SDValue();
1276 SDValue N0 = Op.getOperand(0);
1277 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1280 SDValue N1 = Op.getOperand(1);
1281 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1284 SDValue RV =
1290 // (SDValue) here because the node may reference multiple values
1315 return SDValue();
1321 SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1323 return SDValue();
1327 return SDValue();
1333 return SDValue();
1344 SDValue N0 = Op.getOperand(0);
1345 SDValue N1 = Op.getOperand(1);
1354 return SDValue();
1357 SDValue RV =
1367 return SDValue();
1370 SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1372 return SDValue();
1376 return SDValue();
1382 return SDValue();
1395 return SDValue();
1398 bool DAGCombiner::PromoteLoad(SDValue Op) {
1427 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1430 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1435 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1436 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1462 for (const SDValue &ChildN : N->op_values())
1525 for (const SDValue &ChildN : N->op_values())
1529 SDValue RV = combine(N);
1573 SDValue DAGCombiner::visit(SDNode *N) {
1706 return SDValue();
1709 SDValue DAGCombiner::combine(SDNode *N) {
1710 SDValue RV;
1740 RV = PromoteIntBinOp(SDValue(N, 0));
1745 RV = PromoteIntShiftOp(SDValue(N, 0));
1750 RV = PromoteExtend(SDValue(N, 0));
1753 if (PromoteLoad(SDValue(N, 0)))
1754 RV = SDValue(N, 0);
1763 SDValue N0 = N->getOperand(0);
1764 SDValue N1 = N->getOperand(1);
1768 SDValue Ops[] = {N1, N0};
1772 return SDValue(CSENode, 0);
1781 static SDValue getInputChainForNode(SDNode *N) {
1791 return SDValue();
1794 SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1806 return SDValue();
1815 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1839 for (const SDValue &Op : TF->op_values()) {
1885 for (const SDValue &Op : Ops) {
1935 for (const SDValue &Op : CurNode->op_values())
1956 SDValue Result;
1962 SmallVector<SDValue, 8> PrunedOps;
1964 for (const SDValue &Op : Ops) {
1975 return SDValue();
1979 SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1989 SmallVector<SDValue, 8> Ops;
1995 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2000 static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
2005 SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
2013 SDValue Sel = BO->getOperand(0);
2020 return SDValue();
2022 SDValue CT = Sel.getOperand(1);
2025 return SDValue();
2027 SDValue CF = Sel.getOperand(2);
2030 return SDValue();
2043 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
2047 return SDValue();
2058 return SDValue();
2064 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
2069 return SDValue();
2071 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
2076 return SDValue();
2078 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
2083 static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
2091 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
2092 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
2095 return SDValue();
2100 return SDValue();
2103 SDValue SetCC = Z.getOperand(0);
2108 return SDValue();
2116 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
2117 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
2124 static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2131 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2132 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2135 return SDValue();
2138 SDValue Not = ShiftOp.getOperand(0);
2140 return SDValue();
2144 SDValue ShAmt = ShiftOp.getOperand(1);
2147 return SDValue();
2154 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
2155 if (SDValue NewC =
2159 return SDValue();
2165 SDValue DAGCombiner::visitADDLike(SDNode *N) {
2166 SDValue N0 = N->getOperand(0);
2167 SDValue N1 = N->getOperand(1);
2173 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2206 SDValue Sub =
2215 SDValue Add =
2227 SDValue X = N0.getOperand(0);
2232 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2242 if (SDValue Add0 = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT,
2248 if (SDValue NewSel = foldBinOpIntoSelect(N))
2253 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
2305 SDValue N00 = N0.getOperand(0);
2306 SDValue N01 = N0.getOperand(1);
2307 SDValue N10 = N1.getOperand(0);
2308 SDValue N11 = N1.getOperand(1);
2328 if (SimplifyDemandedBits(SDValue(N, 0)))
2329 return SDValue(N, 0);
2341 SDValue A, Xor;
2361 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2370 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2374 if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
2377 if (SDValue Combined = visitADDLikeCommutative(N1, N0, N))
2380 return SDValue();
2383 SDValue DAGCombiner::visitADD(SDNode *N) {
2384 SDValue N0 = N->getOperand(0);
2385 SDValue N1 = N->getOperand(1);
2389 if (SDValue Combined = visitADDLike(N))
2392 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2395 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2420 return SDValue();
2423 SDValue DAGCombiner::visitADDSAT(SDNode *N) {
2425 SDValue N0 = N->getOperand(0);
2426 SDValue N1 = N->getOperand(1);
2462 return SDValue();
2465 static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2486 return SDValue();
2490 return SDValue();
2494 return SDValue();
2504 return SDValue();
2510 static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2513 return SDValue();
2517 return SDValue();
2525 SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2538 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2547 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2557 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2564 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2574 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2582 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2596 if (SDValue Carry = getAsCarry(TLI, N1))
2601 return SDValue();
2604 SDValue DAGCombiner::visitADDC(SDNode *N) {
2605 SDValue N0 = N->getOperand(0);
2606 SDValue N1 = N->getOperand(1);
2631 return SDValue();
2634 static SDValue flipBoolean(SDValue V, const SDLoc &DL,
2638 SDValue Cst;
2655 * This function returns an empty SDValue in case it cannot flip the boolean
2659 static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2666 return SDValue();
2670 return SDValue();
2691 return SDValue();
2694 SDValue DAGCombiner::visitADDO(SDNode *N) {
2695 SDValue N0 = N->getOperand(0);
2696 SDValue N1 = N->getOperand(1);
2725 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2731 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2734 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2738 return SDValue();
2741 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2744 return SDValue();
2749 SDValue Y = N1.getOperand(0);
2750 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2758 if (SDValue Carry = getAsCarry(TLI, N1))
2762 return SDValue();
2765 SDValue DAGCombiner::visitADDE(SDNode *N) {
2766 SDValue N0 = N->getOperand(0);
2767 SDValue N1 = N->getOperand(1);
2768 SDValue CarryIn = N->getOperand(2);
2781 return SDValue();
2784 SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2785 SDValue N0 = N->getOperand(0);
2786 SDValue N1 = N->getOperand(1);
2787 SDValue CarryIn = N->getOperand(2);
2807 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2814 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2817 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2820 return SDValue();
2845 static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2846 SDValue X, SDValue Carry0, SDValue Carry1,
2849 return SDValue();
2851 return SDValue();
2853 SDValue Z;
2868 return SDValue();
2872 auto cancelDiamond = [&](SDValue A,SDValue B) {
2874 SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
2907 return SDValue();
2936 static SDValue combineCarryDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2937 const TargetLowering &TLI, SDValue Carry0,
2938 SDValue Carry1, SDNode *N) {
2940 return SDValue();
2943 return SDValue();
2945 return SDValue();
2955 return SDValue();
2961 return SDValue();
2962 SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
2966 return SDValue();
2971 return SDValue();
2974 return SDValue();
2977 SDValue Merged =
3000 SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
3004 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
3006 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
3036 return SDValue();
3041 static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
3047 return SDValue();
3050 SDValue DAGCombiner::visitSUB(SDNode *N) {
3051 SDValue N0 = N->getOperand(0);
3052 SDValue N1 = N->getOperand(1);
3058 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3072 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
3075 if (SDValue NewSel = foldBinOpIntoSelect(N))
3140 SDValue NewC =
3148 SDValue N11 = N1.getOperand(1);
3151 SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11});
3161 SDValue NewC =
3171 SDValue NewC =
3205 SDValue A = N1.getOperand(0);
3206 SDValue B = N1.getOperand(1);
3211 SDValue InvB =
3221 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3228 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3241 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3244 if (SDValue V = foldAddSubOfSignBit(N, DAG))
3247 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3252 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3262 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3270 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3276 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3283 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3289 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3300 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3307 SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
3308 SDValue S0 = N1.getOperand(0);
3337 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3352 SDValue ShAmt = N1.getOperand(1);
3356 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3363 if (SDValue Carry = getAsCarry(TLI, N0)) {
3364 SDValue X = N1;
3365 SDValue Zero = DAG.getConstant(0, DL, VT);
3366 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3373 return SDValue();
3376 SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
3377 SDValue N0 = N->getOperand(0);
3378 SDValue N1 = N->getOperand(1);
3400 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
3407 return SDValue();
3410 SDValue DAGCombiner::visitSUBC(SDNode *N) {
3411 SDValue N0 = N->getOperand(0);
3412 SDValue N1 = N->getOperand(1);
3435 return SDValue();
3438 SDValue DAGCombiner::visitSUBO(SDNode *N) {
3439 SDValue N0 = N->getOperand(0);
3440 SDValue N1 = N->getOperand(1);
3474 return SDValue();
3477 SDValue DAGCombiner::visitSUBE(SDNode *N) {
3478 SDValue N0 = N->getOperand(0);
3479 SDValue N1 = N->getOperand(1);
3480 SDValue CarryIn = N->getOperand(2);
3486 return SDValue();
3489 SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
3490 SDValue N0 = N->getOperand(0);
3491 SDValue N1 = N->getOperand(1);
3492 SDValue CarryIn = N->getOperand(2);
3501 return SDValue();
3506 SDValue DAGCombiner::visitMULFIX(SDNode *N) {
3507 SDValue N0 = N->getOperand(0);
3508 SDValue N1 = N->getOperand(1);
3509 SDValue Scale = N->getOperand(2);
3525 return SDValue();
3528 SDValue DAGCombiner::visitMUL(SDNode *N) {
3529 SDValue N0 = N->getOperand(0);
3530 SDValue N1 = N->getOperand(1);
3543 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3559 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1}))
3575 if (SDValue NewSel = foldBinOpIntoSelect(N))
3590 SDValue LogBase2 = BuildLogBase2(N1, DL);
3592 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3633 SDValue Shl =
3635 SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0);
3646 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
3654 SDValue Sh(nullptr, 0), Y(nullptr, 0);
3668 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
3693 if (SDValue RMUL = reassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags()))
3696 return SDValue();
3719 SDValue DAGCombiner::useDivRem(SDNode *Node) {
3721 return SDValue(); // This is a dead node, leave it alone.
3730 return SDValue();
3733 return SDValue();
3739 return SDValue();
3746 return SDValue();
3750 return SDValue();
3753 SDValue Op0 = Node->getOperand(0);
3754 SDValue Op1 = Node->getOperand(1);
3755 SDValue combined;
3774 combined = SDValue(User, 0);
3789 static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
3790 SDValue N0 = N->getOperand(0);
3791 SDValue N1 = N->getOperand(1);
3832 return SDValue();
3835 SDValue DAGCombiner::visitSDIV(SDNode *N) {
3836 SDValue N0 = N->getOperand(0);
3837 SDValue N1 = N->getOperand(1);
3843 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3850 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
3863 if (SDValue V = simplifyDivRem(N, DAG))
3866 if (SDValue NewSel = foldBinOpIntoSelect(N))
3874 if (SDValue V = visitSDIVLike(N0, N1, N)) {
3879 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3880 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3893 if (SDValue DivRem = useDivRem(N))
3896 return SDValue();
3899 SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3923 if (SDValue Res = BuildSDIVPow2(N))
3928 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
3929 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
3931 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
3933 return SDValue();
3936 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
3941 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
3943 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
3945 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
3950 SDValue One = DAG.getConstant(1, DL, VT);
3951 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
3952 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
3953 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
3954 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
3959 SDValue Zero = DAG.getConstant(0, DL, VT);
3960 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
3963 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
3964 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
3974 if (SDValue Op = BuildSDIV(N))
3977 return SDValue();
3980 SDValue DAGCombiner::visitUDIV(SDNode *N) {
3981 SDValue N0 = N->getOperand(0);
3982 SDValue N1 = N->getOperand(1);
3988 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3995 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
4004 if (SDValue V = simplifyDivRem(N, DAG))
4007 if (SDValue NewSel = foldBinOpIntoSelect(N))
4010 if (SDValue V = visitUDIVLike(N0, N1, N)) {
4015 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
4016 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4029 if (SDValue DivRem = useDivRem(N))
4032 return SDValue();
4035 SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
4042 SDValue LogBase2 = BuildLogBase2(N1, DL);
4046 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
4053 SDValue N10 = N1.getOperand(0);
4056 SDValue LogBase2 = BuildLogBase2(N10, DL);
4060 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
4062 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
4072 if (SDValue Op = BuildUDIV(N))
4075 return SDValue();
4079 SDValue DAGCombiner::visitREM(SDNode *N) {
4081 SDValue N0 = N->getOperand(0);
4082 SDValue N1 = N->getOperand(1);
4091 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
4099 if (SDValue V = simplifyDivRem(N, DAG))
4102 if (SDValue NewSel = foldBinOpIntoSelect(N))
4111 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4114 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4121 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4137 SDValue OptimizedDiv =
4145 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
4146 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4154 if (SDValue DivRem = useDivRem(N))
4157 return SDValue();
4160 SDValue DAGCombiner::visitMULHS(SDNode *N) {
4161 SDValue N0 = N->getOperand(0);
4162 SDValue N1 = N->getOperand(1);
4204 return SDValue();
4207 SDValue DAGCombiner::visitMULHU(SDNode *N) {
4208 SDValue N0 = N->getOperand(0);
4209 SDValue N1 = N->getOperand(1);
4235 SDValue LogBase2 = BuildLogBase2(N1, DL);
4236 SDValue SRLAmt = DAG.getNode(
4239 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4260 return SDValue();
4266 SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
4272 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4280 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4286 return SDValue();
4290 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4292 SDValue LoOpt = combine(Lo.getNode());
4300 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4302 SDValue HiOpt = combine(Hi.getNode());
4309 return SDValue();
4312 SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
4313 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
4326 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
4327 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
4340 return SDValue();
4343 SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
4344 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
4352 SDValue Zero = DAG.getConstant(0, DL, VT);
4358 SDValue Zero = DAG.getConstant(0, DL, VT);
4369 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
4370 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
4383 return SDValue();
4386 SDValue DAGCombiner::visitMULO(SDNode *N) {
4387 SDValue N0 = N->getOperand(0);
4388 SDValue N1 = N->getOperand(1);
4411 return SDValue();
4414 SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
4415 SDValue N0 = N->getOperand(0);
4416 SDValue N1 = N->getOperand(1);
4422 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4426 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, SDLoc(N), VT, {N0, N1}))
4451 return SDValue();
4456 SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
4457 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
4467 return SDValue();
4473 SDValue X = N0.getOperand(0);
4474 SDValue Y = N1.getOperand(0);
4482 return SDValue();
4485 return SDValue();
4490 return SDValue();
4495 return SDValue();
4497 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4506 return SDValue();
4509 return SDValue();
4512 return SDValue();
4516 return SDValue();
4518 return SDValue();
4519 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4530 return SDValue();
4531 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4539 return SDValue();
4540 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4557 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4586 return SDValue();
4590 SDValue ShOp = N0.getOperand(1);
4596 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
4609 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
4615 return SDValue();
4619 SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4621 SDValue LL, LR, RL, RR, N0CC, N1CC;
4624 return SDValue();
4639 return SDValue();
4641 return SDValue();
4664 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
4683 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
4695 SDValue One = DAG.getConstant(1, DL, OpVT);
4696 SDValue Two = DAG.getConstant(2, DL, OpVT);
4697 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
4709 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
4710 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
4711 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
4712 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4733 SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
4734 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
4735 SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
4736 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
4737 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4762 return SDValue();
4769 SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4777 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
4799 SDValue NewAdd =
4804 return SDValue(N, 0);
4824 return SDValue();
4843 SDValue SL(N0);
4848 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
4851 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
4852 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
4853 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
4854 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
4861 return SDValue();
4944 if (!SDValue(Load, 0).hasOneUse())
4989 for (SDValue Op : N->op_values()) {
5056 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
5091 SDValue MaskOp = N->getOperand(1);
5097 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
5099 SDValue(FixupNode, 0), MaskOp);
5100 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
5102 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
5107 SDValue Op0 = LogicN->getOperand(0);
5108 SDValue Op1 = LogicN->getOperand(1);
5113 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
5122 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
5123 SDValue(Load, 0), MaskOp);
5124 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
5126 And = SDValue(
5127 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
5128 SDValue NewLoad = ReduceLoadWidth(And.getNode());
5144 SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
5147 SDValue N0 = N->getOperand(0);
5148 SDValue N1 = N->getOperand(1);
5152 return SDValue();
5157 SDValue Y; // Shift amount.
5158 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
5174 SDValue X;
5180 return SDValue();
5186 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
5188 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
5196 static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
5203 return SDValue();
5207 SDValue Not = And->getOperand(0), And1 = And->getOperand(1);
5211 return SDValue();
5216 SDValue Srl = Not.getOperand(0);
5223 return SDValue();
5231 return SDValue();
5236 SDValue X = DAG.getZExtOrTrunc(Srl.getOperand(0), DL, VT);
5238 SDValue Mask = DAG.getConstant(
5240 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask);
5241 SDValue Zero = DAG.getConstant(0, DL, VT);
5242 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
5246 SDValue DAGCombiner::visitAND(SDNode *N) {
5247 SDValue N0 = N->getOperand(0);
5248 SDValue N1 = N->getOperand(1);
5257 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5279 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1}))
5293 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
5297 if (SDValue NewSel = foldBinOpIntoSelect(N))
5301 if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
5306 if (SDValue Shuffle = XformToShuffleWithZero(N))
5309 if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
5321 SDValue N0Op0 = N0.getOperand(0);
5325 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5335 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5417 SDValue NewLoad(Load, 0);
5431 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
5439 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5449 if (SDValue Res = ReduceLoadWidth(N)) {
5453 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 0), Res);
5454 return SDValue(N, 0);
5464 return SDValue(N, 0);
5467 if (SDValue Combined = visitANDLike(N0, N1, N))
5472 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
5484 SDValue SubRHS = N0.getOperand(1);
5496 if (SimplifyDemandedBits(SDValue(N, 0)))
5497 return SDValue(N, 0);
5514 SDValue ExtLoad =
5519 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5525 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
5530 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
5534 if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
5537 return SDValue();
5541 SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5544 return SDValue();
5548 return SDValue();
5550 return SDValue();
5561 return SDValue();
5567 return SDValue();
5574 return SDValue();
5577 return SDValue();
5585 return SDValue();
5587 return SDValue();
5592 return SDValue();
5594 return SDValue();
5597 SDValue N00 = N0->getOperand(0);
5600 return SDValue();
5603 return SDValue();
5608 SDValue N10 = N1->getOperand(0);
5611 return SDValue();
5617 return SDValue();
5623 return SDValue();
5633 return SDValue();
5640 return SDValue();
5643 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
5659 static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
5667 SDValue N0 = N.getOperand(0);
5744 static bool isBSwapHWordPair(SDValue N, MutableArrayRef<SDNode *> Parts) {
5764 static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI,
5765 SelectionDAG &DAG, SDNode *N, SDValue N0,
5766 SDValue N1, EVT VT, EVT ShiftAmountTy) {
5770 return SDValue();
5772 return SDValue();
5775 return SDValue();
5779 return SDValue();
5782 return SDValue();
5783 SDValue Shift0 = N0.getOperand(0);
5784 SDValue Shift1 = N1.getOperand(0);
5786 return SDValue();
5790 return SDValue();
5792 return SDValue();
5794 return SDValue();
5797 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0));
5798 SDValue ShAmt = DAG.getConstant(16, DL, ShiftAmountTy);
5808 SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
5810 return SDValue();
5814 return SDValue();
5816 return SDValue();
5818 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT,
5823 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT,
5837 return SDValue();
5841 return SDValue();
5842 SDValue N00 = N0.getOperand(0);
5843 SDValue N01 = N0.getOperand(1);
5846 return SDValue();
5848 return SDValue();
5852 return SDValue();
5855 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
5856 SDValue(Parts[0], 0));
5860 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
5872 SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
5880 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
5900 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5915 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5920 return SDValue();
5924 static SDValue visitORCommutative(
5925 SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N) {
5937 return SDValue();
5940 SDValue DAGCombiner::visitOR(SDNode *N) {
5941 SDValue N0 = N->getOperand(0);
5942 SDValue N1 = N->getOperand(1);
5951 if (SDValue FoldedVOp = SimplifyVBinOp(N))
6019 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
6020 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
6022 SDValue LegalShuffle =
6034 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1}))
6050 if (SDValue NewSel = foldBinOpIntoSelect(N))
6057 if (SDValue Combined = visitORLike(N0, N1, N))
6060 if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
6064 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
6066 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
6070 if (SDValue ROR = reassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags()))
6080 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT,
6082 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
6088 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
6090 if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
6095 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
6099 if (SDValue Rot = MatchRotate(N0, N1, SDLoc(N)))
6102 if (SDValue Load = MatchLoadCombine(N))
6106 if (SimplifyDemandedBits(SDValue(N, 0)))
6107 return SDValue(N, 0);
6112 if (SDValue Combined = visitADDLike(N))
6115 return SDValue();
6118 static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
6128 static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
6129 SDValue &Mask) {
6142 /// \returns An empty \c SDValue if the needed shift couldn't be extracted.
6162 static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
6163 SDValue ExtractFrom, SDValue &Mask,
6165 assert(OppShift && ExtractFrom && "Empty SDValue");
6173 SDValue OppShiftLHS = OppShift.getOperand(0);
6208 return SDValue();
6215 return SDValue();
6227 return SDValue();
6232 return SDValue();
6252 return SDValue();
6259 return SDValue();
6265 SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
6278 static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
6330 SDValue NegOp1 = Neg.getOperand(1);
6390 SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
6391 SDValue Neg, SDValue InnerPos,
6392 SDValue InnerNeg, unsigned PosOpcode,
6408 return SDValue();
6417 SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
6418 SDValue Neg, SDValue InnerPos,
6419 SDValue InnerNeg, unsigned PosOpcode,
6441 auto IsBinOpImm = [](SDValue Op, unsigned BinOpc, unsigned Imm) {
6477 return SDValue();
6484 SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
6488 return SDValue();
6496 return SDValue();
6502 if (SDValue Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
6508 SDValue LHSShift; // The shift.
6509 SDValue LHSMask; // AND value if any.
6512 SDValue RHSShift; // The shift.
6513 SDValue RHSMask; // AND value if any.
6518 return SDValue();
6530 if (SDValue NewRHSShift =
6535 if (SDValue NewLHSShift =
6541 return SDValue();
6546 return SDValue(); // Shifts must disagree.
6550 return SDValue(); // Requires funnel shift support.
6560 SDValue LHSShiftArg = LHSShift.getOperand(0);
6561 SDValue LHSShiftAmt = LHSShift.getOperand(1);
6562 SDValue RHSShiftArg = RHSShift.getOperand(0);
6563 SDValue RHSShiftAmt = RHSShift.getOperand(1);
6575 SDValue Res;
6585 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
6586 SDValue Mask = AllOnes;
6589 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
6594 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
6608 return SDValue();
6611 SDValue LExtOp0 = LHSShiftAmt;
6612 SDValue RExtOp0 = RHSShiftAmt;
6626 SDValue TryL =
6632 SDValue TryR =
6639 SDValue TryL =
6645 SDValue TryR =
6651 return SDValue();
6699 calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
6749 SDValue NarrowOp = Op->getOperand(0);
6817 static SDValue stripTruncAndExt(SDValue Value) {
6850 SDValue DAGCombiner::MatchStoreCombine(StoreSDNode *N) {
6852 SDValue Chain;
6858 return SDValue();
6867 return SDValue();
6870 return SDValue();
6875 SDValue CombinedValue;
6883 SDValue Trunc = Store->getValue();
6885 return SDValue();
6889 SDValue Value = Trunc.getOperand(0);
6900 return SDValue();
6910 return SDValue();
6920 return SDValue();
6929 return SDValue();
6939 return SDValue();
6950 return SDValue();
6962 return SDValue();
6970 return SDValue();
6982 SDValue NewStore =
7021 SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
7028 return SDValue();
7044 SDValue Chain;
7055 auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
7057 return SDValue();
7063 return SDValue();
7075 SDValue LChain = L->getChain();
7079 return SDValue();
7087 return SDValue();
7112 return SDValue();
7120 return SDValue();
7127 return SDValue();
7134 return SDValue();
7151 return SDValue();
7157 return SDValue();
7165 return SDValue();
7167 SDValue NewLoad = DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD,
7174 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
7179 SDValue ShiftedLoad =
7201 SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
7206 return SDValue();
7212 SDValue X, Y, M;
7213 auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
7216 SDValue Xor = And.getOperand(XorIdx);
7219 SDValue Xor0 = Xor.getOperand(0);
7220 SDValue Xor1 = Xor.getOperand(1);
7234 SDValue N0 = N->getOperand(0);
7235 SDValue N1 = N->getOperand(1);
7238 return SDValue();
7244 return SDValue();
7248 return SDValue();
7256 SDValue NotX = DAG.getNOT(DL, X, VT);
7257 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
7258 SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
7259 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
7263 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
7264 SDValue NotM = DAG.getNOT(DL, M, VT);
7265 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
7270 SDValue DAGCombiner::visitXOR(SDNode *N) {
7271 SDValue N0 = N->getOperand(0);
7272 SDValue N1 = N->getOperand(1);
7277 if (SDValue FoldedVOp = SimplifyVBinOp(N))
7299 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1}))
7311 if (SDValue NewSel = foldBinOpIntoSelect(N))
7315 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags()))
7320 SDValue LHS, RHS, CC;
7340 SDValue SetCC = DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC,
7346 return SDValue(N, 0); // Return N so it doesn't get rechecked!
7357 SDValue V = N0.getOperand(0);
7368 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
7380 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
7408 SDValue X = N0.getOperand(0);
7409 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
7429 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
7438 SDValue A = N0Opcode == ISD::ADD ? N0 : N1;
7439 SDValue S = N0Opcode == ISD::SRA ? N0 : N1;
7441 SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
7442 SDValue S0 = S.getOperand(0);
7482 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
7486 if (SDValue MM = unfoldMaskedMerge(N))
7490 if (SimplifyDemandedBits(SDValue(N, 0)))
7491 return SDValue(N, 0);
7493 if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
7496 return SDValue();
7503 static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG) {
7505 SDValue LogicOp = Shift->getOperand(0);
7507 return SDValue();
7512 return SDValue();
7516 SDValue C1 = Shift->getOperand(1);
7520 auto matchFirstShift = [&](SDValue V, SDValue &ShiftOp,
7546 SDValue X, Y;
7553 return SDValue();
7559 SDValue ShiftSumC = DAG.getConstant(*C0Val + C1Val, DL, ShiftAmtVT);
7560 SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC);
7561 SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1);
7571 SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
7576 return SDValue();
7579 SDValue LHS = N->getOperand(0);
7581 return SDValue();
7588 if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
7597 return SDValue();
7604 return SDValue(); // only shl(add) not sr[al](add).
7611 return SDValue();
7616 SDValue BinOpLHSVal = LHS.getOperand(0);
7625 return SDValue();
7628 return SDValue();
7633 SDValue NewRHS = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(1),
7637 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
7642 SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
7650 SDValue N01 = N->getOperand(0).getOperand(1);
7653 SDValue N00 = N->getOperand(0).getOperand(0);
7654 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
7655 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
7662 return SDValue();
7665 SDValue DAGCombiner::visitRotate(SDNode *N) {
7667 SDValue N0 = N->getOperand(0);
7668 SDValue N1 = N->getOperand(1);
7691 SDValue Bits = DAG.getConstant(Bitsize, dl, AmtVT);
7692 if (SDValue Amt =
7704 if (SimplifyDemandedBits(SDValue(N, 0)))
7705 return SDValue(N, 0);
7710 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
7723 if (SDValue CombinedShift = DAG.FoldConstantArithmetic(
7725 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
7726 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
7733 return SDValue();
7736 SDValue DAGCombiner::visitSHL(SDNode *N) {
7737 SDValue N0 = N->getOperand(0);
7738 SDValue N1 = N->getOperand(1);
7739 if (SDValue V = DAG.simplifyShift(N0, N1))
7748 if (SDValue FoldedVOp = SimplifyVBinOp(N))
7756 SDValue N00 = N0->getOperand(0);
7757 SDValue N01 = N0->getOperand(1);
7763 if (SDValue C =
7774 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1}))
7777 if (SDValue NewSel = foldBinOpIntoSelect(N))
7781 if (DAG.MaskedValueIsZero(SDValue(N, 0),
7788 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
7792 if (SimplifyDemandedBits(SDValue(N, 0)))
7793 return SDValue(N, 0);
7816 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7830 SDValue N0Op0 = N0.getOperand(0);
7831 SDValue InnerShiftAmt = N0Op0.getOperand(1);
7860 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0));
7861 SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT);
7872 SDValue N0Op0 = N0.getOperand(0);
7873 SDValue InnerShiftAmt = N0Op0.getOperand(1);
7886 SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT);
7923 SDValue Shift;
7946 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
7947 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
7960 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
7961 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7971 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7977 if (SDValue NewSHL = visitShiftByConstant(N))
7989 return SDValue();
7996 static SDValue combineShiftToMULH(SDNode *N, SelectionDAG &DAG,
8005 return SDValue();
8010 SDValue ShiftOperand = N->getOperand(0);
8012 return SDValue();
8015 SDValue LeftOp = ShiftOperand.getOperand(0);
8016 SDValue RightOp = ShiftOperand.getOperand(1);
8021 return SDValue();
8033 return SDValue();
8039 return SDValue();
8045 return SDValue();
8052 return SDValue();
8058 SDValue Result = DAG.getNode(MulhOpcode, DL, NarrowVT, LeftOp.getOperand(0),
8064 SDValue DAGCombiner::visitSRA(SDNode *N) {
8065 SDValue N0 = N->getOperand(0);
8066 SDValue N1 = N->getOperand(1);
8067 if (SDValue V = DAG.simplifyShift(N0, N1))
8081 if (SDValue FoldedVOp = SimplifyVBinOp(N))
8087 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1}))
8090 if (SDValue NewSel = foldBinOpIntoSelect(N))
8114 SmallVector<SDValue, 16> ShiftValues;
8127 SDValue ShiftValue;
8164 SDValue Amt = DAG.getConstant(ShiftAmt, DL,
8166 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
8168 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
8183 SDValue Shl = N0.getOperand(0);
8199 SDValue Trunc = DAG.getZExtOrTrunc(Shl.getOperand(0), DL, TruncVT);
8200 SDValue ShiftC = DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).
8202 SDValue Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC);
8211 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
8224 SDValue N0Op0 = N0.getOperand(0);
8230 SDValue Amt = DAG.getConstant(N1C->getZExtValue() + TruncBits, DL,
8232 SDValue SRA =
8240 if (SimplifyDemandedBits(SDValue(N, 0)))
8241 return SDValue(N, 0);
8248 if (SDValue NewSRA = visitShiftByConstant(N))
8253 if (SDValue MULH = combineShiftToMULH(N, DAG, TLI))
8256 return SDValue();
8259 SDValue DAGCombiner::visitSRL(SDNode *N) {
8260 SDValue N0 = N->getOperand(0);
8261 SDValue N1 = N->getOperand(1);
8262 if (SDValue V = DAG.simplifyShift(N0, N1))
8270 if (SDValue FoldedVOp = SimplifyVBinOp(N))
8276 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1}))
8279 if (SDValue NewSel = foldBinOpIntoSelect(N))
8283 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
8309 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
8316 SDValue InnerShift = N0.getOperand(0);
8330 SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
8331 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
8340 SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
8341 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
8343 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(InnerShiftSize,
8346 SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask);
8357 SDValue Mask =
8375 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
8416 SDValue Op = N0.getOperand(0);
8435 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
8441 if (SimplifyDemandedBits(SDValue(N, 0)))
8442 return SDValue(N, 0);
8445 if (SDValue NewSRL = visitShiftByConstant(N))
8449 if (SDValue NarrowLoad = ReduceLoadWidth(N))
8483 if (SDValue MULH = combineShiftToMULH(N, DAG, TLI))
8486 return SDValue();
8489 SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
8491 SDValue N0 = N->getOperand(0);
8492 SDValue N1 = N->getOperand(1);
8493 SDValue N2 = N->getOperand(2);
8504 auto IsUndefOrZero = [](SDValue V) {
8559 SDValue NewPtr =
8562 SDValue Load = DAG.getLoad(
8597 if (SimplifyDemandedBits(SDValue(N, 0)))
8598 return SDValue(N, 0);
8600 return SDValue();
8603 SDValue DAGCombiner::visitABS(SDNode *N) {
8604 SDValue N0 = N->getOperand(0);
8616 return SDValue();
8619 SDValue DAGCombiner::visitBSWAP(SDNode *N) {
8620 SDValue N0 = N->getOperand(0);
8629 return SDValue();
8632 SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
8633 SDValue N0 = N->getOperand(0);
8642 return SDValue();
8645 SDValue DAGCombiner::visitCTLZ(SDNode *N) {
8646 SDValue N0 = N->getOperand(0);
8659 return SDValue();
8662 SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
8663 SDValue N0 = N->getOperand(0);
8669 return SDValue();
8672 SDValue DAGCombiner::visitCTTZ(SDNode *N) {
8673 SDValue N0 = N->getOperand(0);
8686 return SDValue();
8689 SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
8690 SDValue N0 = N->getOperand(0);
8696 return SDValue();
8699 SDValue DAGCombiner::visitCTPOP(SDNode *N) {
8700 SDValue N0 = N->getOperand(0);
8706 return SDValue();
8711 static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS,
8712 SDValue RHS,
8723 static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
8724 SDValue RHS, SDValue True, SDValue False,
8728 return SDValue();
8748 return SDValue();
8763 return SDValue();
8766 return SDValue();
8772 static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG) {
8773 SDValue Cond = N->getOperand(0);
8774 SDValue C1 = N->getOperand(1);
8775 SDValue C2 = N->getOperand(2);
8782 return SDValue();
8786 SDValue X = Cond.getOperand(0);
8787 SDValue CondC = Cond.getOperand(1);
8793 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
8794 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
8800 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
8801 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
8804 return SDValue();
8807 SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
8808 SDValue Cond = N->getOperand(0);
8809 SDValue N1 = N->getOperand(1);
8810 SDValue N2 = N->getOperand(2);
8816 return SDValue();
8821 return SDValue();
8830 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8837 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8879 SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT);
8883 if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8887 return SDValue();
8905 SDValue NotCond =
8912 return SDValue();
8915 SDValue DAGCombiner::visitSELECT(SDNode *N) {
8916 SDValue N0 = N->getOperand(0);
8917 SDValue N1 = N->getOperand(1);
8918 SDValue N2 = N->getOperand(2);
8924 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8932 if (SDValue V = foldSelectOfConstants(N))
8937 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8943 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8954 return SDValue(N, 0); // Don't revisit N.
8970 SDValue Cond0 = N0->getOperand(0);
8971 SDValue Cond1 = N0->getOperand(1);
8972 SDValue InnerSelect =
8983 SDValue Cond0 = N0->getOperand(0);
8984 SDValue Cond1 = N0->getOperand(1);
8985 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(),
8997 SDValue N1_0 = N1->getOperand(0);
8998 SDValue N1_1 = N1->getOperand(1);
8999 SDValue N1_2 = N1->getOperand(2);
9003 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
9008 if (SDValue Combined = visitANDLike(N0, N1_0, N)) {
9016 SDValue N2_0 = N2->getOperand(0);
9017 SDValue N2_1 = N2->getOperand(1);
9018 SDValue N2_2 = N2->getOperand(2);
9022 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
9027 if (SDValue Combined = visitORLike(N0, N2_0, N))
9035 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
9036 SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1);
9043 SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1);
9051 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2,
9079 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
9090 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1,
9099 return SDValue();
9104 static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
9106 SDValue Cond = N->getOperand(0);
9107 SDValue LHS = N->getOperand(1);
9108 SDValue RHS = N->getOperand(2);
9118 return SDValue();
9133 return SDValue();
9145 return SDValue();
9157 SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
9159 SDValue Mask = MSC->getMask();
9160 SDValue Chain = MSC->getChain();
9167 return SDValue();
9170 SDValue DAGCombiner::visitMSTORE(SDNode *N) {
9172 SDValue Mask = MST->getMask();
9173 SDValue Chain = MST->getChain();
9182 return SDValue(N, 0);
9184 return SDValue();
9187 SDValue DAGCombiner::visitMGATHER(SDNode *N) {
9189 SDValue Mask = MGT->getMask();
9196 return SDValue();
9199 SDValue DAGCombiner::visitMLOAD(SDNode *N) {
9201 SDValue Mask = MLD->getMask();
9210 return SDValue(N, 0);
9212 return SDValue();
9217 SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
9218 SDValue Cond = N->getOperand(0);
9219 SDValue N1 = N->getOperand(1);
9220 SDValue N2 = N->getOperand(2);
9226 return SDValue();
9235 SDValue N1Elt = N1.getOperand(i);
9236 SDValue N2Elt = N2.getOperand(i);
9257 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
9265 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT);
9266 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT);
9270 if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
9277 return SDValue();
9280 SDValue DAGCombiner::visitVSELECT(SDNode *N) {
9281 SDValue N0 = N->getOperand(0);
9282 SDValue N1 = N->getOperand(1);
9283 SDValue N2 = N->getOperand(2);
9287 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
9291 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
9300 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
9317 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS,
9320 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
9333 if (SDValue FMinMax =
9360 SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
9361 SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
9363 SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
9370 return SDValue(N, 0); // Don't revisit N.
9385 if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
9389 if (SDValue V = foldVSelectOfConstants(N))
9392 return SDValue();
9395 SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
9396 SDValue N0 = N->getOperand(0);
9397 SDValue N1 = N->getOperand(1);
9398 SDValue N2 = N->getOperand(2);
9399 SDValue N3 = N->getOperand(3);
9400 SDValue N4 = N->getOperand(4);
9408 if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
9423 SDValue SelectOp = DAG.getNode(
9433 return SDValue(N, 0); // Don't revisit N.
9439 SDValue DAGCombiner::visitSETCC(SDNode *N) {
9446 SDValue Combined = SimplifySetCC(
9451 return SDValue();
9456 SDValue NewSetCC = rebuildSetCC(Combined);
9460 return SDValue();
9469 SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
9470 SDValue LHS = N->getOperand(0);
9471 SDValue RHS = N->getOperand(1);
9472 SDValue Carry = N->getOperand(2);
9473 SDValue Cond = N->getOperand(3);
9479 return SDValue();
9488 static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
9491 SDValue N0 = N->getOperand(0);
9510 SDValue Op1 = N0->getOperand(1);
9511 SDValue Op2 = N0->getOperand(2);
9538 return SDValue();
9543 SmallVector<SDValue, 8> Elts;
9552 SDValue Op = N0.getOperand(i);
9575 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
9597 SDValue UseOp = User->getOperand(i);
9636 SDValue OrigLoad, SDValue ExtLoad,
9641 SmallVector<SDValue, 4> Ops;
9644 SDValue SOp = SetCC->getOperand(j);
9657 SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
9658 SDValue N0 = N->getOperand(0);
9684 return SDValue();
9691 !TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
9692 return SDValue();
9696 return SDValue();
9711 return SDValue();
9719 SmallVector<SDValue, 4> Loads;
9720 SmallVector<SDValue, 4> Chains;
9722 SDValue BasePtr = LN0->getBasePtr();
9727 SDValue SplitLoad = DAG.getExtLoad(
9738 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
9739 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
9748 SDValue Trunc =
9752 return SDValue(N, 0); // Return N so it doesn't get rechecked!
9757 SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
9762 return SDValue();
9765 SDValue N0 = N->getOperand(0);
9770 return SDValue();
9773 SDValue N1 = N0->getOperand(0);
9777 return SDValue();
9781 return SDValue();
9786 return SDValue();
9792 return SDValue();
9795 return SDValue();
9800 return SDValue();
9803 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
9808 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
9813 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
9818 if (SDValue(Load, 0).hasOneUse()) {
9819 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
9821 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
9829 return SDValue(N,0); // Return N so it doesn't get rechecked!
9836 SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
9848 return SDValue();
9850 SDValue VSel = Cast->getOperand(0);
9853 return SDValue();
9856 SDValue SetCC = VSel.getOperand(0);
9859 return SDValue();
9862 SDValue A = VSel.getOperand(1);
9863 SDValue B = VSel.getOperand(2);
9864 SDValue CastA, CastB;
9879 static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
9882 SDValue N0, ISD::LoadExtType ExtLoadType) {
9888 return SDValue();
9895 return SDValue();
9897 SDValue ExtLoad =
9901 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
9904 return SDValue(N, 0); // Return N so it doesn't get rechecked!
9910 static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
9912 bool LegalOperations, SDNode *N, SDValue N0,
9927 DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
9932 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
9937 bool NoReplaceTrunc = SDValue(LN0, 0).hasOneUse();
9940 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
9943 SDValue Trunc =
9947 return SDValue(N, 0); // Return N so it doesn't get rechecked!
9950 static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG,
9952 SDNode *N, SDValue N0,
9956 return SDValue();
9960 return SDValue();
9963 return SDValue();
9965 if (!TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
9966 return SDValue();
9969 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru());
9970 SDValue NewLoad = DAG.getMaskedLoad(
9974 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), SDValue(NewLoad.getNode(), 1));
9978 static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
9983 SDValue SetCC = N->getOperand(0);
9986 return SDValue();
9988 SDValue X = SetCC.getOperand(0);
9989 SDValue Ones = SetCC.getOperand(1);
10004 SDValue NotX = DAG.getNOT(DL, X, VT);
10005 SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT);
10011 return SDValue();
10014 SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
10015 SDValue N0 = N->getOperand(0);
10019 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10030 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
10037 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10042 SDValue Op = N0.getOperand(0);
10078 if (SDValue foldedExt =
10083 if (SDValue foldedExt =
10090 if (SDValue ExtLoad = CombineExtLoad(N))
10094 if (SDValue foldedExt = tryToFoldExtOfExtload(
10113 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
10118 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
10122 bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
10126 SDValue TruncAnd =
10131 DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
10133 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
10137 return SDValue(N,0); // Return N so it doesn't get rechecked!
10142 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
10146 SDValue N00 = N0.getOperand(0);
10147 SDValue N01 = N0.getOperand(1);
10176 SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
10194 SDValue ExtTrueVal = (SetCCWidth == 1)
10197 SDValue Zero = DAG.getConstant(0, DL, VT);
10198 if (SDValue SCC =
10210 SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
10221 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
10230 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT);
10239 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT);
10243 return SDValue();
10250 static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
10263 SDValue Op0 = N->getOperand(0);
10264 SDValue Op1 = N->getOperand(1);
10282 static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) {
10286 SDValue CtPop = Extend->getOperand(0);
10288 return SDValue();
10294 return SDValue();
10298 SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT);
10302 SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
10303 SDValue N0 = N->getOperand(0);
10306 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10318 SDValue Op;
10336 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
10343 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10354 SDValue Op = N0.getOperand(0);
10357 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
10365 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
10367 SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT);
10383 SDValue X = N0.getOperand(0).getOperand(0);
10392 if (SDValue foldedExt =
10397 if (SDValue foldedExt =
10404 if (SDValue ExtLoad = CombineExtLoad(N))
10435 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
10441 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
10445 bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
10449 SDValue TruncAnd =
10454 DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
10456 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
10460 return SDValue(N,0); // Return N so it doesn't get rechecked!
10467 if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
10471 if (SDValue foldedExt = tryToFoldExtOfExtload(
10475 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
10484 return SDValue();
10494 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
10503 SDValue VsetCC =
10512 if (SDValue SCC = SimplifySelectCC(
10524 SDValue ShAmt = N0.getOperand(1);
10526 SDValue InnerZExt = N0.getOperand(0);
10532 return SDValue();
10546 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
10549 if (SDValue NewCtPop = widenCtPop(N, DAG))
10552 return SDValue();
10555 SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
10556 SDValue N0 = N->getOperand(0);
10559 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10573 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
10580 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10596 SDValue X = N0.getOperand(0).getOperand(0);
10617 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
10626 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
10629 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
10633 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10646 SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
10650 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
10652 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10665 return SDValue();
10681 SDValue VsetCC =
10690 if (SDValue SCC = SimplifySelectCC(
10697 if (SDValue NewCtPop = widenCtPop(N, DAG))
10700 return SDValue();
10703 SDValue DAGCombiner::visitAssertExt(SDNode *N) {
10705 SDValue N0 = N->getOperand(0);
10706 SDValue N1 = N->getOperand(1);
10721 SDValue BigA = N0.getOperand(0);
10729 SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
10730 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10741 SDValue BigA = N0.getOperand(0);
10749 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10755 return SDValue();
10758 SDValue DAGCombiner::visitAssertAlign(SDNode *N) {
10762 SDValue N0 = N->getOperand(0);
10779 SDValue LHS = N0.getOperand(0);
10780 SDValue RHS = N0.getOperand(1);
10794 return SDValue();
10803 SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
10807 SDValue N0 = N->getOperand(0);
10813 return SDValue();
10827 N0 = SDValue(N, 0);
10832 return SDValue();
10845 return SDValue();
10857 return SDValue();
10864 SDValue SRL = N0;
10873 return SDValue();
10878 if (!LN0) return SDValue();
10884 return SDValue();
10890 return SDValue();
10923 return SDValue();
10930 return SDValue();
10949 SDValue NewPtr =
10953 SDValue Load;
10969 SDValue Result = Load;
10995 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
10997 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
11004 SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
11005 SDValue N0 = N->getOperand(0);
11006 SDValue N1 = N->getOperand(1);
11035 SDValue N00 = N0.getOperand(0);
11057 SDValue N00 = N0.getOperand(0);
11069 if (SimplifyDemandedBits(SDValue(N, 0)))
11070 return SDValue(N, 0);
11074 if (SDValue NarrowLoad = ReduceLoadWidth(N))
11103 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
11110 return SDValue(N, 0); // Return N so it doesn't get rechecked!
11119 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
11125 return SDValue(N, 0); // Return N so it doesn't get rechecked!
11130 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
11135 return SDValue();
11138 SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
11139 SDValue N0 = N->getOperand(0);
11146 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
11149 if (SimplifyDemandedVectorElts(SDValue(N, 0)))
11150 return SDValue(N, 0);
11152 return SDValue();
11155 SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
11156 SDValue N0 = N->getOperand(0);
11163 if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
11166 if (SimplifyDemandedVectorElts(SDValue(N, 0)))
11167 return SDValue(N, 0);
11169 return SDValue();
11172 SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
11173 SDValue N0 = N->getOperand(0);
11188 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
11210 return SDValue();
11234 SDValue EltNo = N0->getOperand(1);
11251 SDValue Cond = N0.getOperand(0);
11252 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
11253 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
11262 SDValue Amt = N0.getOperand(1);
11269 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
11285 SmallVector<SDValue, 8> TruncOps;
11286 for (const SDValue &Op : N0->op_values()) {
11287 SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op);
11301 SDValue BuildVect = N0.getOperand(0);
11315 SmallVector<SDValue, 8> Opnds;
11331 if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
11338 if (SDValue Reduced = ReduceLoadWidth(N))
11347 SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
11361 SDValue V;
11366 SDValue X = N0.getOperand(i);
11386 SmallVector<SDValue, 8> Opnds;
11392 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
11405 SDValue VecSrc = N0.getOperand(0);
11420 SimplifyDemandedBits(SDValue(N, 0)))
11421 return SDValue(N, 0);
11443 SDValue N00 = N0.getOperand(0);
11454 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
11476 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
11477 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
11483 return SDValue();
11487 SDValue Elt = N->getOperand(i);
11495 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
11509 return SDValue();
11524 return SDValue();
11533 static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
11539 return SDValue();
11543 SDValue N0 = N->getOperand(0);
11546 return SDValue();
11564 return SDValue();
11571 SDValue LogicOp0 = N0.getOperand(0);
11576 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0));
11583 return SDValue();
11586 SDValue DAGCombiner::visitBITCAST(SDNode *N) {
11587 SDValue N0 = N->getOperand(0);
11618 SDValue C = DAG.getBitcast(VT, N0);
11645 SDValue Load =
11654 if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
11673 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
11679 SDValue SignBit = DAG.getConstant(
11681 SDValue FlipBit;
11687 SDValue Hi =
11695 SDValue FlipBits =
11726 SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
11749 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11751 SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
11753 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
11755 SDValue XorResult64 = DAG.getNode(
11760 SDValue FlipBit =
11764 SDValue FlipBits =
11774 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11785 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT))
11800 auto PeekThroughBitcast = [&](SDValue Op) {
11803 return SDValue(Op.getOperand(0));
11807 return SDValue();
11813 SDValue SV0 = PeekThroughBitcast(N0->getOperand(0));
11814 SDValue SV1 = PeekThroughBitcast(N0->getOperand(1));
11816 return SDValue();
11825 SDValue LegalShuffle =
11831 return SDValue();
11834 SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
11839 SDValue DAGCombiner::visitFREEZE(SDNode *N) {
11840 SDValue N0 = N->getOperand(0);
11850 return SDValue();
11855 SDValue DAGCombiner::
11860 if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
11868 SmallVector<SDValue, 8> Ops;
11869 for (SDValue Op : BV->op_values()) {
11911 SmallVector<SDValue, 8> Ops;
11920 SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
11943 SmallVector<SDValue, 8> Ops;
11945 for (const SDValue &Op : BV->op_values()) {
11974 SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
11975 SDValue N0 = N->getOperand(0);
11976 SDValue N1 = N->getOperand(1);
11992 return SDValue();
12002 return SDValue();
12005 return SDValue();
12013 auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
12041 SDValue FMA, E;
12054 SDValue A = FMA.getOperand(0);
12055 SDValue B = FMA.getOperand(1);
12056 SDValue C = FMA.getOperand(2).getOperand(0);
12057 SDValue D = FMA.getOperand(2).getOperand(1);
12058 SDValue CDE = DAG.getNode(PreferredFusedOpcode, SL, VT, C, D, E, Flags);
12066 SDValue N00 = N0.getOperand(0);
12081 SDValue N10 = N1.getOperand(0);
12098 SDValue X, SDValue Y, SDValue U, SDValue V, SDValue Z,
12107 SDValue N02 = N0.getOperand(2);
12109 SDValue N020 = N02.getOperand(0);
12126 SDValue X, SDValue Y, SDValue U, SDValue V, SDValue Z,
12137 SDValue N00 = N0.getOperand(0);
12139 SDValue N002 = N00.getOperand(2);
12153 SDValue N12 = N1.getOperand(2);
12155 SDValue N120 = N12.getOperand(0);
12172 SDValue N10 = N1.getOperand(0);
12174 SDValue N102 = N10.getOperand(2);
12186 return SDValue();
12190 SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
12191 SDValue N0 = N->getOperand(0);
12192 SDValue N1 = N->getOperand(1);
12207 return SDValue();
12216 return SDValue();
12219 return SDValue();
12228 auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
12235 auto tryToFoldXYSubZ = [&](SDValue XY, SDValue Z) {
12241 return SDValue();
12246 auto tryToFoldXSubYZ = [&](SDValue X, SDValue YZ) {
12252 return SDValue();
12260 if (SDValue V = tryToFoldXSubYZ(N0, N1))
12263 if (SDValue V = tryToFoldXYSubZ(N0, N1))
12267 if (SDValue V = tryToFoldXYSubZ(N0, N1))
12270 if (SDValue V = tryToFoldXSubYZ(N0, N1))
12277 SDValue N00 = N0.getOperand(0).getOperand(0);
12278 SDValue N01 = N0.getOperand(0).getOperand(1);
12289 SDValue N00 = N0.getOperand(0);
12306 SDValue N10 = N1.getOperand(0);
12327 SDValue N00 = N0.getOperand(0);
12329 SDValue N000 = N00.getOperand(0);
12351 SDValue N00 = N0.getOperand(0);
12353 SDValue N000 = N00.getOperand(0);
12389 SDValue N20 = N1.getOperand(2).getOperand(0);
12390 SDValue N21 = N1.getOperand(2).getOperand(1);
12405 SDValue N02 = N0.getOperand(2);
12407 SDValue N020 = N02.getOperand(0);
12431 SDValue N00 = N0.getOperand(0);
12433 SDValue N002 = N00.getOperand(2);
12458 SDValue N120 = N1.getOperand(2).getOperand(0);
12462 SDValue N1200 = N120.getOperand(0);
12463 SDValue N1201 = N120.getOperand(1);
12485 SDValue CvtSrc = N1.getOperand(0);
12486 SDValue N100 = CvtSrc.getOperand(0);
12487 SDValue N101 = CvtSrc.getOperand(1);
12488 SDValue N102 = CvtSrc.getOperand(2);
12492 SDValue N1020 = N102.getOperand(0);
12493 SDValue N1021 = N102.getOperand(1);
12510 return SDValue();
12516 SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
12517 SDValue N0 = N->getOperand(0);
12518 SDValue N1 = N->getOperand(1);
12530 return SDValue();
12545 return SDValue();
12553 auto FuseFADD = [&](SDValue X, SDValue Y, const SDNodeFlags Flags) {
12564 return SDValue();
12567 if (SDValue FMA = FuseFADD(N0, N1, Flags))
12569 if (SDValue FMA = FuseFADD(N1, N0, Flags))
12576 auto FuseFSUB = [&](SDValue X, SDValue Y, const SDNodeFlags Flags) {
12597 return SDValue();
12600 if (SDValue FMA = FuseFSUB(N0, N1, Flags))
12602 if (SDValue FMA = FuseFSUB(N1, N0, Flags))
12605 return SDValue();
12608 SDValue DAGCombiner::visitFADD(SDNode *N) {
12609 SDValue N0 = N->getOperand(0);
12610 SDValue N1 = N->getOperand(1);
12618 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
12623 if (SDValue FoldedVOp = SimplifyVBinOp(N))
12640 if (SDValue NewSel = foldBinOpIntoSelect(N))
12645 if (SDValue NegN1 = TLI.getCheaperNegatedExpression(
12651 if (SDValue NegN0 = TLI.getCheaperNegatedExpression(
12655 auto isFMulNegTwo = [](SDValue FMul) {
12664 SDValue B = N0.getOperand(0);
12665 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
12670 SDValue B = N1.getOperand(0);
12671 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
12699 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1, Flags);
12713 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12722 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12734 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12743 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12781 if (SDValue Fused = visitFADDForFMACombine(N)) {
12785 return SDValue();
12788 SDValue DAGCombiner::visitFSUB(SDNode *N) {
12789 SDValue N0 = N->getOperand(0);
12790 SDValue N1 = N->getOperand(1);
12798 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
12803 if (SDValue FoldedVOp = SimplifyVBinOp(N))
12810 if (SDValue NewSel = foldBinOpIntoSelect(N))
12835 if (SDValue NegN1 =
12855 if (SDValue NegN1 =
12860 if (SDValue Fused = visitFSUBForFMACombine(N)) {
12865 return SDValue();
12868 SDValue DAGCombiner::visitFMUL(SDNode *N) {
12869 SDValue N0 = N->getOperand(0);
12870 SDValue N1 = N->getOperand(1);
12878 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
12884 if (SDValue FoldedVOp = SimplifyVBinOp(N))
12897 if (SDValue NewSel = foldBinOpIntoSelect(N))
12911 SDValue N00 = N0.getOperand(0);
12912 SDValue N01 = N0.getOperand(1);
12917 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1, Flags);
12926 const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
12927 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1, Flags);
12946 SDValue NegN0 =
12948 SDValue NegN1 =
12960 SDValue Select = N0, X = N1;
12964 SDValue Cond = Select.getOperand(0);
13002 if (SDValue Fused = visitFMULForFMADistributiveCombine(N)) {
13007 return SDValue();
13010 SDValue DAGCombiner::visitFMA(SDNode *N) {
13011 SDValue N0 = N->getOperand(0);
13012 SDValue N1 = N->getOperand(1);
13013 SDValue N2 = N->getOperand(2);
13036 SDValue NegN0 =
13038 SDValue NegN1 =
13093 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0);
13130 if (SDValue Neg = TLI.getCheaperNegatedExpression(
13131 SDValue(N, 0), DAG, LegalOperations, ForCodeSize))
13133 return SDValue();
13143 SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
13150 return SDValue();
13153 SDValue N0 = N->getOperand(0);
13156 return SDValue();
13160 SDValue N1 = N->getOperand(1);
13171 return SDValue();
13188 return SDValue();
13191 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
13192 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags);
13196 SDValue Dividend = U->getOperand(0);
13198 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
13207 return SDValue(N, 0); // N was replaced.
13210 SDValue DAGCombiner::visitFDIV(SDNode *N) {
13211 SDValue N0 = N->getOperand(0);
13212 SDValue N1 = N->getOperand(1);
13220 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
13225 if (SDValue FoldedVOp = SimplifyVBinOp(N))
13232 if (SDValue NewSel = foldBinOpIntoSelect(N))
13235 if (SDValue V = combineRepeatedFPDivisors(N))
13261 if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0), Flags))
13265 if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0),
13273 if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0),
13282 SDValue Sqrt, Y;
13298 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, Y.getOperand(0),
13300 SDValue AAZ =
13302 if (SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
13311 if (SDValue Rsqrt = buildRsqrtEstimate(Sqrt.getOperand(0), Flags)) {
13312 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y, Flags);
13321 if (SDValue RV = BuildDivEstimate(N0, N1, Flags))
13330 SDValue NegN0 =
13332 SDValue NegN1 =
13339 return SDValue();
13342 SDValue DAGCombiner::visitFREM(SDNode *N) {
13343 SDValue N0 = N->getOperand(0);
13344 SDValue N1 = N->getOperand(1);
13350 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
13357 if (SDValue NewSel = foldBinOpIntoSelect(N))
13360 return SDValue();
13363 SDValue DAGCombiner::visitFSQRT(SDNode *N) {
13371 return SDValue();
13373 SDValue N0 = N->getOperand(0);
13375 return SDValue();
13384 SDValue N1 = N->getOperand(1);
13398 SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
13399 SDValue N0 = N->getOperand(0);
13400 SDValue N1 = N->getOperand(1);
13442 return SDValue();
13445 SDValue DAGCombiner::visitFPOW(SDNode *N) {
13448 return SDValue();
13466 return SDValue();
13473 return SDValue();
13497 return SDValue();
13501 return SDValue();
13506 return SDValue();
13510 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags);
13511 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags);
13518 return SDValue();
13521 static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG,
13529 return SDValue();
13540 return SDValue();
13544 SDValue N0 = N->getOperand(0);
13553 return SDValue();
13556 SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
13557 SDValue N0 = N->getOperand(0);
13602 if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
13605 return SDValue();
13608 SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
13609 SDValue N0 = N->getOperand(0);
13641 if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
13644 return SDValue();
13648 static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG) {
13649 SDValue N0 = N->getOperand(0);
13653 return SDValue();
13655 SDValue Src = N0.getOperand(0);
13686 return SDValue();
13689 SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
13690 SDValue N0 = N->getOperand(0);
13704 SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
13705 SDValue N0 = N->getOperand(0);
13719 SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
13720 SDValue N0 = N->getOperand(0);
13721 SDValue N1 = N->getOperand(1);
13746 return SDValue();
13762 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT,
13769 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
13772 return SDValue();
13775 SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
13776 SDValue N0 = N->getOperand(0);
13782 return SDValue();
13797 SDValue In = N0.getOperand(0);
13809 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
13819 return SDValue(N, 0); // Return N so it doesn't get rechecked!
13822 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
13825 return SDValue();
13828 SDValue DAGCombiner::visitFCEIL(SDNode *N) {
13829 SDValue N0 = N->getOperand(0);
13836 return SDValue();
13839 SDValue DAGCombiner::visitFTRUNC(SDNode *N) {
13840 SDValue N0 = N->getOperand(0);
13860 return SDValue();
13863 SDValue DAGCombiner::visitFFLOOR(SDNode *N) {
13864 SDValue N0 = N->getOperand(0);
13871 return SDValue();
13875 SDValue DAGCombiner::visitFNEG(SDNode *N) {
13876 SDValue N0 = N->getOperand(0);
13883 if (SDValue NegN0 =
13903 SDValue Int = N0.getOperand(0);
13940 return SDValue();
13943 static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
13945 SDValue N0 = N->getOperand(0);
13946 SDValue N1 = N->getOperand(1);
13962 return SDValue();
13965 SDValue DAGCombiner::visitFMINNUM(SDNode *N) {
13969 SDValue DAGCombiner::visitFMAXNUM(SDNode *N) {
13973 SDValue DAGCombiner::visitFMINIMUM(SDNode *N) {
13977 SDValue DAGCombiner::visitFMAXIMUM(SDNode *N) {
13981 SDValue DAGCombiner::visitFABS(SDNode *N) {
13982 SDValue N0 = N->getOperand(0);
14000 SDValue Int = N0.getOperand(0);
14021 return SDValue();
14024 SDValue DAGCombiner::visitBRCOND(SDNode *N) {
14025 SDValue Chain = N->getOperand(0);
14026 SDValue N1 = N->getOperand(1);
14027 SDValue N2 = N->getOperand(2);
14049 if (SDValue NewN1 = rebuildSetCC(N1))
14054 return SDValue();
14057 SDValue DAGCombiner::rebuildSetCC(SDValue N) {
14083 SDValue Op0 = N.getOperand(0);
14084 SDValue Op1 = N.getOperand(1);
14087 SDValue AndOp1 = Op0.getOperand(1);
14113 SDValue Tmp = visitXOR(N.getNode());
14128 SDValue Op0 = N->getOperand(0);
14129 SDValue Op1 = N->getOperand(1);
14151 return SDValue();
14156 SDValue DAGCombiner::visitBR_CC(SDNode *N) {
14158 SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3);
14167 SDValue Simp = SimplifySetCC(getSetCCResultType(CondLHS.getValueType()),
14179 return SDValue();
14240 bool &IsLoad, bool &IsMasked, SDValue &Ptr,
14293 SDValue Ptr;
14305 SDValue BasePtr;
14306 SDValue Offset;
14339 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(N)->getValue()
14379 SDValue Op1 = Use.getUser()->getOperand((UI.getOperandNo() + 1) & 1);
14415 SDValue Result;
14418 Result = DAG.getIndexedLoad(SDValue(N, 0), SDLoc(N), BasePtr, Offset, AM);
14421 DAG.getIndexedStore(SDValue(N, 0), SDLoc(N), BasePtr, Offset, AM);
14424 Result = DAG.getIndexedMaskedLoad(SDValue(N, 0), SDLoc(N), BasePtr,
14427 Result = DAG.getIndexedMaskedStore(SDValue(N, 0), SDLoc(N), BasePtr,
14436 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
14437 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
14439 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
14488 SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0));
14489 SDValue NewOp2 = Result.getValue(IsLoad ? 1 : 0);
14491 SDValue NewUse = DAG.getNode(Opcode,
14494 DAG.ReplaceAllUsesOfValueWith(SDValue(OtherUses[i], 0), NewUse);
14506 static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse,
14507 SDValue &BasePtr, SDValue &Offset,
14534 SDValue OtherPtr;
14556 bool &IsMasked, SDValue &Ptr,
14557 SDValue &BasePtr, SDValue &Offset,
14601 SDValue Ptr;
14602 SDValue BasePtr;
14603 SDValue Offset;
14610 SDValue Result;
14612 Result = IsLoad ? DAG.getIndexedLoad(SDValue(N, 0), SDLoc(N), BasePtr,
14614 : DAG.getIndexedStore(SDValue(N, 0), SDLoc(N),
14617 Result = IsLoad ? DAG.getIndexedMaskedLoad(SDValue(N, 0), SDLoc(N),
14619 : DAG.getIndexedMaskedStore(SDValue(N, 0), SDLoc(N),
14628 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
14629 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
14631 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
14638 DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0),
14645 SDValue DAGCombiner::SplitIndexingFromLoad(LoadSDNode *LD) {
14648 SDValue BP = LD->getOperand(1);
14649 SDValue Inc = LD->getOperand(2);
14672 bool DAGCombiner::getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val) {
14697 bool DAGCombiner::extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val) {
14723 SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
14725 return SDValue();
14726 SDValue Chain = LD->getOperand(0);
14730 return SDValue();
14741 return SDValue();
14756 auto ReplaceLd = [&](LoadSDNode *LD, SDValue Val, SDValue Chain) -> SDValue {
14761 return SDValue();
14762 SDValue Idx = SplitIndexingFromLoad(LD);
14763 SDValue Ops[] = {Val, Idx, Chain};
14770 return SDValue();
14792 return SDValue();
14794 SDValue Val;
14817 return SDValue();
14820 SDValue DAGCombiner::visitLOAD(SDNode *N) {
14822 SDValue Chain = LD->getChain();
14823 SDValue Ptr = LD->getBasePtr();
14843 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
14848 return SDValue(N, 0); // Return N so it doesn't get rechecked!
14861 SDValue Undef = DAG.getUNDEF(N->getValueType(0));
14862 SDValue Index;
14874 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef);
14875 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Index);
14876 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain);
14878 return SDValue(N, 0); // Return N so it doesn't get rechecked!
14893 SDValue NewLoad = DAG.getExtLoad(
14906 SDValue BetterChain = FindBetterChain(LD, Chain);
14910 SDValue ReplLoad;
14924 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N),
14934 return SDValue(N, 0);
14939 return SDValue(N, 0);
14941 return SDValue();
15171 SDValue loadSlice() const {
15173 const SDValue &OldBaseAddr = Origin->getBasePtr();
15174 SDValue BaseAddr = OldBaseAddr;
15190 SDValue LastInst =
15475 SmallVector<SDValue, 8> ArgChains;
15480 SDValue SliceInst = LSIt->loadSlice();
15489 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other,
15491 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
15500 CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
15554 SDValue(LD, 1).hasOneUse()) {
15569 static SDValue
15571 SDValue IVal, StoreSDNode *St,
15581 if (!DAG.MaskedValueIsZero(IVal, Mask)) return SDValue();
15589 return SDValue();
15593 return SDValue();
15613 SDValue Ptr = St->getBasePtr();
15633 SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
15636 return SDValue();
15638 SDValue Chain = ST->getChain();
15639 SDValue Value = ST->getValue();
15640 SDValue Ptr = ST->getBasePtr();
15644 return SDValue();
15657 if (SDValue NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
15664 if (SDValue NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
15670 return SDValue();
15674 return SDValue();
15676 SDValue N0 = Value.getOperand(0);
15678 Chain == SDValue(N0.getNode(), 1)) {
15683 return SDValue();
15686 SDValue N1 = Value.getOperand(1);
15692 return SDValue();
15707 return SDValue();
15728 return SDValue();
15730 SDValue NewPtr = DAG.getMemBasePlusOffset(Ptr, PtrOff, SDLoc(LD));
15731 SDValue NewLD =
15735 SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
15738 SDValue NewST =
15752 return SDValue();
15758 SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
15760 SDValue Value = ST->getValue();
15771 return SDValue();
15778 return SDValue();
15785 return SDValue();
15792 return SDValue();
15794 SDValue NewLD =
15798 SDValue NewST =
15810 return SDValue();
15827 SDValue &AddNode,
15828 SDValue &ConstNode) {
15887 SDValue DAGCombiner::getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
15889 SmallVector<SDValue, 8> Chains;
15929 SDValue StoredVal;
15932 SmallVector<SDValue, 8> BuildVector;
15935 SDValue Val = St->getValue();
15961 SmallVector<SDValue, 8> Ops;
15964 SDValue Val = peekThroughBitcasts(St->getValue());
15981 SDValue Vec = Val.getOperand(0);
15982 SDValue Idx = Val.getOperand(1);
16008 SDValue Val = St->getValue();
16033 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
16036 SDValue NewStore;
16046 SDValue ExtendedStoreVal =
16072 SDValue Val = peekThroughBitcasts(St->getValue());
16110 SDValue OtherBC = peekThroughBitcasts(Other->getValue());
16242 for (SDValue Op : N->ops())
16346 SDValue StoredVal = ST->getValue();
16539 SDValue Val = peekThroughBitcasts(St->getValue());
16598 SDValue LoadChain = FirstLoad->getChain();
16726 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
16739 SDValue NewLoad, NewStore;
16744 SDValue StoreOp = NewLoad;
16749 SDValue RotAmt =
16774 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
16775 SDValue(NewLoad.getNode(), 1));
16781 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
16816 SDValue StoredVal = peekThroughBitcasts(St->getValue());
16883 SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
16885 SDValue ReplStore;
16898 SDValue Token = DAG.getNode(ISD::TokenFactor, SL,
16908 SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) {
16909 SDValue Value = ST->getValue();
16911 return SDValue();
16914 return SDValue();
16918 SDValue Chain = ST->getChain();
16919 SDValue Ptr = ST->getBasePtr();
16928 SDValue Tmp;
16936 return SDValue();
16947 return SDValue();
16965 SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, SDLoc(CFP), MVT::i32);
16966 SDValue Hi = DAG.getConstant(Val >> 32, SDLoc(CFP), MVT::i32);
16974 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16978 SDValue St1 = DAG.getStore(Chain, DL, Hi, Ptr,
16985 return SDValue();
16989 SDValue DAGCombiner::visitSTORE(SDNode *N) {
16991 SDValue Chain = ST->getChain();
16992 SDValue Value = ST->getValue();
16993 SDValue Ptr = ST->getBasePtr();
17024 SDValue NewStore =
17037 if (SDValue NewST = TransformFPLoadStorePair(N))
17041 if (SDValue Store = MatchStoreCombine(ST))
17050 return SDValue(ST, 0);
17068 if (SDValue Shorter = DAG.GetDemandedBits(Value, TruncDemandedBits))
17081 return SDValue(N, 0);
17093 Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) {
17125 return SDValue();
17154 return SDValue(N, 0);
17160 return SDValue(N, 0);
17168 if (SDValue NewSt = replaceStoreOfFPConstant(ST))
17172 if (SDValue NewSt = splitMergedValStore(ST))
17178 SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) {
17181 return SDValue();
17183 const BaseIndexOffset LifetimeEndBase(N->getOperand(1), SDValue(),
17187 SmallVector<SDValue, 8> Chains = {N->getOperand(0)};
17189 SDValue Chain = Chains.back();
17219 return SDValue(N, 0);
17224 return SDValue();
17255 SDValue DAGCombiner::splitMergedValStore(StoreSDNode *ST) {
17257 return SDValue();
17262 return SDValue();
17264 SDValue Val = ST->getValue();
17269 return SDValue();
17272 SDValue Op1 = Val.getOperand(0);
17273 SDValue Op2 = Val.getOperand(1);
17274 SDValue Lo, Hi;
17278 return SDValue();
17283 return SDValue();
17289 return SDValue();
17299 return SDValue();
17310 return SDValue();
17322 SDValue Chain = ST->getChain();
17323 SDValue Ptr = ST->getBasePtr();
17325 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
17329 SDValue St1 =
17337 SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) {
17340 SDValue InsertVal = N->getOperand(1);
17341 SDValue Vec = N->getOperand(0);
17353 SDValue X = Vec.getOperand(0);
17354 SDValue Y = Vec.getOperand(1);
17359 SDValue InsertVal0 = InsertVal.getOperand(0);
17365 SmallVector<std::pair<int, SDValue>, 8> ArgWorkList;
17371 SDValue ArgVal;
17384 for (SDValue Op : reverse(ArgVal->ops())) {
17404 SDValue LegalShuffle =
17418 return SDValue();
17420 SDValue SubVec = InsertVal.getOperand(0);
17421 SDValue DestVec = N->getOperand(0);
17428 return SDValue();
17449 return SDValue();
17454 SmallVector<SDValue, 8> ConcatOps(ExtendRatio, DAG.getUNDEF(SubVecVT));
17456 SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps);
17459 SDValue DestVecBC = DAG.getBitcast(ShufVT, DestVec);
17460 SDValue Shuf = DAG.getVectorShuffle(ShufVT, DL, DestVecBC, PaddedSubV, Mask);
17467 SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
17468 SDValue InVec = N->getOperand(0);
17469 SDValue InVal = N->getOperand(1);
17470 SDValue EltNo = N->getOperand(2);
17494 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), InVal);
17498 return SDValue();
17502 return SDValue();
17508 if (SDValue Shuf = combineInsertEltToShuffle(N, Elt))
17523 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
17533 return SDValue();
17538 SmallVector<SDValue, 8> Ops;
17547 return SDValue();
17563 SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
17564 SDValue EltNo,
17576 return SDValue();
17581 return SDValue();
17585 SDValue NewPtr = OriginalLoad->getBasePtr();
17586 SDValue Offset;
17612 SDValue Load;
17613 SDValue Chain;
17637 SDValue From[] = { SDValue(EVE, 0), SDValue(OriginalLoad, 1) };
17638 SDValue To[] = { Load, Chain };
17646 return SDValue(EVE, 0);
17651 static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
17654 SDValue Vec = ExtElt->getOperand(0);
17655 SDValue Index = ExtElt->getOperand(1);
17659 return SDValue();
17663 return SDValue();
17668 SDValue Op0 = Vec.getOperand(0);
17669 SDValue Op1 = Vec.getOperand(1);
17676 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
17677 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
17681 return SDValue();
17684 SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
17685 SDValue VecOp = N->getOperand(0);
17686 SDValue Index = N->getOperand(1);
17699 SDValue Elt = VecOp.getOperand(1);
17712 SDValue InOp = VecOp.getOperand(0);
17736 SDValue Elt = VecOp.getOperand(IndexVal);
17748 return SDValue();
17766 SDValue BCSrc = VecOp.getOperand(0);
17774 SDValue X = BCSrc.getOperand(0);
17792 if (SDValue BO = scalarizeExtractedBinop(N, DAG, LegalOperations))
17811 SDValue SVInVec;
17820 SDValue InOp = SVInVec.getOperand(OrigElt);
17859 return SDValue(N, 0);
17867 return SDValue(N, 0);
17878 return SDValue();
17883 return SDValue();
17887 return SDValue();
17906 return SDValue();
17920 return SDValue();
17931 return SDValue();
17936 return SDValue();
17945 return SDValue();
17970 SDValue NewIdx = DAG.getConstant(Elt % ConcatNumElts, SL,
17973 SDValue ConcatOp = VecOp.getOperand(Elt / ConcatNumElts);
17974 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL,
17983 return SDValue();
17993 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
18001 return SDValue();
18016 SDValue In = N->getOperand(i);
18057 return SDValue();
18061 if (!AllAnyExt && DAG.isSplatValue(SDValue(N, 0), /*AllowUndefs*/ true))
18062 return SDValue();
18067 SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
18071 SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
18075 SDValue Cast = N->getOperand(i);
18079 SDValue In;
18099 return SDValue();
18102 SDValue BV = DAG.getBuildVector(VecVT, DL, Ops);
18114 SDValue DAGCombiner::reduceBuildVecTruncToBitCast(SDNode *N) {
18119 return SDValue();
18128 return SDValue();
18133 auto PeekThroughBitcast = [](SDValue Op) {
18140 SDValue Src;
18142 SDValue In = PeekThroughBitcast(N->getOperand(i));
18147 return SDValue();
18155 return SDValue();
18160 SDValue part = PeekThroughBitcast(In.getOperand(0));
18166 return SDValue();
18169 SDValue ShiftAmtVal = In.getOperand(1);
18171 return SDValue();
18177 return SDValue();
18183 return SDValue();
18188 SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N,
18190 SDValue VecIn1, SDValue VecIn2,
18192 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
18213 SmallVector<SDValue, 2> ConcatOps(NumConcats, DAG.getUNDEF(InVT1));
18217 VecIn2 = SDValue();
18220 return SDValue();
18238 return SDValue();
18245 return SDValue();
18254 return SDValue();
18258 SmallVector<SDValue, 2> ConcatOps(2, DAG.getUNDEF(InVT2));
18266 return SDValue();
18296 SDValue Shuffle = DAG.getVectorShuffle(InVT1, DL, VecIn1, VecIn2, Mask);
18303 static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) {
18311 SDValue Op = BV->getOperand(i);
18317 return SDValue();
18321 return SDValue();
18330 SDValue Zext = BV->getOperand(ZextElt);
18335 return SDValue();
18339 SDValue Extract = Zext.getOperand(0);
18344 return SDValue();
18371 SDValue ZeroVec = DAG.getConstant(0, DL, VecVT);
18373 SDValue Shuf = TLI.buildLegalVectorShuffle(VecVT, DL, Extract.getOperand(0),
18376 return SDValue();
18383 SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
18389 return SDValue();
18391 if (SDValue V = reduceBuildVecToShuffleWithZero(N, DAG))
18396 return SDValue();
18408 SmallVector<SDValue, 8> VecIn;
18409 VecIn.push_back(SDValue());
18412 SDValue Op = N->getOperand(i);
18430 return SDValue();
18431 SDValue ExtractedFromVec = Op.getOperand(0);
18434 return SDValue();
18438 return SDValue();
18443 return SDValue();
18458 return SDValue();
18468 SDValue Vec = VecIn.back();
18487 SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
18489 SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
18524 SmallVector<SDValue, 4> Shuffles;
18527 SDValue VecLeft = VecIn[LeftIdx];
18528 SDValue VecRight =
18529 (LeftIdx + 1) < VecIn.size() ? VecIn[LeftIdx + 1] : SDValue();
18531 if (SDValue Shuffle = createBuildVecShuffle(DL, N, VectorMask, VecLeft,
18535 return SDValue();
18600 SDValue DAGCombiner::convertBuildVecZextToZext(SDNode *N) {
18602 return SDValue();
18607 SDValue Op0 = N->getOperand(0);
18608 auto checkElem = [&](SDValue Op) -> int64_t {
18623 return SDValue();
18626 SDValue In = Op0.getOperand(0).getOperand(0);
18632 return SDValue();
18637 return SDValue();
18647 SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
18663 if (SDValue Splat = cast<BuildVectorSDNode>(N)->getSplatValue()) {
18671 SmallVector<SDValue, 8> Ops(N->getNumOperands(), Splat);
18672 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N),
18682 if (SDValue V = cast<BuildVectorSDNode>(N)->getSplatValue()) {
18689 SDValue Op0 = N->getOperand(0);
18690 auto checkElem = [&](SDValue Op) -> uint64_t {
18716 if (SDValue V = convertBuildVecZextToZext(N))
18719 if (SDValue V = reduceBuildVecExtToExtBuildVec(N))
18722 if (SDValue V = reduceBuildVecTruncToBitCast(N))
18725 if (SDValue V = reduceBuildVecToShuffle(N))
18728 return SDValue();
18731 static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) {
18737 return SDValue();
18741 SmallVector<SDValue, 8> Ops;
18744 SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT);
18749 for (const SDValue &Op : N->ops()) {
18756 return SDValue();
18766 return SDValue();
18776 for (SDValue &Op : Ops) {
18796 static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG) {
18802 return SDValue();
18807 SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT);
18810 for (SDValue Op : N->ops()) {
18820 return SDValue();
18823 SDValue ExtVec = Op.getOperand(0);
18840 return SDValue();
18849 return SDValue();
18861 return SDValue();
18870 static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG) {
18885 return SDValue();
18890 return SDValue();
18894 SmallVector<SDValue, 4> SrcOps;
18895 for (SDValue Op : N->ops()) {
18898 return SDValue();
18915 return SDValue();
18921 return SDValue();
18929 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatSrcVT, SrcOps);
18933 SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
18944 if (std::all_of(std::next(N->op_begin()), N->op_end(), [](const SDValue &Op) {
18947 SDValue In = N->getOperand(0);
18954 SmallVector<SDValue, 4> Ops(In->op_begin(), In->op_end());
18959 SDValue Scalar = peekThroughOneUseBitcasts(In);
18983 return SDValue();
18987 return SDValue();
18991 return SDValue();
18995 return SDValue();
18997 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar);
19006 auto IsBuildVectorOrUndef = [](const SDValue &Op) {
19010 SmallVector<SDValue, 8> Opnds;
19018 for (const SDValue &Op : N->ops())
19027 for (const SDValue &Op : N->ops()) {
19052 if (SDValue V = combineConcatVectorOfScalars(N, DAG))
19057 if (SDValue V = combineConcatVectorOfExtracts(N, DAG))
19060 if (SDValue V = combineConcatVectorOfCasts(N, DAG))
19069 SDValue SingleSource = SDValue();
19074 SDValue Op = N->getOperand(i);
19081 return SDValue();
19086 return SDValue();
19094 return SDValue();
19100 return SDValue();
19106 return SDValue();
19111 static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT) {
19123 return SDValue();
19126 static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract,
19129 SDValue BinOp = Extract->getOperand(0);
19132 return SDValue();
19135 SDValue Bop0 = BinOp.getOperand(0), Bop1 = BinOp.getOperand(1);
19137 return SDValue();
19139 SDValue Index = Extract->getOperand(1);
19142 return SDValue();
19144 SDValue Sub0 = getSubVectorSrc(Bop0, Index, SubVT);
19145 SDValue Sub1 = getSubVectorSrc(Bop1, Index, SubVT);
19151 return SDValue();
19162 static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) {
19166 if (SDValue V = narrowInsertExtractVectorBinOp(Extract, DAG))
19172 return SDValue();
19177 SDValue BinOp = peekThroughBitcasts(Extract->getOperand(0));
19180 return SDValue();
19188 return SDValue();
19197 return SDValue();
19208 return SDValue();
19215 return SDValue();
19221 return SDValue();
19233 SDValue NewExtIndex = DAG.getVectorIdxConstant(ExtBOIdx, DL);
19234 SDValue X = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
19236 SDValue Y = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
19238 SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y,
19246 return SDValue();
19254 return SDValue();
19258 auto GetSubVector = [ConcatOpNum](SDValue V) -> SDValue {
19261 return SDValue();
19263 SDValue SubVecL = GetSubVector(peekThroughBitcasts(BinOp.getOperand(0)));
19264 SDValue SubVecR = GetSubVector(peekThroughBitcasts(BinOp.getOperand(1)));
19273 SDValue IndexC = DAG.getVectorIdxConstant(ExtBOIdx, DL);
19274 SDValue X = SubVecL ? DAG.getBitcast(NarrowBVT, SubVecL)
19278 SDValue Y = SubVecR ? DAG.getBitcast(NarrowBVT, SubVecR)
19282 SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y);
19286 return SDValue();
19292 static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
19295 return SDValue();
19301 return SDValue();
19308 return SDValue();
19323 return SDValue();
19327 return SDValue();
19332 SDValue BaseAddr = Ld->getBasePtr();
19335 SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
19339 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO);
19344 SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
19346 SDValue V = N->getOperand(0);
19354 if (SDValue NarrowLoad = narrowExtractedVectorLoad(N, DAG))
19372 SDValue SrcOp = V.getOperand(0);
19383 SDValue NewIndex = DAG.getVectorIdxConstant(ExtIdx * SrcDestRatio, DL);
19384 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT,
19400 SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
19401 SDValue NewExtract =
19408 SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
19409 SDValue NewExtract =
19448 SDValue NewIndexC = DAG.getVectorIdxConstant(NewExtIdx, DL);
19475 SDValue Src = V->getOperand(IdxVal);
19482 SDValue BuildVec = DAG.getBuildVector(ExtractVT, SDLoc(N),
19494 return SDValue();
19511 if (SDValue NarrowBOp = narrowExtractedVectorBinOp(N, DAG))
19514 if (SimplifyDemandedVectorElts(SDValue(N, 0)))
19515 return SDValue(N, 0);
19517 return SDValue();
19524 static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf,
19526 SDValue N0 = Shuf->getOperand(0), N1 = Shuf->getOperand(1);
19530 return SDValue();
19556 return SDValue();
19560 SDValue X = N0.getOperand(0), Y = N1.getOperand(0);
19562 SDValue Shuf0 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask0);
19563 SDValue Shuf1 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask1);
19569 static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) {
19573 SDValue N0 = N->getOperand(0);
19574 SDValue N1 = N->getOperand(1);
19578 SmallVector<SDValue, 4> Ops;
19615 return SDValue();
19618 return SDValue();
19648 static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN,
19653 SDValue N0 = SVN->getOperand(0);
19654 SDValue N1 = SVN->getOperand(1);
19657 return SDValue();
19663 return SDValue();
19668 return SDValue();
19670 return SDValue();
19679 if (SDValue Splat0 = BV0->getSplatValue())
19682 SmallVector<SDValue, 8> Ops;
19683 SmallSet<SDValue, 16> DuplicateOps;
19685 SDValue Op = DAG.getUNDEF(VT.getScalarType());
19688 SDValue &S = (M < (int)NumElts ? N0 : N1);
19692 SDValue Op0 = S.getOperand(0);
19696 return SDValue();
19706 return SDValue();
19715 for (SDValue &Op : Ops)
19718 for (SDValue &Op : Ops)
19729 static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN,
19738 return SDValue();
19743 SDValue N0 = SVN->getOperand(0);
19778 return SDValue();
19786 static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN,
19793 return SDValue();
19795 SDValue N0 = peekThroughBitcasts(SVN->getOperand(0));
19801 return SDValue();
19803 SDValue N00 = N0.getOperand(0);
19811 return SDValue();
19832 return SDValue();
19839 return SDValue();
19847 static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf,
19850 return SDValue();
19853 return SDValue();
19899 static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf,
19902 return SDValue();
19905 return SDValue();
19930 return SDValue();
19942 return SDValue();
19976 static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf,
19982 SDValue Op0 = Shuf->getOperand(0);
19983 SDValue Op1 = Shuf->getOperand(1);
19990 return SDValue();
20004 return SDValue();
20008 return SDValue();
20020 SDValue NewInsIndex = DAG.getVectorIdxConstant(ShufOp0Index, SDLoc(Shuf));
20029 static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf) {
20033 return SDValue();
20046 return SDValue();
20053 SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
20057 SDValue N0 = N->getOperand(0);
20058 SDValue N1 = N->getOperand(1);
20099 if (SDValue InsElt = replaceShuffleOfInsert(SVN, DAG))
20103 if (SDValue V = combineShuffleOfSplatVal(SVN, DAG))
20106 if (SDValue V = formSplatFromShuffles(SVN, DAG))
20117 SDValue L = N0.getOperand(0), R = N0.getOperand(1);
20120 SDValue Index = DAG.getVectorIdxConstant(SplatIndex, DL);
20121 SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index);
20122 SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index);
20123 SDValue NewBO = DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR,
20125 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO);
20135 SDValue ConvInput = V->getOperand(0);
20144 SDValue Base;
20166 SDValue Splatted = V->getOperand(SplatIndex);
20167 SmallVector<SDValue, 8> Ops(NumElts, Splatted);
20168 SDValue NewBV = DAG.getBuildVector(V->getValueType(0), SDLoc(N), Ops);
20179 if (SimplifyDemandedVectorElts(SDValue(N, 0)))
20180 return SDValue(N, 0);
20184 if (SDValue ShufOp = simplifyShuffleOfShuffle(SVN))
20188 if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations))
20192 if (SDValue V = combineTruncationShuffle(SVN, DAG))
20200 if (SDValue V = partitionShuffleOfConcats(N, DAG))
20221 SDValue UndefVec = DAG.getUNDEF(N0.getOperand(0).getValueType());
20222 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
20231 if (SDValue Res = combineShuffleOfScalars(SVN, DAG, TLI))
20241 SDValue BC0 = peekThroughOneUseBitcasts(N0);
20270 SDValue SV0 = BC0->getOperand(0);
20271 SDValue SV1 = BC0->getOperand(1);
20301 SDValue SV0 = N1->getOperand(0);
20302 SDValue SV1 = N1->getOperand(1);
20324 return SDValue();
20331 SDValue SV0, SV1;
20343 SDValue CurrentVec;
20380 return SDValue();
20411 if (SDValue V = foldShuffleOfConcatUndefs(SVN, DAG))
20414 return SDValue();
20417 SDValue DAGCombiner::visitSCALAR_TO_VECTOR(SDNode *N) {
20418 SDValue InVal = N->getOperand(0);
20426 SDValue InVec = InVal->getOperand(0);
20427 SDValue EltNo = InVal->getOperand(1);
20438 SDValue Val =
20444 SDValue LegalShuffle =
20454 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(N));
20466 return SDValue();
20469 SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
20471 SDValue N0 = N->getOperand(0);
20472 SDValue N1 = N->getOperand(1);
20473 SDValue N2 = N->getOperand(2);
20505 SDValue CN0 = N0.getOperand(0);
20506 SDValue CN1 = N1.getOperand(0);
20512 SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N),
20540 SDValue N0Src = peekThroughBitcasts(N0);
20541 SDValue N1Src = peekThroughBitcasts(N1);
20548 SDValue NewIdx;
20564 SDValue Res = DAG.getBitcast(NewVT, N0Src);
20580 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT,
20593 SmallVector<SDValue, 8> Ops(N0->op_begin(), N0->op_end());
20599 if (SimplifyDemandedVectorElts(SDValue(N, 0)))
20600 return SDValue(N, 0);
20602 return SDValue();
20605 SDValue DAGCombiner::visitFP_TO_FP16(SDNode *N) {
20606 SDValue N0 = N->getOperand(0);
20612 return SDValue();
20615 SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) {
20616 SDValue N0 = N->getOperand(0);
20627 return SDValue();
20630 SDValue DAGCombiner::visitVECREDUCE(SDNode *N) {
20631 SDValue N0 = N->getOperand(0);
20638 SDValue Res =
20657 return SDValue();
20664 SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
20668 SDValue LHS = N->getOperand(0);
20669 SDValue RHS = peekThroughBitcasts(N->getOperand(1));
20675 return SDValue();
20678 return SDValue();
20694 SDValue Elt = RHS.getOperand(EltIdx);
20708 return SDValue();
20721 return SDValue();
20728 return SDValue();
20730 SDValue Zero = DAG.getConstant(0, DL, ClearVT);
20743 if (SDValue S = BuildClearMask(Split))
20746 return SDValue();
20751 static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG) {
20752 SDValue N0 = N->getOperand(0);
20753 SDValue N1 = N->getOperand(1);
20763 SDValue Src0 = DAG.getSplatSourceVector(N0, Index0);
20764 SDValue Src1 = DAG.getSplatSourceVector(N1, Index1);
20770 return SDValue();
20773 SDValue IndexC = DAG.getVectorIdxConstant(Index0, DL);
20774 SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src0, IndexC);
20775 SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src1, IndexC);
20776 SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, X, Y, N->getFlags());
20781 count_if(N0->ops(), [](SDValue V) { return !V.isUndef(); }) == 1 &&
20782 count_if(N1->ops(), [](SDValue V) { return !V.isUndef(); }) == 1) {
20785 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT));
20791 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO);
20796 SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
20800 SDValue LHS = N->getOperand(0);
20801 SDValue RHS = N->getOperand(1);
20802 SDValue Ops[] = {LHS, RHS};
20808 if (SDValue Fold = DAG.FoldConstantVectorArithmetic(
20828 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0),
20830 SDValue UndefV = LHS.getOperand(1);
20845 SDValue X = Shuf0->getOperand(0);
20846 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, X, RHS, Flags);
20855 SDValue X = Shuf1->getOperand(0);
20856 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS, X, Flags);
20870 SDValue X = LHS.getOperand(1);
20871 SDValue Y = RHS.getOperand(1);
20872 SDValue Z = LHS.getOperand(2);
20878 SDValue VecC =
20880 SDValue NarrowBO = DAG.getNode(Opcode, DL, NarrowVT, X, Y);
20886 auto ConcatWithConstantOrUndef = [](SDValue Concat) {
20889 [](const SDValue &Op) {
20907 SmallVector<SDValue, 4> ConcatOps;
20918 if (SDValue V = scalarizeBinOpOfSplats(N, DAG))
20921 return SDValue();
20924 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
20925 SDValue N2) {
20928 SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2,
20939 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0),
20944 SDValue SelectNode = DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC,
20952 return SDValue();
20960 bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
20961 SDValue RHS) {
20967 SDValue Sqrt = RHS;
20969 SDValue CmpLHS;
20978 SDValue Cmp = TheSelect->getOperand(0);
21064 SDValue Addr;
21110 SDValue Load;
21149 SDValue DAGCombiner::foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0,
21150 SDValue N1, SDValue N2, SDValue N3,
21159 return SDValue();
21168 return SDValue();
21173 return SDValue();
21175 return SDValue();
21185 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, ShiftAmtTy);
21186 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt);
21203 return SDValue();
21205 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, ShiftAmtTy);
21206 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt);
21224 SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
21225 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
21228 return SDValue();
21236 return SDValue();
21242 return SDValue();
21247 return SDValue();
21256 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(DAG.getDataLayout()),
21262 SDValue Zero = DAG.getIntPtrConstant(0, DL);
21264 SDValue One = DAG.getIntPtrConstant(EltSize, SDLoc(FV));
21265 SDValue Cond =
21268 SDValue CstOffset = DAG.getSelect(DL, Zero.getValueType(), Cond, One, Zero);
21279 SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
21280 SDValue N2, SDValue N3, ISD::CondCode CC,
21293 if (SDValue SCC = DAG.FoldSetCC(CmpResVT, N0, N1, CC, DL)) {
21302 if (SDValue V =
21306 if (SDValue V = foldSelectCCToShiftAnd(DL, N0, N1, N2, N3, CC))
21317 SDValue AndLHS = N0->getOperand(0);
21324 SDValue ShlAmt =
21327 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt);
21331 SDValue ShrAmt =
21334 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt);
21358 return SDValue();
21360 SDValue Temp, SCC;
21381 return SDValue();
21398 SDValue ValueOnZero = N2;
21399 SDValue Count = N3;
21424 return SDValue();
21428 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
21440 SDValue DAGCombiner::BuildSDIV(SDNode *N) {
21444 return SDValue();
21447 if (SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, Built)) {
21453 return SDValue();
21458 SDValue DAGCombiner::BuildSDIVPow2(SDNode *N) {
21461 return SDValue();
21465 return SDValue();
21468 if (SDValue S = TLI.BuildSDIVPow2(N, C->getAPIntValue(), DAG, Built)) {
21474 return SDValue();
21481 SDValue DAGCombiner::BuildUDIV(SDNode *N) {
21485 return SDValue();
21488 if (SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, Built)) {
21494 return SDValue();
21499 SDValue DAGCombiner::BuildLogBase2(SDValue V, const SDLoc &DL) {
21502 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V);
21503 SDValue Base = DAG.getConstant(EltBits - 1, DL, VT);
21504 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz);
21516 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
21519 return SDValue();
21524 return SDValue();
21530 return SDValue();
21535 if (SDValue Est = TLI.getRecipEstimate(Op, DAG, Enabled, Iterations)) {
21540 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
21545 SDValue MulEst = Est;
21552 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags);
21574 return SDValue();
21583 SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est,
21588 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT);
21592 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags);
21597 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags);
21615 SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est,
21620 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT);
21621 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT);
21630 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags);
21631 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags);
21632 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags);
21637 SDValue LHS;
21655 SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
21658 return SDValue();
21663 return SDValue();
21669 return SDValue();
21676 if (SDValue Est =
21700 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
21701 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
21702 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
21703 SDValue IsDenorm = DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
21707 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
21708 SDValue IsZero = DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
21716 return SDValue();
21719 SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags) {
21723 SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) {
21733 SDValue BasePtr;
21762 return {false /*isvolatile*/, /*isAtomic*/ false, SDValue(),
21863 void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
21864 SmallVectorImpl<SDValue> &Aliases) {
21865 SmallVector<SDValue, 8> Chains; // List of chains to visit.
21877 std::function<bool(SDValue &)> ImproveChain = [&](SDValue &C) -> bool {
21881 C = SDValue();
21923 SDValue Chain = Chains.pop_back_val();
21970 SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
21975 SmallVector<SDValue, 8> Aliases;
22047 if (!SDValue(Chain, 0)->hasOneUse())
22081 SDValue NewChain = STChain->getChain();
22082 SmallVector<SDValue, 8> TFOps;
22085 SDValue BetterChain = FindBetterChain(S, NewChain);
22088 TFOps.push_back(SDValue(S, 0));
22093 SDValue BetterChain = FindBetterChain(St, NewChain);
22094 SDValue NewST;
22108 auto hasImprovedChain = [&](SDValue ST) -> bool {
22115 SDValue TF = DAG.getTokenFactor(SDLoc(STChain), TFOps);
22120 for (const SDValue &Op : TF->ops())
22145 SDValue BetterChain = FindBetterChain(St, St->getChain());