CodeGen/SelectionDAG/DAGCombiner.cpp

1 //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
129   class DAGCombiner {
224     DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
725   DAGCombiner &DC;
728   explicit WorklistRemover(DAGCombiner &dc)
737   DAGCombiner &DC;
740   explicit WorklistInserter(DAGCombiner &dc)
755   ((DAGCombiner*)DC)->AddToWorklist(N);
760   return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
765   return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
770   return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
775   return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
780   return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
787 void DAGCombiner::deleteAndRecombine(SDNode *N) {
816 bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
843 bool DAGCombiner::isOneUseSetCC(SDValue N) const {
896 bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
952 // Helper for DAGCombiner::reassociateOps. Try to reassociate an expression
954 SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
985 SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
1005 SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1037 void DAGCombiner::
1056 bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1079 bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1101 void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1115 SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1152 SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1169 SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1186 SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1251 SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1300 SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1328 bool DAGCombiner::PromoteLoad(SDValue Op) {
1380 bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1408 void DAGCombiner::Run(CombineLevel AtLevel) {
1503 SDValue DAGCombiner::visit(SDNode *N) {
1637 SDValue DAGCombiner::combine(SDNode *N) {
1720 SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1905 SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1931 SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
2089 SDValue DAGCombiner::visitADDLike(SDNode *N) {
2307 SDValue DAGCombiner::visitADD(SDNode *N) {
2330 SDValue DAGCombiner::visitADDSAT(SDNode *N) {
2433 SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2512 SDValue DAGCombiner::visitADDC(SDNode *N) {
2602 SDValue DAGCombiner::visitADDO(SDNode *N) {
2649 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2673 SDValue DAGCombiner::visitADDE(SDNode *N) {
2692 SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2753 static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2844 static SDValue combineCarryDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2908 SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2958 SDValue DAGCombiner::visitSUB(SDNode *N) {
3281 SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
3318 SDValue DAGCombiner::visitSUBC(SDNode *N) {
3346 SDValue DAGCombiner::visitSUBO(SDNode *N) {
3385 SDValue DAGCombiner::visitSUBE(SDNode *N) {
3397 SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
3414 SDValue DAGCombiner::visitMULFIX(SDNode *N) {
3436 SDValue DAGCombiner::visitMUL(SDNode *N) {
3626 SDValue DAGCombiner::useDivRem(SDNode *Node) {
3742 SDValue DAGCombiner::visitSDIV(SDNode *N) {
3805 SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3886 SDValue DAGCombiner::visitUDIV(SDNode *N) {
3943 SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3987 SDValue DAGCombiner::visitREM(SDNode *N) {
4069 SDValue DAGCombiner::visitMULHS(SDNode *N) {
4116 SDValue DAGCombiner::visitMULHU(SDNode *N) {
4175 SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
4221 SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
4252 SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
4295 SDValue DAGCombiner::visitMULO(SDNode *N) {
4323 SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
4367 SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
4530 SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4680 SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4775 bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
4812 bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
4888 bool DAGCombiner::SearchForAndLoads(SDNode *N,
4977 bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
5050 SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
5152 SDValue DAGCombiner::visitAND(SDNode *N) {
5445 SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5670 SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
5724 SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
5792 SDValue DAGCombiner::visitOR(SDNode *N) {
6235 SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
6259 SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
6598 SDValue DAGCombiner::MatchStoreCombine(StoreSDNode *N) {
6770 SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
6950 SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
6990   // InstCombine should have already unfolded this pattern, and DAGCombiner
7019 SDValue DAGCombiner::visitXOR(SDNode *N) {
7302 SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
7373 SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
7396 SDValue DAGCombiner::visitRotate(SDNode *N) {
7454 SDValue DAGCombiner::visitSHL(SDNode *N) {
7704 SDValue DAGCombiner::visitSRA(SDNode *N) {
7896 SDValue DAGCombiner::visitSRL(SDNode *N) {
8124 SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
8199 SDValue DAGCombiner::visitABS(SDNode *N) {
8215 SDValue DAGCombiner::visitBSWAP(SDNode *N) {
8228 SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
8241 SDValue DAGCombiner::visitCTLZ(SDNode *N) {
8258 SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
8268 SDValue DAGCombiner::visitCTTZ(SDNode *N) {
8285 SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
8295 SDValue DAGCombiner::visitCTPOP(SDNode *N) {
8403 SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
8421   // is also a target-independent combine here in DAGCombiner in the other
8511 SDValue DAGCombiner::visitSELECT(SDNode *N) {
8753 SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
8766 SDValue DAGCombiner::visitMSTORE(SDNode *N) {
8783 SDValue DAGCombiner::visitMGATHER(SDNode *N) {
8795 SDValue DAGCombiner::visitMLOAD(SDNode *N) {
8813 SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
8874 SDValue DAGCombiner::visitVSELECT(SDNode *N) {
8989 SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
9033 SDValue DAGCombiner::visitSETCC(SDNode *N) {
9063 SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
9078 /// This function is called by the DAGCombiner when visiting sext/zext/aext
9079 /// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
9229 void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
9251 SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
9351 SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
9431 SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
9474 static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
9505 static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
9609 SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
9898 SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
10154 SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
10303 SDValue DAGCombiner::visitAssertExt(SDNode *N) {
10364 SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
10567 SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
10701 SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(SDNode *N) {
10717 SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) {
10733 SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
11055 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
11146 SDValue DAGCombiner::visitBITCAST(SDNode *N) {
11394 SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
11401 SDValue DAGCombiner::
11520 SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
11738 SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
12040 SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
12132 SDValue DAGCombiner::visitFADD(SDNode *N) {
12311 SDValue DAGCombiner::visitFSUB(SDNode *N) {
12390 bool DAGCombiner::isCheaperToUseNegatedFPOps(SDValue X, SDValue Y) {
12403 SDValue DAGCombiner::visitFMUL(SDNode *N) {
12537 SDValue DAGCombiner::visitFMA(SDNode *N) {
12668 SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
12735 SDValue DAGCombiner::visitFDIV(SDNode *N) {
12839 SDValue DAGCombiner::visitFREM(SDNode *N) {
12856 SDValue DAGCombiner::visitFSQRT(SDNode *N) {
12887 SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
12934 SDValue DAGCombiner::visitFPOW(SDNode *N) {
13045 SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
13106 SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
13194 SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
13209 SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
13224 SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
13280 SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
13333 SDValue DAGCombiner::visitFCEIL(SDNode *N) {
13344 SDValue DAGCombiner::visitFTRUNC(SDNode *N) {
13368 SDValue DAGCombiner::visitFFLOOR(SDNode *N) {
13380 SDValue DAGCombiner::visitFNEG(SDNode *N) {
13468 SDValue DAGCombiner::visitFMINNUM(SDNode *N) {
13472 SDValue DAGCombiner::visitFMAXNUM(SDNode *N) {
13476 SDValue DAGCombiner::visitFMINIMUM(SDNode *N) {
13480 SDValue DAGCombiner::visitFMAXIMUM(SDNode *N) {
13484 SDValue DAGCombiner::visitFABS(SDNode *N) {
13527 SDValue DAGCombiner::visitBRCOND(SDNode *N) {
13556 SDValue DAGCombiner::rebuildSetCC(SDValue N) {
13654 SDValue DAGCombiner::visitBR_CC(SDNode *N) {
13785 bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
14008 bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
14119 SDValue DAGCombiner::SplitIndexingFromLoad(LoadSDNode *LD) {
14146 bool DAGCombiner::getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val) {
14171 bool DAGCombiner::extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val) {
14197 SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
14294 SDValue DAGCombiner::visitLOAD(SDNode *N) {
14867 bool DAGCombiner::SliceUpLoad(SDNode *N) {
15039                                 DAGCombiner *DC) {
15100 SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
15222 SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
15283 bool DAGCombiner::isMulAddWithConstProfitable(SDNode *MulNode,
15344 SDValue DAGCombiner::getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
15364 bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
15521 void DAGCombiner::getStoreMergeCandidates(
15680 bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
15744 bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
16278 SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
16303 SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) {
16384 SDValue DAGCombiner::visitSTORE(SDNode *N) {
16569 SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) {
16646 SDValue DAGCombiner::splitMergedValStore(StoreSDNode *ST) {
16728 SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) {
16854 SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
16942 SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
17062 SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
17322 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
17434 SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N,
17630 SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
17845 SDValue DAGCombiner::convertBuildVecZextToZext(SDNode *N) {
17892 SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
18110 SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
18490 SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
19151 SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
19522 SDValue DAGCombiner::visitSCALAR_TO_VECTOR(SDNode *N) {
19573 SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
19717 SDValue DAGCombiner::visitFP_TO_FP16(SDNode *N) {
19727 SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) {
19742 SDValue DAGCombiner::visitVECREDUCE(SDNode *N) {
19776 SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
19909 SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
20009 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
20045 bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
20234 SDValue DAGCombiner::foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0,
20309 SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
20364 SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
20513 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
20525 SDValue DAGCombiner::BuildSDIV(SDNode *N) {
20543 SDValue DAGCombiner::BuildSDIVPow2(SDNode *N) {
20566 SDValue DAGCombiner::BuildUDIV(SDNode *N) {
20584 SDValue DAGCombiner::BuildLogBase2(SDValue V, const SDLoc &DL) {
20601 SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
20668 SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est,
20700 SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est,
20740 SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
20801 SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags) {
20805 SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) {
20810 bool DAGCombiner::isAlias(SDNode *Op0, SDNode *Op1) const {
20939 void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
21046 SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
21088 bool DAGCombiner::parallelizeChainedStores(StoreSDNode *St) {
21196 bool DAGCombiner::findBetterNeighborChains(StoreSDNode *St) {
21227   DAGCombiner(*this, AA, OptLevel).Run(Level);