Lines Matching refs:DAG

1 //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
9 // This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
10 // both before and after the DAG is legalized.
89                  cl::desc("Enable DAG combiner's use of IR alias analysis"));
93         cl::desc("Enable DAG combiner's use of TBAA"));
98                    cl::desc("Only use DAG-combiner alias analysis in this"
111                     cl::desc("DAG combiner may split indexing from loads"));
115                        cl::desc("DAG combiner enable merging multiple stores "
130     SelectionDAG &DAG;
145     /// due to nodes being deleted from the underlying DAG.
151     /// them) when they are deleted from the underlying DAG. It relies on
161     /// This is used to allow us to reliably add any operands of a DAG node
173     // AA - Used for DAG load/store alias analysis.
191     // failed combine which may have created a DAG node.
225         : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
227       ForCodeSize = DAG.shouldOptForSize();
279     /// Replaces all uses of the results of one DAG node with new values.
283     /// Replaces all uses of the results of one DAG node with new values.
288     /// Replaces all uses of the results of one DAG node with new values.
362     /// target-specific DAG combines.
696     SelectionDAG &getDAG() const { return DAG; }
702       return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
714       return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
799   DAG.DeleteNode(N);
938       Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
939       if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
944       if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
965   if (SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
966     if (SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
968       if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, C1, C2))
969         return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
975       SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
978       return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
1009   LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
1010              To[0].getNode()->dump(&DAG);
1018   DAG.ReplaceAllUsesWith(N, To);
1042   DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1058   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1068   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
1069              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
1082   TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1093   LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);
1094              dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);
1104   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1106   LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
1107              Trunc.getNode()->dump(&DAG); dbgs() << '\n');
1109   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1110   DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1124     return DAG.getExtLoad(ExtType, DL, PVT,
1134       return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1138       return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1143     return DAG.getNode(ExtOpc, DL, PVT, Op);
1149   return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1165   return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1166                      DAG.getValueType(OldVT));
1180   return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1206     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
1218         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1271     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
1288         DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1322     LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG));
1323     return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1357     SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1360     SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1362     LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
1363                Result.getNode()->dump(&DAG); dbgs() << '\n');
1365     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1366     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1396       DAG.DeleteNode(N);
1405 //  Main DAG Combiner implementation
1418   for (SDNode &Node : DAG.allnodes())
1424   HandleSDNode Dummy(DAG.getRoot());
1429     // N is deleted from the DAG, since they too may now be dead or may have a
1436     // If this combine is running after legalizing the DAG, re-legalize any
1440       bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1449     LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
1477     LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG));
1480       DAG.ReplaceAllUsesWith(N, RV.getNode());
1484       DAG.ReplaceAllUsesWith(N, &RV);
1499   DAG.setRoot(Dummy.getValue());
1500   DAG.RemoveDeadNodes();
1640   // If nothing happened, try a target-specific DAG combine.
1648       // Expose the DAG combiner to the target combiner impls.
1650         DagCombineInfo(DAG, Level, false, this);
1686   // version is already present in the DAG.
1695       SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1885       Result = DAG.getEntryNode();
1894         Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
1896         Result = DAG.getTokenFactor(SDLoc(N), Ops);
1918     DAG.ReplaceAllUsesWith(N, Ops.data());
1990   SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
1991                           : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
1997   SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
1998                           : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
2004   SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
2009 static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
2042   SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
2043   SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
2044                        DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
2045   return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
2050 static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2081   SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
2083   return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT));
2114   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2116     if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2117       return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2119     return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
2131       SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N1.getNode(),
2134       return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
2140       SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N1.getNode(),
2143       return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2157         SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2158         return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2166         DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2167       SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2168       return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2182     return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2186     return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2199     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2205     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2211     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2217     return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2224     return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2235       return DAG.getNode(ISD::SUB, DL, VT,
2236                          DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2237                          DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2248       return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
2258       return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2276         return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
2285       SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2286                                 DAG.getAllOnesConstant(DL, VT));
2287       return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
2294     SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2295     return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
2316   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2319   if (SDValue V = foldAddSubOfSignBit(N, DAG))
2324       DAG.haveNoCommonBitsSet(N0, N1))
2325     return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2350     return DAG.getAllOnesConstant(DL, VT);
2352   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2354     if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2355       return DAG.getNode(Opcode, DL, VT, N1, N0);
2357     return DAG.FoldConstantArithmetic(Opcode, DL, VT, N0.getNode(),
2367     if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2368       return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
2419                                  SelectionDAG &DAG, const SDLoc &DL) {
2424   if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits())
2429   return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0));
2441     return DAG.getNode(ISD::SUB, DL, VT, N0,
2442                        DAG.getNode(ISD::SHL, DL, VT,
2446   if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2455     SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2456                               DAG.getAllOnesConstant(DL, VT));
2457     return DAG.getNode(ISD::SUB, DL, VT, N1, Not);
2465     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2466     return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2472     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2473     return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
2482     SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2483     return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2490       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2491                                  DAG.getConstant(1, DL, VT));
2492       return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2499     return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2505       return DAG.getNode(ISD::ADDCARRY, DL,
2506                          DAG.getVTList(VT, Carry.getValueType()), N0,
2507                          DAG.getConstant(0, DL, VT), Carry);
2520     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2521                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2527     return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2531     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2535   if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2536     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2537                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2543                            SelectionDAG &DAG, const TargetLowering &TLI) {
2550     Cst = DAG.getConstant(1, DL, VT);
2553     Cst = DAG.getAllOnesConstant(DL, VT);
2557   return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
2567 static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2571     return flipBoolean(V, SDLoc(V), DAG, TLI);
2598     return flipBoolean(V, SDLoc(V), DAG, TLI);
2613     return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2614                      DAG.getUNDEF(CarryVT));
2617   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2618       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2619     return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
2623     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2627     if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2628       return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2629                        DAG.getConstant(0, DL, CarryVT));
2633       SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2634                                 DAG.getConstant(0, DL, VT), N0.getOperand(0));
2636                        flipBoolean(Sub.getValue(1), DL, DAG, TLI));
2658     SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2659     if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2660       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2667       return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2668                          DAG.getConstant(0, SDLoc(N), VT), Carry);
2682     return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2687     return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2702     return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2708       return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2715     SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2717     return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2718                                     DAG.getConstant(1, DL, VT)),
2719                      DAG.getConstant(0, DL, CarryVT));
2737  * carry is now linearized, other tranforms can kick in and optimize the DAG.
2753 static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2773     Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT);
2782     SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
2784     return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X,
2785                        DAG.getConstant(0, DL, X.getValueType()),
2844 static SDValue combineCarryDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2886       DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0),
2902   DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0));
2904     return DAG.getConstant(0, DL, MVT::i1);
2912     if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
2914       SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
2917                        flipBoolean(Sub.getValue(1), DL, DAG, TLI));
2928     return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2933    * a diamond carry propagation. In which case we try to transform the DAG
2938     if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
2940     if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
2950                              SelectionDAG &DAG, bool LegalOperations) {
2952     return DAG.getConstant(0, DL, VT);
2954     return DAG.getConstant(0, DL, VT);
2977     return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
2978   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2979       DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2981     return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2992     return DAG.getNode(ISD::ADD, DL, VT, N0,
2993                        DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3007           return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
3015     if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
3028     return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
3032     return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
3050     SDValue NewC = DAG.FoldConstantArithmetic(
3053     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
3061       SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
3064       return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
3072     SDValue NewC = DAG.FoldConstantArithmetic(
3075     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
3082     SDValue NewC = DAG.FoldConstantArithmetic(
3085     return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
3093     return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
3099     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
3105     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
3110     return DAG.getNode(ISD::ADD, DL, VT, N0,
3111                        DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
3123           DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT));
3124       return DAG.getNode(ISD::AND, DL, VT, A, InvB);
3132       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3135       return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3139       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3142       return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3152   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3155   if (SDValue V = foldAddSubOfSignBit(N, DAG))
3158   if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3163     SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3164                               DAG.getAllOnesConstant(DL, VT));
3165     return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
3173     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3174     return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
3181     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3182     return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
3187     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3188     return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
3194     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3195     return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
3200     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3201     return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
3211     SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3212     return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
3224             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
3234         return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
3240           return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3248       SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3249                                  DAG.getConstant(1, DL, VT));
3250       return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
3261       SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3262       return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
3270       SDValue Zero = DAG.getConstant(0, DL, VT);
3271       SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3272       return DAG.getNode(ISD::ADDCARRY, DL,
3273                          DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
3298     return DAG.getConstant(0, DL, VT);
3302     return DAG.getConstant(0, DL, VT);
3304   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3305       DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
3307     return DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, N0.getNode(),
3326     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3327                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3331     return CombineTo(N, DAG.getConstant(0, DL, VT),
3332                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3336     return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3340     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3341                      DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3357     return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3358                      DAG.getUNDEF(CarryVT));
3362     return CombineTo(N, DAG.getConstant(0, DL, VT),
3363                      DAG.getConstant(0, DL, CarryVT));
3369     return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
3370                        DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3375     return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
3379     return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3380                      DAG.getConstant(0, DL, CarryVT));
3392     return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
3406       return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
3422     return DAG.getConstant(0, SDLoc(N), VT);
3425   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3426      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3427     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
3431     return DAG.getConstant(0, SDLoc(N), VT);
3443     return DAG.getConstant(0, SDLoc(N), VT);
3478     return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
3482   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3483      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3484     return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
3498     return DAG.getNode(ISD::SUB, DL, VT,
3499                        DAG.getConstant(0, DL, VT), N0);
3503       DAG.isKnownToBeAPowerOfTwo(N1) &&
3508     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3509     return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
3517     return DAG.getNode(ISD::SUB, DL, VT,
3518                        DAG.getConstant(0, DL, VT),
3519                        DAG.getNode(ISD::SHL, DL, VT, N0,
3520                             DAG.getConstant(Log2Val, DL,
3531   if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
3549           DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
3550       SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0);
3552         R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
3561     SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
3563       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
3583       SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
3584       return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
3589   if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
3591       DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
3593       return DAG.getNode(ISD::ADD, SDLoc(N), VT,
3594                          DAG.getNode(ISD::MUL, SDLoc(N0), VT,
3596                          DAG.getNode(ISD::MUL, SDLoc(N1), VT,
3678           SDVTList VTs = DAG.getVTList(VT, VT);
3679           combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
3696 static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
3711   if (DAG.isUndef(Opc, {N0, N1}))
3712     return DAG.getUNDEF(VT);
3717     return DAG.getConstant(0, DL, VT);
3728     return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
3737     return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
3759     return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
3762     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
3765     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3766                          DAG.getConstant(1, DL, VT),
3767                          DAG.getConstant(0, DL, VT));
3769   if (SDValue V = simplifyDivRem(N, DAG))
3777   if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3778     return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
3783     if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
3785       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3786       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3797   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3834     SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
3835     SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
3836     C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
3837     SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
3842     SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
3843                                DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
3847     SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
3849     SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
3851     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
3856     SDValue One = DAG.getConstant(1, DL, VT);
3857     SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
3858     SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
3859     SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
3860     SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
3861     Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
3865     SDValue Zero = DAG.getConstant(0, DL, VT);
3866     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
3869     SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
3870     SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
3877   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3903     if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
3908     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3909                          DAG.getConstant(1, DL, VT),
3910                          DAG.getConstant(0, DL, VT));
3912   if (SDValue V = simplifyDivRem(N, DAG))
3921     if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
3923       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3924       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3935   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3949       DAG.isKnownToBeAPowerOfTwo(N1)) {
3954     SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3956     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
3963         DAG.isKnownToBeAPowerOfTwo(N10)) {
3968       SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3970       SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3972       return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3977   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4001     if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
4005     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
4006                          DAG.getConstant(0, DL, VT), N0);
4008   if (SDValue V = simplifyDivRem(N, DAG))
4017     if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
4018       return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
4020     SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4021     if (DAG.isKnownToBeAPowerOfTwo(N1)) {
4023       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4025       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4028         DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
4030       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4032       return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4036   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4045   if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
4051       if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
4054       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
4055       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4080       return DAG.getConstant(0, DL, VT);
4088     return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
4089                        DAG.getConstant(N0.getScalarValueSizeInBits() - 1, DL,
4094     return DAG.getConstant(0, DL, VT);
4101     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4103       N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
4104       N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
4105       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4106       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4107             DAG.getConstant(SimpleSize, DL,
4109       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4127       return DAG.getConstant(0, DL, VT);
4135     return DAG.getConstant(0, DL, N0.getValueType());
4138     return DAG.getConstant(0, DL, VT);
4142       DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
4145     SDValue SRLAmt = DAG.getNode(
4146         ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
4148     SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4149     return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4157     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4159       N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
4160       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
4161       N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4162       N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4163             DAG.getConstant(SimpleSize, DL,
4165       return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4181     SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4189     SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4199     SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4209     SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4233     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4235       SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
4236       SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
4237       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4239       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4240             DAG.getConstant(SimpleSize, DL,
4242       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4244       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4261     SDValue Zero = DAG.getConstant(0, DL, VT);
4267     SDValue Zero = DAG.getConstant(0, DL, VT);
4276     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4278       SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
4279       SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
4280       Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4282       Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4283             DAG.getConstant(SimpleSize, DL,
4285       Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4287       Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4305   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4306       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4307     return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
4311     return CombineTo(N, DAG.getConstant(0, DL, VT),
4312                      DAG.getConstant(0, DL, CarryVT));
4317       return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
4337     return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
4340   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4341      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4342     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
4348       (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
4349       (N1.isUndef() || DAG.SignBitIsZero(N1))) {
4359       return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
4408     SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4409     return DAG.getNode(HandOpcode, DL, VT, Logic);
4430     SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4431     return DAG.getNode(HandOpcode, DL, VT, Logic);
4442     SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4443     return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
4451     SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4452     return DAG.getNode(HandOpcode, DL, VT, Logic);
4468       SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4469       return DAG.getNode(HandOpcode, DL, VT, Logic);
4503       ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4507       SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
4509       return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
4516       ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4520       SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
4522       return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
4575       SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
4577       return DAG.getSetCC(DL, VT, Or, LR, CC1);
4594       SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
4596       return DAG.getSetCC(DL, VT, And, LR, CC1);
4606     SDValue One = DAG.getConstant(1, DL, OpVT);
4607     SDValue Two = DAG.getConstant(2, DL, OpVT);
4608     SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
4610     return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
4620       SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
4621       SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
4622       SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
4623       SDValue Zero = DAG.getConstant(0, DL, OpVT);
4624       return DAG.getSetCC(DL, VT, Or, Zero, CC1);
4644           SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
4645           SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
4646           SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
4647           SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
4648           SDValue Zero = DAG.getConstant(0, DL, OpVT);
4649           return DAG.getSetCC(DL, VT, And, Zero, CC0);
4670       return DAG.getSetCC(DL, VT, LL, LR, NewCC);
4686     return DAG.getConstant(0, DL, VT);
4706           if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
4711                 DAG.getNode(ISD::ADD, DL0, VT,
4712                             N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
4738         EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
4758           EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
4759           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
4762           SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
4763           SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
4764           SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
4765           SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
4766           return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
4782   ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4836       !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
4933       EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
5003       SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
5006       DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
5008         DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
5019       SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
5022       DAG.UpdateNodeOperands(LogicN, Op0, And);
5028       SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
5030       DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
5033             DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
5039     DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
5092   SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
5094   SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
5102 static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
5107   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5142   SDValue X = DAG.getZExtOrTrunc(Srl.getOperand(0), DL, VT);
5143   EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5144   SDValue Mask = DAG.getConstant(
5146   SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask);
5147   SDValue Zero = DAG.getConstant(0, DL, VT);
5148   SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
5149   return DAG.getZExtOrTrunc(Setcc, DL, VT);
5169       return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
5173       return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
5187     return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
5189   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5190       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5191     return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
5197   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
5199     return DAG.getConstant(0, SDLoc(N), VT);
5213   if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
5228     if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
5229       SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5327         NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
5357       DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 0), Res);
5394         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
5415     if (DAG.MaskedValueIsZero(N1, ExtBits) &&
5419           DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
5438     if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
5542         !DAG.MaskedValueIsZero(
5547   SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
5550     Res = DAG.getNode(ISD::SRL, DL, VT, Res,
5551                       DAG.getConstant(OpSizeInBits - 16, DL,
5707   SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
5712   SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
5714     return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
5716     return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
5717   return DAG.getNode(ISD::OR, DL, VT,
5718                      DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
5719                      DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
5730     return DAG.getAllOnesConstant(DL, VT);
5750         if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
5751             DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
5752           SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5754           return DAG.getNode(ISD::AND, DL, VT, X,
5755                              DAG.getConstant(LHSMask | RHSMask, DL, VT));
5767     SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5769     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
5777     SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N) {
5782       return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(0), N1);
5786       return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(1), N1);
5815       return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
5818       return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
5876                                           Mask, DAG);
5888     return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
5890   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5891      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5892     return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
5904   if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
5910   if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
5930     if (SDValue COR = DAG.FoldConstantArithmetic(
5932       SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
5934       return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
5938   if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
5940   if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
5961       DAG.haveNoCommonBitsSet(N0, N1))
5968 static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
5970       DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
5978 static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
5980   Op = stripConstantMask(DAG, Op, Mask);
6012 static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
6020   ExtractFrom = stripConstantMask(DAG, ExtractFrom, Mask);
6035     return DAG.getNode(ISD::SHL, DL, ShiftedVT, OppShiftLHS,
6036                        DAG.getShiftAmountConstant(1, ShiftedVT, DL));
6114   SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
6115   return DAG.getNode(Opcode, DL, ResVT, OppShiftLHS, NewShiftNode);
6128                            SelectionDAG &DAG) {
6163       KnownBits Known = DAG.computeKnownBits(Neg.getOperand(0));
6185       KnownBits Known = DAG.computeKnownBits(Pos.getOperand(0));
6247   if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
6249     return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
6276       return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot);
6283   matchRotateHalf(DAG, LHS, LHSShift, LHSMask);
6287   matchRotateHalf(DAG, RHS, RHSShift, RHSMask);
6304             extractShiftForRotate(DAG, LHSShift, RHS, RHSMask, DL))
6309             extractShiftForRotate(DAG, RHSShift, LHS, LHSMask, DL))
6344     SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
6349       SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
6353         SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
6354         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
6355                            DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
6358         SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
6359         Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
6360                            DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
6363       Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
6613     *DAG.getContext(), Width * N->getMemoryVT().getSizeInBits());
6673     BaseIndexOffset Ptr = BaseIndexOffset::match(Store, DAG);
6677     else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
6705   bool NeedsBswap = DAG.getDataLayout().isBigEndian() != *IsBigEndian;
6716       TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
6724     CombinedValue = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
6729     CombinedValue = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, CombinedValue);
6732     DAG.getStore(Chain, SDLoc(N),  CombinedValue, FirstStore->getBasePtr(),
6735   // Rely on other DAG combine rules to remove the other individual stores.
6736   DAG.ReplaceAllUsesWith(N, NewStore.getNode());
6780   bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
6831     BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
6835     else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
6858       EVT::getIntegerVT(*DAG.getContext(), (ByteWidth - ZeroExtendedBytes) * 8);
6911       TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
6916   SDValue NewLoad = DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD,
6923     DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
6930           ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad,
6931                         DAG.getShiftAmountConstant(ZeroExtendedBytes * 8, VT,
6934   return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad);
6949 //       patterns are simplified in IR and shouldn't be created in the DAG
7005     SDValue NotX = DAG.getNOT(DL, X, VT);
7006     SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
7007     SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
7008     SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
7009     return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
7012   SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
7013   SDValue NotM = DAG.getNOT(DL, M, VT);
7014   SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
7016   return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
7039     return DAG.getConstant(0, DL, VT);
7049     return DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, N0C, N1C);
7051   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
7052      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
7053     return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
7077         return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
7079         return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
7090     V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V,
7091                     DAG.getConstant(1, DL0, V.getValueType()));
7093     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V);
7102       N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
7103       N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
7105       return DAG.getNode(NewOpcode, DL, VT, N00, N01);
7114       N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
7115       N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
7117       return DAG.getNode(NewOpcode, DL, VT, N00, N01);
7126     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1),
7127                        DAG.getAllOnesConstant(DL, VT));
7133     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
7140     SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
7142     return DAG.getNode(ISD::AND, DL, VT, NotX, N1);
7160           SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
7161           return DAG.getNode(N0Opcode, DL, VT, Not, N0.getOperand(1));
7178             return DAG.getNode(ISD::ABS, DL, VT, S0);
7185     return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
7207     return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
7224   if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
7234 static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG) {
7290   SDValue ShiftSumC = DAG.getConstant(*C0Val + C1Val, DL, ShiftAmtVT);
7291   SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC);
7292   SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1);
7293   return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2);
7319     if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
7364   SDValue NewRHS = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(1),
7368   SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
7370   return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS);
7385       SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
7386       SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
7389       return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
7410     if (DAG.MaskedValueIsZero(N1, ModuloMask))
7419       return DAG.getNode(N->getOpcode(), dl, VT, N0,
7420                          DAG.getConstant(RotAmt, dl, N1.getValueType()));
7428       return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
7434     SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
7435     SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
7441               DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
7442         SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
7443         SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
7446         return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
7457   if (SDValue V = DAG.simplifyShift(N0, N1))
7481           if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
7483             return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
7495     return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
7501   if (DAG.MaskedValueIsZero(SDValue(N, 0),
7503     return DAG.getConstant(0, SDLoc(N), VT);
7509       return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
7526       return DAG.getConstant(0, SDLoc(N), VT);
7537       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7538       return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
7567       return DAG.getConstant(0, SDLoc(N), VT);
7581       SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0));
7582       SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT);
7583       Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1);
7584       return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum);
7607       SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT);
7608       NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL);
7610       return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
7624         return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
7625                            DAG.getConstant(C2 - C1, DL, ShiftVT));
7626       return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
7627                          DAG.getConstant(C1 - C2, DL, ShiftVT));
7648           Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
7649                               DAG.getConstant(c2 - c1, DL, ShiftVT));
7653           Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
7654                               DAG.getConstant(c1 - c2, DL, ShiftVT));
7657         return DAG.getNode(ISD::AND, DL, VT, Shift,
7658                            DAG.getConstant(Mask, DL, VT));
7667     SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
7668     SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
7669     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
7681     SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
7682     SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7685     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
7692     SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
7694       return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
7707   if (SDValue V = DAG.simplifyShift(N0, N1))
7716   if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
7730     return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
7739     EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
7741       ExtVT = EVT::getVectorVT(*DAG.getContext(),
7746       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
7747                          N0.getOperand(0), DAG.getValueType(ExtVT));
7765       ShiftValues.push_back(DAG.getConstant(ShiftSum, DL, ShiftSVT));
7771         ShiftValue = DAG.getBuildVector(ShiftVT, DL, ShiftValues);
7774       return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue);
7787       LLVMContext &Ctx = *DAG.getContext();
7806         SDValue Amt = DAG.getConstant(ShiftAmt, DL,
7808         SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
7810         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
7812         return DAG.getNode(ISD::SIGN_EXTEND, DL,
7828       LLVMContext &Ctx = *DAG.getContext();
7840         SDValue Trunc = DAG.getZExtOrTrunc(Shl.getOperand(0), DL, TruncVT);
7841         SDValue ShiftC = DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).
7843         SDValue Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC);
7844         return DAG.getSExtOrTrunc(Add, DL, VT);
7853       return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
7871         SDValue Amt = DAG.getConstant(N1C->getZExtValue() + TruncBits, DL,
7874             DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt);
7875         return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
7886   if (DAG.SignBitIsZero(N0))
7887     return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
7899   if (SDValue V = DAG.simplifyShift(N0, N1))
7916     return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
7922   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
7924     return DAG.getConstant(0, SDLoc(N), VT);
7936       return DAG.getConstant(0, SDLoc(N), VT);
7948       SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
7949       return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
7968           return DAG.getConstant(0, DL, VT);
7969         SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
7970         SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
7972         return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift);
7979         SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
7980         SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
7982         SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(InnerShiftSize,
7985         SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask);
7986         return DAG.getNode(ISD::TRUNCATE, DL, VT, And);
7997         DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
7999     return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask);
8009       return DAG.getUNDEF(VT);
8014       SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
8016                           DAG.getConstant(ShiftAmt, DL0,
8021       return DAG.getNode(ISD::AND, DL, VT,
8022                          DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
8023                          DAG.getConstant(Mask, DL, VT));
8031       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
8037     KnownBits Known = DAG.computeKnownBits(N0.getOperand(0));
8041     if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
8046     if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
8059         Op = DAG.getNode(ISD::SRL, DL, VT, Op,
8060                   DAG.getConstant(ShAmt, DL,
8066       return DAG.getNode(ISD::XOR, DL, VT,
8067                          Op, DAG.getConstant(1, DL, VT));
8075       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
8135     if (DAG.MaskedValueIsZero(
8150       return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1,
8151                          DAG.getConstant(RotAmt, SDLoc(N), ShAmtTy));
8163       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1,
8164                          DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt,
8167       return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
8168                          DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt,
8178     if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
8179       return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2);
8180     if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
8181       return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2);
8190     return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2);
8204   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8205     return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0);
8210   if (DAG.SignBitIsZero(N0))
8220   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8221     return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
8233   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8234     return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0);
8246   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8247     return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
8251     if (DAG.isKnownNeverZero(N0))
8252       return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8263   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8264     return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8273   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8274     return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
8278     if (DAG.isKnownNeverZero(N0))
8279       return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8290   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8291     return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
8300   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
8301     return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
8307 static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS,
8310   const TargetOptions &Options = DAG.getTarget().Options;
8315          DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS);
8322                                    SelectionDAG &DAG) {
8326   EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
8339       return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS);
8343       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
8354       return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS);
8358       return DAG.getNode(Opcode, DL, VT, LHS, RHS);
8368 static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG) {
8389     SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
8390     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
8391     return DAG.getNode(ISD::OR, DL, VT, Sra, C1);
8396     SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
8397     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
8398     return DAG.getNode(ISD::AND, DL, VT, Sra, C1);
8426       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8428         NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond);
8433       SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
8435         NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond);
8441         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
8447         Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
8461           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
8462         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
8467           Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
8468         return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
8474           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
8475         SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT);
8476         return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC);
8479       if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8502         DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
8505     return DAG.getZExtOrTrunc(NotCond, DL, VT);
8520   if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8526     return DAG.getNode(ISD::OR, DL, VT, N0, N2);
8533     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8535     return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2);
8539     SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
8541     return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1);
8546     return DAG.getNode(ISD::AND, DL, VT, N0, N1);
8558     // to the right anyway if we find the inner select exists in the DAG anyway
8562         TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
8569           DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags);
8571         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
8581       SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(),
8584         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
8599           SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
8600           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1,
8605           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
8618           SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
8619           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, 
8624           return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
8631   if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
8632     SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1);
8646     if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2, TLI))
8648                                                 CC, TLI, DAG))
8674         SDVTList VTs = DAG.getVTList(VT, VT0);
8675         SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
8676         return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0));
8686       SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1,
8700 static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
8747   return DAG.getNode(
8851     SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
8852     return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
8859     SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT);
8860     SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT);
8861     return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC);
8864   if (SDValue V = foldSelectOfConstantsUsingSra(N, DAG))
8881   if (SDValue V = DAG.simplifySelect(N0, N1, N2))
8885   if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
8886     return DAG.getSelect(DL, VT, F, N2, N1);
8909         return DAG.getNode(ISD::ABS, DL, VT, LHS);
8911       SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS,
8912                                   DAG.getConstant(VT.getScalarSizeInBits() - 1,
8914       SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
8917       return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
8926     if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, LHS, RHS, TLI)) {
8928               combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC, TLI, DAG))
8954         SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
8955         SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
8957         SDValue WideSetCC = DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
8958         return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
8979     if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
9013       // coherent the DAG creation, no setcc node is created in this case
9017       SDValue SelectOp = DAG.getNode(
9071     return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
9083                                          SelectionDAG &DAG, bool LegalTypes) {
9098     return DAG.getNode(Opcode, DL, VT, N0);
9120       return DAG.getSelect(DL, VT, N0->getOperand(0),
9121                            DAG.getNode(FoldOpc, DL, VT, Op1),
9122                            DAG.getNode(FoldOpc, DL, VT, Op2));
9148       Elts.push_back(IsZext ? DAG.getConstant(0, DL, SVT) : DAG.getUNDEF(SVT));
9157       Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
9159       Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
9162   return DAG.getBuildVector(VT, DL, Elts);
9242         Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
9246     CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
9300     SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
9301     SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
9321     SDValue SplitLoad = DAG.getExtLoad(
9326     BasePtr = DAG.getMemBasePlusOffset(BasePtr, Stride, DL);
9332   SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
9333   SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
9343       DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
9397   SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
9402   SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
9408   SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
9409                             DAG.getConstant(Mask, DL0, VT));
9414     DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
9416     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
9463     CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
9464     CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
9466     CastA = DAG.getNode(CastOpcode, DL, VT, A);
9467     CastB = DAG.getNode(CastOpcode, DL, VT, B);
9469   return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
9474 static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
9493       DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
9496   DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
9505 static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
9527   SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
9535     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
9539         DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
9545 static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG,
9564   SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru());
9565   SDValue NewLoad = DAG.getMaskedLoad(
9569   DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), SDValue(NewLoad.getNode(), 1));
9573 static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
9597     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9599       SDValue NotX = DAG.getNOT(DL, X, VT);
9600       SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT);
9603       return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount);
9614   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9620     return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
9641     unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
9652         return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
9657         return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
9664         Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
9666         Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
9667       return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
9668                          DAG.getValueType(N0.getValueType()));
9674           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
9679       tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::SEXTLOAD,
9690           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
9708         SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
9714         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
9715                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
9723               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
9727           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
9729           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
9738   if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
9765           return DAG.getSetCC(DL, VT, N00, N01, CC);
9772           SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
9773           return DAG.getSExtOrTrunc(VsetCC, DL, VT);
9791                              ? DAG.getAllOnesConstant(DL, VT)
9792                              : DAG.getBoolConstant(true, DL, VT, N00VT);
9793     SDValue Zero = DAG.getConstant(0, DL, VT);
9806         SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
9807         return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
9814       DAG.SignBitIsZero(N0))
9815     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0);
9826     SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT);
9827     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext);
9835     SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT);
9836     return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
9846 static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
9850     Known = DAG.computeKnownBits(Op);
9870   Known = DAG.computeKnownBits(Op);
9878 static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG) {
9887   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9894   SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT);
9895   return DAG.getNode(ISD::CTPOP, DL, VT, NewZext);
9902   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
9908     return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
9916   if (isTruncateOf(DAG, N0, Op, Known)) {
9925       return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
9951         Op = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
9953         SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
9955         DAG.transferDbgValues(N0, ZExtOrTrunc);
9961       SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
9963       SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
9966       DAG.transferDbgValues(N0, And);
9980     X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
9984     return DAG.getNode(ISD::AND, DL, VT,
9985                        X, DAG.getConstant(Mask, DL, VT));
9990           tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
9995       tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::ZEXTLOAD,
10032         SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
10039         SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
10040                                   ExtLoad, DAG.getConstant(Mask, DL, VT));
10048               DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
10052           DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
10054           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
10070           DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
10073   if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
10090       SDValue VecOnes = DAG.getConstant(1, DL, VT);
10093         SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
10095         return DAG.getNode(ISD::AND, DL, VT, VSetCC, VecOnes);
10103           DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
10105       return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT),
10112             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
10113             DAG.getConstant(0, DL, VT),
10138       ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
10140     return DAG.getNode(N0.getOpcode(), DL, VT,
10141                        DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
10148   if (SDValue NewCtPop = widenCtPop(N, DAG))
10158   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10167     return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
10185     return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
10196     X = DAG.getAnyExtOrTrunc(X, DL, VT);
10199     return DAG.getNode(ISD::AND, DL, VT,
10200                        X, DAG.getConstant(Mask, DL, VT));
10217       SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
10226         DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
10229         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
10246       SDValue ExtLoad = DAG.getExtLoad(ExtType, SDLoc(N),
10250       DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
10273         return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0),
10282         DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0),
10285       return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT);
10291             DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
10292             DAG.getConstant(0, DL, VT),
10297   if (SDValue NewCtPop = widenCtPop(N, DAG))
10329     SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
10330     SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10332     return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
10349       SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
10351       return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
10398       ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShiftAmt);
10400       ExtVT = EVT::getIntegerVT(*DAG.getContext(),
10421     ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
10462           EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(),
10503   if (DAG.getDataLayout().isBigEndian())
10513       DAG.getMemBasePlusOffset(LN0->getBasePtr(), PtrOff, DL, Flags);
10518     Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr,
10522     Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
10529   DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
10542       Result = DAG.getConstant(0, DL, VT);
10544       Result = DAG.getNode(ISD::SHL, DL, VT,
10545                           Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
10551     if (DAG.getDataLayout().isBigEndian())
10558     SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
10559     Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC);
10560     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
10576     return DAG.getUNDEF(VT);
10579   if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
10580     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
10583   if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
10589     return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
10600          (N00Bits - DAG.ComputeNumSignBits(N00)) < EVTBits) &&
10602       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00);
10612       return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT,
10622       return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
10626   if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, EVTBits - 1)))
10627     return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT.getScalarType());
10647         unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
10649           return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0),
10665     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
10681     SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
10694       return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
10706     return DAG.getUNDEF(VT);
10708   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10722     return DAG.getUNDEF(VT);
10724   if (SDValue Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes))
10737   bool isLE = DAG.getDataLayout().isLittleEndian();
10745     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
10748   if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
10749     SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
10760       return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0));
10763       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
10792     EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
10798       EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
10802       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
10803                          DAG.getBitcast(NVT, N0.getOperand(0)),
10804                          DAG.getConstant(Index, DL, IndexTy));
10814       SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
10815       SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
10816       return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1);
10825     KnownBits Known = DAG.computeKnownBits(Amt);
10829       EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
10831       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
10833         Amt = DAG.getZExtOrTrunc(Amt, SL, AmtVT);
10836       return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt);
10847       SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op);
10850     return DAG.getBuildVector(VT, DL, TruncOps);
10879       return DAG.getBuildVector(VT, SDLoc(N), Opnds);
10891     if (SDValue Shorter = DAG.GetDemandedBits(N0, Mask))
10892       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter);
10907         SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
10911         DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
10935       VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
10941       return DAG.getUNDEF(VT);
10948           Opnds.push_back(DAG.getUNDEF(VTs[i]));
10951         SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
10955       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds);
10971       EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
10973       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc,
10974                          DAG.getConstant(Idx, SL, IdxVT));
10992     auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0));
10993     auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
10994     auto VTs = DAG.getVTList(VT, N0->getValueType(1));
10995     return DAG.getNode(N0.getOpcode(), SL, VTs, X, Y, N0.getOperand(2));
11009         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
11036         SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
11037         SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
11038         return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR);
11064   if (DAG.getDataLayout().isBigEndian())
11073       DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) {
11075     unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
11076         VT.getTypeForEVT(*DAG.getContext()));
11080       return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
11087 static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
11090   return DAG.getDataLayout().isBigEndian() ? 1 : 0;
11093 static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
11136     SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0));
11139       return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp);
11151     return DAG.getUNDEF(VT);
11178       SDValue C = DAG.getBitcast(VT, N0);
11186     return DAG.getBitcast(VT, N0.getOperand(0));
11192       TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) ==
11193           TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) &&
11203     if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG,
11206           DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
11209       DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
11214   if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
11233     SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
11239       SDValue SignBit = DAG.getConstant(
11248             DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv,
11249                         DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
11252         FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit);
11256           DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
11258       return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits);
11262       return DAG.getNode(ISD::XOR, DL, VT,
11263                          NewConv, DAG.getConstant(SignBit, DL, VT));
11265     return DAG.getNode(ISD::AND, DL, VT,
11266                        NewConv, DAG.getConstant(~SignBit, DL, VT));
11284     EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
11286       SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
11292         X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X);
11298         X = DAG.getNode(ISD::SRL, DL,
11300                         DAG.getConstant(OrigXWidth-VTWidth, DL,
11303         X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
11309         SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11311         SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
11313         SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
11315         SDValue XorResult64 = DAG.getNode(
11317             DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
11321             DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64,
11322                         DAG.getConstant(SignBit, SDLoc(XorResult64), MVT::i64));
11325             DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
11327         return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits);
11330       X = DAG.getNode(ISD::AND, SDLoc(X), VT,
11331                       X, DAG.getConstant(SignBit, SDLoc(X), VT));
11334       SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
11335       Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
11336                         Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT));
11339       return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
11366         return DAG.getBitcast(VT, Op);
11386         TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG);
11419         Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op);
11420       Ops.push_back(DAG.getBitcast(DstEltVT, Op));
11423     EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
11425     return DAG.getBuildVector(VT, SDLoc(BV), Ops);
11434     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
11442     EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
11460       bool isLE = DAG.getDataLayout().isLittleEndian();
11475         Ops.push_back(DAG.getUNDEF(DstEltVT));
11477         Ops.push_back(DAG.getConstant(NewBits, DL, DstEltVT));
11480     EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
11481     return DAG.getBuildVector(VT, DL, Ops);
11487   EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
11493       Ops.append(NumOutputsPerInput, DAG.getUNDEF(DstEltVT));
11502       Ops.push_back(DAG.getConstant(ThisVal, DL, DstEltVT));
11507     if (DAG.getDataLayout().isBigEndian())
11511   return DAG.getBuildVector(VT, DL, Ops);
11526   const TargetOptions &Options = DAG.getTarget().Options;
11529   bool HasFMAD = (LegalOperations && TLI.isFMADLegalForFAddFSub(DAG, N));
11533       TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) &&
11548   const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo();
11572     return DAG.getNode(PreferredFusedOpcode, SL, VT,
11579     return DAG.getNode(PreferredFusedOpcode, SL, VT,
11589         TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11591       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11592                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11594                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11604         TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11606       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11607                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11609                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11621       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11623                          DAG.getNode(PreferredFusedOpcode, SL, VT,
11634       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11636                          DAG.getNode(PreferredFusedOpcode, SL, VT,
11648       return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
11649                          DAG.getNode(PreferredFusedOpcode, SL, VT,
11650                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
11651                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
11659             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11676       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11677                          DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
11678                          DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
11679                          DAG.getNode(PreferredFusedOpcode, SL, VT,
11680                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
11681                                      DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
11689             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11705             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11724             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11744   const TargetOptions &Options = DAG.getTarget().Options;
11746   bool HasFMAD = (LegalOperations && TLI.isFMADLegalForFAddFSub(DAG, N));
11750       TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) &&
11766   const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo();
11784     return DAG.getNode(PreferredFusedOpcode, SL, VT,
11786                        DAG.getNode(ISD::FNEG, SL, VT, N1), Flags);
11792     return DAG.getNode(PreferredFusedOpcode, SL, VT,
11793                        DAG.getNode(ISD::FNEG, SL, VT,
11803     return DAG.getNode(PreferredFusedOpcode, SL, VT,
11804                        DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
11805                        DAG.getNode(ISD::FNEG, SL, VT, N1), Flags);
11815         TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11817       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11818                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11820                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11822                          DAG.getNode(ISD::FNEG, SL, VT, N1), Flags);
11832         TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11834       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11835                          DAG.getNode(ISD::FNEG, SL, VT,
11836                                      DAG.getNode(ISD::FP_EXTEND, SL, VT,
11838                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11855           TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11857         return DAG.getNode(ISD::FNEG, SL, VT,
11858                            DAG.getNode(PreferredFusedOpcode, SL, VT,
11859                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
11861                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
11879           TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11881         return DAG.getNode(ISD::FNEG, SL, VT,
11882                            DAG.getNode(PreferredFusedOpcode, SL, VT,
11883                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
11885                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
11899       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11901                          DAG.getNode(PreferredFusedOpcode, SL, VT,
11904                                      DAG.getNode(ISD::FNEG, SL, VT,
11915       return DAG.getNode(PreferredFusedOpcode, SL, VT,
11916                          DAG.getNode(ISD::FNEG, SL, VT,
11919                          DAG.getNode(PreferredFusedOpcode, SL, VT,
11920                                      DAG.getNode(ISD::FNEG, SL, VT, N20),
11933             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11935           return DAG.getNode(PreferredFusedOpcode, SL, VT,
11937                              DAG.getNode(PreferredFusedOpcode, SL, VT,
11938                                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11940                                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11942                                          DAG.getNode(ISD::FNEG, SL, VT,
11959             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11961           return DAG.getNode(PreferredFusedOpcode, SL, VT,
11962                              DAG.getNode(ISD::FP_EXTEND, SL, VT,
11964                              DAG.getNode(ISD::FP_EXTEND, SL, VT,
11966                              DAG.getNode(PreferredFusedOpcode, SL, VT,
11967                                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11969                                          DAG.getNode(ISD::FP_EXTEND, SL, VT,
11971                                          DAG.getNode(ISD::FNEG, SL, VT,
11984           TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
11988         return DAG.getNode(PreferredFusedOpcode, SL, VT,
11989                            DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)),
11991                            DAG.getNode(PreferredFusedOpcode, SL, VT,
11992                                        DAG.getNode(ISD::FNEG, SL, VT,
11993                                                    DAG.getNode(ISD::FP_EXTEND, SL,
11995                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
12014           TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
12018         return DAG.getNode(PreferredFusedOpcode, SL, VT,
12019                            DAG.getNode(ISD::FNEG, SL, VT,
12020                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
12022                            DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
12023                            DAG.getNode(PreferredFusedOpcode, SL, VT,
12024                                        DAG.getNode(ISD::FNEG, SL, VT,
12025                                                    DAG.getNode(ISD::FP_EXTEND, SL,
12027                                        DAG.getNode(ISD::FP_EXTEND, SL, VT,
12049   const TargetOptions &Options = DAG.getTarget().Options;
12059       TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) &&
12081           return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
12084           return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
12085                              DAG.getNode(ISD::FNEG, SL, VT, Y), Flags);
12104           return DAG.getNode(PreferredFusedOpcode, SL, VT,
12105                              DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
12108           return DAG.getNode(PreferredFusedOpcode, SL, VT,
12109                              DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
12110                              DAG.getNode(ISD::FNEG, SL, VT, Y), Flags);
12114           return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
12115                              DAG.getNode(ISD::FNEG, SL, VT, Y), Flags);
12117           return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
12139   const TargetOptions &Options = DAG.getTarget().Options;
12149     return DAG.getNode(ISD::FADD, DL, VT, N0, N1, Flags);
12153     return DAG.getNode(ISD::FADD, DL, VT, N1, N0, Flags);
12166       TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) == 2)
12167     return DAG.getNode(
12169         TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
12173       TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize) == 2)
12174     return DAG.getNode(
12176         TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize), Flags);
12188     SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
12189     return DAG.getNode(ISD::FSUB, DL, VT, N1, Add, Flags);
12194     SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags);
12195     return DAG.getNode(ISD::FSUB, DL, VT, N0, Add, Flags);
12206       return DAG.getConstantFP(0.0, DL, VT);
12210       return DAG.getConstantFP(0.0, DL, VT);
12222       SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1, Flags);
12223       return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC, Flags);
12236           SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12237                                        DAG.getConstantFP(1.0, DL, VT), Flags);
12238           return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP, Flags);
12245           SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
12246                                        DAG.getConstantFP(2.0, DL, VT), Flags);
12247           return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP, Flags);
12257           SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12258                                        DAG.getConstantFP(1.0, DL, VT), Flags);
12259           return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP, Flags);
12266           SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
12267                                        DAG.getConstantFP(2.0, DL, VT), Flags);
12268           return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP, Flags);
12277           return DAG.getNode(ISD::FMUL, DL, VT,
12278                              N1, DAG.getConstantFP(3.0, DL, VT), Flags);
12287           return DAG.getNode(ISD::FMUL, DL, VT,
12288                              N0, DAG.getConstantFP(3.0, DL, VT), Flags);
12297         return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0),
12298                            DAG.getConstantFP(4.0, DL, VT), Flags);
12318   const TargetOptions &Options = DAG.getTarget().Options;
12328     return DAG.getNode(ISD::FSUB, DL, VT, N0, N1, Flags);
12344       return DAG.getConstantFP(0.0f, DL, VT);
12355       if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize))
12356         return TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
12358         return DAG.getNode(ISD::FNEG, DL, VT, N1, Flags);
12367       return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1), Flags);
12370       return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0), Flags);
12374   if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize))
12375     return DAG.getNode(
12377         TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
12392           TLI.isNegatibleForFree(X, DAG, LegalOperations, ForCodeSize))
12394             TLI.isNegatibleForFree(Y, DAG, LegalOperations, ForCodeSize))
12410   const TargetOptions &Options = DAG.getTarget().Options;
12422     return DAG.getNode(ISD::FMUL, DL, VT, N0, N1, Flags);
12427     return DAG.getNode(ISD::FMUL, DL, VT, N1, N0, Flags);
12449         SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1, Flags);
12450         return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts, Flags);
12458       const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
12459       SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1, Flags);
12460       return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts, Flags);
12466     return DAG.getNode(ISD::FADD, DL, VT, N0, N0, Flags);
12471       return DAG.getNode(ISD::FNEG, DL, VT, N0);
12476         TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
12478         TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
12479     return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1, Flags);
12518           return DAG.getNode(ISD::FNEG, DL, VT,
12519                    DAG.getNode(ISD::FABS, DL, VT, X));
12521           return DAG.getNode(ISD::FABS, DL, VT, X);
12545   const TargetOptions &Options = DAG.getTarget().Options;
12555     return DAG.getNode(ISD::FMA, DL, VT, N0, N1, N2);
12561         TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
12563         TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
12564     return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2, Flags);
12575     return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2);
12577     return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2);
12582     return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2);
12589       return DAG.getNode(ISD::FMUL, DL, VT, N0,
12590                          DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1),
12598       return DAG.getNode(ISD::FMA, DL, VT,
12600                          DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1),
12611       return DAG.getNode(ISD::FADD, DL, VT, N0, N2);
12615       SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0);
12618       return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg);
12626       return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0),
12627                          DAG.getNode(ISD::FNEG, DL, VT, N1, Flags), N2);
12634       return DAG.getNode(ISD::FMUL, DL, VT, N0,
12635                          DAG.getNode(ISD::FADD, DL, VT, N1,
12636                                      DAG.getConstantFP(1.0, DL, VT), Flags),
12642       return DAG.getNode(ISD::FMUL, DL, VT, N0,
12643                          DAG.getNode(ISD::FADD, DL, VT, N1,
12644                                      DAG.getConstantFP(-1.0, DL, VT), Flags),
12652       TLI.isNegatibleForFree(SDValue(N, 0), DAG, LegalOperations,
12654     return DAG.getNode(ISD::FNEG, DL, VT,
12655                        TLI.getNegatedExpression(SDValue(N, 0), DAG,
12672   bool UnsafeMath = DAG.getTarget().Options.UnsafeFPMath;
12692   if (VT.isVector() && DAG.isSplatValue(N1))
12716   SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
12717   SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags);
12723       SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
12742   const TargetOptions &Options = DAG.getTarget().Options;
12752     return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1, Flags);
12776         return DAG.getNode(ISD::FMUL, DL, VT, N0,
12777                            DAG.getConstantFP(Recip, DL, VT), Flags);
12784         return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12789         RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV);
12791         return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12797         RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1));
12799         return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12817           RV = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, RV, OtherOp, Flags);
12819           return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
12831     return DAG.getNode(
12833         TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize),
12834         TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
12848     return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1, N->getFlags());
12858   if (!DAG.getTarget().Options.UnsafeFPMath &&
12863   if (TLI.isFsqrtCheap(N0, DAG))
12895     return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1);
12903         return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
12906         return DAG.getNode(ISD::FNEG, SDLoc(N), VT,
12907                            DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0));
12916     return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1);
12920     return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
12924     return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1));
12929     return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0));
12959     if (!DAG.getLibInfo().has(LibFunc_cbrt) ||
12960         (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) &&
12961          DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT)))
12964     return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0), Flags);
12989     if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT))
12999     SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags);
13000     SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags);
13004     return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt, Flags);
13010 static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG,
13015   const Function &F = DAG.getMachineFunction().getFunction();
13028       !DAG.getTarget().Options.NoSignedZerosFPMath)
13036     return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0));
13040     return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0));
13052     return DAG.getConstantFP(0.0, SDLoc(N), VT);
13055   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
13059     return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0);
13066     if (DAG.SignBitIsZero(N0))
13067       return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0);
13080           DAG.getConstantFP(-1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
13082       return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
13094           DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
13096       return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
13100   if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
13113     return DAG.getConstantFP(0.0, SDLoc(N), VT);
13116   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
13120     return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0);
13127     if (DAG.SignBitIsZero(N0))
13128       return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0);
13140           DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
13142       return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
13146   if (SDValue FTrunc = foldFPToIntToFP(N, DAG, TLI))
13153 static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG) {
13177   const fltSemantics &sem = DAG.EVTToAPFloatSemantics(N0.getValueType());
13185       return DAG.getNode(ExtOp, SDLoc(N), VT, Src);
13188       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src);
13189     return DAG.getBitcast(VT, Src);
13200     return DAG.getUNDEF(VT);
13204     return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0);
13206   return FoldIntToFPToInt(N, DAG);
13215     return DAG.getUNDEF(VT);
13219     return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0);
13221   return FoldIntToFPToInt(N, DAG);
13232     return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1);
13258     if (DAG.getTarget().Options.UnsafeFPMath || N0IsTrunc) {
13260       return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0),
13261                          DAG.getIntPtrConstant(NIsTrunc && N0IsTrunc, DL));
13267     SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT,
13270     return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT,
13291     return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0);
13296     return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0));
13305       return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT,
13307     return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In);
13314     SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
13320               DAG.getNode(ISD::FP_ROUND, SDLoc(N0),
13322                           DAG.getIntPtrConstant(1, SDLoc(N0))),
13339     return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0);
13350     return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0);
13374     return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0);
13386     return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0);
13388   if (TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize))
13389     return TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
13395       (DAG.getTarget().Options.NoSignedZerosFPMath ||
13397     return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1),
13420       Int = DAG.getNode(ISD::XOR, DL0, IntVT, Int,
13421                         DAG.getConstant(SignMask, DL0, IntVT));
13423       return DAG.getBitcast(VT, Int);
13436         return DAG.getNode(
13438             DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0.getOperand(1)),
13446 static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
13457     return DAG.getConstantFP(Op(C0, C1), SDLoc(N), VT);
13463     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
13469   return visitFMinMax(DAG, N, minnum);
13473   return visitFMinMax(DAG, N, maxnum);
13477   return visitFMinMax(DAG, N, minimum);
13481   return visitFMinMax(DAG, N, maximum);
13490     return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
13499     return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0));
13517       Int = DAG.getNode(ISD::AND, DL, IntVT, Int,
13518                         DAG.getConstant(SignMask, DL, IntVT));
13520       return DAG.getBitcast(N->getValueType(0), Int);
13543     return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other,
13550       return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain, NewN1, N2);
13594           return DAG.getSetCC(DL, getSetCCResultType(Op0.getValueType()),
13595                               Op0, DAG.getConstant(0, DL, Op0.getValueType()),
13644       return DAG.getSetCC(SDLoc(TheXor), SetCCVT, Op0, Op1,
13672     return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other,
13683                                     SelectionDAG &DAG,
13733   return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM,
13734                                    VT.getTypeForEVT(*DAG.getContext()), AS);
13806   if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
13906     if (!canFoldInAddressingMode(Ptr.getNode(), Use, DAG, TLI))
13916       Result = DAG.getIndexedLoad(SDValue(N, 0), SDLoc(N), BasePtr, Offset, AM);
13919           DAG.getIndexedStore(SDValue(N, 0), SDLoc(N), BasePtr, Offset, AM);
13922       Result = DAG.getIndexedMaskedLoad(SDValue(N, 0), SDLoc(N), BasePtr,
13925       Result = DAG.getIndexedMaskedStore(SDValue(N, 0), SDLoc(N), BasePtr,
13930   LLVM_DEBUG(dbgs() << "\nReplacing.4 "; N->dump(&DAG); dbgs() << "\nWith: ";
13931              Result.getNode()->dump(&DAG); dbgs() << '\n');
13934     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
13935     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
13937     DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
13986     SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0));
13989     SDValue NewUse = DAG.getNode(Opcode,
13992     DAG.ReplaceAllUsesOfValueWith(SDValue(OtherUses[i], 0), NewUse);
13997   DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(IsLoad ? 1 : 0));
14030     if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
14056             if (!canFoldInAddressingMode(Use, UseUse, DAG, TLI))
14081           Result = IsLoad ? DAG.getIndexedLoad(SDValue(N, 0), SDLoc(N), BasePtr,
14083                           : DAG.getIndexedStore(SDValue(N, 0), SDLoc(N),
14086           Result = IsLoad ? DAG.getIndexedMaskedLoad(SDValue(N, 0), SDLoc(N),
14088                           : DAG.getIndexedMaskedStore(SDValue(N, 0), SDLoc(N),
14092         LLVM_DEBUG(dbgs() << "\nReplacing.5 "; N->dump(&DAG);
14093                    dbgs() << "\nWith: "; Result.getNode()->dump(&DAG);
14097           DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
14098           DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
14100           DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
14107         DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0),
14133     Inc = DAG.getConstant(*ConstInc->getConstantIntValue(), SDLoc(Inc),
14139   return DAG.getNode(Opc, SDLoc(LD), BP.getSimpleValueType(), BP, Inc);
14156     Val = DAG.getNode(ISD::FTRUNC, SDLoc(ST), STMemType, Val);
14161     Val = DAG.getNode(ISD::TRUNCATE, SDLoc(ST), STMemType, Val);
14165     Val = DAG.getBitcast(STMemType, Val);
14181       Val = DAG.getBitcast(LDType, Val);
14184       Val = DAG.getNode(ISD::ANY_EXTEND, SDLoc(LD), LDType, Val);
14187       Val = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(LD), LDType, Val);
14190       Val = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(LD), LDType, Val);
14211   BaseIndexOffset BasePtrLD = BaseIndexOffset::match(LD, DAG);
14212   BaseIndexOffset BasePtrST = BaseIndexOffset::match(ST, DAG);
14214   if (!BasePtrST.equalBaseIndex(BasePtrLD, DAG, Offset))
14221   if (DAG.getDataLayout().isBigEndian())
14256           DAG.getConstant(APInt::getLowBitsSet(STType.getSizeInBits(),
14259       auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask);
14279         Val = DAG.getNode(ISD::TRUNCATE, SDLoc(LD), LDMemType, Val);
14313         LLVM_DEBUG(dbgs() << "\nReplacing.6 "; N->dump(&DAG);
14314                    dbgs() << "\nWith chain: "; Chain.getNode()->dump(&DAG);
14317         DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
14335         SDValue Undef = DAG.getUNDEF(N->getValueType(0));
14343           Index = DAG.getUNDEF(N->getValueType(1));
14344         LLVM_DEBUG(dbgs() << "\nReplacing.7 "; N->dump(&DAG);
14345                    dbgs() << "\nWith: "; Undef.getNode()->dump(&DAG);
14348         DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef);
14349         DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Index);
14350         DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain);
14364     if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
14366         SDValue NewLoad = DAG.getExtLoad(
14387         ReplLoad = DAG.getLoad(N->getValueType(0), SDLoc(LD),
14390         ReplLoad = DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD),
14397       SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N),
14452           !LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
14462       const TargetLowering &TLI = LS.DAG->getTargetLoweringInfo();
14521   // The DAG from which Origin came from.
14523   SelectionDAG *DAG;
14526               unsigned Shift = 0, SelectionDAG *DAG = nullptr)
14527       : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
14559     assert(DAG && "Missing context");
14560     LLVMContext &Ctxt = *DAG->getContext();
14576     if (!Origin || !Inst || !DAG)
14583     const TargetLowering &TLI = DAG->getTargetLoweringInfo();
14619   /// \pre DAG != nullptr.
14621     assert(DAG && "Missing context.");
14622     bool IsBigEndian = DAG->getDataLayout().isBigEndian();
14655       BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr,
14656                               DAG->getConstant(Offset, DL, ArithType));
14664         DAG->getLoad(SliceType, SDLoc(Origin), Origin->getChain(), BaseAddr,
14672           DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst);
14686     assert(DAG && "Missing context");
14687     const TargetLowering &TLI = DAG->getTargetLoweringInfo();
14699     const TargetRegisterInfo *TRI = DAG->getSubtarget().getRegisterInfo();
14707     unsigned RequiredAlignment = DAG->getDataLayout().getABITypeAlignment(
14708         ResVT.getTypeForEVT(*DAG->getContext()));
14774   const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
14915     LoadedSlice LS(User, LD, Shift, &DAG);
14956   SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other,
14958   DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
15042   SelectionDAG &DAG = DC->getDAG();
15048   if (!DAG.MaskedValueIsZero(IVal, Mask)) return SDValue();
15054   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15058       !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
15066     IVal = DAG.getNode(ISD::SRL, DL, IVal.getValueType(), IVal,
15067                        DAG.getConstant(ByteShift*8, DL,
15075   if (DAG.getDataLayout().isLittleEndian())
15083     Ptr = DAG.getMemBasePlusOffset(Ptr, StOffset, DL);
15088   IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal);
15091   return DAG
15160     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
15168       NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
15186       if (DAG.getDataLayout().isBigEndian())
15190       Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
15191       if (NewAlign < DAG.getDataLayout().getABITypeAlignment(NewVTTy))
15194       SDValue NewPtr = DAG.getMemBasePlusOffset(Ptr, PtrOff, SDLoc(LD));
15196           DAG.getLoad(NewVT, SDLoc(N0), LD->getChain(), NewPtr,
15199       SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
15200                                    DAG.getConstant(NewImm, SDLoc(Value),
15203           DAG.getStore(Chain, SDLoc(N), NewVal, NewPtr,
15210       DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1));
15237     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
15246     Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
15247     unsigned ABIAlign = DAG.getDataLayout().getABITypeAlignment(IntVTTy);
15252         DAG.getLoad(IntVT, SDLoc(Value), LD->getChain(), LD->getBasePtr(),
15256         DAG.getStore(ST->getChain(), SDLoc(N), NewLD, ST->getBasePtr(),
15262     DAG.ReplaceAllUsesOfValueWith(Value.getValue(1), NewLD.getValue(1));
15334           DAG.isConstantIntBuildVectorOrConstantInt(OtherOp->getOperand(1)) &&
15361   return DAG.getTokenFactor(StoreDL, Chains);
15371   // The latest Node in the DAG.
15382     StoreTy = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
15384     StoreTy = EVT::getIntegerVT(*DAG.getContext(), SizeInBits);
15399                 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
15404               Val = DAG.getConstant(C->getAPIntValue()
15410           Val = DAG.getBitcast(MemVT, Val);
15414       StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS
15434             Val = DAG.getBitcast(MemVT, Val);
15440             Val = DAG.getNode(OpC, SDLoc(Val), MemVT, Vec, Idx);
15447       StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS
15460     bool IsLE = DAG.getDataLayout().isLittleEndian();
15486     StoredVal = DAG.getConstant(StoreInt, DL, StoreTy);
15495     NewStore = DAG.getStore(NewChain, DL, StoredVal, FirstInChain->getBasePtr(),
15500         TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType());
15504         DAG.getConstant(C->getAPIntValue().zextOrTrunc(LegalizedStoreSize), DL,
15506     NewStore = DAG.getTruncStore(
15526   BaseIndexOffset BasePtr = BaseIndexOffset::match(St, DAG);
15547     LBasePtr = BaseIndexOffset::match(Ld, DAG);
15578         BaseIndexOffset LPtr = BaseIndexOffset::match(OtherLd, DAG);
15592         if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
15613     Ptr = BaseIndexOffset::match(Other, DAG);
15614     return (BasePtr.equalBaseIndex(Ptr, DAG, Offset));
15676 // the DAG. Any store candidate may depend on another candidate
15725   // Search through DAG. We can stop early if we find a store node.
15755   bool NoVectors = DAG.getMachineFunction().getFunction().hasFnAttribute(
15782   // Find potential store merge candidates by searching through chain sub-DAG
15838     LLVMContext &Context = *DAG.getContext();
15839     const DataLayout &DL = DAG.getDataLayout();
15876               TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
15888                 TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
15908                 TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
15972               EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
15980               TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) &&
16042       BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld, DAG);
16047         if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
16111             TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
16125             TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
16139               TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) &&
16233             DAG.getLoad(JointMemOpVT, LoadDL, FirstLoad->getChain(),
16236         NewStore = DAG.getStore(
16241             TLI.getTypeToTransformTo(*DAG.getContext(), JointMemOpVT);
16242         NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy,
16246         NewStore = DAG.getTruncStore(NewStoreChain, StoreDL, NewLoad,
16256         DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
16284     ReplStore = DAG.getTruncStore(BetterChain, SL, ST->getValue(),
16288     ReplStore = DAG.getStore(BetterChain, SL, ST->getValue(), ST->getBasePtr(),
16293   SDValue Token = DAG.getNode(ISD::TokenFactor, SL,
16336       Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF().
16339       return DAG.getStore(Chain, DL, Tmp, Ptr, ST->getMemOperand());
16348       Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
16350       return DAG.getStore(Chain, DL, Tmp,
16360       SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, SDLoc(CFP), MVT::i32);
16361       SDValue Hi = DAG.getConstant(Val >> 32, SDLoc(CFP), MVT::i32);
16362       if (DAG.getDataLayout().isBigEndian())
16369       SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16371       Ptr = DAG.getMemBasePlusOffset(Ptr, 4, DL);
16373       SDValue St1 = DAG.getStore(Chain, DL, Hi, Ptr,
16376       return DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
16404                                      DAG, *ST->getMemOperand())) {
16405       return DAG.getStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
16416     if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
16419             DAG.getTruncStore(Chain, SDLoc(N), Value, Ptr, ST->getPointerInfo(),
16462     if (SDValue Shorter = DAG.GetDemandedBits(Value, TruncDemandedBits))
16463       return DAG.getTruncStore(Chain, SDLoc(N), Shorter, Ptr, ST->getMemoryVT(),
16506         const BaseIndexOffset STBase = BaseIndexOffset::match(ST, DAG);
16507         const BaseIndexOffset ChainBase = BaseIndexOffset::match(ST1, DAG);
16514         if (STBase.contains(DAG, STBitSize, ChainBase, ChainBitSize)) {
16528     return DAG.getTruncStore(Chain, SDLoc(N), Value.getOperand(0),
16601       const BaseIndexOffset StoreBase = BaseIndexOffset::match(ST, DAG);
16604       if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
16709   EVT VT = EVT::getIntegerVT(*DAG.getContext(), HalfValBitSize);
16710   Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0));
16711   Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0));
16716   SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
16718   Ptr = DAG.getMemBasePlusOffset(Ptr, HalfValBitSize / 8, DL);
16721       DAG.getStore(St0, DL, Hi, Ptr,
16797                                           Y, NewMask, DAG);
16834   EVT ShufVT = EVT::getVectorVT(*DAG.getContext(), SubVecEltVT, NumMaskVals);
16841   SmallVector<SDValue, 8> ConcatOps(ExtendRatio, DAG.getUNDEF(SubVecVT));
16843   SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps);
16846   SDValue DestVecBC = DAG.getBitcast(ShufVT, DestVec);
16847   SDValue Shuf = DAG.getVectorShuffle(ShufVT, DL, DestVecBC, PaddedSubV, Mask);
16851   return DAG.getBitcast(VT, Shuf);
16866       return DAG.getUNDEF(VT);
16880       return DAG.getBuildVector(VT, DL, Ops);
16902       SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16905       return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()),
16924     Ops.append(NumElts, DAG.getUNDEF(InVal.getValueType()));
16935     Ops[Elt] = OpVT.isInteger() ? DAG.getAnyExtOrTrunc(InVal, DL, OpVT) : InVal;
16939   return DAG.getBuildVector(VT, DL, Ops);
16950   unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
16951       VecEltVT.getTypeForEVT(*DAG.getContext()));
16971     Offset = DAG.getConstant(PtrOff, DL, PtrType);
16974     Offset = DAG.getZExtOrTrunc(EltNo, DL, PtrType);
16975     Offset = DAG.getNode(
16977         DAG.getConstant(VecEltVT.getStoreSize(), DL, PtrType));
16982   NewPtr = DAG.getMemBasePlusOffset(NewPtr, Offset, DL);
16999     Load = DAG.getExtLoad(ExtType, SDLoc(EVE), ResultVT,
17005     Load = DAG.getLoad(VecEltVT, SDLoc(EVE), OriginalLoad->getChain(), NewPtr,
17010       Load = DAG.getNode(ISD::TRUNCATE, SDLoc(EVE), ResultVT, Load);
17012       Load = DAG.getBitcast(ResultVT, Load);
17017   DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
17029 static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
17031   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17054     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
17055     SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
17056     return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1);
17068     return DAG.getUNDEF(ScalarVT);
17078     return VecVT.isInteger() ? DAG.getAnyExtOrTrunc(Elt, DL, ScalarVT) : Elt;
17089       return DAG.getSExtOrTrunc(InOp, DL, ScalarVT);
17098     return DAG.getUNDEF(ScalarVT);
17117   // mess of scalar and vector code if we reduce only part of the DAG to scalar.
17121     bool IsLE = DAG.getDataLayout().isLittleEndian();
17127       return DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, BCSrc);
17147         return DAG.getAnyExtOrTrunc(X, DL, ScalarVT);
17152   if (SDValue BO = scalarizeExtractedBinop(N, DAG, LegalOperations))
17168       return DAG.getUNDEF(ScalarVT);
17183         InOp = DAG.getSExtOrTrunc(InOp, DL, ScalarVT);
17196       EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
17197       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, SVInVec,
17198                          DAG.getConstant(OrigElt, DL, IndexTy));
17305       Index = DAG.getConstant(Elt, DL, Index.getValueType());
17316     return DAG.getUNDEF(LVT);
17388   bool isLE = DAG.getDataLayout().isLittleEndian();
17391   SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
17392                                DAG.getConstant(0, DL, SourceType);
17405       In = DAG.getUNDEF(SourceType);
17416   EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
17426   SDValue BV = DAG.getBuildVector(VecVT, DL, Ops);
17431   return DAG.getBitcast(VT, BV);
17438   MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
17439   SDValue ZeroIdx = DAG.getConstant(0, DL, IdxTy);
17460       SmallVector<SDValue, 2> ConcatOps(NumConcats, DAG.getUNDEF(InVT1));
17462       ConcatOps[1] = VecIn2 ? VecIn2 : DAG.getUNDEF(InVT1);
17463       VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
17472         VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1,
17473                              DAG.getConstant(NumElems, DL, IdxTy));
17474         VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx);
17493           VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1,
17494                                DAG.getUNDEF(InVT1), VecIn2, ZeroIdx);
17505       SmallVector<SDValue, 2> ConcatOps(2, DAG.getUNDEF(InVT2));
17507       VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
17512       // mismatches, we don't create any DAG nodes.
17540   VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(InVT1);
17543   SDValue Shuffle = DAG.getVectorShuffle(InVT1, DL, VecIn1, VecIn2, Mask);
17545     Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx);
17550 static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) {
17618   SDValue ZeroVec = DAG.getConstant(0, DL, VecVT);
17619   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17621                                              ZeroVec, ShufMask, DAG);
17624   return DAG.getBitcast(VT, Shuf);
17638   if (SDValue V = reduceBuildVecToShuffleWithZero(N, DAG))
17714     MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
17729       EVT SplitVT = EVT::getVectorVT(*DAG.getContext(),
17732         SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
17733                                      DAG.getConstant(SplitSize, DL, IdxTy));
17734         SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec,
17735                                      DAG.getConstant(0, DL, IdxTy));
17786     Shuffles.push_back(VT.isInteger() ? DAG.getConstant(0, DL, VT)
17787                                       : DAG.getConstantFP(0.0, DL, VT));
17813     Shuffles.push_back(DAG.getUNDEF(VT));
17817       Shuffles[CurSize] = DAG.getUNDEF(VT);
17835           DAG.getVectorShuffle(VT, DL, Shuffles[Left], Shuffles[Right], Mask);
17873   EVT InVT = EVT::getVectorVT(*DAG.getContext(), InSVT, NumElems);
17886   In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InVT, In,
17888   return DAG.getNode(FoundZeroExtend ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, DL,
17897     return DAG.getUNDEF(VT);
17913         EVT NewVT = EVT::getVectorVT(*DAG.getContext(),
17917           SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N),
17919           return DAG.getBitcast(VT, Concat);
17929       return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V);
17957       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), N->getValueType(0),
17973 static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) {
17974   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17985   EVT SVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
17986   SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT);
18016     ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT);
18024           Op = DAG.getBitcast(SVT, Op);
18029   EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SVT,
18031   return DAG.getBitcast(VT, DAG.getBuildVector(VecVT, DL, Ops));
18038 static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG) {
18044   SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT);
18105   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18106   return TLI.buildLegalVectorShuffle(VT, SDLoc(N), DAG.getBitcast(VT, SV0),
18107                                      DAG.getBitcast(VT, SV1), Mask, DAG);
18118     return DAG.getUNDEF(VT);
18132       Ops.resize(NumOps, DAG.getUNDEF(Ops[0].getValueType()));
18133       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops);
18170       EVT NVT = EVT::getVectorVT(*DAG.getContext(), SclTy, VNTNumElms);
18174       SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar);
18175       return DAG.getBitcast(VT, Res);
18209         Opnds.append(NumElts, DAG.getUNDEF(MinVT));
18218                 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i)));
18225     return DAG.getBuildVector(VT, SDLoc(N), Opnds);
18229   if (SDValue V = combineConcatVectorOfScalars(N, DAG))
18234     if (SDValue V = combineConcatVectorOfExtracts(N, DAG))
18237   // Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR
18303                                               SelectionDAG &DAG) {
18304   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18332   return DAG.getNode(BinOpcode, SDLoc(Extract), SubVT, Sub0, Sub1,
18338 static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) {
18342   if (SDValue V = narrowInsertExtractVectorBinOp(Extract, DAG))
18352   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18382   EVT NarrowBVT = EVT::getVectorVT(*DAG.getContext(), WideBVT.getScalarType(),
18398     SDValue NewExtIndex = DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT);
18399     SDValue X = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
18401     SDValue Y = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
18403     SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y,
18405     return DAG.getBitcast(VT, NarrowBinOp);
18438     SDValue IndexC = DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT);
18439     SDValue X = SubVecL ? DAG.getBitcast(NarrowBVT, SubVecL)
18440                         : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
18443     SDValue Y = SubVecR ? DAG.getBitcast(NarrowBVT, SubVecR)
18444                         : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT,
18447     SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y);
18448     return DAG.getBitcast(VT, NarrowBinOp);
18457 static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
18459   if (DAG.getDataLayout().isBigEndian())
18470   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18481   SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
18482   MachineFunction &MF = DAG.getMachineFunction();
18485   SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO);
18486   DAG.makeEquivalentMemoryOrdering(Ld, NewLd);
18496     return DAG.getUNDEF(NVT);
18499     if (SDValue NarrowLoad = narrowExtractedVectorLoad(N, DAG))
18510       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, V.getOperand(0),
18526       EVT NewExtVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
18531         SDValue NewIndex = DAG.getIntPtrConstant(IndexValScaled, DL);
18532         SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT,
18534         return DAG.getBitcast(NVT, NewExtract);
18541         EVT NewExtVT = EVT::getVectorVT(*DAG.getContext(),
18547           SDValue NewIndex = DAG.getIntPtrConstant(IndexValScaled, DL);
18548           SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT,
18550           return DAG.getBitcast(NVT, NewExtract);
18587       MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
18588       SDValue NewIndexC = DAG.getConstant(NewExtIdx, DL, IdxTy);
18589       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT,
18607                                       : EVT::getVectorVT(*DAG.getContext(),
18620               Src = DAG.getNode(ISD::TRUNCATE, SDLoc(N), InVT, Src);
18621             return DAG.getBitcast(NVT, Src);
18625           SDValue BuildVec = DAG.getBuildVector(
18627           return DAG.getBitcast(NVT, BuildVec);
18651         return DAG.getBitcast(NVT, V.getOperand(1));
18652       return DAG.getNode(
18654           DAG.getBitcast(N->getOperand(0).getValueType(), V.getOperand(0)),
18659   if (SDValue NarrowBOp = narrowExtractedVectorBinOp(N, DAG))
18673                                          SelectionDAG &DAG) {
18699   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18700   EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
18710   SDValue Shuf0 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask0);
18711   SDValue Shuf1 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask1);
18712   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1);
18717 static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) {
18739     N0 = DAG.getVectorShuffle(ConcatVT, SDLoc(N), N0.getOperand(0),
18742     N1 = DAG.getUNDEF(ConcatVT);
18743     return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1);
18754       Ops.push_back(DAG.getUNDEF(ConcatVT));
18777   return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops);
18797                                        SelectionDAG &DAG,
18833     SDValue Op = DAG.getUNDEF(VT.getScalarType());
18841         Op = Idx == 0 ? Op0 : DAG.getUNDEF(Op0.getValueType());
18868                ? DAG.getZExtOrTrunc(Op, SDLoc(SVN), SVT)
18869                : DAG.getSExtOrTrunc(Op, SDLoc(SVN), SVT);
18870   return DAG.getBuildVector(VT, SDLoc(SVN), Ops);
18878                                             SelectionDAG &DAG,
18882   bool IsBigEndian = DAG.getDataLayout().isBigEndian();
18914     EVT OutSVT = EVT::getIntegerVT(*DAG.getContext(), EltSizeInBits * Scale);
18915     EVT OutVT = EVT::getVectorVT(*DAG.getContext(), OutSVT, NumElts / Scale);
18921         return DAG.getBitcast(VT,
18922                               DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG,
18935                                         SelectionDAG &DAG) {
18937   bool IsBigEndian = DAG.getDataLayout().isBigEndian();
18985     return DAG.getBitcast(VT, N00);
18996                                         SelectionDAG &DAG) {
19040   return DAG.getVectorShuffle(Splat->getValueType(0), SDLoc(Splat),
19074                                       SelectionDAG &DAG) {
19117   SDValue NewInsIndex = DAG.getConstant(ShufOp0Index, SDLoc(Shuf),
19119   return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Shuf), Op0.getValueType(),
19158   assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG");
19162     return DAG.getUNDEF(VT);
19174     return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), NewMask);
19179     return DAG.getCommutedVectorShuffle(*SVN);
19194       return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask);
19197   if (SDValue InsElt = replaceShuffleOfInsert(SVN, DAG))
19201   if (SDValue V = combineShuffleOfSplatVal(SVN, DAG))
19215       SDValue Index = DAG.getIntPtrConstant(SplatIndex, DL);
19216       SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index);
19217       SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index);
19218       SDValue NewBO = DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR,
19220       SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO);
19222       return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask);
19263       SDValue NewBV = DAG.getBuildVector(V->getValueType(0), SDLoc(N), Ops);
19268         NewBV = DAG.getBitcast(VT, NewBV);
19283   if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations))
19287   if (SDValue V = combineTruncationShuffle(SVN, DAG))
19295     if (SDValue V = partitionShuffleOfConcats(N, DAG))
19316       SDValue UndefVec = DAG.getUNDEF(N0.getOperand(0).getValueType());
19317       SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
19319       return DAG.getVectorShuffle(VT, SDLoc(N), NewCat, N1, NewMask);
19326     if (SDValue Res = combineShuffleOfScalars(SVN, DAG, TLI))
19385           SV0 = DAG.getBitcast(ScaleVT, SV0);
19386           SV1 = DAG.getBitcast(ScaleVT, SV1);
19387           return DAG.getBitcast(
19388               VT, DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask));
19413       return DAG.getCommutedVectorShuffle(*SVN);
19499       return DAG.getUNDEF(VT);
19502       SV0 = DAG.getUNDEF(VT);
19504       SV1 = DAG.getUNDEF(VT);
19513     return TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, Mask, DAG);
19516   if (SDValue V = foldShuffleOfConcatUndefs(SVN, DAG))
19542             DAG.getNode(ISD::TRUNCATE, SDLoc(InVal), VT.getScalarType(), InVal);
19543         return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val);
19549                                       DAG.getUNDEF(InVecT), NewMask, DAG);
19557             MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
19558             SDValue ZeroIdx = DAG.getConstant(0, SDLoc(N), IdxTy);
19560                 EVT::getVectorVT(*DAG.getContext(), InVecT.getVectorElementType(),
19562             return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), SubVT,
19600     return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0));
19615       SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N),
19617       return DAG.getBitcast(VT, NewINSERT);
19627     return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0),
19635     return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0,
19657       MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
19658       LLVMContext &Ctx = *DAG.getContext();
19664         NewIdx = DAG.getConstant(InsIdx * Scale, DL, IdxVT);
19669           NewIdx = DAG.getConstant(InsIdx / Scale, DL, IdxVT);
19673         SDValue Res = DAG.getBitcast(NewVT, N0Src);
19674         Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, Res, N1Src, NewIdx);
19675         return DAG.getBitcast(VT, Res);
19690       SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT,
19693       return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N0.getNode()),
19707     return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops);
19734       return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0),
19750     SDValue Res = DAG.getNode(
19752         DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
19754       Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res);
19765         DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits())
19766       return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), N0);
19823       if (DAG.getDataLayout().isBigEndian())
19837     EVT ClearSVT = EVT::getIntegerVT(*DAG.getContext(), NumSubBits);
19838     EVT ClearVT = EVT::getVectorVT(*DAG.getContext(), ClearSVT, NumSubElts);
19842     SDValue Zero = DAG.getConstant(0, DL, ClearVT);
19843     return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL,
19844                                                    DAG.getBitcast(ClearVT, LHS),
19863 static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG) {
19869   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19875   SDValue Src0 = DAG.getSplatSourceVector(N0, Index0);
19876   SDValue Src1 = DAG.getSplatSourceVector(N1, Index1);
19886       DAG.getConstant(Index0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
19887   SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, N0, IndexC);
19888   SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, N1, IndexC);
19889   SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, X, Y, N->getFlags());
19898     SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT));
19900     return DAG.getBuildVector(VT, DL, Ops);
19905   return DAG.getBuildVector(VT, DL, Ops);
19920   if (SDValue Fold = DAG.FoldConstantVectorArithmetic(
19940       SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0),
19943       return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask());
19964           DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT));
19965       SDValue NarrowBO = DAG.getNode(Opcode, DL, NarrowVT, X, Y);
19966       return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z);
19995         ConcatOps.push_back(DAG.getNode(Opcode, DL, NarrowVT, LHS.getOperand(i),
19999       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
20003   if (SDValue V = scalarizeBinOpOfSplats(N, DAG))
20024       SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0),
20029       SDValue SelectNode = DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC,
20044 /// the DAG combiner loop to avoid it being looked at.
20132     // folding this will induce a cycle into the DAG.  If not, this is safe to
20165       Addr = DAG.getSelect(SDLoc(TheSelect),
20187       Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect),
20207       Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect),
20212       Load = DAG.getExtLoad(
20270       SDValue ShiftAmt = DAG.getConstant(ShCt, DL, ShiftAmtTy);
20271       SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt);
20275         Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
20280         Shift = DAG.getNOT(DL, Shift, AType);
20282       return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
20290   SDValue ShiftAmt = DAG.getConstant(ShCt, DL, ShiftAmtTy);
20291   SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt);
20295     Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
20300     Shift = DAG.getNOT(DL, Shift, AType);
20302   return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
20337   const DataLayout &TD = DAG.getDataLayout();
20341   SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(DAG.getDataLayout()),
20347   SDValue Zero = DAG.getIntPtrConstant(0, DL);
20349   SDValue One = DAG.getIntPtrConstant(EltSize, SDLoc(FV));
20351       DAG.getSetCC(DL, getSetCCResultType(N0.getValueType()), N0, N1, CC);
20353   SDValue CstOffset = DAG.getSelect(DL, Zero.getValueType(), Cond, One, Zero);
20355   CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, CstOffset);
20357   return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
20359                          DAG.getMachineFunction()), Alignment);
20378   if (SDValue SCC = DAG.FoldSetCC(CmpResVT, N0, N1, CC, DL)) {
20410           DAG.getConstant(AndMask.countLeadingZeros(), SDLoc(AndLHS),
20412         SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt);
20417           DAG.getConstant(ShCt, SDLoc(Shl),
20419         SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt);
20421         return DAG.getNode(ISD::AND, DL, VT, Shr, N3);
20448       SCC = DAG.getSetCC(DL, CmpResVT, N0, N1, CC);
20450         Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), VT);
20452         Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC);
20454       SCC = DAG.getSetCC(SDLoc(N0), MVT::i1, N0, N1, CC);
20455       Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC);
20469     return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
20470                        DAG.getConstant(ShCt, SDLoc(Temp),
20497           return DAG.getNode(ISD::CTTZ, DL, VT, N0);
20504           return DAG.getNode(ISD::CTLZ, DL, VT, N0);
20517     DagCombineInfo(DAG, Level, false, this);
20522 /// a DAG expression to select that will generate the same value by multiplying
20528   if (DAG.getMachineFunction().getFunction().hasMinSize())
20532   if (SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, Built)) {
20542 /// DAG expression that will generate the same value by right shifting.
20553   if (SDValue S = TLI.BuildSDIVPow2(N, C->getAPIntValue(), DAG, Built)) {
20562 /// Given an ISD::UDIV node expressing a divide by constant, return a DAG
20569   if (DAG.getMachineFunction().getFunction().hasMinSize())
20573   if (SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, Built)) {
20587   SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V);
20588   SDValue Base = DAG.getConstant(EltBits - 1, DL, VT);
20589   SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz);
20612   MachineFunction &MF = DAG.getMachineFunction();
20620   if (SDValue Est = TLI.getRecipEstimate(Op, DAG, Enabled, Iterations)) {
20625       SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
20633           MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags);
20637         SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags);
20640         NewEst = DAG.getNode(ISD::FSUB, DL, VT,
20644         NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags);
20647         Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags);
20652       Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags);
20673   SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT);
20677   SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags);
20678   HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags);
20682     SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags);
20683     NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags);
20684     NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags);
20685     Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags);
20690     Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags);
20705   SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT);
20706   SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT);
20715     SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags);
20716     SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags);
20717     SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags);
20725       LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags);
20728       LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags);
20731     Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags);
20751   MachineFunction &MF = DAG.getMachineFunction();
20762       TLI.getSqrtEstimate(Op, DAG, Enabled, Iterations, UseOneConstNR,
20777         DenormalMode DenormMode = DAG.getDenormalMode(VT);
20780           const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
20782           SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
20783           SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
20784           SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
20785           SDValue IsDenorm = DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
20786           Est = DAG.getNode(SelOpcode, DL, VT, IsDenorm, FPZero, Est);
20789           SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
20790           SDValue IsZero = DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
20791           Est = DAG.getNode(SelOpcode, DL, VT, IsZero, FPZero, Est);
20874                                        DAG, IsAlias))
20912                    : DAG.getSubtarget().useAA();
20915       CombinerAAOnlyFunc != DAG.getMachineFunction().getName())
21058     return DAG.getEntryNode();
21065   return DAG.getTokenFactor(SDLoc(N), Aliases);
21102   const BaseIndexOffset BasePtr = BaseIndexOffset::match(St, DAG);
21124     const BaseIndexOffset Ptr = BaseIndexOffset::match(Chain, DAG);
21127     if (!BasePtr.equalBaseIndex(Ptr, DAG, Offset))
21156     S = cast<StoreSDNode>(DAG.UpdateNodeOperands(
21166     NewST = DAG.getTruncStore(BetterChain, SDLoc(St), St->getValue(),
21170     NewST = DAG.getStore(BetterChain, SDLoc(St), St->getValue(),
21185   SDValue TF = DAG.getTokenFactor(SDLoc(STChain), TFOps);
21200   const BaseIndexOffset BasePtr = BaseIndexOffset::match(St, DAG);