Lines Matching refs:SDValue
1100 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
1176 SDValue New;
1187 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1188 New = SDValue(
1196 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1241 const SDValue Op, KnownBits &Known,
1882 static bool isCMN(SDValue Op, ISD::CondCode CC) {
1887 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
1888 SelectionDAG &DAG, SDValue Chain,
1898 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1934 const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
2004 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
2005 ISD::CondCode CC, SDValue CCOp,
2021 SDValue SubOp0 = RHS.getOperand(0);
2031 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
2034 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
2052 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
2070 SDValue O0 = Val->getOperand(0);
2071 SDValue O1 = Val->getOperand(1);
2110 /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
2114 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
2115 AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
2120 SDValue LHS = Val->getOperand(0);
2121 SDValue RHS = Val->getOperand(1);
2137 SDValue ExtraCmp;
2159 SDValue LHS = Val->getOperand(0);
2166 SDValue RHS = Val->getOperand(1);
2213 SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
2216 SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
2226 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
2231 return SDValue();
2233 return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
2240 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
2241 auto isSupportedExtend = [&](SDValue V) {
2274 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2275 SDValue &AArch64cc, SelectionDAG &DAG,
2344 SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2352 SDValue Cmp;
2379 SDValue SExt =
2405 static std::pair<SDValue, SDValue>
2406 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2409 SDValue Value, Overflow;
2411 SDValue LHS = Op.getOperand(0);
2412 SDValue RHS = Op.getOperand(1);
2446 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2447 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2460 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2463 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2476 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2490 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2491 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2499 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2520 SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
2524 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
2525 SmallVector<SDValue, 2> Ops(Op->op_begin() + Offset, Op->op_end());
2527 SDValue Result;
2534 static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
2535 SDValue Sel = Op.getOperand(0);
2536 SDValue Other = Op.getOperand(1);
2549 return SDValue();
2551 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2552 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2554 SDValue Value, Overflow;
2556 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2574 SDValue LHS = Sel.getOperand(0);
2575 SDValue RHS = Sel.getOperand(1);
2576 SDValue TVal = Sel.getOperand(2);
2577 SDValue FVal = Sel.getOperand(3);
2600 SDValue CCVal;
2601 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
2614 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2619 return SDValue();
2650 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
2653 return SDValue();
2658 SDValue Value, Overflow;
2662 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2663 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2668 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2681 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
2707 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
2717 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
2720 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
2726 return SDValue();
2739 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
2740 SDValue Result;
2747 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
2768 SDValue Cv =
2779 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
2787 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
2790 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
2819 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
2825 SDValue In = Op.getOperand(0);
2847 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2853 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
2868 return SDValue();
2885 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2890 SDValue Arg = Op.getOperand(0);
2906 SDValue Callee =
2915 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2919 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2922 return SDValue();
2929 return SDValue(
2952 static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
2976 for (const SDValue &Elt : N->op_values()) {
2995 static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
3008 SmallVector<SDValue, 8> Ops;
3051 SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
3059 SDValue Chain = Op.getOperand(0);
3060 SDValue FPCR_64 = DAG.getNode(
3064 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
3065 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
3067 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
3069 SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
3074 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
3112 return SDValue();
3121 SDValue Op0;
3122 SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
3133 SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
3134 SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
3143 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
3149 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
3154 default: return SDValue(); // Don't custom lower most intrinsics.
3162 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
3249 SDValue Reinterpret =
3251 SDValue Mask = getPTrue(DAG, dl, InVT, AArch64SVEPredPattern::all);
3252 SDValue MaskReinterpret =
3258 SDValue Scalar = Op.getOperand(2);
3279 SDValue FnOp = Op.getOperand(1);
3280 SDValue IncomingFPOp = Op.getOperand(2);
3314 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
3319 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
3325 SDValue Value = ST->getValue();
3334 SDValue Undef = DAG.getUNDEF(MVT::i16);
3335 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
3338 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
3340 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
3343 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
3353 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
3359 SDValue Value = StoreNode->getValue();
3389 SDValue Lo =
3393 SDValue Hi = DAG.getNode(
3398 SDValue Result = DAG.getMemIntrinsicNode(
3406 SDValue Lo =
3409 SDValue Hi =
3412 SDValue Result = DAG.getMemIntrinsicNode(
3419 return SDValue();
3422 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
3430 return SDValue();
3682 SDValue AArch64TargetLowering::LowerFormalArguments(
3683 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3685 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3692 DenseMap<unsigned, SDValue> CopiedRegs;
3728 SmallVector<SDValue, 16> ArgValues;
3743 SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
3749 SDValue ArgValue;
3819 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3918 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
3954 SDValue &Chain) const {
3961 SmallVector<SDValue, 8> MemOps;
3980 SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
3984 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
3985 SDValue Store = DAG.getStore(
4012 SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
4016 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
4018 SDValue Store = DAG.getStore(
4037 SDValue AArch64TargetLowering::LowerCallResult(
4038 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
4040 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
4041 SDValue ThisVal) const {
4047 DenseMap<unsigned, SDValue> CopiedRegs;
4067 SDValue Val = CopiedRegs.lookup(VA.getLocReg());
4120 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
4122 const SmallVectorImpl<SDValue> &OutVals,
4264 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
4268 SmallVector<SDValue, 8> ArgChains;
4290 ArgChains.push_back(SDValue(L, 1));
4304 SDValue
4306 SmallVectorImpl<SDValue> &InVals) const {
4310 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4312 SDValue Chain = CLI.Chain;
4313 SDValue Callee = CLI.Callee;
4450 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
4453 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4455 SmallVector<SDValue, 8> MemOpChains;
4461 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
4469 SDValue Arg = OutVals[i];
4517 SDValue SpillSlot = DAG.getFrameIndex(
4540 SDValue &Bits =
4542 [=](const std::pair<unsigned, SDValue> &Elt) {
4565 SDValue DstAddr;
4585 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
4601 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
4609 SDValue SizeNode =
4611 SDValue Cpy = DAG.getMemcpy(
4626 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
4637 SDValue InFlag;
4680 std::vector<SDValue> Ops;
4728 SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
4752 IsThisReturn ? OutVals[0] : SDValue());
4766 SDValue
4767 AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4770 const SmallVectorImpl<SDValue> &OutVals,
4784 SDValue Flag;
4785 SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
4791 SDValue Arg = OutVals[realRVLocIdx];
4821 SDValue &Bits =
4823 [=](const std::pair<unsigned, SDValue> &Elt) {
4834 SmallVector<SDValue, 4> RetOps(1, Chain);
4847 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
4885 SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
4892 SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
4898 SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
4905 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
4913 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
4918 SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
4926 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
4942 SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4947 SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
4948 SDValue Lo = getTargetNode(N, Ty, DAG,
4950 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
4956 SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
4961 SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
4965 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
4981 SDValue Result;
4999 /// return an SDValue containing the final node.
5025 SDValue
5026 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
5036 SDValue TLVPAddr =
5038 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
5042 SDValue Chain = DAG.getEntryNode();
5043 SDValue FuncTLVGet = DAG.getLoad(
5067 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
5078 SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
5079 SDValue ThreadBase,
5083 SDValue TPOff, Addr;
5092 SDValue Var = DAG.getTargetGlobalAddress(
5094 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
5104 SDValue HiVar = DAG.getTargetGlobalAddress(
5106 SDValue LoVar = DAG.getTargetGlobalAddress(
5109 Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
5113 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
5124 SDValue HiVar = DAG.getTargetGlobalAddress(
5126 SDValue LoVar = DAG.getTargetGlobalAddress(
5129 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
5132 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
5144 SDValue HiVar = DAG.getTargetGlobalAddress(
5146 SDValue MiVar = DAG.getTargetGlobalAddress(
5149 SDValue LoVar = DAG.getTargetGlobalAddress(
5152 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
5155 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
5158 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
5184 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
5189 SDValue Chain = DAG.getEntryNode();
5194 SDValue Glue = Chain.getValue(1);
5199 SDValue
5200 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
5224 SDValue TPOff;
5229 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
5250 SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
5259 SDValue HiVar = DAG.getTargetGlobalAddress(
5261 SDValue LoVar = DAG.getTargetGlobalAddress(
5265 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
5268 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
5275 SDValue SymAddr =
5286 SDValue
5287 AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
5291 SDValue Chain = DAG.getEntryNode();
5295 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
5299 SDValue TLSArray =
5308 SDValue TLSIndexHi =
5310 SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
5312 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
5313 SDValue TLSIndex =
5321 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
5323 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
5330 SDValue TGAHi = DAG.getTargetGlobalAddress(
5332 SDValue TGALo = DAG.getTargetGlobalAddress(
5337 SDValue Addr =
5338 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
5345 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
5361 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5362 SDValue Chain = Op.getOperand(0);
5364 SDValue LHS = Op.getOperand(2);
5365 SDValue RHS = Op.getOperand(3);
5366 SDValue Dest = Op.getOperand(4);
5396 return SDValue();
5400 SDValue Value, Overflow;
5405 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
5427 SDValue Test = LHS.getOperand(0);
5443 SDValue Test = LHS.getOperand(0);
5470 SDValue CCVal;
5471 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
5481 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
5484 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5485 SDValue BR1 =
5488 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5496 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
5501 SDValue In1 = Op.getOperand(0);
5502 SDValue In2 = Op.getOperand(1);
5512 SDValue VecVal1, VecVal2;
5547 SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
5557 SDValue Sel =
5570 SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
5573 return SDValue();
5576 return SDValue();
5586 SDValue Val = Op.getOperand(0);
5595 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
5596 SDValue UaddLV = DAG.getNode(
5606 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
5607 SDValue UaddLV = DAG.getNode(
5637 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
5645 SDValue Chain;
5648 SDValue LHS = Op.getOperand(OpNo + 0);
5649 SDValue RHS = Op.getOperand(OpNo + 1);
5655 SDValue TVal = DAG.getConstant(1, dl, VT);
5656 SDValue FVal = DAG.getConstant(0, dl, VT);
5673 SDValue CCVal;
5674 SDValue Cmp = getAArch64Cmp(
5680 SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
5690 SDValue Cmp;
5698 SDValue Res;
5702 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5715 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5716 SDValue CS1 =
5719 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5725 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
5726 SDValue RHS, SDValue TVal,
5727 SDValue FVal, const SDLoc &dl,
5862 SDValue CCVal;
5863 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
5873 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
5899 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5900 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
5905 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5913 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
5916 SDValue LHS = Op.getOperand(0);
5917 SDValue RHS = Op.getOperand(1);
5918 SDValue TVal = Op.getOperand(2);
5919 SDValue FVal = Op.getOperand(3);
5924 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
5926 SDValue CCVal = Op->getOperand(0);
5927 SDValue TVal = Op->getOperand(1);
5928 SDValue FVal = Op->getOperand(2);
5933 SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal);
5935 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC);
5944 return SDValue();
5947 SDValue Value, Overflow;
5949 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
5957 SDValue LHS, RHS;
5970 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
5985 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
5990 SDValue JT = Op.getOperand(1);
5991 SDValue Entry = Op.getOperand(2);
5998 SDValue(Dest, 0));
6001 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
6018 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
6030 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
6036 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
6044 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
6050 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
6059 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
6068 SDValue Chain = Op.getOperand(0);
6069 SDValue VAList = Op.getOperand(1);
6071 SmallVector<SDValue, 4> MemOps;
6074 SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
6081 SDValue GRTop, GRTopAddr;
6098 SDValue VRTop, VRTopAddr;
6112 SDValue GROffsAddr =
6119 SDValue VROffsAddr =
6128 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
6140 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
6157 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
6164 SDValue Chain = Op.getOperand(0);
6165 SDValue Addr = Op.getOperand(1);
6170 SDValue VAList =
6202 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
6207 SDValue APStore =
6213 SDValue WideFP =
6216 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
6218 SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
6226 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
6234 SDValue FrameAddr =
6247 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
6277 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
6284 SDValue FrameAddr =
6286 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
6291 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
6301 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
6302 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
6315 SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
6321 SDValue ShOpLo = Op.getOperand(0);
6322 SDValue ShOpHi = Op.getOperand(1);
6323 SDValue ShAmt = Op.getOperand(2);
6328 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
6330 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
6334 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
6336 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
6341 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
6344 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
6345 SDValue LoForNormalShift =
6351 SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6352 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
6357 SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
6358 SDValue HiForBigShift =
6363 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
6366 SDValue Ops[2] = { Lo, Hi };
6372 SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
6378 SDValue ShOpLo = Op.getOperand(0);
6379 SDValue ShOpHi = Op.getOperand(1);
6380 SDValue ShAmt = Op.getOperand(2);
6383 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
6385 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
6389 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
6391 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
6396 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
6398 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
6399 SDValue HiForNormalShift =
6402 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
6407 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
6412 SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
6413 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
6414 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
6417 SDValue Ops[2] = { Lo, Hi };
6469 static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
6470 SDValue Operand, SelectionDAG &DAG,
6488 return SDValue();
6491 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
6498 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
6509 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
6517 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
6518 SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ);
6530 return SDValue();
6533 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
6537 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
6548 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
6557 return SDValue();
6784 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
6786 SDValue Result;
6941 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
6954 static unsigned getExtFactor(SDValue &V) {
6961 static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
6973 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
6982 SDValue Vec;
6989 SDValue ShuffleVec;
6996 ShuffleSourceInfo(SDValue Vec)
7000 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7007 SDValue V = Op.getOperand(i);
7017 return SDValue();
7021 SDValue SourceVec = V.getOperand(0);
7036 return SDValue();
7083 return SDValue();
7099 SDValue VEXTSrc1 =
7102 SDValue VEXTSrc2 =
7136 SDValue Entry = Op.getOperand(i);
7164 return SDValue();
7167 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
7171 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
7173 SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
7437 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
7440 SDValue V0 = Op.getOperand(0);
7441 SDValue V1 = Op.getOperand(1);
7446 return SDValue();
7451 return SDValue();
7467 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
7468 SDValue RHS, SelectionDAG &DAG,
7499 SDValue OpLHS, OpRHS;
7539 SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
7570 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
7573 SDValue V1 = Op.getOperand(0);
7574 SDValue V2 = Op.getOperand(1);
7580 SmallVector<SDValue, 8> TBLMask;
7595 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
7596 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
7598 SDValue Shuffle;
7645 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
7658 SDValue V1 = Op.getOperand(0);
7659 SDValue V2 = Op.getOperand(1);
7680 auto getScaledOffsetDup = [](SDValue BitCast, int &LaneC, MVT &CastVT) {
7688 SDValue Extract = BitCast.getOperand(0);
7779 if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
7786 SDValue DstVec = DstIsLeft ? V1 : V2;
7787 SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
7789 SDValue SrcVec = V1;
7795 SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
7833 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
7839 SDValue SplatVal = Op.getOperand(0);
7857 SDValue ID = DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, dl,
7883 SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
7889 return SDValue();
7893 return SDValue();
7896 SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
7897 SDValue Idx128 = Op.getOperand(2);
7902 SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
7905 return DAG.getNode(ISD::BITCAST, DL, VT, SDValue(DUPQ, 0));
7912 SDValue One = DAG.getConstant(1, DL, MVT::i64);
7913 SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
7916 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
7917 SDValue SV = DAG.getNode(AArch64ISD::INDEX_VECTOR,
7922 SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128);
7923 SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
7924 SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
7927 SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
7955 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7966 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
7972 return SDValue();
7976 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
7978 const SDValue *LHS = nullptr) {
8005 SDValue Mov;
8020 return SDValue();
8024 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
8026 const SDValue *LHS = nullptr) {
8045 SDValue Mov;
8060 return SDValue();
8064 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
8084 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
8091 return SDValue();
8095 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
8106 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
8112 return SDValue();
8116 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
8137 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
8143 return SDValue();
8149 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
8186 static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
8190 return SDValue();
8194 SDValue And;
8195 SDValue Shift;
8197 SDValue FirstOp = N->getOperand(0);
8199 SDValue SecondOp = N->getOperand(1);
8216 return SDValue();
8224 return SDValue();
8230 return SDValue();
8245 return SDValue();
8251 return SDValue();
8253 SDValue X = And.getOperand(0);
8254 SDValue Y = Shift.getOperand(0);
8257 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Shift.getOperand(1));
8268 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
8271 if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
8276 SDValue LHS = Op.getOperand(0);
8290 SDValue NewOp;
8311 static SDValue NormalizeBuildVector(SDValue Op,
8321 SmallVector<SDValue, 16> Ops;
8322 for (SDValue Lane : Op->ops()) {
8342 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG) {
8349 SDValue NewOp;
8380 return SDValue();
8383 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
8405 if (SDValue V = ConstantBuildVector(Op, DAG))
8428 SDValue Value;
8429 SDValue ConstantValue;
8431 SDValue V = Op.getOperand(i);
8477 SDValue V = Op.getOperand(i);
8481 SDValue N0 = N->getOperand(0);
8515 SDValue LHS =
8516 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
8518 SDValue RHS =
8519 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
8544 SDValue Lane = Value.getOperand(1);
8558 SmallVector<SDValue, 8> Ops;
8569 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
8584 SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue),
8594 SDValue V = Op.getOperand(i);
8595 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
8609 return SDValue();
8614 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
8629 SDValue Vec = DAG.getUNDEF(VT);
8630 SDValue Op0 = Op.getOperand(0);
8651 SDValue V = Op.getOperand(i);
8654 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
8663 return SDValue();
8666 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8674 return SDValue();
8686 return SDValue();
8691 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
8694 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec,
8700 SDValue
8701 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
8709 return SDValue();
8721 return SDValue();
8726 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
8738 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
8752 return SDValue();
8764 return SDValue();
8767 SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
8777 return SDValue();
8783 return SDValue();
8831 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
8851 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
8862 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
8878 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
8886 SDValue Zero = DAG.getConstant(0, dl, OpVT);
8887 SDValue One = DAG.getConstant(1, dl, OpVT);
8888 SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
8903 SDValue Shift = Op.getOperand(0);
8912 SDValue Sub = Shift->getOperand(0);
8916 SDValue Xor = Sub.getOperand(1);
8920 SDValue ExtendOpA = Xor.getOperand(0);
8921 SDValue ExtendOpB = Sub.getOperand(0);
8930 SDValue OpA = ExtendOpA.getOperand(0);
8931 SDValue OpB = ExtendOpB.getOperand(0);
8950 SDValue ResultURHADD = DAG.getNode(RHADDOpc, DL, VT, OpA, OpB);
8955 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
9002 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
9003 SDValue NegShiftLeft =
9010 return SDValue();
9013 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
9029 return SDValue();
9031 SDValue Fcmeq;
9056 return SDValue();
9068 return SDValue();
9070 SDValue Cmeq;
9108 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
9117 SDValue LHS = Op.getOperand(0);
9118 SDValue RHS = Op.getOperand(1);
9125 SDValue Cmp =
9139 SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
9143 return SDValue();
9156 SDValue Cmp =
9159 return SDValue();
9162 SDValue Cmp2 =
9165 return SDValue();
9178 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
9180 SDValue VecOp = ScalarOp.getOperand(0);
9186 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
9219 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
9223 return SDValue();
9228 SDValue RHS = Op.getOperand(2);
9236 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
9240 return SDValue();
9245 SDValue RHS = Op.getOperand(2);
9253 SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
9254 SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
9257 SDValue Callee = DAG.getTargetExternalSymbol("__chkstk", PtrVT, 0);
9266 Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
9281 SDValue
9282 AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
9289 SDValue Chain = Op.getOperand(0);
9290 SDValue Size = Op.getOperand(1);
9297 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
9304 SDValue Ops[2] = {SP, Chain};
9312 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
9321 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
9323 SDValue Ops[2] = {SP, Chain};
9327 SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
9512 const SDValue &Base = Mem->getBasePtr();
9589 bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
10104 SDValue AArch64TargetLowering::LowerSVEStructLoad(unsigned Intrinsic,
10105 ArrayRef<SDValue> LoadOps,
10128 SDValue PseudoLoad = DAG.getNode(Opcode, DL, NodeTys, LoadOps);
10129 SmallVector<SDValue, 4> PseudoLoadOps;
10131 PseudoLoadOps.push_back(SDValue(PseudoLoad.getNode(), I));
10389 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
10393 return SDValue();
10397 SDValue Shift = N->getOperand(0);
10398 SDValue Ones = N->getOperand(1);
10401 return SDValue();
10407 return SDValue();
10413 static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
10416 SDValue N0 = N->getOperand(0);
10417 SDValue N1 = N->getOperand(1);
10427 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
10430 SDValue Cmp =
10435 SDValue(Cmp.getNode(), 1));
10437 return SDValue();
10440 static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
10444 return SDValue();
10446 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
10452 SDValue
10458 return SDValue(N,0); // Lower SDIV as SDIV
10464 return SDValue();
10467 SDValue N0 = N->getOperand(0);
10469 SDValue Zero = DAG.getConstant(0, DL, VT);
10470 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
10473 SDValue CCVal;
10474 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL);
10475 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
10476 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp);
10483 SDValue SRA =
10495 static bool IsSVECntIntrinsic(SDValue S) {
10508 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
10512 return SDValue();
10516 return SDValue();
10518 SDValue N0 = N->getOperand(0);
10528 return SDValue();
10548 return SDValue();
10553 return SDValue();
10578 return SDValue();
10593 return SDValue();
10598 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0,
10601 SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0;
10602 SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal;
10603 SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1);
10616 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
10633 return SDValue();
10643 return SDValue();
10650 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
10652 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
10653 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
10655 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
10659 return SDValue();
10662 static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
10666 if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
10671 return SDValue();
10675 return SDValue();
10680 SDValue N0 = N->getOperand(0);
10685 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
10691 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
10698 return SDValue();
10703 static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
10707 return SDValue();
10710 return SDValue();
10712 SDValue Op = N->getOperand(0);
10715 return SDValue();
10717 SDValue ConstVec = Op->getOperand(1);
10719 return SDValue();
10724 return SDValue();
10729 return SDValue();
10733 return SDValue();
10740 return SDValue();
10746 return SDValue();
10756 return SDValue();
10765 SDValue FixConv =
10778 static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
10782 return SDValue();
10784 SDValue Op = N->getOperand(0);
10789 return SDValue();
10791 SDValue ConstVec = N->getOperand(1);
10793 return SDValue();
10798 return SDValue();
10803 return SDValue();
10807 return SDValue();
10813 return SDValue();
10819 return SDValue();
10829 return SDValue();
10832 SDValue ConvInput = Op.getOperand(0);
10847 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,
10869 static SDValue tryCombineToEXTR(SDNode *N,
10878 return SDValue();
10880 SDValue LHS;
10884 return SDValue();
10886 SDValue RHS;
10890 return SDValue();
10895 return SDValue();
10898 return SDValue();
10909 static SDValue tryCombineToBSL(SDNode *N,
10916 return SDValue();
10918 SDValue N0 = N->getOperand(0);
10920 return SDValue();
10922 SDValue N1 = N->getOperand(1);
10924 return SDValue();
10949 return DAG.getNode(AArch64ISD::BSP, DL, VT, SDValue(BVN0, 0),
10953 return SDValue();
10956 static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
10963 return SDValue();
10965 if (SDValue Res = tryCombineToEXTR(N, DCI))
10968 if (SDValue Res = tryCombineToBSL(N, DCI))
10971 return SDValue();
11001 static SDValue performSVEAndCombine(SDNode *N,
11004 return SDValue();
11007 SDValue Src = N->getOperand(0);
11012 SDValue UnpkOp = Src->getOperand(0);
11013 SDValue Dup = N->getOperand(1);
11016 return SDValue();
11039 SDValue And = DAG.getNode(ISD::AND, DL,
11045 SDValue Mask = N->getOperand(1);
11048 return SDValue();
11078 return SDValue();
11084 return SDValue();
11087 static SDValue performANDCombine(SDNode *N,
11090 SDValue LHS = N->getOperand(0);
11093 return SDValue();
11101 return SDValue();
11110 SDValue NewOp;
11113 if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
11115 (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
11120 if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
11122 (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
11127 return SDValue();
11130 static SDValue performSRLCombine(SDNode *N,
11135 return SDValue();
11140 SDValue N0 = N->getOperand(0);
11143 SDValue N1 = N->getOperand(1);
11144 SDValue N00 = N0.getOperand(0);
11155 return SDValue();
11158 static SDValue performConcatVectorsCombine(SDNode *N,
11163 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
11179 SDValue N00 = N0->getOperand(0);
11180 SDValue N10 = N1->getOperand(0);
11201 return SDValue();
11216 SDValue N00 = N0->getOperand(0);
11217 SDValue N01 = N0->getOperand(1);
11218 SDValue N10 = N1->getOperand(0);
11219 SDValue N11 = N1->getOperand(1);
11228 SDValue N00Source = N00->getOperand(0);
11229 SDValue N01Source = N01->getOperand(0);
11230 SDValue N10Source = N10->getOperand(0);
11231 SDValue N11Source = N11->getOperand(0);
11268 return SDValue();
11269 SDValue RHS = N1->getOperand(0);
11273 return SDValue();
11286 static SDValue tryCombineFixedPointConvert(SDNode *N,
11292 return SDValue();
11303 SDValue Op1 = N->getOperand(1);
11306 SDValue IID = N->getOperand(0);
11307 SDValue Shift = N->getOperand(2);
11308 SDValue Vec = Op1.getOperand(0);
11309 SDValue Lane = Op1.getOperand(1);
11326 SDValue Convert =
11330 return SDValue();
11348 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
11366 return SDValue();
11371 return SDValue();
11383 static bool isEssentiallyExtractHighSubvector(SDValue N) {
11394 const SDValue *Opnd0;
11395 const SDValue *Opnd1;
11401 const SDValue *Cmp;
11425 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
11468 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
11481 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
11483 SDValue LHS = Op->getOperand(0);
11484 SDValue RHS = Op->getOperand(1);
11491 return SDValue();
11499 return SDValue();
11501 SDValue CCVal;
11502 SDValue Cmp;
11531 static SDValue performAddSubLongCombine(SDNode *N,
11535 return SDValue();
11541 return SDValue();
11545 SDValue LHS = N->getOperand(0);
11546 SDValue RHS = N->getOperand(1);
11550 return SDValue();
11559 return SDValue();
11565 return SDValue();
11580 static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
11584 return SDValue();
11586 SDValue LHS = N->getOperand(1);
11587 SDValue RHS = N->getOperand(2);
11598 return SDValue();
11602 return SDValue();
11609 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
11621 return SDValue();
11627 return SDValue();
11674 return SDValue();
11680 static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
11681 SDValue AndN = N->getOperand(2);
11683 return SDValue();
11687 return SDValue();
11693 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
11703 static SDValue LowerSVEIntReduction(SDNode *N, unsigned Opc,
11710 SDValue Pred = N->getOperand(1);
11711 SDValue Data = N->getOperand(2);
11717 return SDValue();
11721 SDValue Reduce = DAG.getNode(Opc, dl, OutputVT, Pred, Data);
11722 SDValue Zero = DAG.getConstant(0, dl, MVT::i64);
11723 SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Reduce, Zero);
11728 return SDValue();
11731 static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
11733 SDValue Op1 = N->getOperand(1);
11734 SDValue Op2 = N->getOperand(2);
11746 static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
11748 SDValue Scalar = N->getOperand(3);
11754 SDValue Passthru = N->getOperand(1);
11755 SDValue Pred = N->getOperand(2);
11760 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
11769 return SDValue();
11776 SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
11777 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2));
11778 SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3),
11781 SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2);
11785 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
11789 return SDValue();
11791 SDValue Comparator = N->getOperand(3);
11797 SDValue Pred = N->getOperand(1);
11798 SDValue Imm;
11818 return SDValue();
11832 return SDValue();
11839 return SDValue();
11841 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
11846 return SDValue();
11849 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
11860 SDValue TVal = DAG.getConstant(1, DL, OutVT);
11861 SDValue FVal = DAG.getConstant(0, DL, OutVT);
11864 SDValue Test = DAG.getNode(AArch64ISD::PTEST, DL, MVT::Other, Pg, Op);
11868 SDValue CC = DAG.getConstant(getInvertedCondCode(Cond), DL, MVT::i32);
11869 SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test);
11873 static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc,
11877 SDValue Pred = N->getOperand(1);
11878 SDValue VecToReduce = N->getOperand(2);
11881 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
11885 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
11890 static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc,
11894 SDValue Pred = N->getOperand(1);
11895 SDValue InitVal = N->getOperand(2);
11896 SDValue VecToReduce = N->getOperand(3);
11901 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
11905 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce);
11913 static SDValue performIntrinsicCombine(SDNode *N,
12092 return SDValue();
12095 static SDValue performExtendCombine(SDNode *N,
12108 SDValue NewABD = tryCombineLongOpWithDup(IID, ABDNode, DCI, DAG);
12110 return SDValue();
12142 return SDValue();
12150 return SDValue();
12154 SDValue Src = N->getOperand(0);
12157 return SDValue();
12162 return SDValue();
12173 SDValue Lo, Hi;
12190 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
12191 SDValue SplatVal, unsigned NumVecElts) {
12201 SDValue BasePtr = St.getBasePtr();
12205 SDValue NewST1 =
12219 SDValue OffsetPtr =
12260 static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
12265 return SDValue();
12272 SDValue Ops[] = { N->getOperand(0), // Chain
12277 SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
12278 SDValue LoadChain = SDValue(Load.getNode(), 1);
12286 static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
12293 return SDValue();
12300 SDValue PassThru = DAG.getConstant(0, DL, LoadVT);
12301 SDValue L = DAG.getMaskedLoad(LoadVT, DL, MINode->getChain(),
12308 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
12316 static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
12324 return SDValue();
12330 SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
12331 SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
12332 SDValue LoadChain = SDValue(Load.getNode(), 1);
12340 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
12342 SDValue Data = N->getOperand(2);
12345 SDValue InputVT = DAG.getValueType(DataVT);
12349 return SDValue();
12354 SDValue SrcNew;
12360 SDValue Ops[] = { N->getOperand(0), // Chain
12370 static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
12373 SDValue Data = N->getOperand(2);
12379 return SDValue();
12406 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
12407 SDValue StVal = St.getValue();
12412 return SDValue();
12421 return SDValue();
12424 return SDValue();
12430 return SDValue();
12435 return SDValue();
12442 return SDValue();
12446 SDValue EltVal = StVal.getOperand(I);
12448 return SDValue();
12463 SDValue SplatVal =
12473 static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
12474 SDValue StVal = St.getValue();
12480 return SDValue();
12485 return SDValue();
12490 return SDValue();
12496 SDValue SplatVal;
12500 return SDValue();
12506 return SDValue();
12511 return SDValue();
12514 return SDValue();
12521 return SDValue();
12526 static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
12532 return SDValue();
12534 SDValue StVal = S->getValue();
12538 return SDValue();
12543 if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
12551 return SDValue();
12555 return SDValue();
12560 return SDValue();
12569 return SDValue();
12574 if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
12582 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
12584 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
12586 SDValue BasePtr = S->getBasePtr();
12587 SDValue NewST1 =
12590 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
12599 static SDValue performPostLD1Combine(SDNode *N,
12603 return SDValue();
12609 return SDValue();
12615 return SDValue();
12618 SDValue Lane;
12623 return SDValue();
12630 return SDValue();
12639 return SDValue();
12642 SDValue Addr = LD->getOperand(1);
12643 SDValue Vector = N->getOperand(0);
12653 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
12674 SmallVector<SDValue, 8> Ops;
12686 SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
12691 SDValue NewResults[] = {
12692 SDValue(LD, 0), // The result of load
12693 SDValue(UpdN.getNode(), 2) // Chain
12696 DCI.CombineTo(N, SDValue(UpdN.getNode(), 0)); // Dup/Inserted Result
12697 DCI.CombineTo(User, SDValue(UpdN.getNode(), 1)); // Write back register
12701 return SDValue();
12706 static bool performTBISimplification(SDValue Addr,
12721 static SDValue performSTORECombine(SDNode *N,
12725 if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
12730 return SDValue(N, 0);
12732 return SDValue();
12738 static SDValue performNEONPostLDSTCombine(SDNode *N,
12742 return SDValue();
12745 SDValue Addr = N->getOperand(AddrOpIdx);
12826 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
12836 SmallVector<SDValue, 8> Ops;
12856 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
12861 std::vector<SDValue> NewResults;
12863 NewResults.push_back(SDValue(UpdN.getNode(), i));
12865 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1));
12867 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
12871 return SDValue();
12877 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
13056 SDValue performCONDCombine(SDNode *N,
13065 return SDValue();
13074 return SDValue();
13085 return SDValue();
13087 SDValue AddValue = AndNode->getOperand(0);
13090 return SDValue();
13094 SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
13095 SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
13096 SDValue SubsInputValue = SubsNode->getOperand(1);
13103 return SDValue();
13110 return SDValue();
13115 return SDValue();
13121 SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
13123 SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
13126 return SDValue(N, 0);
13130 static SDValue performBRCONDCombine(SDNode *N,
13138 return SDValue();
13140 if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
13142 SDValue Chain = N->getOperand(0);
13143 SDValue Dest = N->getOperand(1);
13144 SDValue CCVal = N->getOperand(2);
13145 SDValue Cmp = N->getOperand(3);
13150 return SDValue();
13154 return SDValue();
13159 return SDValue();
13161 SDValue LHS = Cmp.getOperand(0);
13162 SDValue RHS = Cmp.getOperand(1);
13167 return SDValue();
13173 return SDValue();
13177 return SDValue();
13180 SDValue BR;
13189 return SDValue();
13196 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
13269 static SDValue performTBZCombine(SDNode *N,
13274 SDValue TestSrc = N->getOperand(1);
13275 SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
13278 return SDValue();
13300 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
13301 SDValue N0 = N->getOperand(0);
13306 return SDValue();
13313 return SDValue();
13315 SDValue IfTrue = N->getOperand(1);
13316 SDValue IfFalse = N->getOperand(2);
13317 SDValue SetCC =
13329 static SDValue performSelectCombine(SDNode *N,
13332 SDValue N0 = N->getOperand(0);
13336 return SDValue();
13352 return SDValue();
13356 return SDValue();
13365 return SDValue();
13374 SDValue LHS =
13376 SDValue RHS =
13378 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
13382 SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
13390 static SDValue performNVCASTCombine(SDNode *N) {
13394 return SDValue();
13400 static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
13406 return SDValue();
13411 return SDValue();
13416 return SDValue();
13425 return SDValue();
13436 return SDValue();
13442 return SDValue();
13445 SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
13452 static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset,
13457 SDValue Shift = DAG.getConstant(Log2_32(BitWidth / 8), DL, MVT::i64);
13458 SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift);
13491 static bool isValidImmForSVEVecImmAddrMode(SDValue Offset,
13498 static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
13501 const SDValue Src = N->getOperand(2);
13511 return SDValue();
13516 return SDValue();
13520 SDValue Base = N->getOperand(4);
13523 SDValue Offset = N->getOperand(5);
13562 return SDValue();
13572 return SDValue();
13580 SDValue InputVT = DAG.getValueType(SrcVT);
13585 SDValue SrcNew;
13592 SDValue Ops[] = {N->getOperand(0), // Chain
13602 static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
13613 return SDValue();
13617 SDValue Base = N->getOperand(3);
13620 SDValue Offset = N->getOperand(4);
13665 return SDValue();
13680 SDValue OutVT = DAG.getValueType(RetVT);
13685 SDValue Ops[] = {N->getOperand(0), // Chain
13689 SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops);
13690 SDValue LoadChain = SDValue(Load.getNode(), 1);
13703 static SDValue
13707 return SDValue();
13710 SDValue Src = N->getOperand(0);
13727 SDValue ExtOp = Src->getOperand(0);
13739 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(),
13808 return SDValue();
13815 return SDValue();
13820 SmallVector<SDValue, 5> Ops;
13824 SDValue ExtLoad = DAG.getNode(NewOpc, SDLoc(N), VTs, Ops);
13829 return SDValue(N, 0);
13835 static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG) {
13837 SDValue Offset = N->getOperand(OffsetPos);
13841 return SDValue();
13846 SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
13858 static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG,
13863 return SDValue();
13866 SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
13877 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
13921 return SDValue(N, 0);
14080 SDValue Chain = N->getOperand(0);
14081 SDValue Src1 = N->getOperand(2);
14082 SDValue Idx = N->getOperand(3);
14087 SDValue Val =
14094 SDValue Chain = N->getOperand(0);
14095 SDValue Tuple = N->getOperand(2);
14096 SDValue Idx = N->getOperand(3);
14097 SDValue Vec = N->getOperand(4);
14110 SmallVector<SDValue, 4> Opnds;
14120 SDValue Concat =
14128 SDValue Chain = N->getOperand(0);
14130 SmallVector<SDValue, 4> Opnds;
14139 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Opnds);
14146 SDValue Chain = N->getOperand(0);
14147 SDValue Mask = N->getOperand(2);
14148 SDValue BasePtr = N->getOperand(3);
14149 SDValue LoadOps[] = {Chain, Mask, BasePtr};
14152 SDValue Result =
14163 return SDValue();
14171 SDValue &Chain) const {
14177 SDValue TCChain = Chain;
14211 bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
14212 SDValue &Offset,
14235 bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
14236 SDValue &Offset,
14240 SDValue Ptr;
14258 SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
14261 SDValue Ptr;
14282 static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
14285 SDValue Op = N->getOperand(0);
14291 Op = SDValue(
14301 SmallVectorImpl<SDValue> &Results,
14305 SDValue Lo, Hi;
14309 SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
14310 SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
14314 static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
14316 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N);
14317 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
14324 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
14325 SDValue In = N->getOperand(0);
14353 SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0));
14358 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
14360 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i64);
14361 SDValue VHi = DAG.getAnyExtOrTrunc(
14366 SDValue RegClass =
14368 SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
14369 SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
14370 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
14371 return SDValue(
14376 SmallVectorImpl<SDValue> &Results,
14385 SDValue Ops[] = {
14420 SDValue Lo = DAG.getTargetExtractSubreg(SubReg1, SDLoc(N), MVT::i64,
14421 SDValue(CmpSwap, 0));
14422 SDValue Hi = DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
14423 SDValue(CmpSwap, 0));
14426 Results.push_back(SDValue(CmpSwap, 1)); // Chain out
14432 SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
14442 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
14443 Results.push_back(SDValue(CmpSwap, 3));
14447 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
14459 Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
14463 Results.push_back(LowerCTPOP(SDValue(N, 0), DAG));
14492 assert(SDValue(N, 0).getValueType() == MVT::i128 &&
14501 SDValue Result = DAG.getMemIntrinsicNode(
14507 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
14817 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
14987 static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
15055 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
15063 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
15071 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
15077 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
15082 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
15088 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
15093 SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
15094 SDValue Op, SelectionDAG &DAG) const {
15108 SDValue MergedValues[2] = {Result, Load->getChain()};
15113 SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
15114 SDValue Op, SelectionDAG &DAG) const {
15129 SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
15130 SDValue Op, SelectionDAG &DAG) const {
15135 SDValue Val = Op.getOperand(0);
15165 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
15176 SmallVector<SDValue, 4> Operands = {Pg};
15177 for (const SDValue &V : Op->op_values()) {
15194 SmallVector<SDValue, 4> Operands = {Pg};
15195 for (const SDValue &V : Op->op_values()) {