Lines Matching refs:SDValue
1877 static bool isSRL16(const SDValue &Op) {
1885 static bool isSRA16(const SDValue &Op) {
1893 static bool isSHL16(const SDValue &Op) {
1905 static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
2038 SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
2039 MVT LocVT, MVT ValVT, SDValue Val) const {
2052 SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
2054 SDValue Val) const {
2069 SDValue ARMTargetLowering::LowerCallResult(
2070 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2072 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
2073 SDValue ThisVal) const {
2093 SDValue Val;
2097 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2102 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2111 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
2158 SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
2159 SDValue Arg, const SDLoc &dl,
2164 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
2173 SDValue Chain, SDValue &Arg,
2176 SDValue &StackPtr,
2177 SmallVectorImpl<SDValue> &MemOpChains,
2179 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2201 SDValue
2203 SmallVectorImpl<SDValue> &InVals) const {
2207 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2209 SDValue Chain = CLI.Chain;
2210 SDValue Callee = CLI.Callee;
2285 SDValue StackPtr =
2289 SmallVector<SDValue, 8> MemOpChains;
2297 SDValue Arg = OutVals[realArgIdx];
2332 SDValue Mask =
2342 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2344 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2394 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
2395 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
2396 SDValue Load =
2413 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
2414 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
2415 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
2416 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
2417 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
2419 SDValue AlignNode =
2423 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2440 SDValue InFlag;
2477 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2491 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2543 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2548 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2595 std::vector<SDValue> Ops;
2632 SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
2652 isThisReturn ? OutVals[0] : SDValue());
2713 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2739 SDValue Ptr = Ld->getBasePtr();
2757 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
2760 const SmallVectorImpl<SDValue> &OutVals,
2848 SDValue Arg = OutVals[realArgIdx];
2893 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2926 SDValue
2927 ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2930 const SmallVectorImpl<SDValue> &OutVals,
2942 SDValue Flag;
2943 SmallVector<SDValue, 4> RetOps;
2969 SDValue Arg = OutVals[realRVLocIdx];
2984 SDValue ZE = Arg.getOperand(0);
2986 SDValue BC = ZE.getOperand(0);
3012 SDValue Mask =
3024 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3026 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
3048 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
3103 bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3109 SDValue TCChain = Chain;
3132 SDValue UseChain = UI->getOperand(0);
3190 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3192 SDValue WriteValue = Op->getOperand(2);
3198 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
3200 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
3202 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
3212 SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3218 SDValue Res;
3236 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV),
3253 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3261 SDValue CPAddr;
3274 SDValue Result = DAG.getLoad(
3279 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
3285 /// SDValue containing the final node.
3307 SDValue
3308 ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3316 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3320 SDValue Chain = DAG.getEntryNode();
3321 SDValue FuncTLVGet = DAG.getLoad(
3344 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
3352 SDValue
3353 ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3357 SDValue Chain = DAG.getEntryNode();
3362 SDValue Ops[] = {Chain,
3369 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
3372 SDValue TEB = CurrentTEB.getValue(0);
3377 SDValue TLSArray =
3385 SDValue TLSIndex =
3390 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
3392 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
3399 SDValue Offset = DAG.getLoad(
3409 SDValue
3421 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3426 SDValue Chain = Argument.getValue(1);
3428 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3444 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3450 SDValue
3456 SDValue Offset;
3457 SDValue Chain = DAG.getEntryNode();
3460 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3479 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3502 SDValue
3503 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3550 static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3570 return SDValue();
3576 return SDValue();
3583 return SDValue();
3599 return SDValue();
3612 return SDValue();
3622 return SDValue();
3636 SDValue CPAddr = DAG.getTargetConstantPool(CPVal, PtrVT, Align(4));
3655 SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3668 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3678 if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
3683 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3685 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3693 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3694 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3698 SDValue RelAddr;
3701 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
3706 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3712 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
3713 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
3726 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, Align(4));
3734 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3750 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
3751 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
3759 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3775 SDValue Result;
3791 SDValue
3792 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3794 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
3800 SDValue
3801 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3807 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3814 SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3815 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3822 return SDValue(); // Don't custom lower most intrinsics.
3827 SDValue Chain = Op.getOperand(0);
3835 SDValue ReturnAddress =
3838 SDValue Callee =
3840 SDValue RegisterMask = DAG.getRegisterMask(Mask);
3842 return SDValue(
3848 return SDValue(
3856 SDValue
3857 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3862 default: return SDValue(); // Don't custom lower most intrinsics.
3868 const SDValue &Operand = Op.getOperand(1);
3870 SDValue SRA =
3872 SDValue XOR = DAG.getNode(ISD::XOR, dl, VTy, SRA, Operand);
3873 SDValue SHL =
3875 SDValue OR =
3877 SDValue Result = DAG.getNode(ISD::CTLZ, dl, VTy, OR);
3883 const SDValue &Operand = Op.getOperand(1);
3886 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
3888 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
3890 SDValue Constant0 = DAG.getConstant(0, dl, VTy);
3891 SDValue Constant1 = DAG.getConstant(1, dl, VTy);
3892 SDValue Constant31 = DAG.getConstant(31, dl, VTy);
3893 SDValue SRAHi = DAG.getNode(ISD::SRA, dl, VTy, Hi, Constant31);
3894 SDValue XORHi = DAG.getNode(ISD::XOR, dl, VTy, SRAHi, Hi);
3895 SDValue SHLHi = DAG.getNode(ISD::SHL, dl, VTy, XORHi, Constant1);
3896 SDValue ORHi = DAG.getNode(ISD::OR, dl, VTy, SHLHi, Constant1);
3897 SDValue CLSHi = DAG.getNode(ISD::CTLZ, dl, VTy, ORHi);
3898 SDValue CheckLo =
3900 SDValue HiIsZero =
3902 SDValue AdjustedLo =
3904 SDValue CLZAdjustedLo = DAG.getNode(ISD::CTLZ, dl, VTy, AdjustedLo);
3905 SDValue Result =
3915 SDValue CPAddr;
3923 SDValue Result = DAG.getLoad(
3928 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3953 return SDValue();
3995 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4032 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4059 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4067 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4073 SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4075 SDValue &Root,
4089 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4091 SDValue ArgValue2;
4097 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4119 const SDLoc &dl, SDValue &Chain,
4151 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
4153 SmallVector<SDValue, 4> MemOps;
4159 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
4160 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4173 const SDLoc &dl, SDValue &Chain,
4193 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4210 SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4211 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4218 SDValue Val = Parts[0];
4225 return SDValue();
4228 SDValue ARMTargetLowering::LowerFormalArguments(
4229 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4231 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4243 SmallVector<SDValue, 16> ArgValues;
4244 SDValue ArgValue;
4303 SDValue ArgValue1 =
4306 SDValue ArgValue2;
4309 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4415 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4437 static bool isFloatingPointZero(SDValue Op) {
4443 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
4452 SDValue BitcastOp = Op->getOperand(0);
4462 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4463 SDValue &ARMcc, SelectionDAG &DAG,
4540 SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
4560 SDValue Shift = DAG.getNode(ARMISD::LSLS, dl,
4564 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
4565 Shift.getValue(1), SDValue());
4603 SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4607 SDValue Cmp;
4619 SDValue
4620 ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
4642 std::pair<SDValue, SDValue>
4643 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4644 SDValue &ARMcc) const {
4647 SDValue Value, OverflowCmp;
4648 SDValue LHS = Op.getOperand(0);
4649 SDValue RHS = Op.getOperand(1);
4712 SDValue
4713 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
4716 return SDValue();
4718 SDValue Value, OverflowCmp;
4719 SDValue ARMcc;
4721 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4724 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4725 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4728 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
4735 static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,
4742 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL,
4748 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
4759 SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
4763 return SDValue();
4765 SDValue LHS = Op.getOperand(0);
4766 SDValue RHS = Op.getOperand(1);
4771 SDValue Value;
4772 SDValue Overflow;
4796 static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4800 return SDValue();
4802 return SDValue();
4808 return SDValue();
4818 SDValue Add =
4825 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
4826 SDValue Cond = Op.getOperand(0);
4827 SDValue SelectTrue = Op.getOperand(1);
4828 SDValue SelectFalse = Op.getOperand(2);
4836 return SDValue();
4838 SDValue Value, OverflowCmp;
4839 SDValue ARMcc;
4841 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4863 SDValue True;
4864 SDValue False;
4875 SDValue ARMcc = Cond.getOperand(2);
4876 SDValue CCR = Cond.getOperand(3);
4877 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
4944 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
4945 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
4946 SDValue Cmp, SelectionDAG &DAG) const {
4953 SDValue TrueLow = TrueVal.getValue(0);
4954 SDValue TrueHigh = TrueVal.getValue(1);
4955 SDValue FalseLow = FalseVal.getValue(0);
4956 SDValue FalseHigh = FalseVal.getValue(1);
4958 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
4960 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
4984 static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
4985 const SDValue TrueVal, const SDValue FalseVal,
4986 const ISD::CondCode CC, const SDValue K) {
4994 static bool isUpperSaturate(const SDValue LHS, const SDValue RHS,
4995 const SDValue TrueVal, const SDValue FalseVal,
4996 const ISD::CondCode CC, const SDValue K) {
5020 static bool isSaturatingConditional(const SDValue &Op, SDValue &V,
5022 SDValue LHS1 = Op.getOperand(0);
5023 SDValue RHS1 = Op.getOperand(1);
5024 SDValue TrueVal1 = Op.getOperand(2);
5025 SDValue FalseVal1 = Op.getOperand(3);
5028 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5032 SDValue LHS2 = Op2.getOperand(0);
5033 SDValue RHS2 = Op2.getOperand(1);
5034 SDValue TrueVal2 = Op2.getOperand(2);
5035 SDValue FalseVal2 = Op2.getOperand(3);
5040 SDValue *K1 = isa<ConstantSDNode>(LHS1) ? &LHS1 : isa<ConstantSDNode>(RHS1)
5043 SDValue *K2 = isa<ConstantSDNode>(LHS2) ? &LHS2 : isa<ConstantSDNode>(RHS2)
5046 SDValue K2Tmp = isa<ConstantSDNode>(TrueVal2) ? TrueVal2 : FalseVal2;
5047 SDValue V1Tmp = (K1 && *K1 == LHS1) ? RHS1 : LHS1;
5048 SDValue V2Tmp = (K2 && *K2 == LHS2) ? RHS2 : LHS2;
5049 SDValue V2 = (K2Tmp == TrueVal2) ? FalseVal2 : TrueVal2;
5055 SDValue V2TmpReg = V2Tmp;
5066 const SDValue *LowerCheckOp =
5072 const SDValue *UpperCheckOp =
5119 static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
5120 SDValue &SatK)
5122 SDValue LHS = Op.getOperand(0);
5123 SDValue RHS = Op.getOperand(1);
5125 SDValue TrueVal = Op.getOperand(2);
5126 SDValue FalseVal = Op.getOperand(3);
5128 SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS)
5136 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5138 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5163 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5168 SDValue SatValue;
5187 SDValue LowerSatConstant;
5190 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
5193 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
5200 SDValue LHS = Op.getOperand(0);
5201 SDValue RHS = Op.getOperand(1);
5203 SDValue TrueVal = Op.getOperand(2);
5204 SDValue FalseVal = Op.getOperand(3);
5251 SDValue ARMcc;
5252 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5292 SDValue ARMcc;
5293 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5294 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5326 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5327 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5328 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5329 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5331 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
5333 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5341 static bool canChangeToInt(SDValue Op, bool &SeenZero,
5362 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5374 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5375 SDValue &RetVal1, SDValue &RetVal2) {
5385 SDValue Ptr = Ld->getBasePtr();
5392 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
5405 SDValue
5406 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5407 SDValue Chain = Op.getOperand(0);
5409 SDValue LHS = Op.getOperand(2);
5410 SDValue RHS = Op.getOperand(3);
5411 SDValue Dest = Op.getOperand(4);
5427 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5428 SDValue ARMcc;
5434 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5435 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5440 SDValue LHS1, LHS2;
5441 SDValue RHS1, RHS2;
5449 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5453 return SDValue();
5456 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5457 SDValue Chain = Op.getOperand(0);
5458 SDValue Cond = Op.getOperand(1);
5459 SDValue Dest = Op.getOperand(2);
5472 return SDValue();
5475 SDValue Value, OverflowCmp;
5476 SDValue ARMcc;
5484 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5490 return SDValue();
5493 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5494 SDValue Chain = Op.getOperand(0);
5496 SDValue LHS = Op.getOperand(2);
5497 SDValue RHS = Op.getOperand(3);
5498 SDValue Dest = Op.getOperand(4);
5524 return SDValue();
5527 SDValue Value, OverflowCmp;
5528 SDValue ARMcc;
5538 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5545 SDValue ARMcc;
5546 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5547 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5555 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5562 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5563 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5564 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5566 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
5567 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
5570 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
5576 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5577 SDValue Chain = Op.getOperand(0);
5578 SDValue Table = Op.getOperand(1);
5579 SDValue Index = Op.getOperand(2);
5584 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
5587 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
5612 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5643 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5649 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
5662 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
5663 SDValue Result;
5672 SDValue Result =
5682 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5727 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5747 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5749 SDValue Tmp0 = Op.getOperand(0);
5750 SDValue Tmp1 = Op.getOperand(1);
5761 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
5783 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff),
5786 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
5789 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
5810 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
5811 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5823 SDValue Lo = Tmp0.getValue(0);
5824 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
5829 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
5835 return SDValue();
5841 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5842 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
5853 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
5864 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
5886 static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
5894 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
5911 static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
5913 SDValue Op = BC->getOperand(0);
5924 return SDValue();
5931 return SDValue();
5940 return SDValue();
5945 SDValue ExtractSrc = Op.getOperand(0);
5949 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
5959 SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
5963 SDValue Op = N->getOperand(0);
5982 return SDValue();
5988 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
5991 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
5993 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
6001 SDValue Cvt;
6014 return SDValue();
6023 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6026 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
6028 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
6034 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6040 SDValue ShOpLo = Op.getOperand(0);
6041 SDValue ShOpHi = Op.getOperand(1);
6042 SDValue ShAmt = Op.getOperand(2);
6043 SDValue ARMcc;
6044 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6049 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6051 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
6052 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6054 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
6055 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6056 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6057 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6059 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift,
6062 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
6063 SDValue HiBigShift = Opc == ISD::SRA
6067 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6069 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
6072 SDValue Ops[2] = { Lo, Hi };
6078 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6084 SDValue ShOpLo = Op.getOperand(0);
6085 SDValue ShOpHi = Op.getOperand(1);
6086 SDValue ShAmt = Op.getOperand(2);
6087 SDValue ARMcc;
6088 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6091 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6093 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
6094 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
6095 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6097 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6099 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
6100 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6102 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
6105 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6107 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
6108 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
6111 SDValue Ops[2] = { Lo, Hi };
6115 SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
6122 SDValue Chain = Op.getOperand(0);
6123 SDValue Ops[] = {Chain,
6126 SDValue FPSCR =
6129 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
6131 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
6133 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
6138 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6145 SDValue X = N->getOperand(0);
6146 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
6147 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
6153 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6155 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6163 SDValue WidthMinus1 =
6166 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
6173 SDValue Bits;
6176 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6180 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6188 return SDValue();
6190 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
6194 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6206 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
6213 SmallVector<SDValue, 8> Ops;
6230 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6251 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6265 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6280 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6287 return SDValue();
6316 SDValue NegatedCount = DAG.getNode(
6323 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6330 return SDValue();
6338 SDValue ShAmt = N->getOperand(1);
6346 return SDValue();
6365 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6368 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6375 Hi = SDValue(Lo.getNode(), 1);
6381 return SDValue();
6385 return SDValue();
6388 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6390 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6405 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6411 SDValue Op0 = Op.getOperand(0);
6412 SDValue Op1 = Op.getOperand(1);
6413 SDValue CC = Op.getOperand(2);
6426 return SDValue();
6431 return SDValue();
6442 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
6443 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
6444 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
6446 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6447 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
6457 return SDValue();
6486 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6488 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6490 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6498 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6500 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6502 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6531 SDValue AndOp;
6544 SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
6557 SDValue SingleOp;
6568 SDValue Result;
6585 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6586 SDValue LHS = Op.getOperand(0);
6587 SDValue RHS = Op.getOperand(1);
6588 SDValue Carry = Op.getOperand(2);
6589 SDValue Cond = Op.getOperand(3);
6602 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
6604 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
6605 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
6606 SDValue ARMcc = DAG.getConstant(
6608 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6609 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
6610 Cmp.getValue(1), SDValue());
6618 static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6636 return SDValue();
6659 return SDValue();
6693 if (type == OtherModImm) return SDValue();
6705 return SDValue();
6720 return SDValue();
6724 return SDValue();
6735 return SDValue();
6768 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
6789 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
6790 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
6800 return SDValue();
6805 return SDValue();
6820 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
6821 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
6830 return SDValue();
6839 return SDValue();
6842 SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
6844 if (NewVal != SDValue()) {
6846 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
6852 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
6861 if (NewVal != SDValue()) {
6863 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
6869 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
6875 return SDValue();
7274 static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7278 return SDValue();
7283 return SDValue();
7292 return SDValue();
7296 return SDValue();
7297 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7298 SDValue Op1 = BV.getOperand(1).getOperand(0).getOperand(0);
7300 return SDValue();
7304 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7311 return SDValue();
7313 return SDValue();
7316 SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
7327 static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7331 return SDValue();
7336 return SDValue();
7344 return SDValue();
7345 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7348 return SDValue();
7352 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7359 return SDValue();
7367 // instruction, return an SDValue of such a constant (will become a MOV
7369 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7373 return SDValue();
7383 return SDValue();
7386 static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7406 return SDValue();
7410 SDValue FirstOp = Op.getOperand(0);
7416 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
7424 SDValue V = Op.getOperand(i);
7433 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
7436 SDValue V = Op.getOperand(i);
7448 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7468 SDValue Val =
7473 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
7483 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
7491 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
7511 // Map of the number of times a particular SDValue appears in the
7513 DenseMap<SDValue, unsigned> ValueCounts;
7514 SDValue Value;
7516 SDValue V = Op.getOperand(i);
7552 SDValue N;
7585 SmallVector<SDValue, 3> Ops;
7595 SmallVector<SDValue, 8> Ops;
7603 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
7609 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
7619 return SDValue();
7625 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7630 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
7632 if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
7638 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7641 SDValue Lower =
7645 SDValue Upper = DAG.getBuildVector(
7661 SmallVector<SDValue, 8> Ops;
7664 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
7675 SDValue Vec = DAG.getUNDEF(VT);
7677 SDValue V = Op.getOperand(i);
7680 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
7686 return SDValue();
7691 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7699 SDValue Vec;
7706 SDValue ShuffleVec;
7713 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7715 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7722 SDValue V = Op.getOperand(i);
7728 return SDValue();
7732 return SDValue();
7736 SDValue SourceVec = V.getOperand(0);
7750 return SDValue();
7782 return SDValue();
7792 return SDValue();
7796 return SDValue();
7812 SDValue VEXTSrc1 =
7815 SDValue VEXTSrc2 =
7848 SDValue Entry = Op.getOperand(i);
7878 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
7882 SDValue Shuffle = buildLegalVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
7885 return SDValue();
7974 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
7975 SDValue RHS, SelectionDAG &DAG,
7987 SDValue OpLHS, OpRHS;
8032 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
8036 SDValue V1 = Op.getOperand(0);
8037 SDValue V2 = Op.getOperand(1);
8040 SmallVector<SDValue, 8> VTBLMask;
8053 static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
8056 SDValue OpLHS = Op.getOperand(0);
8083 static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
8088 SDValue AllOnes =
8092 SDValue AllZeroes =
8099 SDValue RecastV1;
8110 SDValue PredAsVector =
8118 static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8127 SDValue V1 = Op.getOperand(0);
8130 SDValue cast = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, V1);
8131 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast);
8132 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit,
8145 SDValue PredAsVector = PromoteMVEPredVector(dl, V1, VT, DAG);
8149 SDValue Shuffled = DAG.getVectorShuffle(NewVT, dl, PredAsVector,
8158 static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,
8167 return SDValue();
8174 SDValue Parts[4];
8206 SDValue Input = Op->getOperand(0);
8211 SDValue BitCast = DAG.getBitcast(MVT::v4i32, Input);
8219 return SDValue();
8229 SDValue NewShuffle = DAG.getVectorShuffle(
8231 SDValue BitCast = DAG.getBitcast(MVT::v4i32, NewShuffle);
8240 SDValue NewVec = DAG.getBuildVector(MVT::v4i32, dl, Parts);
8244 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8246 SDValue V1 = Op.getOperand(0);
8247 SDValue V2 = Op.getOperand(1);
8354 SDValue SubV1 = V1->getOperand(0);
8355 SDValue SubV2 = V1->getOperand(1);
8369 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
8417 SmallVector<SDValue, 8> Ops;
8427 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
8435 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8439 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8442 return SDValue();
8445 static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8453 SDValue Conv =
8459 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32,
8461 SDValue BFI = DAG.getNode(ARMISD::BFI, dl, MVT::i32, Conv, Ext,
8466 SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8469 SDValue Lane = Op.getOperand(2);
8471 return SDValue();
8473 SDValue Elt = Op.getOperand(1);
8493 SDValue VecIn = Op.getOperand(0);
8498 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt);
8499 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn);
8500 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT,
8508 static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8516 SDValue Conv =
8521 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv,
8526 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8529 SDValue Lane = Op.getOperand(1);
8531 return SDValue();
8533 SDValue Vec = Op.getOperand(0);
8547 static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8549 SDValue V1 = Op.getOperand(0);
8550 SDValue V2 = Op.getOperand(1);
8563 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8564 SDValue NewV2 = PromoteMVEPredVector(dl, V2, Op2VT, DAG);
8577 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
8578 auto ExractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
8582 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
8599 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
8610 SDValue Val = DAG.getUNDEF(MVT::v2f64);
8611 SDValue Op0 = Op.getOperand(0);
8612 SDValue Op1 = Op.getOperand(1);
8624 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
8626 SDValue V1 = Op.getOperand(0);
8627 SDValue V2 = Op.getOperand(1);
8639 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8647 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
8649 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
8755 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
8777 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
8801 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
8812 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
8813 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1));
8815 SDValue extLoad =
8817 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad);
8839 SmallVector<SDValue, 8> Ops;
8873 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
8911 return SDValue();
8920 SDValue Op0;
8921 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
8938 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
8939 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
8948 static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
8979 static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
8983 SDValue N2;
9018 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9025 SDValue N0 = Op.getOperand(0);
9026 SDValue N1 = Op.getOperand(1);
9027 SDValue N2, N3;
9054 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9062 SDValue N0 = Op.getOperand(0);
9063 SDValue N1 = Op.getOperand(1);
9064 SDValue N2, N3;
9098 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9131 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
9136 SDValue Carry = Op.getOperand(2);
9140 SDValue Result;
9175 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
9181 SDValue Arg = Op.getOperand(0);
9195 SDValue SRet;
9224 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
9231 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
9236 SDValue LoadSin =
9240 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
9242 SDValue LoadCos =
9250 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9252 SDValue &Chain) const {
9267 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
9291 SDValue
9297 return SDValue();
9307 return SDValue();
9312 return SDValue();
9318 return SDValue(N, 0);
9325 return SDValue();
9327 return SDValue(N, 0);
9330 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9336 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
9342 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
9344 SDValue Op = N->getOperand(1);
9347 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
9349 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
9356 SDValue Op, SelectionDAG &DAG, bool Signed,
9357 SmallVectorImpl<SDValue> &Results) const {
9365 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode());
9367 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
9369 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
9370 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
9377 static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
9396 SDValue Load = DAG.getExtLoad(
9400 SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Load);
9407 void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
9416 SDValue Result = DAG.getMemIntrinsicNode(
9419 SDValue Lo = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 0 : 1);
9420 SDValue Hi = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 1 : 0);
9421 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
9426 static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
9438 SDValue Build = ST->getValue();
9440 SmallVector<SDValue, 16> Ops;
9448 SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
9455 static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG,
9466 SDValue Lo = DAG.getNode(
9470 SDValue Hi = DAG.getNode(
9484 return SDValue();
9487 static bool isZeroVector(SDValue N) {
9493 static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
9496 SDValue Mask = N->getMask();
9497 SDValue PassThru = N->getPassThru();
9505 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
9507 SDValue NewLoad = DAG.getMaskedLoad(
9511 SDValue Combo = NewLoad;
9520 static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
9523 return SDValue();
9539 SDValue Op0 = Op->getOperand(0);
9553 SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0);
9558 SDValue Res;
9561 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9563 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9565 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9567 SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9569 SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
9570 SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags());
9573 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9575 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9586 static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
9589 return SDValue();
9593 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
9597 return SDValue();
9604 SmallVectorImpl<SDValue> &Results,
9610 SDValue Ops[] = { N->getOperand(0), // Chain
9619 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
9626 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
9628 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32);
9629 SDValue VHi = DAG.getAnyExtOrTrunc(
9635 SDValue RegClass =
9637 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
9638 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
9639 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
9640 return SDValue(
9645 SmallVectorImpl<SDValue> & Results,
9649 SDValue Ops[] = {N->getOperand(1),
9662 SDValue Lo =
9664 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
9665 SDValue Hi =
9667 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
9669 Results.push_back(SDValue(CmpSwap, 2));
9672 SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
9675 SDValue Chain = Op.getOperand(0);
9676 SDValue LHS = Op.getOperand(1);
9677 SDValue RHS = Op.getOperand(2);
9690 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS,
9703 SDValue True = DAG.getConstant(1, dl, VT);
9704 SDValue False = DAG.getConstant(0, dl, VT);
9705 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
9706 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
9707 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
9708 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
9717 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
9822 case ARMISD::WIN__DBZCHK: return SDValue();
9826 static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
9842 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
9845 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
9849 SDValue LongMul = DAG.getNode(Opc, dl,
9860 SmallVectorImpl<SDValue> &Results,
9862 SDValue Res;
9883 Res = LowerDivRem(SDValue(N, 0), DAG);
9890 Res = LowerSADDSUBSAT(SDValue(N, 0), DAG, Subtarget);
9898 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
11400 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
11417 SDValue &CC, bool &Invert,
11418 SDValue &OtherOp,
11424 SDValue N1 = N->getOperand(1);
11425 SDValue N2 = N->getOperand(2);
11488 // @returns The new node, or SDValue() on failure.
11490 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
11495 SDValue NonConstantVal;
11496 SDValue CCOp;
11500 return SDValue();
11503 SDValue TrueVal = OtherOp;
11504 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
11516 SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
11518 SDValue N0 = N->getOperand(0);
11519 SDValue N1 = N->getOperand(1);
11521 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
11524 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
11526 return SDValue();
11541 static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
11547 return SDValue();
11551 return SDValue();
11560 SmallVector<SDValue, 8> Ops;
11569 static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
11577 return SDValue();
11579 SDValue N00 = N0.getOperand(0);
11580 SDValue N10 = N1.getOperand(0);
11585 return SDValue();
11591 return SDValue();
11599 SmallVector<SDValue, 8> Ops;
11611 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT,
11621 static SDValue
11622 AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
11630 return SDValue();
11635 return SDValue();
11645 return SDValue();
11646 SDValue Vec = N0->getOperand(0)->getOperand(0);
11657 SDValue ExtVec0 = N0->getOperand(i);
11658 SDValue ExtVec1 = N1->getOperand(i);
11663 return SDValue();
11672 return SDValue();
11677 return SDValue();
11685 return SDValue();
11694 SmallVector<SDValue, 8> Ops;
11714 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
11719 static SDValue findMUL_LOHI(SDValue V) {
11723 return SDValue();
11726 static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
11730 return SDValue();
11736 SDValue Mul = AddcNode->getOperand(0);
11737 SDValue Lo = AddcNode->getOperand(1);
11742 return SDValue();
11745 SDValue SRA = AddeNode->getOperand(0);
11746 SDValue Hi = AddeNode->getOperand(1);
11751 return SDValue();
11755 return SDValue();
11757 return SDValue();
11760 return SDValue();
11765 SDValue Op0;
11766 SDValue Op1;
11787 return SDValue();
11789 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
11792 SDValue HiMLALResult(SMLAL.getNode(), 1);
11793 SDValue LoMLALResult(SMLAL.getNode(), 0);
11795 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
11796 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
11799 SDValue resNode(AddcNode, 0);
11803 static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
11840 return SDValue();
11842 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0);
11843 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1);
11847 return SDValue();
11863 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0);
11864 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1);
11868 return SDValue();
11872 SDValue MULOp = findMUL_LOHI(AddeSubeOp0);
11873 if (MULOp == SDValue())
11877 if (MULOp == SDValue())
11878 return SDValue();
11885 SDValue *HiAddSub = nullptr;
11886 SDValue *LoMul = nullptr;
11887 SDValue *LowAddSub = nullptr;
11891 return SDValue();
11911 return SDValue();
11917 return SDValue();
11923 SmallVector<SDValue, 8> Ops;
11942 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops);
11943 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode);
11945 return SDValue(AddeSubeNode, 0);
11949 return SDValue();
11955 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode),
11959 SDValue HiMLALResult(MLALNode.getNode(), 1);
11960 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult);
11962 SDValue LoMLALResult(MLALNode.getNode(), 0);
11963 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult);
11966 return SDValue(AddeSubeNode, 0);
11969 static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
11984 return SDValue();
11988 SDValue AddHi;
12002 return SDValue();
12009 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1),
12011 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
12015 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1));
12016 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0));
12019 return SDValue(AddeNode, 0);
12021 return SDValue();
12024 static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
12027 return SDValue();
12043 return SDValue();
12046 static SDValue PerformAddcSubcCombine(SDNode *N,
12053 SDValue LHS = N->getOperand(0);
12054 SDValue RHS = N->getOperand(1);
12058 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2));
12063 SDValue RHS = N->getOperand(1);
12076 return SDValue();
12079 static SDValue PerformAddeSubeCombine(SDNode *N,
12084 SDValue RHS = N->getOperand(1);
12104 return SDValue();
12107 static SDValue PerformVSELECTCombine(SDNode *N,
12120 return SDValue();
12123 return SDValue();
12124 SDValue XOR = N->getOperand(0);
12133 return SDValue();
12136 SDValue Cond = XOR->getOperand(0);
12137 SDValue LHS = N->getOperand(1);
12138 SDValue RHS = N->getOperand(2);
12143 static SDValue PerformABSCombine(SDNode *N,
12146 SDValue res;
12151 return SDValue();
12154 return SDValue();
12162 static SDValue PerformADDECombine(SDNode *N,
12170 if (DCI.isBeforeLegalize()) return SDValue();
12179 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
12183 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
12187 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
12189 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
12195 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
12197 return SDValue();
12200 static SDValue PerformADDVecReduce(SDNode *N,
12204 return SDValue();
12206 SDValue N0 = N->getOperand(0);
12207 SDValue N1 = N->getOperand(1);
12215 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
12216 SDValue NB) {
12218 return SDValue();
12219 SDValue VecRed = NB->getOperand(0);
12221 NB->getOperand(1) != SDValue(VecRed.getNode(), 1))
12222 return SDValue();
12225 SmallVector<SDValue, 4> Ops;
12232 SDValue Red = DCI.DAG.getNode(OpcodeA, dl,
12235 SDValue(Red.getNode(), 1));
12238 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
12240 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
12242 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
12244 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
12246 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
12248 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
12250 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
12252 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
12254 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
12256 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
12258 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
12260 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
12262 return SDValue();
12280 SDValue N1 = N->getOperand(0);
12321 static SDValue PerformSHLSimplify(SDNode *N,
12326 return SDValue();
12345 return SDValue();
12351 return SDValue();
12364 return SDValue();
12369 return SDValue();
12376 return SDValue();
12379 return SDValue();
12381 SDValue SHL = N->getOperand(0);
12386 return SDValue();
12395 return SDValue();
12407 return SDValue();
12411 SDValue X = SHL.getOperand(0);
12412 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X,
12415 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1));
12426 static SDValue PerformADDCombine(SDNode *N,
12429 SDValue N0 = N->getOperand(0);
12430 SDValue N1 = N->getOperand(1);
12433 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
12436 if (SDValue Result = PerformADDVecReduce(N, DCI, Subtarget))
12440 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
12449 static SDValue PerformSUBCombine(SDNode *N,
12452 SDValue N0 = N->getOperand(0);
12453 SDValue N1 = N->getOperand(1);
12457 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI))
12461 return SDValue();
12466 SDValue VDup = N->getOperand(1);
12468 return SDValue();
12470 SDValue VMov = N->getOperand(0);
12475 return SDValue();
12478 SDValue Negate = DCI.DAG.getNode(ISD::SUB, dl, MVT::i32,
12499 static SDValue PerformVMULCombine(SDNode *N,
12503 return SDValue();
12506 SDValue N0 = N->getOperand(0);
12507 SDValue N1 = N->getOperand(1);
12514 return SDValue();
12519 return SDValue();
12523 SDValue N00 = N0->getOperand(0);
12524 SDValue N01 = N0->getOperand(1);
12530 static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
12534 return SDValue();
12536 SDValue N0 = N->getOperand(0);
12537 SDValue N1 = N->getOperand(1);
12539 auto IsSignExt = [&](SDValue Op) {
12541 return SDValue();
12545 return SDValue();
12547 auto IsZeroExt = [&](SDValue Op) {
12554 return SDValue();
12556 SDValue And = Op;
12560 return SDValue();
12561 SDValue Mask = And->getOperand(1);
12567 return SDValue();
12573 return SDValue();
12577 if (SDValue Op0 = IsSignExt(N0)) {
12578 if (SDValue Op1 = IsSignExt(N1)) {
12579 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
12580 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
12584 if (SDValue Op0 = IsZeroExt(N0)) {
12585 if (SDValue Op1 = IsZeroExt(N1)) {
12586 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
12587 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
12592 return SDValue();
12595 static SDValue PerformMULCombine(SDNode *N,
12605 return SDValue();
12608 return SDValue();
12613 return SDValue();
12617 return SDValue();
12623 SDValue V = N->getOperand(0);
12626 SDValue Res;
12647 return SDValue();
12669 return SDValue();
12678 return SDValue();
12681 static SDValue CombineANDShift(SDNode *N,
12686 return SDValue();
12689 return SDValue();
12693 return SDValue();
12698 return SDValue();
12702 return SDValue();
12705 return SDValue();
12711 return SDValue();
12715 return SDValue();
12735 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
12746 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
12759 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
12772 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
12782 return SDValue();
12785 static SDValue PerformANDCombine(SDNode *N,
12796 return SDValue();
12806 SDValue Val = isVMOVModifiedImm((~SplatBits).getZExtValue(),
12810 SDValue Input =
12812 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
12820 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI))
12823 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
12828 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
12831 return SDValue();
12835 static SDValue PerformORCombineToSMULWBT(SDNode *OR,
12841 return SDValue();
12843 SDValue SRL = OR->getOperand(0);
12844 SDValue SHL = OR->getOperand(1);
12851 return SDValue();
12857 return SDValue();
12860 if (SRL.getOperand(0) != SDValue(SMULLOHI, 0) ||
12861 SHL.getOperand(0) != SDValue(SMULLOHI, 1))
12862 return SDValue();
12870 SDValue OpS16 = SMULLOHI->getOperand(0);
12871 SDValue OpS32 = SMULLOHI->getOperand(1);
12888 return SDValue();
12890 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16);
12891 DAG.ReplaceAllUsesOfValueWith(SDValue(OR, 0), Res);
12892 return SDValue(OR, 0);
12895 static SDValue PerformORCombineToBFI(SDNode *N,
12900 return SDValue();
12903 SDValue N0 = N->getOperand(0);
12904 SDValue N1 = N->getOperand(1);
12918 return SDValue();
12920 SDValue N00 = N0.getOperand(0);
12925 SDValue MaskOp = N0.getOperand(1);
12928 return SDValue();
12931 return SDValue();
12932 SDValue Res;
12938 return SDValue();
12950 return SDValue(N, 0);
12956 return SDValue();
12967 return SDValue();
12977 return SDValue(N, 0);
12984 return SDValue();
12994 return SDValue(N, 0);
13003 SDValue ShAmt = N00.getOperand(1);
13007 return SDValue();
13015 return SDValue(N, 0);
13018 return SDValue();
13038 static ARMCC::CondCodes getVCMPCondCode(SDValue N) {
13047 static bool CanInvertMVEVCMP(SDValue N) {
13052 static SDValue PerformORCombine_i1(SDNode *N,
13059 SDValue N0 = N->getOperand(0);
13060 SDValue N1 = N->getOperand(1);
13062 auto IsFreelyInvertable = [&](SDValue V) {
13070 return SDValue();
13072 SDValue NewN0 = DCI.DAG.getLogicalNOT(DL, N0, VT);
13073 SDValue NewN1 = DCI.DAG.getLogicalNOT(DL, N1, VT);
13074 SDValue And = DCI.DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1);
13079 static SDValue PerformORCombine(SDNode *N,
13089 return SDValue();
13103 SDValue Val =
13107 SDValue Input =
13109 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
13117 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
13119 if (SDValue Result = PerformORCombineToSMULWBT(N, DCI, Subtarget))
13123 SDValue N0 = N->getOperand(0);
13124 SDValue N1 = N->getOperand(1);
13134 return SDValue();
13156 SDValue Result = DAG.getNode(ARMISD::VBSL, dl, CanonicalVT,
13169 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
13173 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13176 return SDValue();
13179 static SDValue PerformXORCombine(SDNode *N,
13186 return SDValue();
13190 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
13193 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13199 SDValue N0 = N->getOperand(0);
13200 SDValue N1 = N->getOperand(1);
13208 SmallVector<SDValue, 4> Ops;
13218 return SDValue();
13224 static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
13227 SDValue From = N->getOperand(1);
13253 static SDValue FindBFIToCombineWith(SDNode *N) {
13257 SDValue From = ParseBFI(N, ToMask, FromMask);
13258 SDValue To = N->getOperand(0);
13263 SDValue V = To;
13267 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
13278 return SDValue();
13294 return SDValue();
13297 static SDValue PerformBFICombine(SDNode *N,
13299 SDValue N1 = N->getOperand(1);
13305 return SDValue();
13322 SDValue CombineBFI = FindBFIToCombineWith(N);
13323 if (CombineBFI == SDValue())
13324 return SDValue();
13328 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
13331 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
13351 return SDValue();
13356 static SDValue PerformVMOVRRDCombine(SDNode *N,
13360 SDValue InDouble = N->getOperand(0);
13375 SDValue BasePtr = LD->getBasePtr();
13376 SDValue NewLD1 =
13380 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
13383 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr,
13388 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
13391 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
13395 return SDValue();
13400 static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
13402 SDValue Op0 = N->getOperand(0);
13403 SDValue Op1 = N->getOperand(1);
13413 return SDValue();
13416 static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
13417 SDValue Op0 = N->getOperand(0);
13430 SDValue Copy = Op0->getOperand(0);
13433 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1)};
13434 SDValue NewCopy =
13444 SDValue Load =
13447 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
13457 return SDValue(N, 0);
13459 return SDValue();
13462 static SDValue PerformVMOVrhCombine(SDNode *N,
13464 SDValue N0 = N->getOperand(0);
13471 SDValue Load =
13474 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
13485 return SDValue();
13504 static SDValue PerformBUILD_VECTORCombine(SDNode *N,
13513 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
13520 return SDValue();
13522 SmallVector<SDValue, 8> Ops;
13525 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
13531 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
13536 static SDValue
13555 return SDValue();
13564 return SDValue();
13575 SDValue Elt = N->getOperand(Idx);
13588 return SDValue();
13596 return SDValue();
13603 SDValue Vec = DAG.getUNDEF(VecVT);
13606 SDValue V = N->getOperand(Idx);
13618 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
13627 static SDValue
13630 SDValue Op = N->getOperand(0);
13641 return SDValue();
13644 static SDValue
13648 SDValue Op = N->getOperand(0);
13663 return SDValue();
13666 static SDValue PerformVCMPCombine(SDNode *N,
13670 return SDValue();
13673 SDValue Op0 = N->getOperand(0);
13674 SDValue Op1 = N->getOperand(1);
13696 return SDValue();
13701 static SDValue PerformInsertEltCombine(SDNode *N,
13709 return SDValue();
13715 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
13716 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
13720 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
13725 static SDValue PerformExtractEltCombine(SDNode *N,
13727 SDValue Op0 = N->getOperand(0);
13733 SDValue X = Op0->getOperand(0);
13745 return SDValue();
13750 static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
13761 SDValue Op0 = N->getOperand(0);
13762 SDValue Op1 = N->getOperand(1);
13767 return SDValue();
13768 SDValue Concat0Op1 = Op0.getOperand(1);
13769 SDValue Concat1Op1 = Op1.getOperand(1);
13771 return SDValue();
13778 return SDValue();
13780 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
13805 static SDValue CombineBaseUpdate(SDNode *N,
13812 SDValue Addr = N->getOperand(AddrOpIdx);
13911 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
13971 SmallVector<SDValue, 8> Ops;
13992 SDValue &StVal = Ops[Ops.size()-2];
13997 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
14001 SmallVector<SDValue, 5> NewResults;
14003 NewResults.push_back(SDValue(UpdN.getNode(), i));
14008 SDValue &LdVal = NewResults[0];
14012 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs+1)); // chain
14014 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
14018 return SDValue();
14021 static SDValue PerformVLDCombine(SDNode *N,
14024 return SDValue();
14029 static SDValue PerformMVEVLDCombine(SDNode *N,
14032 return SDValue();
14035 SDValue Addr = N->getOperand(2);
14044 return SDValue();
14047 return SDValue();
14108 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14125 SmallVector<SDValue, 8> Ops;
14133 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, VecTy,
14137 SmallVector<SDValue, 5> NewResults;
14139 NewResults.push_back(SDValue(UpdN.getNode(), i));
14141 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
14143 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
14148 return SDValue();
14204 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
14206 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
14218 DCI.CombineTo(User, SDValue(VLDDup.getNode(), ResNo));
14223 std::vector<SDValue> VLDDupResults;
14225 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n));
14226 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs));
14234 static SDValue PerformVDUPLANECombine(SDNode *N,
14237 SDValue Op = N->getOperand(0);
14246 SDValue Extract = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), ExtractVT,
14254 return SDValue(N, 0);
14261 return SDValue();
14271 return SDValue();
14277 static SDValue PerformVDUPCombine(SDNode *N,
14281 SDValue Op = N->getOperand(0);
14296 return SDValue();
14304 SDValue Ops[] = { LD->getOperand(0), LD->getOperand(1),
14307 SDValue VLDDup = DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys,
14310 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1));
14314 return SDValue();
14317 static SDValue PerformLOADCombine(SDNode *N,
14326 return SDValue();
14332 static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
14334 SDValue StVal = St->getValue();
14337 return SDValue();
14347 return SDValue();
14352 return SDValue();
14363 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
14371 return SDValue();
14373 SDValue Shuff = DAG.getVectorShuffle(
14386 return SDValue();
14393 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
14394 SmallVector<SDValue, 8> Chains;
14395 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
14397 SDValue BasePtr = St->getBasePtr();
14402 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
14404 SDValue Ch =
14417 static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
14420 return SDValue();
14421 SDValue Trunc = St->getValue();
14423 return SDValue();
14427 return SDValue();
14442 return SDValue();
14469 return SDValue();
14474 SDValue Ch = St->getChain();
14475 SDValue BasePtr = St->getBasePtr();
14486 SmallVector<SDValue, 4> Stores;
14489 SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
14491 SDValue Extract =
14496 SDValue FPTrunc =
14502 SDValue Store = DAG.getTruncStore(
14512 static SDValue PerformSTORECombine(SDNode *N,
14517 return SDValue();
14518 SDValue StVal = St->getValue();
14522 if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG))
14526 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
14530 return SDValue();
14539 SDValue BasePtr = St->getBasePtr();
14540 SDValue NewST1 = DAG.getStore(
14545 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
14561 SDValue IntVec = StVal.getOperand(0);
14564 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
14565 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
14568 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
14583 return SDValue();
14595 static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
14598 return SDValue();
14600 SDValue Op = N->getOperand(0);
14603 return SDValue();
14605 SDValue ConstVec = Op->getOperand(1);
14607 return SDValue();
14619 return SDValue();
14626 return SDValue();
14632 SDValue FixConv = DAG.getNode(
14652 static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG,
14655 return SDValue();
14657 SDValue Op = N->getOperand(0);
14661 return SDValue();
14663 SDValue ConstVec = N->getOperand(1);
14665 return SDValue();
14677 return SDValue();
14684 return SDValue();
14688 SDValue ConvInput = Op.getOperand(0);
14702 static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
14705 return SDValue();
14709 SDValue N0 = N->getOperand(0);
14726 return SDValue();
14727 SDValue A = N0->getOperand(0);
14730 return SDValue();
14733 SDValue &A, SDValue &B) {
14736 SDValue ExtA = N0->getOperand(0);
14737 SDValue ExtB = N0->getOperand(1);
14747 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
14748 SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
14750 SDValue(Node.getNode(), 1));
14753 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
14755 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
14757 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
14759 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
14762 SDValue A, B;
14771 return SDValue();
14774 static SDValue PerformVMOVNCombine(SDNode *N,
14776 SDValue Op0 = N->getOperand(0);
14777 SDValue Op1 = N->getOperand(1);
14801 return SDValue(N, 0);
14804 return SDValue(N, 0);
14806 return SDValue();
14809 static SDValue PerformVQMOVNCombine(SDNode *N,
14811 SDValue Op0 = N->getOperand(0);
14823 return SDValue(N, 0);
14824 return SDValue();
14827 static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
14829 SDValue Op0 = N->getOperand(0);
14830 SDValue Op1 = N->getOperand(1);
14837 SDValue Merge = DAG.getMergeValues({Op0, Op1}, DL);
14839 return SDValue();
14845 SDValue NewShift = DAG.getNode(NewOpcode, DL, N->getVTList(), Op0, Op1,
14852 return SDValue();
14856 SDValue ARMTargetLowering::PerformIntrinsicCombine(SDNode *N,
14900 return SDValue();
14906 return SDValue();
14912 return SDValue();
15025 return SDValue();
15042 return SDValue();
15063 SmallVector<SDValue, 4> Ops;
15069 SDValue val = DAG.getNode(Opc, dl, {MVT::i32, MVT::i32}, Ops);
15075 return SDValue();
15083 static SDValue PerformShiftCombine(SDNode *N,
15091 SDValue N1 = N->getOperand(1);
15093 SDValue N0 = N->getOperand(0);
15105 return SDValue();
15110 SDValue N0 = N->getOperand(0);
15113 return SDValue();
15117 return SDValue();
15121 return SDValue();
15126 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
15138 return SDValue();
15140 return SDValue();
15165 return SDValue();
15172 static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
15173 SDValue N0 = N->getOperand(0);
15175 return SDValue();
15179 return SDValue();
15183 return SDValue();
15199 return SDValue();
15204 SDValue Ch = LD->getChain();
15205 SDValue BasePtr = LD->getBasePtr();
15212 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
15218 SmallVector<SDValue, 4> Loads;
15219 SmallVector<SDValue, 4> Chains;
15222 SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
15224 SDValue NewLoad =
15229 Chains.push_back(SDValue(NewLoad.getNode(), 1));
15234 SmallVector<SDValue, 4> Extends;
15237 SDValue LoadBC =
15239 SDValue FPExt = DAG.getNode(ARMISD::VCVTL, DL, MVT::v4f32, LoadBC,
15247 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
15248 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
15254 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
15256 SDValue N0 = N->getOperand(0);
15264 SDValue Vec = N0.getOperand(0);
15265 SDValue Lane = N0.getOperand(1);
15291 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
15294 return SDValue();
15297 static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
15300 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
15303 return SDValue();
15308 static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
15311 SDValue N0 = N->getOperand(0);
15313 return SDValue();
15316 return SDValue();
15355 SDValue VQMOVN =
15358 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
15396 SDValue VQMOVN =
15399 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
15404 return SDValue();
15407 static const APInt *isPowerOf2Constant(SDValue V) {
15415 SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
15429 SDValue Op0 = CMOV->getOperand(0);
15430 SDValue Op1 = CMOV->getOperand(1);
15433 SDValue CmpZ = CMOV->getOperand(4);
15437 return SDValue();
15440 SDValue And = CmpZ->getOperand(0);
15442 return SDValue();
15445 return SDValue();
15446 SDValue X = And->getOperand(0);
15457 return SDValue();
15461 return SDValue();
15462 SDValue Y = Op1->getOperand(0);
15465 return SDValue();
15471 return SDValue();
15477 return SDValue();
15480 SDValue V = Y;
15511 static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
15518 return SDValue();
15520 return SDValue();
15527 return SDValue();
15533 return SDValue();
15541 return SDValue();
15545 return SDValue();
15548 static SDValue PerformHWLoopCombine(SDNode *N,
15564 SDValue Cond;
15567 SDValue Chain = N->getOperand(0);
15568 SDValue Dest;
15581 return SDValue();
15584 return SDValue();
15587 SDValue Int = SearchLoopIntrinsic(Cond, CC, Imm, Negate);
15589 return SDValue();
15615 SDValue Elements = Int.getOperand(2);
15620 SDValue OtherTarget = Br->getOperand(1);
15623 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
15624 SDValue NewBrOps[] = { Br->getOperand(0), Dest };
15625 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps);
15626 DAG.ReplaceAllUsesOfValueWith(SDValue(Br, 0), NewBr);
15630 SDValue Res;
15633 SDValue Ops[] = { Chain, Elements, Dest };
15640 SDValue Ops[] = { Chain, Elements, OtherTarget };
15646 SDValue Size = DAG.getTargetConstant(
15648 SDValue Args[] = { Int.getOperand(0), Elements, Size, };
15649 SDValue LoopDec = DAG.getNode(ARMISD::LOOP_DEC, dl,
15654 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
15662 SDValue(LoopDec.getNode(), 1), Chain);
15664 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
15667 return SDValue();
15671 SDValue
15673 SDValue Cmp = N->getOperand(4);
15676 return SDValue();
15680 SDValue LHS = Cmp.getOperand(0);
15681 SDValue RHS = Cmp.getOperand(1);
15682 SDValue Chain = N->getOperand(0);
15683 SDValue BB = N->getOperand(1);
15684 SDValue ARMcc = N->getOperand(2);
15707 return SDValue();
15711 SDValue
15713 SDValue Cmp = N->getOperand(4);
15716 return SDValue();
15720 SDValue LHS = Cmp.getOperand(0);
15721 SDValue RHS = Cmp.getOperand(1);
15722 SDValue FalseVal = N->getOperand(0);
15723 SDValue TrueVal = N->getOperand(1);
15724 SDValue ARMcc = N->getOperand(2);
15730 SDValue R = PerformCMOVToBFICombine(N, DAG);
15752 SDValue Res;
15757 SDValue ARMcc;
15758 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
15779 return SDValue();
15788 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
15801 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
15803 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
15806 SDValue Carry =
15815 SDValue Sub =
15817 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
15818 Sub.getValue(1), SDValue());
15829 SDValue Sub =
15831 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
15832 Sub.getValue(1), SDValue());
15863 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
15872 KnownBits Known = DAG.computeKnownBits(SDValue(N,0));
15888 static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG,
15890 SDValue Src = N->getOperand(0);
15915 return SDValue();
15918 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
16004 return SDValue();
16011 return SDValue();
16021 return SDValue();
16031 return SDValue();
16041 return SDValue();
16049 return SDValue();
16058 return SDValue();
16097 return SDValue();
16222 bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
16395 bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
16747 bool isSEXTLoad, SDValue &Base,
16748 SDValue &Offset, bool &isInc,
16806 bool isSEXTLoad, SDValue &Base,
16807 SDValue &Offset, bool &isInc,
16832 SDValue &Base, SDValue &Offset,
16887 ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
16888 SDValue &Offset,
16895 SDValue Ptr;
16948 SDValue &Base,
16949 SDValue &Offset,
16953 SDValue Ptr;
17034 void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
17049 SDValue LHS = Op.getOperand(0);
17050 SDValue RHS = Op.getOperand(1);
17099 const SDValue &SrcSV = Op.getOperand(0);
17134 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
17182 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
17183 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
17216 SDValue Op, const APInt &OriginalDemandedBits,
17441 void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
17443 std::vector<SDValue>&Ops,
17445 SDValue Result;
17647 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
17670 const SDValue Dividend = Op->getOperand(0);
17671 const SDValue Divisor = Op->getOperand(1);
17672 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor);
17673 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor);
17674 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
17676 SDValue Values[2] = {Div, Rem};
17682 SDValue InChain = DAG.getEntryNode();
17688 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
17701 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
17707 SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
17727 SDValue InChain = DAG.getEntryNode();
17731 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
17742 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
17750 SDValue
17751 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
17756 SDValue Chain = Op.getOperand(0);
17757 SDValue Size = Op.getOperand(1);
17763 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
17771 SDValue Ops[2] = { SP, Chain };
17775 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
17778 SDValue Flag;
17785 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
17788 SDValue Ops[2] = { NewSP, Chain };
17792 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
17794 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
17811 SDValue Result = DAG.getNode(ISD::FP_EXTEND,
17826 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
17851 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
17854 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
17877 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
17878 SDValue Result;
17884 void ARMTargetLowering::lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
17889 SDValue Hi, Lo, Tmp;
17908 SDValue(Lo.getNode(), 1));