Lines Matching defs:DL
1175 SDLoc DL(Op);
1181 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
1182 TLO.DAG.getConstant(NewImm, DL, VT));
1187 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1189 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
1308 MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1631 DebugLoc DL = MI.getDebugLoc();
1650 BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
1651 BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
1663 BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
2008 const SDLoc &DL, SelectionDAG &DAG) {
2016 LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
2017 RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
2031 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
2034 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
2035 return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
2126 SDLoc DL(Val);
2139 ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
2142 ExtraCC, DL, DAG);
2150 return emitComparison(LHS, RHS, CC, DL, DAG);
2152 return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
2410 SDLoc DL(Op);
2444 LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
2445 RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
2446 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2447 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2448 DAG.getConstant(0, DL, MVT::i64));
2453 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
2460 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2461 DAG.getConstant(32, DL, MVT::i64));
2462 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
2463 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2464 DAG.getConstant(31, DL, MVT::i64));
2468 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2476 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2477 DAG.getConstant(32, DL, MVT::i64));
2480 DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2481 DAG.getConstant(0, DL, MVT::i64),
2488 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2490 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2491 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2492 DAG.getConstant(63, DL, MVT::i64));
2496 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2499 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2502 DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2503 DAG.getConstant(0, DL, MVT::i64),
2514 Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2682 SDLoc DL(Op);
2703 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
2704 DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
2925 SDLoc DL(Op);
2927 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
2928 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
2930 DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, OpVT, Op,
2931 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
3120 SDLoc DL(Op);
3128 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
3136 return DAG.getNode(N0->getOpcode(), DL, VT,
3137 DAG.getNode(NewOpc, DL, VT,
3138 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
3139 DAG.getNode(NewOpc, DL, VT,
3140 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
3143 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
3145 return DAG.getNode(AArch64ISD::PTRUE, DL, VT,
3146 DAG.getTargetConstant(Pattern, DL, MVT::i32));
3319 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
3335 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
3338 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
3340 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
3342 Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc);
3343 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
3344 Trunc, DAG.getConstant(0, DL, MVT::i64));
3346 return DAG.getStore(ST->getChain(), DL, ExtractTrunc,
3684 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3777 ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
3792 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
3799 ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue,
3800 DAG.getConstant(32, DL, RegVT));
3801 ArgValue = DAG.getZExtOrTrunc(ArgValue, DL, VA.getValVT());
3849 ExtType, DL, VA.getLocVT(), Chain, FIN,
3860 DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, MachinePointerInfo());
3864 ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(),
3878 saveVarArgRegisters(CCInfo, DAG, DL, Chain);
3918 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
3919 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain);
3953 const SDLoc &DL,
3984 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
3986 Val.getValue(1), DL, Val, FIN,
3994 DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
4016 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
4019 Val.getValue(1), DL, Val, FIN,
4022 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
4023 DAG.getConstant(16, DL, PtrVT));
4031 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
4039 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
4070 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
4082 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
4085 Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val,
4086 DAG.getConstant(32, DL, VA.getLocVT()));
4091 Val = DAG.getZExtOrTrunc(Val, DL, VA.getValVT());
4308 SDLoc &DL = CLI.DL;
4448 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
4450 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
4461 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
4479 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
4482 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
4487 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
4488 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
4490 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
4494 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
4495 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
4496 DAG.getConstant(32, DL, VA.getLocVT()));
4502 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4505 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
4520 Chain, DL, Arg, SpillSlot,
4546 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
4585 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
4586 PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
4601 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
4603 DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
4610 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
4612 Chain, DL, DstAddr, Arg, SizeNode,
4624 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
4626 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
4633 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
4639 Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
4652 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
4653 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
4656 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
4663 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
4675 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
4676 DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
4688 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
4728 SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
4734 Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
4742 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
4743 DAG.getIntPtrConstant(CalleePopBytes, DL, true),
4744 InFlag, DL);
4750 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
4771 const SDLoc &DL, SelectionDAG &DAG) const {
4801 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
4802 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
4806 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
4810 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4814 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4815 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
4816 DAG.getConstant(32, DL, VA.getLocVT()));
4827 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
4836 Chain = DAG.getCopyToReg(Chain, DL, RetVal.first, RetVal.second, Flag);
4847 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
4851 Chain = DAG.getCopyToReg(Chain, DL, RetValReg, Val, Flag);
4878 return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
4916 SDLoc DL(N);
4921 return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
4929 SDLoc DL(N);
4933 AArch64ISD::WrapperLarge, DL, Ty,
4945 SDLoc DL(N);
4950 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
4951 return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo);
4959 SDLoc DL(N);
4962 return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym);
4990 SDLoc DL(GN);
4992 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
5031 SDLoc DL(Op);
5037 DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
5038 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
5044 PtrMemVT, DL, Chain, DescAddr,
5051 FuncTLVGet = DAG.getZExtOrTrunc(FuncTLVGet, DL, PtrVT);
5067 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
5069 DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
5072 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
5080 const SDLoc &DL,
5093 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
5094 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
5096 DAG.getTargetConstant(0, DL, MVT::i32)),
5105 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
5107 GV, DL, PtrVT, 0,
5109 Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
5111 DAG.getTargetConstant(0, DL, MVT::i32)),
5113 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
5115 DAG.getTargetConstant(0, DL, MVT::i32)),
5125 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G1);
5127 GV, DL, PtrVT, 0,
5129 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
5130 DAG.getTargetConstant(16, DL, MVT::i32)),
5132 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
5133 DAG.getTargetConstant(0, DL, MVT::i32)),
5135 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
5145 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_G2);
5147 GV, DL, PtrVT, 0,
5150 GV, DL, PtrVT, 0,
5152 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
5153 DAG.getTargetConstant(32, DL, MVT::i32)),
5155 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
5156 DAG.getTargetConstant(16, DL, MVT::i32)),
5158 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
5159 DAG.getTargetConstant(0, DL, MVT::i32)),
5161 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
5185 const SDLoc &DL,
5193 DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr});
5196 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
5226 SDLoc DL(Op);
5229 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
5232 return LowerELFTLSLocalExec(GV, ThreadBase, DL, DAG);
5234 TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
5235 TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
5255 TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
5260 GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
5262 GV, DL, MVT::i64, 0,
5265 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
5266 DAG.getTargetConstant(0, DL, MVT::i32)),
5268 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
5269 DAG.getTargetConstant(0, DL, MVT::i32)),
5276 DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
5279 TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
5283 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
5293 SDLoc DL(Op);
5300 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL));
5301 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
5312 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
5314 DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo);
5315 TLSIndex = DAG.getLoad(MVT::i32, DL, Chain, TLSIndex, MachinePointerInfo());
5320 TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex);
5321 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
5322 DAG.getConstant(3, DL, PtrVT));
5323 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
5324 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
5331 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
5333 GV, DL, PtrVT, 0,
5338 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
5339 DAG.getTargetConstant(0, DL, MVT::i32)),
5341 Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo);
5499 SDLoc DL(Op);
5506 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
5508 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL));
5516 VecVal1 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
5518 VecVal2 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
5521 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
5522 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
5547 SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
5552 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
5553 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
5554 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
5558 DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
5561 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
5563 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
5565 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
5567 return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
5587 SDLoc DL(Op);
5592 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
5593 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
5595 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
5597 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
5598 DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
5601 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
5604 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Val);
5606 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
5608 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
5609 DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
5611 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV);
5620 Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
5630 ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
5631 DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
5920 SDLoc DL(Op);
5921 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
5929 SDLoc DL(Op);
5933 SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal);
5935 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC);
5936 return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal);
5949 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
5951 return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
5964 RHS = DAG.getConstant(0, DL, CCVal.getValueType());
5967 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
5989 SDLoc DL(Op);
5995 DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
5997 return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0),
6035 SDLoc DL(Op);
6038 FR = DAG.getZExtOrTrunc(FR, DL, getPointerMemTy(DAG.getDataLayout()));
6040 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
6049 SDLoc DL(Op);
6055 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
6066 SDLoc DL(Op);
6075 MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
6084 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT));
6087 GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
6088 DAG.getConstant(GPRSize, DL, PtrVT));
6090 MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
6099 VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
6100 DAG.getConstant(16, DL, PtrVT));
6103 VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
6104 DAG.getConstant(FPRSize, DL, PtrVT));
6106 MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
6113 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT));
6115 Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32), GROffsAddr,
6120 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT));
6122 Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32), VROffsAddr,
6125 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
6144 SDLoc DL(Op);
6151 return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
6152 DAG.getConstant(VaListSize, DL, MVT::i32),
6163 SDLoc DL(Op);
6171 DAG.getLoad(PtrMemVT, DL, Chain, Addr, MachinePointerInfo(V));
6173 VAList = DAG.getZExtOrTrunc(VAList, DL, PtrVT);
6180 VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
6181 DAG.getConstant(Align->value() - 1, DL, PtrVT));
6182 VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
6183 DAG.getConstant(-(int64_t)Align->value(), DL, PtrVT));
6202 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
6203 DAG.getConstant(ArgSize, DL, PtrVT));
6204 VANext = DAG.getZExtOrTrunc(VANext, DL, PtrMemVT);
6208 DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V));
6214 DAG.getLoad(MVT::f64, DL, APStore, VAList, MachinePointerInfo());
6216 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
6217 DAG.getIntPtrConstant(1, DL));
6220 return DAG.getMergeValues(Ops, DL);
6223 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
6232 SDLoc DL(Op);
6235 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
6237 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
6241 FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr,
6252 SDLoc DL(Op);
6282 SDLoc DL(Op);
6285 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
6286 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
6288 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset);
6298 SDLoc DL(Op);
6302 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
6303 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
6304 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
6310 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
6500 SDLoc DL(Operand);
6509 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
6511 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
6512 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
6517 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
6518 SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ);
6520 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
6522 Estimate = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL,
6539 SDLoc DL(Operand);
6548 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
6550 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
6946 SDLoc DL(V64Reg);
6948 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy),
6949 V64Reg, DAG.getConstant(0, DL, MVT::i32));
6966 SDLoc DL(V128Reg);
6968 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg);
7438 SDLoc DL(Op);
7455 V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0,
7456 DAG.getConstant(0, DL, MVT::i64));
7459 V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1,
7460 DAG.getConstant(0, DL, MVT::i64));
7462 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
7575 SDLoc DL(Op);
7584 TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32));
7595 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
7596 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
7601 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst);
7603 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
7604 DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
7605 DAG.getBuildVector(IndexVT, DL,
7609 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst);
7611 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
7612 DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
7613 DAG.getBuildVector(IndexVT, DL,
7619 // Shuffle = DAG.getNode(AArch64ISD::TBL2, DL, IndexVT, V1Cst, V2Cst,
7620 // DAG.getBuildVector(IndexVT, DL, &TBLMask[0],
7623 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
7624 DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32), V1Cst,
7625 V2Cst, DAG.getBuildVector(IndexVT, DL,
7629 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
7885 SDLoc DL(Op);
7896 SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
7902 SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
7904 DAG.getMachineNode(AArch64::DUP_ZZI_Q, DL, MVT::nxv2i64, V, CI);
7905 return DAG.getNode(ISD::BITCAST, DL, VT, SDValue(DUPQ, 0));
7912 SDValue One = DAG.getConstant(1, DL, MVT::i64);
7913 SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
7916 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
7918 DL, MVT::nxv2i64, Zero, One);
7919 SV = DAG.getNode(ISD::AND, DL, MVT::nxv2i64, SV, SplatOne);
7922 SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128);
7923 SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
7924 SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
7927 SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
7928 return DAG.getNode(ISD::BITCAST, DL, VT, TBL);
8192 SDLoc DL(N);
8257 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Shift.getOperand(1));
8690 SDLoc DL(Op);
8694 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec,
8725 SDLoc DL(Op);
8734 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec,
8947 SDLoc DL(Op);
8950 SDValue ResultURHADD = DAG.getNode(RHADDOpc, DL, VT, OpA, OpB);
8958 SDLoc DL(Op);
8974 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
8975 DAG.getConstant(Cnt, DL, MVT::i32));
8976 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
8977 DAG.getConstant(Intrinsic::aarch64_neon_ushl, DL,
8992 return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
8993 DAG.getConstant(Cnt, DL, MVT::i32));
9002 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
9004 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9005 DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0),
9178 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
9181 auto Rdx = DAG.getNode(Op, DL, VecOp.getSimpleValueType(), VecOp);
9182 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarOp.getValueType(), Rdx,
9183 DAG.getConstant(0, DL, MVT::i64));
9332 SDLoc DL(Op);
9334 return DAG.getZExtOrTrunc(DAG.getVScale(DL, MVT::i64, MulImm.sextOrSelf(64)),
9335 DL, VT);
9369 auto &DL = I.getModule()->getDataLayout();
9391 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
9416 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
9433 Info.align = DL.getABITypeAlign(PtrTy->getElementType());
9444 Info.align = DL.getABITypeAlign(PtrTy->getElementType());
9472 Info.align = DL.getABITypeAlign(PtrTy->getElementType());
9484 Info.align = DL.getABITypeAlign(PtrTy->getElementType());
9563 const DataLayout &DL = F->getParent()->getDataLayout();
9567 isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) &&
9627 auto &DL = Ext->getModule()->getDataLayout();
9635 countTrailingZeros(DL.getTypeStoreSizeInBits(IdxTy).getFixedSize()) - 3;
9803 const DataLayout &DL) const {
9804 return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
9816 VectorType *VecTy, const DataLayout &DL) const {
9818 unsigned VecSize = DL.getTypeSizeInBits(VecTy);
9819 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
9854 const DataLayout &DL = LI->getModule()->getDataLayout();
9861 if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VTy, DL))
9864 unsigned NumLoads = getNumInterleavedAccesses(VTy, DL);
9873 FixedVectorType::get(DL.getIntPtrType(EltTy), FVTy->getNumElements());
9987 const DataLayout &DL = SI->getModule()->getDataLayout();
9992 if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(SubVecTy, DL))
9995 unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
10004 Type *IntTy = DL.getIntPtrType(EltTy);
10107 const SDLoc &DL) const {
10128 SDValue PseudoLoad = DAG.getNode(Opcode, DL, NodeTys, LoadOps);
10132 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, PseudoLoadOps);
10221 bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL,
10247 uint64_t NumBits = DL.getTypeSizeInBits(Ty);
10279 int AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL,
10289 if (isLegalAddressingMode(DL, AM, Ty, AS))
10418 SDLoc DL(N);
10427 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
10431 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
10432 N0.getOperand(0), DAG.getConstant(0, DL, VT));
10433 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg,
10434 DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
10466 SDLoc DL(N);
10469 SDValue Zero = DAG.getConstant(0, DL, VT);
10470 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
10474 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL);
10475 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
10476 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp);
10484 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64));
10492 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
10596 SDLoc DL(N);
10598 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0,
10599 DAG.getConstant(ShiftAmt, DL, MVT::i64));
10603 SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1);
10608 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res);
10611 return DAG.getNode(ISD::SHL, DL, VT, Res,
10612 DAG.getConstant(TrailingZeroes, DL, MVT::i64));
10646 SDLoc DL(N);
10650 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
10652 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
10653 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
10655 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
10761 SDLoc DL(N);
10766 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy,
10767 DAG.getConstant(IntrinsicOpcode, DL, MVT::i32),
10768 Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32));
10771 FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv);
10831 SDLoc DL(N);
10835 ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
10840 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
10841 DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
10842 DAG.getConstant(C, DL, MVT::i32));
10872 SDLoc DL(N);
10905 return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS,
10906 DAG.getConstant(ShiftRHS, DL, MVT::i64));
10913 SDLoc DL(N);
10949 return DAG.getNode(AArch64ISD::BSP, DL, VT, SDValue(BVN0, 0),
11018 SDLoc DL(N);
11035 Dup = DAG.getNode(AArch64ISD::DUP, DL,
11037 DAG.getConstant(Mask.zextOrTrunc(32), DL, MVT::i32));
11039 SDValue And = DAG.getNode(ISD::AND, DL,
11042 return DAG.getNode(Opc, DL, N->getValueType(0), And);
11142 SDLoc DL(N);
11149 return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
11152 return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
11312 SDLoc DL(N);
11327 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift);
11328 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane);
11732 SDLoc DL(N);
11738 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
11739 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
11742 return DAG.getNode(AArch64ISD::INDEX_VECTOR, DL, N->getValueType(0),
11799 SDLoc DL(N);
11816 Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
11830 Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
11841 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
11842 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred,
11853 SDLoc DL(Op);
11860 SDValue TVal = DAG.getConstant(1, DL, OutVT);
11861 SDValue FVal = DAG.getConstant(0, DL, OutVT);
11864 SDValue Test = DAG.getNode(AArch64ISD::PTEST, DL, MVT::Other, Pg, Op);
11868 SDValue CC = DAG.getConstant(getInvertedCondCode(Cond), DL, MVT::i32);
11869 SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test);
11870 return DAG.getZExtOrTrunc(Res, DL, VT);
11875 SDLoc DL(N);
11881 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
11885 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
11886 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
11892 SDLoc DL(N);
11901 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
11902 InitVal = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ReduceVT,
11905 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce);
11909 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce,
12167 SDLoc DL(N);
12168 Src = DAG.getNode(N->getOpcode(), DL, SrcVT, Src);
12178 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src,
12179 DAG.getConstant(0, DL, MVT::i64));
12180 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src,
12181 DAG.getConstant(InNVT.getVectorMinNumElements(), DL, MVT::i64));
12182 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, Lo);
12183 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, Hi);
12187 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
12200 SDLoc DL(&St);
12206 DAG.getStore(St.getChain(), DL, SplatVal, BasePtr, PtrInfo,
12220 DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
12221 DAG.getConstant(BaseOffset + Offset, DL, MVT::i64));
12222 NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr,
12261 SDLoc DL(N);
12277 SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
12281 Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
12283 return DAG.getMergeValues({ Load, LoadChain }, DL);
12287 SDLoc DL(N);
12300 SDValue PassThru = DAG.getConstant(0, DL, LoadVT);
12301 SDValue L = DAG.getMaskedLoad(LoadVT, DL, MINode->getChain(),
12308 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
12309 return DAG.getMergeValues(Ops, DL);
12320 SDLoc DL(N);
12331 SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
12335 Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0));
12337 return DAG.getMergeValues({Load, LoadChain}, DL);
12341 SDLoc DL(N);
12356 SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data);
12358 SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data);
12367 return DAG.getNode(AArch64ISD::ST1_PRED, DL, N->getValueType(0), Ops);
12371 SDLoc DL(N);
12382 Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data);
12385 return DAG.getMaskedStore(MINode->getChain(), DL, Data, MINode->getOperand(4),
12453 SDLoc DL(&St);
12464 DAG.getCopyFromReg(DAG.getEntryNode(), DL, ZeroReg, ZeroVT);
12577 SDLoc DL(S);
12582 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
12583 DAG.getConstant(0, DL, MVT::i64));
12584 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
12585 DAG.getConstant(NumElts, DL, MVT::i64));
12588 DAG.getStore(S->getChain(), DL, SubVector0, BasePtr, S->getPointerInfo(),
12590 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
12591 DAG.getConstant(8, DL, MVT::i64));
12592 return DAG.getStore(NewST1.getValue(0), DL, SubVector1, OffsetPtr,
13290 SDLoc DL(N);
13291 return DAG.getNode(NewOpc, DL, MVT::Other, N->getOperand(0), NewTestSrc,
13292 DAG.getConstant(Bit, DL, MVT::i64), N->getOperand(3));
13373 SDLoc DL(N0);
13375 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0));
13377 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1));
13378 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
13382 SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
13383 Mask = DAG.getNode(ISD::BITCAST, DL,
13386 return DAG.getSelect(DL, ResVT, Mask, N->getOperand(1), N->getOperand(2));
13444 SDLoc DL(GN);
13445 SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
13446 return DAG.getNode(ISD::SUB, DL, MVT::i64, Result,
13447 DAG.getConstant(MinOffset, DL, MVT::i64));
13453 SDLoc DL, unsigned BitWidth) {
13457 SDValue Shift = DAG.getConstant(Log2_32(BitWidth / 8), DL, MVT::i64);
13458 SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift);
13460 return DAG.getNode(ISD::SHL, DL, MVT::nxv2i64, Offset, SplatShift);
13506 SDLoc DL(N);
13530 getScaledOffsetForBitWidth(DAG, Offset, DL, SrcElVT.getSizeInBits());
13569 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
13588 SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Src);
13590 SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Src);
13599 return DAG.getNode(Opcode, DL, VTs, Ops);
13609 SDLoc DL(N);
13626 Offset = getScaledOffsetForBitWidth(DAG, Offset, DL,
13672 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0);
13689 SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops);
13693 Load = DAG.getNode(ISD::TRUNCATE, DL, RetVT, Load.getValue(0));
13698 Load = DAG.getNode(ISD::BITCAST, DL, RetVT, Load.getValue(0));
13700 return DAG.getMergeValues({Load, LoadChain}, DL);
13709 SDLoc DL(N);
13739 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(),
13742 return DAG.getNode(SOpc, DL, N->getValueType(0), Ext);
13844 SDLoc DL(N);
13845 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset);
13850 return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops);
13870 SDLoc DL(N);
13871 Ops[1] = DAG.getConstant(Intrinsic::aarch64_sve_prfb_gather_uxtw_index, DL,
13874 return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops);
14079 SDLoc DL(N);
14088 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, Src1,
14089 DAG.getConstant(IdxConst * NumLanes, DL, MVT::i32));
14090 return DAG.getMergeValues({Val, Chain}, DL);
14093 SDLoc DL(N);
14116 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, Vec.getValueType(), Tuple,
14117 DAG.getConstant(I * NumLanes, DL, MVT::i32)));
14121 DAG.getNode(ISD::CONCAT_VECTORS, DL, Tuple.getValueType(), Opnds);
14122 return DAG.getMergeValues({Concat, Chain}, DL);
14127 SDLoc DL(N);
14139 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Opnds);
14140 return DAG.getMergeValues({Concat, Chain}, DL);
14145 SDLoc DL(N);
14153 LowerSVEStructLoad(IntrinsicID, LoadOps, N->getValueType(0), DAG, DL);
14154 return DAG.getMergeValues({Result, Chain}, DL);
14284 SDLoc DL(N);
14292 DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32,
14294 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
14296 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
14297 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
14315 SDLoc DL(N);
14316 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N);
14317 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
14318 DAG.getNode(ISD::SRL, DL, MVT::i128, N,
14319 DAG.getConstant(64, DL, MVT::i64)));
14332 SDLoc DL(N);
14353 SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0));
14354 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half));
14526 SDLoc DL(N);
14527 auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2));
14528 auto V = DAG.getNode(AArch64ISD::CLASTA_N, DL, MVT::i32,
14530 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
14534 SDLoc DL(N);
14535 auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2));
14536 auto V = DAG.getNode(AArch64ISD::CLASTB_N, DL, MVT::i32,
14538 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
14542 SDLoc DL(N);
14543 auto V = DAG.getNode(AArch64ISD::LASTA, DL, MVT::i32,
14545 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
14549 SDLoc DL(N);
14550 auto V = DAG.getNode(AArch64ISD::LASTB, DL, MVT::i32,
14552 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
14662 const DataLayout &DL = M->getDataLayout();
14663 IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(EltTy));
14701 const DataLayout &DL = M->getDataLayout();
14702 IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
14905 AArch64TargetLowering::getVaListSizeInBits(const DataLayout &DL) const {
14907 return getPointerTy(DL).getSizeInBits();
14909 return 3 * getPointerTy(DL).getSizeInBits() + 2 * 32;
14987 static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
15051 return DAG.getNode(AArch64ISD::PTRUE, DL, MaskVT,
15052 DAG.getTargetConstant(PgPattern, DL, MVT::i64));
15055 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
15060 return getPTrue(DAG, DL, PredTy, AArch64SVEPredPattern::all);
15063 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
15065 return getPredicateForFixedLengthVector(DAG, DL, VT);
15067 return getPredicateForScalableVector(DAG, DL, VT);
15076 SDLoc DL(V);
15077 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
15078 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
15087 SDLoc DL(V);
15088 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
15089 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
15097 SDLoc DL(Op);
15102 ContainerVT, DL, Load->getChain(), Load->getBasePtr(), Load->getOffset(),
15103 getPredicateForFixedLengthVector(DAG, DL, VT), DAG.getUNDEF(ContainerVT),
15109 return DAG.getMergeValues(MergedValues, DL);
15117 SDLoc DL(Op);
15123 Store->getChain(), DL, NewValue, Store->getBasePtr(), Store->getOffset(),
15124 getPredicateForFixedLengthVector(DAG, DL, VT), Store->getMemoryVT(),
15134 SDLoc DL(Op);
15144 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val);
15145 Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val);
15150 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val);
15151 Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val);
15156 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val);
15157 Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
15169 SDLoc DL(Op);
15170 auto Pg = getPredicateForVector(DAG, DL, VT);
15188 auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands);
15201 return DAG.getNode(NewOp, DL, VT, Operands);