Lines Matching refs:SDValue
1520 SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
1522 SDValue Chain,
1537 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
1543 SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
1550 SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
1551 const SDLoc &SL, SDValue Val,
1581 SDValue SITargetLowering::lowerKernargMemParameter(
1582 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
1599 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
1600 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, 4,
1604 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
1605 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
1607 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
1615 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
1616 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
1620 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
1624 SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
1625 const SDLoc &SL, SDValue Chain,
1642 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1643 SDValue ArgValue;
1673 SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
2131 SDValue SITargetLowering::LowerFormalArguments(
2132 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2134 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2230 SmallVector<SDValue, 16> Chains;
2256 SDValue Arg =
2276 SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
2290 SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
2375 SDValue
2376 SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2379 const SmallVectorImpl<SDValue> &OutVals,
2405 SDValue Flag;
2406 SmallVector<SDValue, 48> RetOps;
2412 SDValue ReturnAddrReg = CreateLiveInRegister(
2415 SDValue ReturnAddrVirtualReg = DAG.getRegister(
2430 SDValue Arg = OutVals[RealRVLocIdx];
2485 SDValue SITargetLowering::LowerCallResult(
2486 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
2488 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
2489 SDValue ThisVal) const {
2501 SDValue Val;
2547 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
2548 SmallVectorImpl<SDValue> &MemOpChains,
2549 SDValue Chain) const {
2601 SDValue InputReg;
2619 SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2649 SDValue InputReg;
2657 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
2665 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
2687 SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2708 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
2710 const SmallVectorImpl<SDValue> &OutVals,
2789 SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
2790 SmallVectorImpl<SDValue> &InVals) const {
2794 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
2796 SDValue Chain = CLI.Chain;
2797 SDValue Callee = CLI.Callee;
2860 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2861 SmallVector<SDValue, 8> MemOpChains;
2897 SmallVector<SDValue, 4> CopyFromChains;
2900 SDValue ScratchRSrcReg
2912 SDValue Arg = OutVals[i];
2942 SDValue DstAddr;
2948 SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
2984 SDValue SizeNode =
2986 SDValue Cpy =
2995 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo,
3012 SDValue InFlag;
3020 SDValue PhysReturnAddrReg;
3026 SDValue ReturnAddrReg = CreateLiveInRegister(
3047 std::vector<SDValue> Ops;
3095 SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
3110 IsThisReturn ? OutVals[0] : SDValue());
3115 SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl(
3116 SDValue Op, SelectionDAG &DAG) const {
3122 SDValue Tmp1 = Op;
3123 SDValue Tmp2 = Op.getValue(1);
3124 SDValue Tmp3 = Op.getOperand(2);
3125 SDValue Chain = Tmp1.getOperand(0);
3133 SDValue Size = Tmp2.getOperand(1);
3134 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
3143 SDValue ScaledSize = DAG.getNode(
3159 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
3164 SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
3171 SDValue Size = Op.getOperand(1);
4326 SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
4332 SDValue Lo, Hi;
4336 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
4338 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
4346 SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
4352 SDValue Lo0, Hi0;
4354 SDValue Lo1, Hi1;
4359 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
4361 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
4367 SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
4373 SDValue Lo0, Hi0;
4375 SDValue Lo1, Hi1;
4377 SDValue Lo2, Hi2;
4382 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Lo2,
4384 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Hi2,
4391 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
4397 SDValue Result = LowerLOAD(Op, DAG);
4467 return SDValue();
4470 static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
4482 SmallVector<SDValue, 4> Elts;
4484 for (SDValue &Elt : Elts)
4497 SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode,
4500 ArrayRef<SDValue> Ops,
4517 SDValue Load
4525 SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
4530 SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
4532 ArrayRef<SDValue> Ops) const {
4558 SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
4565 static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI,
4575 SDValue LHS = N->getOperand(1);
4576 SDValue RHS = N->getOperand(2);
4593 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS,
4600 static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI,
4609 SDValue Src0 = N->getOperand(1);
4610 SDValue Src1 = N->getOperand(2);
4623 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0,
4630 static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
4633 SDValue Src = N->getOperand(1);
4654 return SDValue();
4668 SmallVectorImpl<SDValue> &Results,
4672 if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
4677 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
4685 SDValue Src0 = N->getOperand(1);
4686 SDValue Src1 = N->getOperand(2);
4688 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32,
4697 SDValue Src0 = N->getOperand(1);
4698 SDValue Src1 = N->getOperand(2);
4715 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
4724 if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
4742 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
4743 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
4752 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT,
4765 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4767 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32,
4778 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4780 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32,
4792 static SDNode *findUser(SDValue Value, unsigned Opcode) {
4858 SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND,
4863 SDValue Target = BRCOND.getOperand(2);
4894 SmallVector<SDValue, 4> Ops;
4907 SDValue Ops[] = {
4908 SDValue(Result, 0),
4917 SDValue Ops[] = {
4921 SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops);
4925 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
4929 SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg);
4936 SDValue(Result, i - 1),
4937 SDValue());
4939 DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0));
4944 SDValue(Intr, Intr->getNumValues() - 1),
4950 SDValue SITargetLowering::LowerRETURNADDR(SDValue Op,
4975 SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG,
4976 SDValue Op,
4985 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
4989 SDValue Src = Op.getOperand(0);
4996 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src);
4997 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
5001 SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
5020 SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
5023 SDValue LHS = Op.getOperand(0);
5024 SDValue RHS = Op.getOperand(1);
5033 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32);
5034 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt);
5035 SDValue Overflow = DAG.getSetCC(SL, MVT::i1,
5043 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS);
5044 SDValue Top = DAG.getNode(isSigned ? ISD::MULHS : ISD::MULHU,
5047 SDValue Sign = isSigned
5051 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE);
5056 SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
5058 SDValue Chain = Op.getOperand(0);
5068 SDValue QueuePtr = CreateLiveInRegister(
5070 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
5071 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
5072 QueuePtr, SDValue());
5073 SDValue Ops[] = {
5082 SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
5084 SDValue Chain = Op.getOperand(0);
5098 SDValue Ops[] = {
5105 SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
5120 SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16);
5121 SDValue ApertureReg = SDValue(
5123 SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32);
5132 SDValue QueuePtr = CreateLiveInRegister(
5139 SDValue Ptr = DAG.getObjectPtrOffset(DL, QueuePtr, StructOffset);
5151 SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
5156 SDValue Src = ASC->getOperand(0);
5157 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
5169 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
5170 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
5171 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
5185 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
5187 SDValue NonNull
5190 SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG);
5191 SDValue CvtPtr
5219 SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
5221 SDValue Vec = Op.getOperand(0);
5222 SDValue Ins = Op.getOperand(1);
5223 SDValue Idx = Op.getOperand(2);
5232 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
5240 SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
5242 SDValue Vec = Op.getOperand(0);
5243 SDValue InsVal = Op.getOperand(1);
5244 SDValue Idx = Op.getOperand(2);
5258 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
5260 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
5262 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
5265 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
5266 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
5270 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16,
5277 SDValue Concat = InsertLo ?
5285 return SDValue();
5294 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
5298 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
5301 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
5303 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
5304 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
5308 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
5309 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT,
5312 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS);
5316 SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
5321 SDValue Vec = Op.getOperand(0);
5322 SDValue Idx = Op.getOperand(1);
5334 if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI))
5341 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
5344 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
5346 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
5347 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
5350 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
5362 SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
5382 SmallVector<SDValue, 4> Pieces;
5388 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL,
5400 SDValue Vec0 = SVN->getOperand(VecIdx0);
5401 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
5404 SDValue Vec1 = SVN->getOperand(VecIdx1);
5405 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
5414 SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
5424 SDValue Lo = DAG.getBuildVector(HalfVT, SL,
5426 SDValue Hi = DAG.getBuildVector(HalfVT, SL,
5429 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo);
5430 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi);
5432 SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi });
5439 SDValue Lo = Op.getOperand(0);
5440 SDValue Hi = Op.getOperand(1);
5445 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
5452 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
5460 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi);
5473 static SDValue
5508 SDValue PtrLo =
5510 SDValue PtrHi;
5520 SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
5521 SDValue Op,
5535 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(),
5546 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT,
5561 SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
5562 const SDLoc &DL, SDValue V) const {
5571 // A Null SDValue creates a glue result.
5574 return SDValue(M0, 0);
5577 SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG,
5578 SDValue Op,
5582 SDValue Param = lowerKernargMemParameter(
5589 static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
5598 static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
5607 static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
5608 ArrayRef<SDValue> Elts) {
5634 SmallVector<SDValue, 16> VecElts(NumElts);
5636 SDValue Elt = Elts[i];
5649 static bool parseCachePolicy(SDValue CachePolicy, SelectionDAG &DAG,
5650 SDValue *GLC, SDValue *SLC, SDValue *DLC) {
5671 static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
5672 SDValue Src, int ExtraElts) {
5675 SmallVector<SDValue, 8> Elts;
5682 SDValue Undef = DAG.getUNDEF(SrcVT.getScalarType());
5692 static SDValue constructRetValue(SelectionDAG &DAG,
5713 SDValue Data(Result, 0);
5714 SDValue TexFail;
5717 SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32);
5720 SDValue(Result, 0), ZeroIdx);
5723 SDValue(Result, 0), ZeroIdx);
5727 SDValue(Result, 0),
5744 return DAG.getMergeValues({Data, TexFail, SDValue(Result, 1)}, DL);
5749 return DAG.getMergeValues({Data, SDValue(Result, 1)}, DL);
5752 static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
5753 SDValue *LWE, bool &IsTexFail) {
5770 static void packImageA16AddressToDwords(SelectionDAG &DAG, SDValue Op,
5772 SmallVectorImpl<SDValue> &PackedAddrs,
5777 SDValue Addr = Op.getOperand(I);
5799 SDValue SITargetLowering::lowerImage(SDValue Op,
5820 SDValue VData;
5833 SDValue VData2 = Op.getOperand(3);
5901 SmallVector<SDValue, 4> VAddrs;
6001 SDValue VAddr;
6005 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1);
6006 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1);
6008 SDValue Unorm;
6020 SDValue TFE;
6021 SDValue LWE;
6022 SDValue TexFail = Op.getOperand(CtrlIdx);
6045 SDValue Undef = DAG.getUNDEF(Op.getValueType());
6064 SDValue GLC;
6065 SDValue SLC;
6066 SDValue DLC;
6078 SmallVector<SDValue, 26> Ops;
6082 for (const SDValue &Addr : VAddrs)
6137 SmallVector<SDValue, 1> Elt;
6138 DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1);
6139 return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL);
6148 return SDValue(NewNode, 0);
6151 SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
6152 SDValue Offset, SDValue CachePolicy,
6167 SDValue Ops[] = {
6191 SmallVector<SDValue, 4> Loads;
6204 SDValue Ops[] = {
6233 SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
6290 return SDValue();
6303 SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
6304 SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq,
6395 SDValue GLC;
6396 SDValue DLC = DAG.getTargetConstant(0, DL, MVT::i1);
6418 return SDValue();
6450 SDValue Numerator = Op.getOperand(1);
6451 SDValue Denominator = Op.getOperand(2);
6458 SDValue Src0 = Param->isAllOnesValue() ? Numerator : Denominator;
6517 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32,
6526 return SDValue(DAG.getMachineNode(AMDGPU::SI_IF_BREAK, DL, VT,
6537 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0,
6546 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
6547 SDValue SrcVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32,
6550 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec,
6563 SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0,
6581 static unsigned getBufferOffsetForMMO(SDValue VOffset,
6582 SDValue SOffset,
6583 SDValue Offset,
6584 SDValue VIndex = SDValue()) {
6622 SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
6631 SDValue Chain = M->getOperand(0);
6632 SDValue M0 = M->getOperand(2);
6633 SDValue Value = M->getOperand(3);
6669 SDValue Ops[] = {
6714 SDValue Ops[] = {
6730 SDValue Ops[] = {
6734 SDValue(), // voffset -- will be set by setBufferOffsets
6735 SDValue(), // soffset -- will be set by setBufferOffsets
6736 SDValue(), // offset -- will be set by setBufferOffsets
6772 SDValue Ops[] = {
6792 SDValue Ops[] = {
6819 SDValue Ops[] = {
6843 SDValue Ops[] = {
6867 SDValue Ops[] = {
6901 SDValue Ops[] = {
6906 SDValue(), // voffset -- will be set by setBufferOffsets
6907 SDValue(), // soffset -- will be set by setBufferOffsets
6908 SDValue(), // offset -- will be set by setBufferOffsets
6976 SDValue Ops[] = {
7050 SDValue Ops[] = {
7117 SDValue Ops[] = {
7123 SDValue(), // voffset -- will be set by setBufferOffsets
7124 SDValue(), // soffset -- will be set by setBufferOffsets
7125 SDValue(), // offset -- will be set by setBufferOffsets
7142 SDValue Ops[] = {
7163 SDValue Ops[] = {
7185 SDValue Ops[] = {
7201 return SDValue();
7207 SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
7209 ArrayRef<SDValue> Ops, EVT MemVT,
7232 NewOp = DAG.getMergeValues({ Extract, SDValue(NewOp.getNode(), 1) }, DL);
7237 SDValue SITargetLowering::handleD16VData(SDValue VData,
7251 SDValue IntVData = DAG.getNode(ISD::BITCAST, DL, IntStoreVT, VData);
7255 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, EquivStoreVT, IntVData);
7263 SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
7266 SDValue Chain = Op.getOperand(0);
7272 SDValue Src0 = Op.getOperand(4);
7273 SDValue Src1 = Op.getOperand(5);
7276 return SDValue();
7279 SDValue Undef = DAG.getUNDEF(MVT::f32);
7280 const SDValue Ops[] = {
7293 return SDValue(DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops), 0);
7300 return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other,
7303 return SDValue();
7306 SDValue VData = Op.getOperand(2);
7317 SDValue Ops[] = {
7337 SDValue VData = Op.getOperand(2);
7342 SDValue Ops[] = {
7362 SDValue VData = Op.getOperand(2);
7367 SDValue Ops[] = {
7388 SDValue VData = Op.getOperand(2);
7397 SDValue Ops[] = {
7402 SDValue(), // voffset -- will be set by setBufferOffsets
7403 SDValue(), // soffset -- will be set by setBufferOffsets
7404 SDValue(), // offset -- will be set by setBufferOffsets
7432 SDValue VData = Op.getOperand(2);
7446 SDValue Ops[] = {
7476 SDValue VData = Op.getOperand(2);
7491 SDValue Ops[] = {
7523 SDValue Ops[] = {
7528 SDValue(), // voffset -- will be set by setBufferOffsets
7529 SDValue(), // soffset -- will be set by setBufferOffsets
7530 SDValue(), // offset -- will be set by setBufferOffsets
7550 SDValue Ops[] = {
7569 return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other,
7588 std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
7589 SDValue Offset, SelectionDAG &DAG) const {
7592 SDValue N0 = Offset;
7596 N0 = SDValue();
7623 SDValue Ops[] = { N0, OverflowVal };
7632 return {N0, SDValue(C1, 0)};
7636 // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
7638 unsigned SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
7639 SelectionDAG &DAG, SDValue *Offsets,
7654 SDValue N0 = CombinedOffset.getOperand(0);
7655 SDValue N1 = CombinedOffset.getOperand(1);
7673 SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
7675 ArrayRef<SDValue> Ops,
7682 SDValue BufferLoad = DAG.getMemIntrinsicNode(Opc, DL, ResList,
7685 SDValue LoadVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, BufferLoad);
7692 SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
7694 SDValue Ops[],
7699 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]);
7703 ArrayRef<SDValue> OpsRef = makeArrayRef(&Ops[0], 9);
7708 static SDValue getLoadExtOrTrunc(SelectionDAG &DAG,
7709 ISD::LoadExtType ExtType, SDValue Op,
7728 SDValue SITargetLowering::widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const {
7731 return SDValue();
7738 return SDValue();
7746 return SDValue();
7754 SDValue Ptr = Ld->getBasePtr();
7755 SDValue NewLoad = DAG.getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD,
7771 SDValue Cvt = NewLoad;
7798 SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
7806 return SDValue();
7811 SDValue Chain = Load->getChain();
7812 SDValue BasePtr = Load->getBasePtr();
7817 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain,
7821 SDValue Ops[] = {
7829 SmallVector<SDValue, 3> Elts;
7831 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD,
7837 SDValue Ops[] = {
7846 return SDValue();
7853 SDValue Ops[2];
7881 return SDValue();
7899 return SDValue();
7919 return SDValue();
7927 SDValue Ops[2];
7934 return SDValue();
7942 return SDValue();
7950 return SDValue();
7966 return SDValue();
7969 SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
7974 SDValue Cond = Op.getOperand(0);
7976 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
7977 SDValue One = DAG.getConstant(1, DL, MVT::i32);
7979 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1));
7980 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2));
7982 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero);
7983 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero);
7985 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1);
7987 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One);
7988 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One);
7990 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1);
7992 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi});
7998 SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
8001 SDValue LHS = Op.getOperand(0);
8002 SDValue RHS = Op.getOperand(1);
8012 return SDValue();
8037 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
8044 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
8048 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
8049 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
8070 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
8071 EVT VT, SDValue A, SDValue B, SDValue C,
8072 SDValue GlueChain, SDNodeFlags Flags) {
8092 SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
8093 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
8097 SDValue Src0 = Op.getOperand(0);
8098 SDValue Src1 = Op.getOperand(1);
8100 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
8101 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
8103 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1);
8104 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1);
8106 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32);
8107 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag);
8113 SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
8115 SDValue LHS = Op.getOperand(1);
8116 SDValue RHS = Op.getOperand(2);
8118 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS);
8121 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32);
8124 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32);
8126 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
8131 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT);
8133 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One);
8139 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1);
8141 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0);
8148 static const SDValue getSPDenormModeValue(int SPDenormMode, SelectionDAG &DAG,
8159 SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
8160 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
8171 SDValue LHS = Op.getOperand(0);
8172 SDValue RHS = Op.getOperand(1);
8174 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
8178 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT,
8180 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT,
8184 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32,
8186 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32,
8192 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32);
8205 const SDValue EnableDenormValue =
8211 const SDValue EnableDenormValue = DAG.getConstant(FP_DENORM_FLUSH_NONE,
8218 SDValue Ops[3] = {
8220 SDValue(EnableDenorm, 0),
8221 SDValue(EnableDenorm, 1)
8227 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0,
8230 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp,
8233 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled,
8236 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul,
8239 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32,
8242 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3,
8248 const SDValue DisableDenormValue =
8255 const SDValue DisableDenormValue =
8263 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
8264 SDValue(DisableDenorm, 0), DAG.getRoot());
8268 SDValue Scale = NumeratorScaled.getValue(1);
8269 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32,
8275 SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
8280 SDValue X = Op.getOperand(0);
8281 SDValue Y = Op.getOperand(1);
8283 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
8287 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X);
8289 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0);
8291 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0);
8293 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One);
8295 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp);
8297 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One);
8299 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X);
8301 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1);
8302 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3);
8304 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64,
8307 SDValue Scale;
8313 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
8316 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
8317 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y);
8318 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0);
8319 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1);
8321 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi);
8322 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi);
8324 SDValue Scale0Hi
8326 SDValue Scale1Hi
8329 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ);
8330 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ);
8336 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64,
8342 SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
8357 SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
8400 return SDValue();
8408 return SDValue();
8412 return SDValue();
8420 return SDValue();
8436 return SDValue();
8442 SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
8445 SDValue Arg = Op.getOperand(0);
8446 SDValue TrigVal;
8452 SDValue OneOver2Pi = DAG.getConstantFP(0.5 * numbers::inv_pi, DL, VT);
8455 SDValue MulVal = DAG.getNode(ISD::FMUL, DL, VT, Arg, OneOver2Pi, Flags);
8471 SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
8483 SDValue ChainIn = Op.getOperand(0);
8484 SDValue Addr = Op.getOperand(1);
8485 SDValue Old = Op.getOperand(2);
8486 SDValue New = Op.getOperand(3);
8491 SDValue NewOld = DAG.getBuildVector(VecType, DL, {New, Old});
8492 SDValue Ops[] = { ChainIn, Addr, NewOld };
8502 SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N,
8507 return SDValue();
8512 SDValue Src = N->getOperand(0);
8521 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src);
8533 return SDValue();
8550 SDValue SITargetLowering::performSHLPtrCombine(SDNode *N,
8554 SDValue N0 = N->getOperand(0);
8555 SDValue N1 = N->getOperand(1);
8561 return SDValue();
8565 return SDValue();
8569 return SDValue();
8580 return SDValue();
8586 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1);
8587 SDValue COffset = DAG.getConstant(Offset, SL, MVT::i32);
8597 SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
8599 SDValue Ptr = N->getBasePtr();
8605 SDValue NewPtr = performSHLPtrCombine(Ptr.getNode(), N->getAddressSpace(),
8608 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
8611 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
8615 return SDValue();
8629 SDValue SITargetLowering::splitBinaryBitConstantOp(
8632 unsigned Opc, SDValue LHS,
8648 return SDValue();
8653 static bool isBoolSGPR(SDValue V) {
8690 static uint32_t getPermuteMask(SelectionDAG &DAG, SDValue V) {
8733 SDValue SITargetLowering::performAndCombine(SDNode *N,
8736 return SDValue();
8740 SDValue LHS = N->getOperand(0);
8741 SDValue RHS = N->getOperand(1);
8746 if (SDValue Split
8766 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
8771 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE,
8773 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(LHS), VT, Ext,
8785 return SDValue();
8801 SDValue X = LHS.getOperand(0);
8802 SDValue Y = RHS.getOperand(0);
8804 return SDValue();
8808 return SDValue();
8813 return SDValue();
8916 return SDValue();
8919 SDValue SITargetLowering::performOrCombine(SDNode *N,
8922 SDValue LHS = N->getOperand(0);
8923 SDValue RHS = N->getOperand(1);
8930 SDValue Src = LHS.getOperand(0);
8932 return SDValue();
8937 return SDValue();
8948 return SDValue();
8957 return SDValue();
9006 return SDValue();
9018 SDValue ExtSrc = RHS.getOperand(0);
9022 SDValue LowLHS, HiBits;
9024 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc);
9029 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
9037 if (SDValue Split
9042 return SDValue();
9045 SDValue SITargetLowering::performXorCombine(SDNode *N,
9049 return SDValue();
9051 SDValue LHS = N->getOperand(0);
9052 SDValue RHS = N->getOperand(1);
9056 if (SDValue Split
9061 return SDValue();
9121 SDValue SITargetLowering::performZeroExtendCombine(SDNode *N,
9125 return SDValue();
9129 return SDValue();
9131 SDValue Src = N->getOperand(0);
9133 return SDValue();
9138 SDValue BCSrc = Src.getOperand(0);
9144 return SDValue();
9147 SDValue SITargetLowering::performSignExtendInRegCombine(SDNode *N,
9150 SDValue Src = N->getOperand(0);
9159 SDValue Ops[] = {
9174 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(Opc, SDLoc(N),
9181 return SDValue();
9184 SDValue SITargetLowering::performClassCombine(SDNode *N,
9187 SDValue Mask = N->getOperand(1);
9198 return SDValue();
9201 SDValue SITargetLowering::performRcpCombine(SDNode *N,
9204 SDValue N0 = N->getOperand(0);
9223 bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
9321 SDValue SrcOp = Op.getOperand(i);
9342 SDValue Src = Op.getOperand(0);
9345 SDValue TruncSrc = Src.getOperand(0);
9386 SDValue SITargetLowering::getCanonicalConstantFP(
9412 static bool vectorEltWillFoldAway(SDValue Op) {
9416 SDValue SITargetLowering::performFCanonicalizeCombine(
9420 SDValue N0 = N->getOperand(0);
9444 SDValue NewElts[2];
9445 SDValue Lo = N0.getOperand(0);
9446 SDValue Hi = N0.getOperand(1);
9451 SDValue Op = N0.getOperand(I);
9492 SDValue Canon0 = DAG.getNode(ISD::FCANONICALIZE, SL, VT,
9494 SDValue Canon1 = getCanonicalConstantFP(DAG, SL, VT, CRHS->getValueAPF());
9501 return isCanonicalized(DAG, N0) ? N0 : SDValue();
9525 SDValue SITargetLowering::performIntMed3ImmCombine(
9527 SDValue Op0, SDValue Op1, bool Signed) const {
9530 return SDValue();
9534 return SDValue();
9538 return SDValue();
9541 return SDValue();
9548 Op0.getOperand(0), SDValue(K0, 0), SDValue(K1, 0));
9555 SDValue Tmp1 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(0));
9556 SDValue Tmp2 = DAG.getNode(ExtOp, SL, NVT, Op0->getOperand(1));
9557 SDValue Tmp3 = DAG.getNode(ExtOp, SL, NVT, Op1);
9559 SDValue Med3 = DAG.getNode(Med3Opc, SL, NVT, Tmp1, Tmp2, Tmp3);
9563 static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
9575 SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
9577 SDValue Op0,
9578 SDValue Op1) const {
9581 return SDValue();
9585 return SDValue();
9589 return SDValue();
9610 SDValue Var = Op0.getOperand(0);
9612 return SDValue();
9621 Var, SDValue(K0, 0), SDValue(K1, 0));
9625 return SDValue();
9628 SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
9634 SDValue Op0 = N->getOperand(0);
9635 SDValue Op1 = N->getOperand(1);
9672 if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, true))
9677 if (SDValue Med3 = performIntMed3ImmCombine(DAG, SDLoc(N), Op0, Op1, false))
9690 if (SDValue Res = performFPMed3ImmCombine(DAG, SDLoc(N), Op0, Op1))
9694 return SDValue();
9697 static bool isClampZeroToOne(SDValue A, SDValue B) {
9710 SDValue SITargetLowering::performFMed3Combine(SDNode *N,
9719 SDValue Src0 = N->getOperand(0);
9720 SDValue Src1 = N->getOperand(1);
9721 SDValue Src2 = N->getOperand(2);
9751 return SDValue();
9754 SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
9756 SDValue Src0 = N->getOperand(0);
9757 SDValue Src1 = N->getOperand(1);
9760 return SDValue();
9793 SDValue Idx = N->getOperand(N->getNumOperands() - 1);
9797 SDValue Vec = N->getOperand(0);
9807 SDValue SITargetLowering::performExtractVectorEltCombine(
9809 SDValue Vec = N->getOperand(0);
9819 SDValue Idx = N->getOperand(1);
9820 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9833 SDValue Idx = N->getOperand(1);
9852 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9854 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9870 SDValue Idx = N->getOperand(1);
9871 SDValue V;
9873 SDValue IC = DAG.getVectorIdxConstant(I, SL);
9874 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
9884 return SDValue();
9903 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
9906 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
9909 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
9913 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, EltVT.changeTypeToInteger(), Srl);
9918 return SDValue();
9921 SDValue
9924 SDValue Vec = N->getOperand(0);
9925 SDValue Idx = N->getOperand(2);
9932 return SDValue();
9936 SDValue Ins = N->getOperand(1);
9939 SmallVector<SDValue, 16> Ops;
9941 SDValue IC = DAG.getConstant(I, SL, IdxVT);
9942 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
9943 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ);
9976 SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
9980 return SDValue();
9983 SDValue Op0 = N->getOperand(0);
9984 SDValue Op1 = N->getOperand(1);
9987 return SDValue();
9993 return SDValue();
9995 SDValue Op2 = Op1.getOperand(1);
9998 return SDValue();
10007 return SDValue();
10010 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1);
10014 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL,
10016 SDValue N0, SDValue N1, SDValue N2,
10020 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2);
10024 SDValue SITargetLowering::performAddCombine(SDNode *N,
10029 SDValue LHS = N->getOperand(0);
10030 SDValue RHS = N->getOperand(1);
10039 SDValue MulLHS = LHS.getOperand(0);
10040 SDValue MulRHS = LHS.getOperand(1);
10041 SDValue AddRHS = RHS;
10059 return SDValue();
10062 if (SDValue V = reassociateScalarOps(N, DAG)) {
10067 return SDValue();
10088 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
10096 SDValue Args[] = { LHS, RHS.getOperand(0), RHS.getOperand(2) };
10100 return SDValue();
10103 SDValue SITargetLowering::performSubCombine(SDNode *N,
10109 return SDValue();
10112 SDValue LHS = N->getOperand(0);
10113 SDValue RHS = N->getOperand(1);
10129 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
10139 return SDValue();
10140 SDValue Args[] = { LHS.getOperand(0), RHS, LHS.getOperand(2) };
10143 return SDValue();
10146 SDValue SITargetLowering::performAddCarrySubCarryCombine(SDNode *N,
10150 return SDValue();
10154 return SDValue();
10157 SDValue LHS = N->getOperand(0);
10165 SDValue Args[] = { LHS.getOperand(0), LHS.getOperand(1), N->getOperand(2) };
10168 return SDValue();
10171 SDValue SITargetLowering::performFAddCombine(SDNode *N,
10174 return SDValue();
10180 SDValue LHS = N->getOperand(0);
10181 SDValue RHS = N->getOperand(1);
10188 SDValue A = LHS.getOperand(0);
10192 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
10200 SDValue A = RHS.getOperand(0);
10204 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
10210 return SDValue();
10213 SDValue SITargetLowering::performFSubCombine(SDNode *N,
10216 return SDValue();
10228 SDValue LHS = N->getOperand(0);
10229 SDValue RHS = N->getOperand(1);
10232 SDValue A = LHS.getOperand(0);
10236 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
10237 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
10247 SDValue A = RHS.getOperand(0);
10251 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
10257 return SDValue();
10260 SDValue SITargetLowering::performFMACombine(SDNode *N,
10267 return SDValue();
10271 SDValue Op1 = N->getOperand(0);
10272 SDValue Op2 = N->getOperand(1);
10273 SDValue FMA = N->getOperand(2);
10278 return SDValue();
10291 return SDValue();
10293 SDValue Vec1 = Op1.getOperand(0);
10294 SDValue Idx1 = Op1.getOperand(1);
10295 SDValue Vec2 = Op2.getOperand(0);
10297 SDValue FMAOp1 = FMA.getOperand(0);
10298 SDValue FMAOp2 = FMA.getOperand(1);
10299 SDValue FMAAcc = FMA.getOperand(2);
10303 return SDValue();
10309 return SDValue();
10311 SDValue Vec3 = FMAOp1.getOperand(0);
10312 SDValue Vec4 = FMAOp2.getOperand(0);
10313 SDValue Idx2 = FMAOp1.getOperand(1);
10318 return SDValue();
10321 return SDValue();
10324 return SDValue();
10332 return SDValue();
10335 SDValue SITargetLowering::performSetCCCombine(SDNode *N,
10340 SDValue LHS = N->getOperand(0);
10341 SDValue RHS = N->getOperand(1);
10401 return SDValue();
10410 return SDValue();
10428 return SDValue();
10431 SDValue SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
10437 SDValue Src = N->getOperand(0);
10438 SDValue Shift = N->getOperand(0);
10474 return SDValue(N, 0);
10478 if (SDValue DemandedSrc =
10482 return SDValue();
10485 SDValue SITargetLowering::performClampCombine(SDNode *N,
10489 return SDValue();
10503 return SDValue(CSrc, 0);
10507 SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
10510 return SDValue();
10541 if (SDValue Widended = widenLoad(cast<LoadSDNode>(N), DCI))
10592 SDValue Src = N->getOperand(0);
10618 SDValue Src = N->getOperand(0);
10623 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src);
10753 SmallVector<SDValue, 12> Ops;
10773 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(NewNode, 1));
10780 SDValue(NewNode, 0));
10794 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
10795 DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op);
10811 static bool isFrameIndexOp(SDValue Op) {
10825 SDValue SrcVal = Node->getOperand(2);
10833 SDValue VReg = DAG.getRegister(
10837 SDValue ToVReg
10839 SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
10840 SDValue ToResultReg
10841 = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0),
10849 SmallVector<SDValue, 8> Ops;
10857 Ops.push_back(SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL,
10889 SDValue Src0 = Node->getOperand(0);
10890 SDValue Src1 = Node->getOperand(1);
10891 SDValue Src2 = Node->getOperand(2);
10903 SDValue UndefReg = DAG.getRegister(MRI.createVirtualRegister(RC), VT);
10905 SDValue ImpDef = DAG.getCopyToReg(DAG.getEntryNode(), SDLoc(Node),
10906 UndefReg, Src0, SDValue());
10926 SmallVector<SDValue, 4> Ops = { Src0, Src1, Src2 };
11020 static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
11022 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32);
11023 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0);
11028 SDValue Ptr) const {
11034 const SDValue Ops0[] = {
11042 SDValue SubRegHi = SDValue(DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL,
11046 const SDValue Ops1[] = {
11062 SDValue Ptr, uint32_t RsrcDword1,
11064 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr);
11065 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr);
11067 PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi,
11072 SDValue DataLo = buildSMovImm32(DAG, DL,
11074 SDValue DataHi = buildSMovImm32(DAG, DL, RsrcDword2And3 >> 32);
11076 const SDValue Ops[] = {
11218 void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
11220 std::vector<SDValue> &Ops,
11234 bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
11268 bool SITargetLowering::checkAsmConstraintVal(SDValue Op,
11301 bool SITargetLowering::checkAsmConstraintValA(SDValue Op,
11544 bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
11674 ComputeConstraintToUse(TC, SDValue());