Lines Matching refs:SDValue
55 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
61 unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
814 SDValue AMDGPUTargetLowering::getNegatedExpression(
815 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
823 return SDValue();
914 bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1116 SDValue AMDGPUTargetLowering::LowerReturn(
1117 SDValue Chain, CallingConv::ID CallConv,
1120 const SmallVectorImpl<SDValue> &OutVals,
1143 SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1147 SmallVector<SDValue, 8> ArgChains;
1169 ArgChains.push_back(SDValue(L, 1));
1179 SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1180 SmallVectorImpl<SDValue> &InVals,
1182 SDValue Callee = CLI.Callee;
1206 SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1207 SmallVectorImpl<SDValue> &InVals) const {
1211 SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1222 SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1264 SmallVectorImpl<SDValue> &Results,
1288 SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
1289 SDValue Op,
1311 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode());
1312 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1333 return SDValue();
1336 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1338 SmallVector<SDValue, 8> Args;
1343 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0));
1344 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1));
1346 SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi });
1356 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1359 SmallVector<SDValue, 8> Args;
1369 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
1370 SDValue LHS, SDValue RHS,
1371 SDValue True, SDValue False,
1372 SDValue CC,
1375 return SDValue();
1409 return SDValue();
1430 return SDValue();
1439 return SDValue();
1442 std::pair<SDValue, SDValue>
1443 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1446 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1448 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1449 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1451 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1452 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1457 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1460 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1461 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1465 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1468 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1469 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1491 std::pair<SDValue, SDValue>
1492 AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1499 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
1501 SDValue Hi = DAG.getNode(
1507 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1517 SDValue Ops[2];
1522 SDValue BasePtr = Load->getBasePtr();
1529 SDValue Lo, Hi;
1539 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1542 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, Size);
1543 SDValue HiLoad =
1548 SDValue Join;
1561 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1568 SDValue AMDGPUTargetLowering::WidenVectorLoad(SDValue Op,
1573 SDValue BasePtr = Load->getBasePtr();
1583 SDValue WideLoad = DAG.getExtLoad(
1593 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1596 SDValue Val = Store->getValue();
1605 SDValue Chain = Store->getChain();
1606 SDValue BasePtr = Store->getBasePtr();
1611 SDValue Lo, Hi;
1617 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());
1624 SDValue LoStore =
1627 SDValue HiStore =
1637 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
1641 SDValue LHS = Op.getOperand(0);
1642 SDValue RHS = Op.getOperand(1);
1648 return SDValue();
1652 return SDValue();
1663 SDValue jq = DAG.getConstant(1, DL, IntVT);
1678 SDValue ia = LHS;
1681 SDValue ib = RHS;
1684 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
1687 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
1689 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
1696 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
1707 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
1710 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
1721 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
1727 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
1730 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
1735 SDValue InRegSize
1740 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
1748 void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
1750 SmallVectorImpl<SDValue> &Results) const {
1758 SDValue One = DAG.getConstant(1, DL, HalfVT);
1759 SDValue Zero = DAG.getConstant(0, DL, HalfVT);
1762 SDValue LHS = Op.getOperand(0);
1763 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1764 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One);
1766 SDValue RHS = Op.getOperand(1);
1767 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1768 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One);
1773 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1776 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
1777 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
1795 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
1796 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
1797 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
1800 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
1801 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
1803 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
1805 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
1806 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
1809 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
1810 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
1811 SDValue Rcp64 = DAG.getBitcast(VT,
1814 SDValue Zero64 = DAG.getConstant(0, DL, VT);
1815 SDValue One64 = DAG.getConstant(1, DL, VT);
1816 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
1819 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
1820 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
1821 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
1822 SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1824 SDValue Mulhi1_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1827 SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
1829 SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
1831 SDValue Add1_HiNc = DAG.getNode(ISD::ADD, DL, HalfVT, Rcp_Hi, Mulhi1_Hi);
1832 SDValue Add1 = DAG.getBitcast(VT,
1835 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
1836 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
1837 SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1839 SDValue Mulhi2_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1842 SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
1844 SDValue Add2_HiC = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_HiNc,
1846 SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add2_HiC,
1848 SDValue Add2 = DAG.getBitcast(VT,
1850 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
1852 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
1854 SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero);
1855 SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One);
1856 SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
1858 SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
1860 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
1861 SDValue Sub1 = DAG.getBitcast(VT,
1864 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
1865 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
1867 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
1869 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
1876 SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
1878 SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
1880 SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1882 SDValue Sub2 = DAG.getBitcast(VT,
1885 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
1887 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
1889 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
1891 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
1894 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
1896 SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
1898 SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1900 SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
1902 SDValue Sub3 = DAG.getBitcast(VT,
1908 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
1909 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
1911 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
1912 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
1922 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
1923 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
1925 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
1926 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
1929 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
1930 SDValue DIV_Lo = Zero;
1936 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
1938 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
1947 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
1948 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
1953 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
1957 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
1963 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
1969 SmallVector<SDValue, 2> Results;
1975 if (SDValue Res = LowerDIVREM24(Op, DAG, false))
1979 SDValue X = Op.getOperand(0);
1980 SDValue Y = Op.getOperand(1);
1986 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y);
1989 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y);
1990 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z);
1995 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z);
1996 SDValue R =
2001 SDValue One = DAG.getConstant(1, DL, VT);
2002 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
2018 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2023 SDValue LHS = Op.getOperand(0);
2024 SDValue RHS = Op.getOperand(1);
2026 SDValue Zero = DAG.getConstant(0, DL, VT);
2027 SDValue NegOne = DAG.getConstant(-1, DL, VT);
2030 if (SDValue Res = LowerDIVREM24(Op, DAG, true))
2040 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
2041 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
2042 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
2044 SDValue Res[2] = {
2051 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
2052 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
2053 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
2054 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2062 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
2063 SDValue Rem = Div.getValue(1);
2071 SDValue Res[2] = {
2079 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
2082 SDValue X = Op.getOperand(0);
2083 SDValue Y = Op.getOperand(1);
2087 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
2088 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
2089 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
2094 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2096 SDValue Src = Op.getOperand(0);
2102 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2104 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2105 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
2110 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
2111 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2112 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2114 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
2119 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2124 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
2128 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
2134 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2136 SDValue Src = Op.getOperand(0);
2140 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2141 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2143 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2147 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
2149 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2154 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
2155 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
2158 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
2161 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
2162 const SDValue FractMask
2165 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
2166 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
2167 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
2172 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
2174 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2175 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2177 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
2178 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
2183 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2185 SDValue Src = Op.getOperand(0);
2190 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
2191 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
2195 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
2196 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
2198 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
2201 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
2205 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
2210 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
2222 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2224 SDValue X = Op.getOperand(0);
2227 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2231 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2233 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2235 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2236 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2237 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2239 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);
2244 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
2246 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);
2251 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2253 SDValue Src = Op.getOperand(0);
2259 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2261 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2262 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
2267 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
2268 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2269 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2271 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
2276 SDValue AMDGPUTargetLowering::LowerFLOG(SDValue Op, SelectionDAG &DAG,
2281 SDValue Operand = Op.getOperand(0);
2282 SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand);
2283 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2289 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
2292 SDValue Src = Op.getOperand(0);
2294 const SDValue K = DAG.getConstantFP(numbers::log2e, SL, VT);
2295 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags());
2307 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
2309 SDValue Src = Op.getOperand(0);
2327 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2329 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2330 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2332 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
2333 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
2338 SDValue HiOrLo = isCtlzOpc(Op.getOpcode()) ? Hi : Lo;
2339 SDValue Hi0orLo0 = DAG.getSetCC(SL, SetCCVT, HiOrLo, Zero, ISD::SETEQ);
2341 SDValue OprLo = DAG.getNode(ISDOpc, SL, MVT::i32, Lo);
2342 SDValue OprHi = DAG.getNode(ISDOpc, SL, MVT::i32, Hi);
2344 const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32);
2345 SDValue Add, NewOpr;
2361 SDValue LoOrHi = isCtlzOpc(Op.getOpcode()) ? Lo : Hi;
2362 SDValue Lo0OrHi0 = DAG.getSetCC(SL, SetCCVT, LoOrHi, Zero, ISD::SETEQ);
2363 SDValue SrcIsZero = DAG.getNode(ISD::AND, SL, SetCCVT, Lo0OrHi0, Hi0orLo0);
2367 // SDValue SrcIsZero = DAG.getSetCC(SL, SetCCVT, Src,
2370 const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32);
2381 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
2403 SDValue Src = Op.getOperand(0);
2404 SDValue L = Src;
2406 SDValue S;
2408 const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64);
2411 SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S);
2419 SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32);
2420 SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64);
2421 SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L);
2424 SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32);
2425 SDValue E = DAG.getSelect(SL, MVT::i32,
2430 SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64,
2434 SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U,
2437 SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64,
2440 SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32,
2444 SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64);
2445 SDValue RCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETUGT);
2446 SDValue TCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETEQ);
2448 SDValue One = DAG.getConstant(1, SL, MVT::i32);
2450 SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One);
2452 SDValue R = DAG.getSelect(SL, MVT::i32,
2462 SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R);
2466 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
2469 SDValue Src = Op.getOperand(0);
2471 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2473 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
2475 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
2478 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
2481 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
2483 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
2489 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
2493 SDValue Src = Op.getOperand(0);
2502 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src);
2511 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2512 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2513 SDValue FPRound =
2526 SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
2530 SDValue Src = Op.getOperand(0);
2539 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src);
2549 SDValue Src = Op.getOperand(0);
2551 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2552 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2553 SDValue FPRound =
2566 SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
2570 SDValue Src = Op.getOperand(0);
2572 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2574 SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
2576 SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
2579 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
2581 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul);
2584 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc);
2586 SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL,
2588 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
2590 SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi});
2595 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
2597 SDValue N0 = Op.getOperand(0);
2605 return SDValue();
2614 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
2615 SDValue One = DAG.getConstant(1, DL, MVT::i32);
2616 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
2617 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
2621 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2630 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2635 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
2639 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
2643 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
2648 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2653 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
2655 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
2659 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2662 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
2663 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
2664 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
2667 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
2668 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
2672 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
2674 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
2685 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2694 SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
2696 SDValue Src = Op.getOperand(0);
2704 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2705 SDValue FpToInt32 =
2714 return SDValue();
2717 SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
2719 SDValue Src = Op.getOperand(0);
2727 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2728 SDValue FpToInt32 =
2737 return SDValue();
2740 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2748 SDValue Src = Op.getOperand(0);
2753 SmallVector<SDValue, 8> Args;
2756 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
2767 static bool isU24(SDValue Op, SelectionDAG &DAG) {
2771 static bool isI24(SDValue Op, SelectionDAG &DAG) {
2778 static SDValue simplifyI24(SDNode *Node24,
2784 SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0);
2785 SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1);
2798 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG);
2799 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG);
2808 return SDValue(Node24, 0);
2810 return SDValue(Node24, 0);
2812 return SDValue();
2816 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
2859 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
2862 return SDValue();
2866 return SDValue();
2884 SDValue Ops[2];
2895 return SDValue();
2899 return SDValue();
2903 SDValue NewLoad
2907 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
2909 return SDValue(N, 0);
2914 SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
2917 return SDValue();
2921 return SDValue();
2947 return SDValue();
2951 return SDValue();
2954 SDValue Val = SN->getValue();
2959 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
2961 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
2972 SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
2975 SDValue N0 = N->getOperand(0);
2980 SDValue N1 = N->getOperand(1);
2984 SDValue Src = N0.getOperand(0);
2987 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
2992 return SDValue();
2995 SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
3009 SDValue Src = N->getOperand(1);
3010 return Src.isUndef() ? Src : SDValue();
3013 return SDValue();
3019 SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
3021 unsigned Opc, SDValue LHS,
3024 SDValue Lo, Hi;
3027 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
3028 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
3030 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
3031 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
3038 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
3042 SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
3048 return SDValue();
3050 SDValue LHS = N->getOperand(0);
3064 SDValue X = LHS->getOperand(0);
3070 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
3083 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
3089 return SDValue();
3097 return SDValue();
3099 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
3101 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
3102 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
3104 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
3106 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
3110 SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
3113 return SDValue();
3117 return SDValue();
3125 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
3126 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
3129 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
3135 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
3136 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
3138 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
3142 return SDValue();
3145 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
3149 return SDValue();
3152 SDValue LHS = N->getOperand(0);
3172 return SDValue();
3175 return SDValue();
3180 SDValue One = DAG.getConstant(1, SL, MVT::i32);
3181 SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
3183 SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, LHS);
3184 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecOp, One);
3186 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
3187 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
3189 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
3194 SDValue AMDGPUTargetLowering::performTruncateCombine(
3199 SDValue Src = N->getOperand(0);
3203 SDValue Vec = Src.getOperand(0);
3205 SDValue Elt0 = Vec.getOperand(0);
3224 SDValue BV = stripBitcast(Src.getOperand(0));
3227 SDValue SrcElt = BV.getOperand(1);
3250 SDValue Amt = Src.getOperand(1);
3260 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
3269 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
3276 return SDValue();
3283 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
3284 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
3296 SDValue Mul = DAG.getNode(MulOpc, SL,
3303 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
3309 return SDValue();
3313 return SDValue();
3318 SDValue N0 = N->getOperand(0);
3319 SDValue N1 = N->getOperand(1);
3331 SDValue Mul;
3342 return SDValue();
3350 SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
3355 return SDValue();
3360 SDValue N0 = N->getOperand(0);
3361 SDValue N1 = N->getOperand(1);
3364 return SDValue();
3369 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
3374 SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
3379 return SDValue();
3384 SDValue N0 = N->getOperand(0);
3385 SDValue N1 = N->getOperand(1);
3388 return SDValue();
3393 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
3398 SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
3403 if (SDValue V = simplifyI24(N, DCI))
3406 SDValue N0 = N->getOperand(0);
3407 SDValue N1 = N->getOperand(1);
3416 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
3417 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
3421 static bool isNegativeOne(SDValue Val) {
3427 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
3428 SDValue Op,
3435 return SDValue();
3440 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
3454 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
3455 SDValue LHS, SDValue RHS,
3459 return SDValue();
3463 SDValue CmpLHS = Cond.getOperand(0);
3486 return SDValue();
3489 static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
3492 SDValue Cond,
3493 SDValue N1,
3494 SDValue N2) {
3498 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
3511 static SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
3512 SDValue N) {
3514 SDValue Cond = N.getOperand(0);
3515 SDValue LHS = N.getOperand(1);
3516 SDValue RHS = N.getOperand(2);
3537 SDValue NewLHS = LHS.getOperand(0);
3538 SDValue NewRHS = RHS;
3555 return SDValue();
3560 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
3567 return SDValue();
3571 SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
3573 if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
3576 SDValue Cond = N->getOperand(0);
3578 return SDValue();
3581 SDValue LHS = Cond.getOperand(0);
3582 SDValue RHS = Cond.getOperand(1);
3583 SDValue CC = Cond.getOperand(2);
3585 SDValue True = N->getOperand(1);
3586 SDValue False = N->getOperand(2);
3600 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
3605 SDValue MinMax
3629 bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
3660 SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
3663 SDValue N0 = N->getOperand(0);
3676 return SDValue();
3680 return SDValue();
3687 return SDValue();
3690 SDValue LHS = N0.getOperand(0);
3691 SDValue RHS = N0.getOperand(1);
3703 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
3705 return SDValue(); // Op got folded away.
3714 SDValue LHS = N0.getOperand(0);
3715 SDValue RHS = N0.getOperand(1);
3724 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
3726 return SDValue(); // Op got folded away.
3734 return SDValue();
3737 SDValue LHS = N0.getOperand(0);
3738 SDValue MHS = N0.getOperand(1);
3739 SDValue RHS = N0.getOperand(2);
3753 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
3755 return SDValue(); // Op got folded away.
3771 SDValue LHS = N0.getOperand(0);
3772 SDValue RHS = N0.getOperand(1);
3777 return SDValue();
3779 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3780 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3783 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
3785 return SDValue(); // Op got folded away.
3791 SDValue Ops[3];
3795 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
3797 return SDValue(); // Op got folded away.
3812 SDValue CvtSrc = N0.getOperand(0);
3820 return SDValue();
3824 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3828 SDValue CvtSrc = N0.getOperand(0);
3837 return SDValue();
3840 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3849 SDValue Src = N0.getOperand(0);
3853 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
3858 return SDValue();
3862 SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
3865 SDValue N0 = N->getOperand(0);
3868 return SDValue();
3874 SDValue Src = N0.getOperand(0);
3878 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
3883 return SDValue();
3887 SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
3891 return SDValue();
3899 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
3916 SDValue Src = N->getOperand(0);
3924 SmallVector<SDValue, 8> CastedElts;
3927 SDValue Elt = Src.getOperand(I);
3943 SDValue Src = N->getOperand(0);
3948 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3959 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3998 if (SDValue V = simplifyI24(N, DCI))
4000 return SDValue();
4027 SDValue BitsFrom = N->getOperand(0);
4073 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
4108 return SDValue();
4115 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
4151 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
4160 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
4167 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
4169 SDValue Chain,
4170 SDValue ArgVal,
4175 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
4176 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, 4,
4181 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
4187 SDValue V = Arg.isRegister() ?
4376 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
4391 return SDValue();
4394 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
4412 return SDValue();
4416 const SDValue Op, KnownBits &Known,
4559 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4623 bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,