Lines Matching refs:SDValue
39 SDValue R = SDValue();
181 SetScalarizedVector(SDValue(N, ResNo), R);
184 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
185 SDValue LHS = GetScalarizedVector(N->getOperand(0));
186 SDValue RHS = GetScalarizedVector(N->getOperand(1));
191 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
192 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
193 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
194 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
199 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
200 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
201 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
202 SDValue Op2 = N->getOperand(2);
207 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
210 SDValue Chain = N->getOperand(0);
214 SmallVector<SDValue, 4> Opers(NumOpers);
221 SDValue Oper = N->getOperand(i);
229 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
234 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
238 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
244 SDValue ScalarLHS, ScalarRHS;
249 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
266 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
268 SDValue OtherVal = DAG.getNode(
269 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
270 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
273 return SDValue(ScalarNode, ResNo);
276 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
278 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
282 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
283 SDValue Op = N->getOperand(0);
293 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
295 SDValue InOp = N->getOperand(0);
303 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
309 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
311 SDValue Op = GetScalarizedVector(N->getOperand(0));
316 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
317 SDValue Op = GetScalarizedVector(N->getOperand(0));
322 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
325 SDValue Op = N->getOperand(1);
333 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
336 SDValue Result = DAG.getLoad(
345 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
349 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
352 SDValue Op = N->getOperand(0);
372 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
375 SDValue LHS = GetScalarizedVector(N->getOperand(0));
380 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
382 SDValue Op = N->getOperand(0);
407 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
411 SDValue InOp = N->getOperand(0);
417 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
418 SDValue Cond = N->getOperand(0);
432 SDValue LHS = GetScalarizedVector(N->getOperand(1));
485 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
486 SDValue LHS = GetScalarizedVector(N->getOperand(1));
492 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
493 SDValue LHS = GetScalarizedVector(N->getOperand(2));
500 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
504 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
506 SDValue Arg = N->getOperand(2).getOperand(0);
513 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
517 SDValue LHS = N->getOperand(0);
518 SDValue RHS = N->getOperand(1);
536 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
553 SDValue Res = SDValue();
634 ReplaceValueWith(SDValue(N, 0), Res);
640 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
641 SDValue Elt = GetScalarizedVector(N->getOperand(0));
648 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
651 SDValue Elt = GetScalarizedVector(N->getOperand(0));
652 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
661 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
664 SDValue Elt = GetScalarizedVector(N->getOperand(1));
665 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
670 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
675 // Do our own replacement and return SDValue() to tell the caller that we
677 ReplaceValueWith(SDValue(N, 0), Res);
678 return SDValue();
682 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
683 SmallVector<SDValue, 8> Ops(N->getNumOperands());
691 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
693 SDValue Res = GetScalarizedVector(N->getOperand(0));
704 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
705 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
715 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
722 SDValue LHS = GetScalarizedVector(N->getOperand(0));
723 SDValue RHS = GetScalarizedVector(N->getOperand(1));
729 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
744 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
764 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
765 SDValue Elt = GetScalarizedVector(N->getOperand(0));
766 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
772 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
775 SDValue Elt = GetScalarizedVector(N->getOperand(1));
776 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
782 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
786 // Do our own replacement and return SDValue() to tell the caller that we
788 ReplaceValueWith(SDValue(N, 0), Res);
789 return SDValue();
792 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
793 SDValue Res = GetScalarizedVector(N->getOperand(0));
810 SDValue Lo, Hi;
976 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
981 SDValue &Ptr) {
986 SDValue BytesIncrement = DAG.getVScale(
998 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
999 SDValue &Hi) {
1000 SDValue LHSLo, LHSHi;
1002 SDValue RHSLo, RHSHi;
1012 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1013 SDValue &Hi) {
1014 SDValue Op0Lo, Op0Hi;
1016 SDValue Op1Lo, Op1Hi;
1018 SDValue Op2Lo, Op2Hi;
1028 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1029 SDValue LHSLo, LHSHi;
1031 SDValue RHSLo, RHSHi;
1034 SDValue Op2 = N->getOperand(2);
1043 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1044 SDValue &Hi) {
1051 SDValue InOp = N->getOperand(0);
1103 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1104 SDValue &Hi) {
1109 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1112 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1116 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1117 SDValue &Hi) {
1130 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1133 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1137 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1138 SDValue &Hi) {
1139 SDValue Vec = N->getOperand(0);
1140 SDValue Idx = N->getOperand(1);
1153 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1154 SDValue &Hi) {
1155 SDValue Vec = N->getOperand(0);
1156 SDValue SubVec = N->getOperand(1);
1157 SDValue Idx = N->getOperand(2);
1183 SDValue StackPtr =
1189 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1193 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1210 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1211 SDValue &Hi) {
1218 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1219 SDValue &Hi) {
1220 SDValue LHSLo, LHSHi;
1224 SDValue RHSLo, RHSHi;
1225 SDValue RHS = N->getOperand(1);
1237 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1238 SDValue &Hi) {
1239 SDValue LHSLo, LHSHi;
1253 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1254 SDValue &Hi) {
1256 SDValue N0 = N->getOperand(0);
1259 SDValue InLo, InHi;
1291 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1292 SDValue &Hi) {
1294 SDValue Chain = N->getOperand(0);
1299 SmallVector<SDValue, 4> OpsLo(NumOps);
1300 SmallVector<SDValue, 4> OpsHi(NumOps);
1308 SDValue Op = N->getOperand(i);
1309 SDValue OpLo = Op;
1310 SDValue OpHi = Op;
1340 ReplaceValueWith(SDValue(N, 1), Chain);
1343 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1344 SDValue Chain = N->getOperand(0);
1350 SmallVector<SDValue, 8> Scalars;
1351 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1361 SmallVector<SDValue, 8> Chains;
1367 SDValue Operand = N->getOperand(j);
1377 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1391 ReplaceValueWith(SDValue(N, 1), Chain);
1399 SDValue &Lo, SDValue &Hi) {
1407 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1424 Lo = SDValue(LoNode, ResNo);
1425 Hi = SDValue(HiNode, ResNo);
1431 SetSplitVector(SDValue(N, OtherNo),
1432 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1434 SDValue OtherVal = DAG.getNode(
1436 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1437 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1441 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1442 SDValue &Hi) {
1443 SDValue Vec = N->getOperand(0);
1444 SDValue Elt = N->getOperand(1);
1445 SDValue Idx = N->getOperand(2);
1482 SDValue StackPtr =
1488 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1493 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1520 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1521 SDValue &Hi) {
1529 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1530 SDValue &Hi) {
1537 SDValue Ch = LD->getChain();
1538 SDValue Ptr = LD->getBasePtr();
1539 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1548 SDValue Value, NewChain;
1551 ReplaceValueWith(SDValue(LD, 1), NewChain);
1572 ReplaceValueWith(SDValue(LD, 1), Ch);
1576 SDValue &Lo, SDValue &Hi) {
1582 SDValue Ch = MLD->getChain();
1583 SDValue Ptr = MLD->getBasePtr();
1584 SDValue Offset = MLD->getOffset();
1586 SDValue Mask = MLD->getMask();
1587 SDValue PassThru = MLD->getPassThru();
1592 SDValue MaskLo, MaskHi;
1608 SDValue PassThruLo, PassThruHi;
1649 ReplaceValueWith(SDValue(MLD, 1), Ch);
1654 SDValue &Lo, SDValue &Hi) {
1659 SDValue Ch = MGT->getChain();
1660 SDValue Ptr = MGT->getBasePtr();
1661 SDValue Mask = MGT->getMask();
1662 SDValue PassThru = MGT->getPassThru();
1663 SDValue Index = MGT->getIndex();
1664 SDValue Scale = MGT->getScale();
1668 SDValue MaskLo, MaskHi;
1678 SDValue PassThruLo, PassThruHi;
1684 SDValue IndexHi, IndexLo;
1695 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1699 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1710 ReplaceValueWith(SDValue(MGT, 1), Ch);
1714 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1724 SDValue LL, LH, RL, RH;
1741 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1742 SDValue &Hi) {
1768 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1769 SDValue &Hi) {
1802 SDValue NewSrc =
1817 SDValue &Lo, SDValue &Hi) {
1819 SDValue Inputs[4];
1831 SDValue &Output = High ? Hi : Lo;
1882 SmallVector<SDValue, 16> SVOps;
1913 SDValue Op0 = Inputs[InputUsed[0]];
1915 SDValue Op1 = InputUsed[1] == -1U ?
1925 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1928 SDValue Chain = N->getOperand(0);
1929 SDValue Ptr = N->getOperand(1);
1930 SDValue SV = N->getOperand(2);
1942 ReplaceValueWith(SDValue(N, 1), Chain);
1956 SDValue Res = SDValue();
2065 ReplaceValueWith(SDValue(N, 0), Res);
2069 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2074 SDValue Mask = N->getOperand(0);
2075 SDValue Src0 = N->getOperand(1);
2076 SDValue Src1 = N->getOperand(2);
2081 SDValue Lo, Hi;
2090 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2095 SDValue LoSelect =
2097 SDValue HiSelect =
2103 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2105 SDValue Lo, Hi;
2108 SDValue VecOp = N->getOperand(OpNo);
2141 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2145 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2148 SDValue Lo, Hi;
2164 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2169 ReplaceValueWith(SDValue(N, 1), Ch);
2178 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2182 SDValue Lo, Hi;
2194 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2197 SDValue Idx = N->getOperand(1);
2199 SDValue Lo, Hi;
2221 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2222 SDValue Vec = N->getOperand(0);
2223 SDValue Idx = N->getOperand(1);
2229 SDValue Lo, Hi;
2235 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2236 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2243 return SDValue();
2259 SDValue StackPtr =
2264 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2273 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2284 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2285 SDValue Lo, Hi;
2294 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2300 SDValue Ch = MGT->getChain();
2301 SDValue Ptr = MGT->getBasePtr();
2302 SDValue Index = MGT->getIndex();
2303 SDValue Scale = MGT->getScale();
2304 SDValue Mask = MGT->getMask();
2305 SDValue PassThru = MGT->getPassThru();
2308 SDValue MaskLo, MaskHi;
2319 SDValue PassThruLo, PassThruHi;
2325 SDValue IndexHi, IndexLo;
2336 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2337 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2340 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2341 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2351 ReplaceValueWith(SDValue(MGT, 1), Ch);
2353 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2355 ReplaceValueWith(SDValue(MGT, 0), Res);
2356 return SDValue();
2359 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2362 SDValue Ch = N->getChain();
2363 SDValue Ptr = N->getBasePtr();
2364 SDValue Offset = N->getOffset();
2366 SDValue Mask = N->getMask();
2367 SDValue Data = N->getValue();
2371 SDValue DataLo, DataHi;
2379 SDValue MaskLo, MaskHi;
2395 SDValue Lo, Hi, Res;
2430 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2432 SDValue Ch = N->getChain();
2433 SDValue Ptr = N->getBasePtr();
2434 SDValue Mask = N->getMask();
2435 SDValue Index = N->getIndex();
2436 SDValue Scale = N->getScale();
2437 SDValue Data = N->getValue();
2443 SDValue DataLo, DataHi;
2451 SDValue MaskLo, MaskHi;
2461 SDValue IndexHi, IndexLo;
2467 SDValue Lo;
2472 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2479 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2484 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2490 SDValue Ch = N->getChain();
2491 SDValue Ptr = N->getBasePtr();
2496 SDValue Lo, Hi;
2525 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2533 SmallVector<SDValue, 32> Elts;
2535 for (const SDValue &Op : N->op_values()) {
2546 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2565 SDValue InVec = N->getOperand(OpNo);
2596 SDValue InLoVec, InHiVec;
2609 SDValue HalfLo;
2610 SDValue HalfHi;
2611 SDValue Chain;
2627 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2635 SDValue Res = DAG.getNode(
2640 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2651 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2656 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2668 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2677 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2680 SDValue Lo, Hi;
2695 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2697 ReplaceValueWith(SDValue(N, 1), NewChain);
2706 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2725 SDValue Res = SDValue();
2905 SetWidenedVector(SDValue(N, ResNo), Res);
2908 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2912 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2913 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2914 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2918 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2922 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2923 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2927 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
2931 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2932 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2933 SDValue InOp3 = N->getOperand(2);
2941 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
2942 SmallVectorImpl<SDValue> &ConcatOps,
2974 SDValue VecOp = DAG.getUNDEF(NextVT);
2984 SDValue undefVec = DAG.getUNDEF(VT);
2986 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3010 SDValue UndefVal = DAG.getUNDEF(MaxVT);
3018 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3034 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3035 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3045 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3046 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3049 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3060 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3062 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3075 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3077 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3089 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3124 SmallVector<SDValue, 4> InOps;
3127 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3128 SmallVector<SDValue, 16> Chains;
3137 SDValue Oper = N->getOperand(i);
3155 SmallVector<SDValue, 4> EOps;
3158 SDValue Op = InOps[i];
3168 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3181 SmallVector<SDValue, 4> EOps;
3184 SDValue Op = InOps[i];
3194 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3203 SDValue NewChain;
3208 ReplaceValueWith(SDValue(N, 1), NewChain);
3213 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3218 SDValue WideLHS, WideRHS;
3235 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3252 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3254 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3255 SDValue OtherVal = DAG.getNode(
3256 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3257 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3260 return SDValue(WideNode, ResNo);
3263 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3264 SDValue InOp = N->getOperand(0);
3308 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3310 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3317 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3328 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3333 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3344 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3345 SDValue InOp = N->getOperand(1);
3347 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3362 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3363 SmallVector<SDValue, 32> OpChains;
3373 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3374 ReplaceValueWith(SDValue(N, 1), NewChain);
3379 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3381 SDValue InOp = N->getOperand(0);
3406 SmallVector<SDValue, 16> Ops;
3408 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3432 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3443 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3445 SDValue InOp = GetWidenedVector(N->getOperand(0));
3446 SDValue ShOp = N->getOperand(1);
3450 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3452 SDValue InOp = GetWidenedVector(N->getOperand(0));
3453 SDValue ShOp = N->getOperand(1);
3469 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3472 SDValue InOp = GetWidenedVector(N->getOperand(0));
3476 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3482 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3487 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3488 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3492 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3493 SDValue InOp = N->getOperand(0);
3513 SDValue NInOp = GetPromotedInteger(InOp);
3568 SDValue NewVec;
3575 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3589 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3602 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3609 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3622 SDValue UndefVal = DAG.getUNDEF(InVT);
3623 SmallVector<SDValue, 16> Ops(NumConcat);
3671 SmallVector<SDValue, 16> Ops(WidenNumElts);
3674 SDValue InOp = N->getOperand(i);
3681 SDValue UndefVal = DAG.getUNDEF(EltVT);
3687 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3691 SDValue InOp = N->getOperand(0);
3692 SDValue Idx = N->getOperand(1);
3712 SmallVector<SDValue, 16> Ops(WidenNumElts);
3720 SDValue UndefVal = DAG.getUNDEF(EltVT);
3726 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3727 SDValue InOp = GetWidenedVector(N->getOperand(0));
3733 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3744 SDValue Value, NewChain;
3746 ReplaceValueWith(SDValue(LD, 0), Value);
3747 ReplaceValueWith(SDValue(LD, 1), NewChain);
3748 return SDValue();
3751 SDValue Result;
3752 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
3761 SDValue NewChain;
3769 ReplaceValueWith(SDValue(N, 1), NewChain);
3774 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3777 SDValue Mask = N->getMask();
3779 SDValue PassThru = GetWidenedVector(N->getPassThru());
3789 SDValue Res = DAG.getMaskedLoad(
3795 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3799 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3802 SDValue Mask = N->getMask();
3804 SDValue PassThru = GetWidenedVector(N->getPassThru());
3805 SDValue Scale = N->getScale();
3816 SDValue Index = N->getIndex();
3821 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3823 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3829 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3833 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3863 static inline EVT getSETCCOperandType(SDValue N) {
3871 static inline bool isSETCCorConvertedSETCC(SDValue N) {
3897 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3905 SDValue Mask;
3906 SmallVector<SDValue, 4> Ops;
3939 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
3945 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3960 SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3962 SDValue Cond = N->getOperand(0);
3965 return SDValue();
3968 return SDValue();
3974 return SDValue();
3979 return SDValue();
3987 return SDValue();
3996 return SDValue();
4004 return SDValue();
4008 SDValue VSelOp1 = N->getOperand(1);
4009 SDValue VSelOp2 = N->getOperand(2);
4021 SDValue Mask;
4029 SDValue SETCC0 = Cond->getOperand(0);
4030 SDValue SETCC1 = Cond->getOperand(1);
4061 return SDValue();
4066 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4070 SDValue Cond1 = N->getOperand(0);
4073 if (SDValue Res = WidenVSELECTAndMask(N))
4088 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4089 SDValue Res = ModifyToType(SplitSelect, WidenVT);
4097 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4098 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4104 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4105 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4106 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4112 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4117 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4125 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4126 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4142 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4149 SDValue InOp1 = N->getOperand(0);
4159 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4160 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4165 SDValue InOp2 = N->getOperand(1);
4184 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
4195 SDValue Chain = N->getOperand(0);
4196 SDValue LHS = N->getOperand(1);
4197 SDValue RHS = N->getOperand(2);
4198 SDValue CC = N->getOperand(3);
4202 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
4203 SmallVector<SDValue, 8> Chains(NumElts);
4205 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4207 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4218 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4219 ReplaceValueWith(SDValue(N, 1), NewChain);
4230 SDValue Res = SDValue();
4314 ReplaceValueWith(SDValue(N, 0), Res);
4318 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4322 SDValue InOp = N->getOperand(0);
4378 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4385 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4391 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4404 SDValue Res;
4414 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4428 SmallVector<SDValue, 16> Ops(NumElts);
4430 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4431 SmallVector<SDValue, 32> OpChains;
4438 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4439 ReplaceValueWith(SDValue(N, 1), NewChain);
4450 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4452 SDValue InOp = GetWidenedVector(N->getOperand(0));
4464 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4481 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4491 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4512 SmallVector<SDValue, 16> Ops(NumElts);
4518 SDValue InOp = N->getOperand(i);
4530 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4531 SDValue InOp = GetWidenedVector(N->getOperand(0));
4536 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4537 SDValue InOp = GetWidenedVector(N->getOperand(0));
4542 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4550 SmallVector<SDValue, 16> StChain;
4562 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4566 SDValue Mask = MST->getMask();
4568 SDValue StVal = MST->getValue();
4602 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4605 SDValue DataOp = MG->getPassThru();
4606 SDValue Mask = MG->getMask();
4607 SDValue Scale = MG->getScale();
4610 SDValue Index = GetWidenedVector(MG->getIndex());
4613 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4615 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4617 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4618 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4619 return SDValue();
4622 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4624 SDValue DataOp = MSC->getValue();
4625 SDValue Mask = MSC->getMask();
4626 SDValue Index = MSC->getIndex();
4627 SDValue Scale = MSC->getScale();
4650 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4657 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4658 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4659 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4675 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4682 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4691 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
4692 SDValue Chain = N->getOperand(0);
4693 SDValue LHS = GetWidenedVector(N->getOperand(1));
4694 SDValue RHS = GetWidenedVector(N->getOperand(2));
4695 SDValue CC = N->getOperand(3);
4704 SmallVector<SDValue, 8> Scalars(NumElts);
4705 SmallVector<SDValue, 8> Chains(NumElts);
4708 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4710 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4721 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4722 ReplaceValueWith(SDValue(N, 1), NewChain);
4727 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4729 SDValue Op = GetWidenedVector(N->getOperand(0));
4734 SDValue NeutralElem;
4783 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4790 SDValue Cond = GetWidenedVector(N->getOperand(0));
4791 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4792 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4795 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4881 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4882 SmallVectorImpl<SDValue> &LdOps,
4891 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4909 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4923 SDValue Chain = LD->getChain();
4924 SDValue BasePtr = LD->getBasePtr();
4937 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4946 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4954 SmallVector<SDValue, 16> ConcatOps(NumConcat);
4955 SDValue UndefVal = DAG.getUNDEF(NewVT);
4963 SmallVector<SDValue, 16> LdOps;
4974 SDValue L;
5005 SmallVector<SDValue, 16> ConcatOps(End);
5025 SmallVector<SDValue, 16> WidenOps(NumOps);
5046 SmallVector<SDValue, 16> WidenOps(NumOps);
5047 SDValue UndefVal = DAG.getUNDEF(LdTy);
5058 SDValue
5059 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
5070 SDValue Chain = LD->getChain();
5071 SDValue BasePtr = LD->getBasePtr();
5081 SmallVector<SDValue, 16> Ops(WidenNumElts);
5089 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
5097 SDValue UndefVal = DAG.getUNDEF(EltVT);
5104 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
5109 SDValue Chain = ST->getChain();
5110 SDValue BasePtr = ST->getBasePtr();
5113 SDValue ValOp = GetWidenedVector(ST->getValue());
5134 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
5149 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5153 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
5169 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
5173 SDValue Chain = ST->getChain();
5174 SDValue BasePtr = ST->getBasePtr();
5177 SDValue ValOp = GetWidenedVector(ST->getValue());
5194 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
5201 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
5202 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
5213 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5230 SmallVector<SDValue, 16> Ops(NumConcat);
5231 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5245 SmallVector<SDValue, 16> Ops(WidenNumElts);
5253 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :