Lines Matching refs:RISCVISD
821 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
2593 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2755 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
2757 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
2767 if (Opc == RISCVISD::FCVT_WU_RV64)
2806 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
2815 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
2819 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
2823 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
2825 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
2898 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
2910 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
2916 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
2939 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
2944 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
2949 Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL);
2953 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
2959 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
2960 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
2964 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
2997 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
3002 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
3008 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3020 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3025 RISCVISD::SETCC_VL, DL, MaskVT,
3043 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3049 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3053 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3062 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3069 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3101 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
3122 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
3139 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3151 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3327 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
3415 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
3467 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
3472 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
3543 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
3544 : RISCVISD::VMV_V_X_VL;
3585 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
3713 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
3739 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
3796 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
3797 : RISCVISD::VMV_V_X_VL;
3915 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
3940 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
3959 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
3971 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
3976 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
3979 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
4005 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
4018 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
4025 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
4068 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT,
4085 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT,
4285 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
4288 DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift,
4421 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
4471 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
4472 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
4525 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
4527 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
4532 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
4538 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
4546 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
4828 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
4836 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
5015 unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
5016 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
5022 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
5031 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
5177 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
5404 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
5429 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
5432 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
5436 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5441 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
5445 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
5448 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5454 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
5492 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
5520 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
5523 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
5529 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
5532 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
5538 ? RISCVISD::VFMAX_VL
5539 : RISCVISD::VFMIN_VL;
5551 return RISCVISD::NODE##_VL;
5554 return RISCVISD::NODE##_VL;
5629 return RISCVISD::CTLZ_VL;
5632 return RISCVISD::CTTZ_VL;
5635 return RISCVISD::VFMADD_VL;
5637 return RISCVISD::STRICT_VFMADD_VL;
5641 return RISCVISD::VMAND_VL;
5642 return RISCVISD::AND_VL;
5646 return RISCVISD::VMOR_VL;
5647 return RISCVISD::OR_VL;
5651 return RISCVISD::VMXOR_VL;
5652 return RISCVISD::XOR_VL;
5655 return RISCVISD::VMERGE_VL;
5657 return RISCVISD::SRA_VL;
5659 return RISCVISD::SRL_VL;
5661 return RISCVISD::FSQRT_VL;
5663 return RISCVISD::VSEXT_VL;
5665 return RISCVISD::VZEXT_VL;
5667 return RISCVISD::VFCVT_RTZ_X_F_VL;
5669 return RISCVISD::VFCVT_RTZ_XU_F_VL;
5672 return RISCVISD::VFMIN_VL;
5675 return RISCVISD::VFMAX_VL;
5684 assert(Opcode > RISCVISD::FIRST_NUMBER &&
5685 Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
5687 static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
5689 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
5693 if (Opcode >= RISCVISD::ADD_VL && Opcode <= RISCVISD::VFMAX_VL)
5695 if (Opcode == RISCVISD::FCOPYSIGN_VL)
5697 if (Opcode >= RISCVISD::VWMUL_VL && Opcode <= RISCVISD::VFWSUB_W_VL)
5699 if (Opcode == RISCVISD::SETCC_VL)
5701 if (Opcode >= RISCVISD::STRICT_FADD_VL && Opcode <= RISCVISD::STRICT_FDIV_VL)
5703 if (Opcode == RISCVISD::VMERGE_VL)
5710 assert(Opcode > RISCVISD::FIRST_NUMBER &&
5711 Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
5713 static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
5715 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
5719 if (Opcode >= RISCVISD::TRUNCATE_VECTOR_VL && Opcode <= RISCVISD::SETCC_VL)
5721 if (Opcode >= RISCVISD::VRGATHER_VX_VL && Opcode <= RISCVISD::VFIRST_VL)
5723 if (Opcode >= RISCVISD::STRICT_FADD_VL &&
5724 Opcode <= RISCVISD::STRICT_VFROUND_NOEXCEPT_VL)
5892 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
5898 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::bf16, NewOp0);
5905 DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
5913 DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
5978 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
5990 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
5995 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
6016 SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
6026 SDValue Res = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
6084 return DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0);
6087 DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0);
6101 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, Op0);
6107 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, FloatVal);
6248 RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
6251 RVVOpc = RISCVISD::VFCVT_RTZ_XU_F_VL;
6254 RVVOpc = RISCVISD::SINT_TO_FP_VL;
6257 RVVOpc = RISCVISD::UINT_TO_FP_VL;
6260 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_X_F_VL;
6263 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL;
6266 RVVOpc = RISCVISD::STRICT_SINT_TO_FP_VL;
6269 RVVOpc = RISCVISD::STRICT_UINT_TO_FP_VL;
6304 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
6315 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op)
6333 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
6343 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32,
6820 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
6845 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
6846 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNHi, AddrLo);
6870 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
6944 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
6947 DAG.getNode(RISCVISD::ADD_TPREL, DL, Ty, MNHi, TPReg, AddrAdd);
6948 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNAdd, AddrLo);
7200 // sequence or RISCVISD::SELECT_CC node (branch-based select).
7205 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV);
7208 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV);
7215 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
7221 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV));
7232 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
7233 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
7266 // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
7276 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
7280 // then merge the SETCC node into the lowered RISCVISD::SELECT_CC to take
7333 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
7350 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
7354 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
7528 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
7532 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
7603 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
7605 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
7608 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal,
7680 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
7682 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
7686 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
7688 Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT,
7710 // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
7748 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
7783 ? RISCVISD::STRICT_FP_EXTEND_VL
7784 : RISCVISD::STRICT_VFNCVT_ROD_VL;
7792 ? RISCVISD::STRICT_FP_EXTEND_VL
7793 : RISCVISD::STRICT_FP_ROUND_VL;
7852 unsigned ConvOpc = IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::FP_ROUND_VL;
7862 IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::VFNCVT_ROD_VL;
7988 VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
8019 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8024 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8040 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8044 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8094 DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL);
8222 SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
8364 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
8366 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
8369 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
8371 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
8390 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
8394 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
8504 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
8505 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
8506 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
8507 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
8508 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
8509 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
8510 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
8511 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
8526 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
8544 IntNo == Intrinsic::riscv_zip ? RISCVISD::ZIP : RISCVISD::UNZIP;
8553 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
8556 return DAG.getNode(RISCVISD::CLMUL, DL, XLenVT, Op.getOperand(1),
8561 IntNo == Intrinsic::riscv_clmulh ? RISCVISD::CLMULH : RISCVISD::CLMULR;
8582 SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
8593 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
8600 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
8629 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
8634 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
8636 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
8639 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal,
8643 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, Op.getSimpleValueType(),
9055 return RISCVISD::VECREDUCE_ADD_VL;
9058 return RISCVISD::VECREDUCE_UMAX_VL;
9061 return RISCVISD::VECREDUCE_SMAX_VL;
9064 return RISCVISD::VECREDUCE_UMIN_VL;
9067 return RISCVISD::VECREDUCE_SMIN_VL;
9070 return RISCVISD::VECREDUCE_AND_VL;
9073 return RISCVISD::VECREDUCE_OR_VL;
9076 return RISCVISD::VECREDUCE_XOR_VL;
9078 return RISCVISD::VECREDUCE_FADD_VL;
9080 return RISCVISD::VECREDUCE_SEQ_FADD_VL;
9082 return RISCVISD::VECREDUCE_FMAX_VL;
9084 return RISCVISD::VECREDUCE_FMIN_VL;
9130 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
9131 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
9132 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9140 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9148 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9262 // RISCVISD node.
9276 return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
9279 return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
9288 ? RISCVISD::VECREDUCE_FMIN_VL
9289 : RISCVISD::VECREDUCE_FMAX_VL;
9432 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL);
9496 SubVec = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, InterSubVT, AlignedExtract,
9748 SDValue EvenWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
9750 SDValue OddWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
9832 DAG.getNode(RISCVISD::ADD_VL, DL, IdxVT, Idx, VLMax, Idx, OddMask, VL);
9837 Interleaved = DAG.getNode(RISCVISD::VRGATHEREI16_VV_VL, DL, ConcatVT,
9860 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
9865 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
9899 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
9928 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
9946 SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, DAG.getUNDEF(IntVT),
9949 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
9950 SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
10225 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
10291 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
10295 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
10299 DAG.getNode(RISCVISD::VMAND_VL, DL, MaskVT, OrderMask1, OrderMask2, VL);
10302 Res = DAG.getNode(RISCVISD::STRICT_FSETCCS_VL, DL,
10306 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
10307 : RISCVISD::STRICT_FSETCCS_VL;
10345 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
10347 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
10349 SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
10372 SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag,
10396 SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1,
10451 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
10529 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10534 SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10537 SDValue Result = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Src, Splat,
10563 SDValue AllOneMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
10570 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
10575 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
10577 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, AllOneMask, VL);
10585 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
10586 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Temp, Op2, VL);
10594 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
10595 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Op1, Temp, VL);
10603 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
10604 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op2, VL);
10612 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
10613 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op1, VL);
10651 unsigned RISCVISDExtOpc = RISCVISDOpc == RISCVISD::SINT_TO_FP_VL
10652 ? RISCVISD::VSEXT_VL
10653 : RISCVISD::VZEXT_VL;
10660 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
10663 RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, XLenVT);
10664 SDValue OneSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
10666 Src = DAG.getNode(RISCVISD::VMERGE_VL, DL, IntVT, Src, OneSplat,
10686 DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterimFVT, Src, Mask, VL);
10708 Result = DAG.getNode(RISCVISD::FP_ROUND_VL, DL, DstVT, Src, Mask, VL);
10728 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterimIVT,
10730 Result = DAG.getNode(RISCVISD::SETCC_VL, DL, DstVT,
10744 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, InterimIVT,
10785 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10788 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10791 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1,
10794 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10797 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10800 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2,
10827 RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1),
10865 SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
10868 SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
10871 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, IndicesVT, Op1, SplatOne,
10881 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
10920 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
10933 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
10936 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IndicesVT, Mask, EVL);
10939 SDValue VecLenSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
10941 SDValue VRSUB = DAG.getNode(RISCVISD::SUB_VL, DL, IndicesVT, VecLenSplat, VID,
10949 RISCVISD::SETCC_VL, DL, ContainerVT,
11263 SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
11314 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
11331 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
11336 return RISCVISD::SLLW;
11338 return RISCVISD::SRAW;
11340 return RISCVISD::SRLW;
11342 return RISCVISD::DIVW;
11344 return RISCVISD::DIVUW;
11346 return RISCVISD::REMUW;
11348 return RISCVISD::ROLW;
11350 return RISCVISD::RORW;
11362 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
11412 unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
11413 : RISCVISD::STRICT_FCVT_WU_RV64;
11429 unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
11470 DAG.getNode(RISCVISD::FCVT_W_RV64, DL, MVT::i64, Op0,
11494 DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
11538 SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
11615 unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
11749 SDValue Abs = DAG.getNode(RISCVISD::ABSW, DL, MVT::i64, Src);
11784 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
11788 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
11793 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
11797 SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL,
11817 case RISCVISD::BREV8: {
11872 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
11876 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11880 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
11883 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
11911 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
11912 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
11913 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
11914 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
11915 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
11916 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
11917 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
11918 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
11930 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
11948 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
11974 unsigned Opc = IntNo == Intrinsic::riscv_clmulh ? RISCVISD::CLMULH
11975 : RISCVISD::CLMULR;
11987 SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
12001 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
12008 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
12010 SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
12012 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
12191 return RISCVISD::VECREDUCE_ADD_VL;
12193 return RISCVISD::VECREDUCE_UMAX_VL;
12195 return RISCVISD::VECREDUCE_SMAX_VL;
12197 return RISCVISD::VECREDUCE_UMIN_VL;
12199 return RISCVISD::VECREDUCE_SMIN_VL;
12201 return RISCVISD::VECREDUCE_AND_VL;
12203 return RISCVISD::VECREDUCE_OR_VL;
12205 return RISCVISD::VECREDUCE_XOR_VL;
12207 return RISCVISD::VECREDUCE_FADD_VL;
12209 return RISCVISD::VECREDUCE_FMAX_VL;
12211 return RISCVISD::VECREDUCE_FMIN_VL;
12247 if (ScalarV.getOpcode() != RISCVISD::VFMV_S_F_VL &&
12248 ScalarV.getOpcode() != RISCVISD::VMV_S_X_VL &&
12249 ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
12372 Slct.getOpcode() != RISCVISD::SELECT_CC) ||
12381 unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
12401 if (Slct.getOpcode() == RISCVISD::SELECT_CC)
12402 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
12654 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
12681 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
12716 if (N0.getOpcode() != RISCVISD::CZERO_EQZ ||
12717 N1.getOpcode() != RISCVISD::CZERO_NEZ ||
12738 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0),
12740 SDValue NewN1 = DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0),
12780 // RISCVISD:::SLLW and we can't recover it to use a BSET instruction.
12796 if (N0.getOpcode() == RISCVISD::SLLW &&
12800 return DAG.getNode(RISCVISD::ROLW, DL, MVT::i64,
12906 N0.getOpcode() != RISCVISD::VZEXT_VL)
12993 if (Src.getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
12995 return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, SDLoc(N), VT,
13057 case RISCVISD::VSEXT_VL:
13058 case RISCVISD::VZEXT_VL:
13067 return OrigOperand.getOpcode() == RISCVISD::VMV_V_X_VL;
13085 unsigned ExtOpc = *SExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
13093 case RISCVISD::VSEXT_VL:
13094 case RISCVISD::VZEXT_VL:
13096 case RISCVISD::VMV_V_X_VL:
13097 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT,
13133 case RISCVISD::ADD_VL:
13134 case RISCVISD::VWADD_W_VL:
13135 case RISCVISD::VWADDU_W_VL:
13136 return IsSExt ? RISCVISD::VWADD_VL : RISCVISD::VWADDU_VL;
13138 case RISCVISD::MUL_VL:
13139 return IsSExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
13141 case RISCVISD::SUB_VL:
13142 case RISCVISD::VWSUB_W_VL:
13143 case RISCVISD::VWSUBU_W_VL:
13144 return IsSExt ? RISCVISD::VWSUB_VL : RISCVISD::VWSUBU_VL;
13153 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) &&
13155 return RISCVISD::VWMULSU_VL;
13163 case RISCVISD::ADD_VL:
13164 return IsSExt ? RISCVISD::VWADD_W_VL : RISCVISD::VWADDU_W_VL;
13166 case RISCVISD::SUB_VL:
13167 return IsSExt ? RISCVISD::VWSUB_W_VL : RISCVISD::VWSUBU_W_VL;
13218 case RISCVISD::VZEXT_VL:
13223 case RISCVISD::VSEXT_VL:
13228 case RISCVISD::VMV_V_X_VL: {
13283 case RISCVISD::ADD_VL:
13284 case RISCVISD::MUL_VL:
13285 case RISCVISD::VWADD_W_VL:
13286 case RISCVISD::VWADDU_W_VL:
13287 case RISCVISD::SUB_VL:
13288 case RISCVISD::VWSUB_W_VL:
13289 case RISCVISD::VWSUBU_W_VL:
13308 case RISCVISD::VWADD_W_VL:
13309 case RISCVISD::VWADDU_W_VL:
13310 case RISCVISD::VWSUB_W_VL:
13311 case RISCVISD::VWSUBU_W_VL:
13314 Opc == RISCVISD::VWADDU_W_VL || Opc == RISCVISD::VWSUBU_W_VL;
13371 case RISCVISD::ADD_VL:
13372 case RISCVISD::MUL_VL:
13373 case RISCVISD::VWADD_W_VL:
13374 case RISCVISD::VWADDU_W_VL:
13377 case RISCVISD::SUB_VL:
13378 case RISCVISD::VWSUB_W_VL:
13379 case RISCVISD::VWSUBU_W_VL:
13562 case RISCVISD::ADD_VL:
13563 case RISCVISD::SUB_VL:
13570 case RISCVISD::MUL_VL:
13576 case RISCVISD::VWADD_W_VL:
13577 case RISCVISD::VWSUB_W_VL:
13581 case RISCVISD::VWADDU_W_VL:
13582 case RISCVISD::VWSUBU_W_VL:
13721 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD;
13723 Opcode = RISCVISD::TH_LDD;
13739 unsigned Opcode = (MemVT == MVT::i32) ? RISCVISD::TH_SWD : RISCVISD::TH_SDD;
13901 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
13905 IsSigned ? RISCVISD::VFCVT_X_F_VL : RISCVISD::VFCVT_XU_F_VL;
13909 IsSigned ? RISCVISD::VFCVT_RM_X_F_VL : RISCVISD::VFCVT_RM_XU_F_VL;
13928 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
13930 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
13981 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
13983 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
13996 if (Opc == RISCVISD::FCVT_WU_RV64)
14021 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0));
14033 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
14034 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
14035 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
14036 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
14037 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
14038 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
14039 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
14040 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
14050 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
14051 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
14052 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
14053 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
14054 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
14055 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
14056 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
14057 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
14076 if (V.getOpcode() == RISCVISD::FNEG_VL && V.getOperand(1) == Mask &&
14124 if (Op0.getOpcode() != RISCVISD::FP_EXTEND_VL ||
14125 Op1.getOpcode() != RISCVISD::FP_EXTEND_VL)
14143 case RISCVISD::VFMADD_VL:
14144 NewOpc = RISCVISD::VFWMADD_VL;
14146 case RISCVISD::VFNMSUB_VL:
14147 NewOpc = RISCVISD::VFWNMSUB_VL;
14149 case RISCVISD::VFNMADD_VL:
14150 NewOpc = RISCVISD::VFWNMADD_VL;
14152 case RISCVISD::VFMSUB_VL:
14153 NewOpc = RISCVISD::VFWMSUB_VL;
14183 if (Op0.getOpcode() != RISCVISD::FP_EXTEND_VL ||
14184 Op1.getOpcode() != RISCVISD::FP_EXTEND_VL)
14201 return DAG.getNode(RISCVISD::VFWMUL_VL, SDLoc(N), N->getValueType(0), Op0,
14220 bool IsAdd = N->getOpcode() == RISCVISD::FADD_VL;
14224 Op0.getOpcode() == RISCVISD::FP_EXTEND_VL &&
14228 (Op1.getOpcode() == RISCVISD::FP_EXTEND_VL && Op1.hasOneUse());
14257 Opc = Op0IsExtend ? RISCVISD::VFWADD_VL : RISCVISD::VFWADD_W_VL;
14259 Opc = Op0IsExtend ? RISCVISD::VFWSUB_VL : RISCVISD::VFWSUB_W_VL;
14950 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
14958 if (N->getOpcode() == RISCVISD::ADD_VL) {
14966 case RISCVISD::VWMUL_VL:
14967 case RISCVISD::VWMULU_VL:
14968 case RISCVISD::VWMULSU_VL:
15002 unsigned Opc = RISCVISD::VWMACC_VL + MulOp.getOpcode() - RISCVISD::VWMUL_VL;
15003 static_assert(RISCVISD::VWMACC_VL + 1 == RISCVISD::VWMACCU_VL,
15005 static_assert(RISCVISD::VWMACC_VL + 2 == RISCVISD::VWMACCSU_VL,
15007 static_assert(RISCVISD::VWMUL_VL + 1 == RISCVISD::VWMULU_VL,
15009 static_assert(RISCVISD::VWMUL_VL + 2 == RISCVISD::VWMULSU_VL,
15152 case RISCVISD::SplitF64: {
15156 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
15183 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
15198 case RISCVISD::SLLW:
15199 case RISCVISD::SRAW:
15200 case RISCVISD::SRLW:
15201 case RISCVISD::RORW:
15202 case RISCVISD::ROLW: {
15210 case RISCVISD::CLZW:
15211 case RISCVISD::CTZW: {
15217 case RISCVISD::FMV_W_X_RV64: {
15222 if (Op0.getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64)
15226 case RISCVISD::FMV_X_ANYEXTH:
15227 case RISCVISD::FMV_X_ANYEXTW_RV64: {
15234 if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
15235 Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
15236 (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
15237 Op0->getOpcode() == RISCVISD::FMV_H_X)) {
15251 unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
15322 case RISCVISD::TRUNCATE_VECTOR_VL: {
15330 if (V.getOpcode() != RISCVISD::TRUNCATE_VECTOR_VL)
15338 return V.getOperand(1).getOpcode() == RISCVISD::VMSET_VL &&
15376 case RISCVISD::CZERO_EQZ:
15377 case RISCVISD::CZERO_NEZ:
15385 unsigned NewOpc = N->getOpcode() == RISCVISD::CZERO_EQZ
15386 ? RISCVISD::CZERO_NEZ
15387 : RISCVISD::CZERO_EQZ;
15394 case RISCVISD::SELECT_CC: {
15437 return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
15490 SDValue NewSel = DAG.getNode(RISCVISD::SELECT_CC, DL, VT, LHS, RHS, CC,
15497 case RISCVISD::BR_CC: {
15504 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
15726 case RISCVISD::SRA_VL:
15727 case RISCVISD::SRL_VL:
15728 case RISCVISD::SHL_VL: {
15730 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
15735 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
15749 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
15753 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
15760 case RISCVISD::ADD_VL:
15764 case RISCVISD::SUB_VL:
15765 case RISCVISD::VWADD_W_VL:
15766 case RISCVISD::VWADDU_W_VL:
15767 case RISCVISD::VWSUB_W_VL:
15768 case RISCVISD::VWSUBU_W_VL:
15769 case RISCVISD::MUL_VL:
15771 case RISCVISD::VFMADD_VL:
15772 case RISCVISD::VFNMADD_VL:
15773 case RISCVISD::VFMSUB_VL:
15774 case RISCVISD::VFNMSUB_VL:
15775 case RISCVISD::STRICT_VFMADD_VL:
15776 case RISCVISD::STRICT_VFNMADD_VL:
15777 case RISCVISD::STRICT_VFMSUB_VL:
15778 case RISCVISD::STRICT_VFNMSUB_VL:
15780 case RISCVISD::FMUL_VL:
15782 case RISCVISD::FADD_VL:
15783 case RISCVISD::FSUB_VL:
15866 if (Val.getOpcode() == RISCVISD::VMV_X_S ||
15910 case RISCVISD::VFMV_V_F_VL: {
15918 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL);
15921 case RISCVISD::VMV_V_X_VL: {
15941 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL);
15945 case RISCVISD::VFMV_S_F_VL: {
15964 case RISCVISD::VMV_S_X_VL: {
15989 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
15993 case RISCVISD::VMV_X_S: {
16000 return DAG.getNode(RISCVISD::VMV_X_S, DL, N->getSimpleValueType(0), Vec);
16288 case RISCVISD::SELECT_CC: {
16299 case RISCVISD::CZERO_EQZ:
16300 case RISCVISD::CZERO_NEZ:
16306 case RISCVISD::REMUW: {
16316 case RISCVISD::DIVUW: {
16326 case RISCVISD::SLLW: {
16335 case RISCVISD::CTZW: {
16342 case RISCVISD::CLZW: {
16349 case RISCVISD::BREV8:
16350 case RISCVISD::ORC_B: {
16354 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
16361 case RISCVISD::READ_VLENB: {
16373 case RISCVISD::FCLASS: {
16418 case RISCVISD::SELECT_CC: {
16426 case RISCVISD::CZERO_EQZ:
16427 case RISCVISD::CZERO_NEZ:
16431 case RISCVISD::ABSW: {
16439 case RISCVISD::SLLW:
16440 case RISCVISD::SRAW:
16441 case RISCVISD::SRLW:
16442 case RISCVISD::DIVW:
16443 case RISCVISD::DIVUW:
16444 case RISCVISD::REMUW:
16445 case RISCVISD::ROLW:
16446 case RISCVISD::RORW:
16447 case RISCVISD::FCVT_W_RV64:
16448 case RISCVISD::FCVT_WU_RV64:
16449 case RISCVISD::STRICT_FCVT_W_RV64:
16450 case RISCVISD::STRICT_FCVT_WU_RV64:
16455 case RISCVISD::VMV_X_S: {
16517 if (Ptr.getOpcode() == RISCVISD::LLA) {
16526 if (Ptr.getOpcode() != RISCVISD::ADD_LO ||
16527 Ptr.getOperand(0).getOpcode() != RISCVISD::HI)
17750 Val = DAG.getNode(RISCVISD::FMV_H_X, DL, VA.getValVT(), Val);
17756 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
17816 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, LocVT, Val);
17822 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
17897 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
18396 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
18563 SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
18570 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
18601 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
18670 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
18716 unsigned RetOpc = RISCVISD::RET_GLUE;
18729 RetOpc = RISCVISD::SRET_GLUE;
18731 RetOpc = RISCVISD::MRET_GLUE;
18775 // The copy must be used by a RISCVISD::RET_GLUE, and nothing else.
18778 if (Node->getOpcode() != RISCVISD::RET_GLUE)
18795 case RISCVISD::NODE: \
18796 return "RISCVISD::" #NODE;
18798 switch ((RISCVISD::NodeType)Opcode) {
18799 case RISCVISD::FIRST_NUMBER: