Lines Matching defs:DL
2041 const SDLoc &DL) const {
2044 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2159 EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
2220 const DataLayout &DL) const {
2223 Align TyAlign = DL.getABITypeAlign(Ty);
3351 : FuncInfo(FuncInfo), DL(Loc), DAG(DAG), Subtarget(Subtarget),
3371 const SDLoc &DL;
3441 LiveGPRs.push_back(DAG.getCopyFromReg(Chain, DL, GPR, MVT::i64));
3446 ALVal = DAG.getCopyFromReg(Chain, DL, AL, MVT::i8);
3450 DAG.getCopyFromReg(Chain, DL, XMMReg, MVT::v4f32));
3461 SDValue FIN = DAG.getNode(ISD::ADD, DL,
3463 RSFIN, DAG.getIntPtrConstant(Offset, DL));
3465 DAG.getStore(Val.getValue(1), DL, Val, FIN,
3479 DAG.getIntPtrConstant(FuncInfo->getRegSaveFrameIndex(), DL));
3481 DAG.getIntPtrConstant(FuncInfo->getVarArgsFPOffset(), DL));
3484 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, DL,
3489 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3527 SDValue RegVal = DAG.getCopyFromReg(Chain, DL, FR.VReg, FR.VT);
3530 Chain = DAG.getCopyToReg(Chain, DL, FR.VReg, RegVal);
3839 SDLoc &dl = CLI.DL;
4923 static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
4930 RHS = DAG.getConstant(0, DL, RHS.getValueType());
4943 RHS = DAG.getConstant(0, DL, RHS.getValueType());
5962 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
5985 return Builder(DAG, DL, Ops);
5994 SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
5996 Subs.push_back(Builder(DAG, DL, SubOps));
5998 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
6203 static SDValue getExtendInVec(unsigned Opcode, const SDLoc &DL, EVT VT,
6217 In = extractSubVector(In, 0, DAG, DL,
6225 return DAG.getNode(Opcode, DL, VT, In);
8050 SDLoc DL(Op);
8057 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
8058 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
8160 SDLoc DL(Op);
8161 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
8162 DAG.getIntPtrConstant(InsertPSMask, DL, true));
8233 SDLoc DL(Ptr);
8234 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
8235 DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
8302 const SDLoc &DL, SelectionDAG &DAG,
8358 return VT.isInteger() ? DAG.getConstant(0, DL, VT)
8359 : DAG.getConstantFP(0.0, DL, VT);
8409 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
8414 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
8465 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
8466 : DAG.getConstantFP(0.0, DL, VT);
8467 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
8479 EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
8482 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
8483 HalfLD, DAG.getIntPtrConstant(0, DL));
8502 X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
8551 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, isAfterLegalize)) {
8554 SDValue Broadcast = DAG.getNode(Opcode, DL, BroadcastVT, RepeatLoad);
8567 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
8580 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
8930 SDLoc DL(Op);
8985 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
8988 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
8989 DAG.getIntPtrConstant(Idx, DL));
9223 const SDLoc &DL, SelectionDAG &DAG,
9231 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
9232 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
9233 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
9234 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
9243 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
9245 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
9249 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
9252 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
9255 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
9429 SDLoc DL(BV);
9435 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
9450 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
9618 SDLoc DL(BV);
9655 return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
9686 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
9705 SDLoc DL(Op);
9752 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType());
9765 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
9766 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
9767 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
9782 SDLoc DL(Op);
9796 return getOnesVector(VT, DAG, DL);
9806 SDLoc &DL, SelectionDAG &DAG,
9831 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
9833 return extractSubVector(NewSrcVec, 0, DAG, DL, SizeInBits);
9896 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
9902 DL, IndicesVec,
9903 getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL),
9904 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),
9905 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),
9913 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
9914 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
9915 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
9916 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
9918 ISD::CONCAT_VECTORS, DL, VT,
9919 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
9920 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
9922 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
9923 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
9924 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
9925 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
9926 auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
9933 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),
9934 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
9935 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
9939 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,
9952 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));
9961 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9963 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9967 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
9968 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
9972 DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),
9973 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
9974 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
9988 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
9990 return extract256BitVector(Res, 0, DAG, DL);
9996 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});
9998 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});
10000 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
10003 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
10004 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
10008 DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),
10009 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
10010 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
10047 ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec)
10048 : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec);
10103 SDLoc DL(V);
10105 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
10959 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
10961 return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
10997 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
11013 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
11040 PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);
11046 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
11047 DAG.getBuildVector(I8VT, DL, PSHUFBMask)));
11055 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
11067 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
11072 Subtarget, DAG, DL);
11073 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
11075 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
11080 ArrayRef<int> TargetMask, const SDLoc &DL,
11137 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
11138 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
11165 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
11171 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
11176 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
11181 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
11185 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
11192 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
11210 V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1),
11213 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1);
11294 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
11345 return DAG.getNode(X86ISD::VTRUNC, DL, VT, Src);
11478 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
11513 Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1),
11527 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
11543 Zero = DAG.getConstantFP(0.0, DL, EltVT);
11546 AllOnes = DAG.getConstantFP(AllOnesValue, DL, EltVT);
11550 Zero = DAG.getConstant(0, DL, EltVT);
11551 AllOnes = DAG.getAllOnesConstant(DL, EltVT);
11571 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
11574 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
11583 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
11588 SDValue Zero = DAG.getConstant(0, DL, EltVT);
11589 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
11597 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
11598 V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask);
11599 V2 = DAG.getNode(X86ISD::ANDNP, DL, VT, V1Mask, V2);
11600 return DAG.getNode(ISD::OR, DL, VT, V1, V2);
11666 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
11680 V1 = getZeroVector(VT, Subtarget, DAG, DL);
11682 V2 = getZeroVector(VT, Subtarget, DAG, DL);
11699 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
11700 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
11711 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11712 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
11720 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11721 DAG.getTargetConstant(LoMask, DL, MVT::i8));
11722 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11723 DAG.getTargetConstant(HiMask, DL, MVT::i8));
11725 MVT::v16i16, DL, Lo, Hi,
11737 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11744 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11751 lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
11786 : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
11793 DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask),
11805 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11814 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11827 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
11857 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11858 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
11866 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
11923 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
11924 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);
11930 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11987 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
11989 DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
12002 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);
12021 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12044 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
12047 if (SDValue UnpackPerm = lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask,
12051 DL, VT, V1, V2, Mask, Subtarget, DAG))
12054 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
12059 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
12060 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
12061 return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
12113 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
12140 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
12141 DAG.getTargetConstant(ShlAmt, DL, MVT::i8));
12142 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
12143 DAG.getTargetConstant(SrlAmt, DL, MVT::i8));
12144 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
12149 DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1),
12150 DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
12264 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
12286 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
12287 DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
12302 DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
12303 DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
12305 DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
12306 DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
12308 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
12321 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
12337 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
12338 DAG.getTargetConstant(Rotation, DL, MVT::i8));
12342 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
12378 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
12379 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
12380 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
12381 DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
12384 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
12385 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
12386 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
12387 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
12393 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
12394 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
12396 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
12397 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
12398 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
12399 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
12491 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
12520 V = DAG.getNode(Opcode, DL, ShiftVT, V,
12521 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
12649 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
12654 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
12655 DAG.getTargetConstant(BitLen, DL, MVT::i8),
12656 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
12659 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
12661 DAG.getTargetConstant(BitLen, DL, MVT::i8),
12662 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
12677 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
12706 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);
12719 InputV = getExtendInVec(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND, DL,
12732 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
12734 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
12739 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
12741 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
12745 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,
12747 getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG)));
12758 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
12759 DAG.getTargetConstant(EltBits, DL, MVT::i8),
12760 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
12767 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
12768 DAG.getTargetConstant(EltBits, DL, MVT::i8),
12769 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
12771 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
12783 PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);
12787 AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);
12791 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
12792 DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));
12802 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);
12816 : getZeroVector(InputVT, Subtarget, DAG, DL);
12818 InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext);
12839 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12909 return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt,
12946 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
12995 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13029 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
13031 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
13056 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL,
13064 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
13076 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
13079 V2 = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
13081 V2Index * EltVT.getSizeInBits() / 8, DL, MVT::i8));
13092 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
13134 Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar,
13135 DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));
13137 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
13138 DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
13166 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
13208 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
13211 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
13212 DAG.getIntPtrConstant(0, DL));
13220 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
13304 DL, VT, V, BroadcastIdx, Subtarget, DAG))
13328 SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
13337 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SVT,
13344 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
13371 V = extract128BitVector(V, ExtractIdx, DAG, DL);
13375 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
13384 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
13391 V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL);
13397 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V));
13494 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
13506 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
13507 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
13520 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13569 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
13570 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
13579 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
13619 VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL,
13620 DL, VT, V1, V2),
13634 static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13644 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
13655 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
13656 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13660 X86ISD::SHUFP, DL, MVT::v2f64,
13663 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13671 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13677 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13684 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13695 X86ISD::MOVSD, DL, MVT::v2f64, V2,
13696 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
13699 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
13704 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
13708 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
13709 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13718 static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13728 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
13741 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13742 getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
13750 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13754 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
13761 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13767 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13774 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
13779 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
13786 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,
13790 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
13798 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, Mask,
13808 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
13816 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
13842 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
13843 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13881 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
13882 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13893 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
13894 getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
13902 static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13914 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
13921 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
13923 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
13929 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
13930 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13937 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
13939 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
13944 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
13945 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13949 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13959 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13963 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
13968 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13972 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13981 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
13983 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
13987 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13991 return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
13998 static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14009 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
14018 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
14035 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
14036 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
14040 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
14044 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
14051 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
14058 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
14062 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
14067 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
14074 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,
14078 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
14090 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, Mask,
14094 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
14106 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
14127 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
14139 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14140 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
14146 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14147 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
14172 V = DAG.getNode(ShufWOp, DL, VT, V,
14173 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
14175 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
14176 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
14308 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
14325 FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
14327 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
14352 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
14353 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
14364 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
14586 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14587 getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
14589 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14590 getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
14594 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
14595 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
14606 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14607 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
14614 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14615 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
14623 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14648 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
14649 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
14656 V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1),
14657 DAG.getBuildVector(ShufVT, DL, V1Mask));
14659 V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2),
14660 DAG.getBuildVector(ShufVT, DL, V2Mask));
14665 V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2);
14685 static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14695 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
14703 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,
14708 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
14713 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
14718 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14722 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14727 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
14733 return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,
14742 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
14748 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
14755 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14762 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
14766 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
14771 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14775 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14780 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
14785 lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
14789 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
14799 SmallVector<SDValue, 8> DWordClearOps(4, DAG.getConstant(0, DL, MVT::i32));
14801 DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);
14802 SDValue DWordClearMask = DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);
14803 V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1),
14805 V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2),
14808 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, V1, V2);
14811 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, Result, Result);
14817 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
14825 return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
14831 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
14835 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
14841 SDValue MaskNode = getConstVector(Mask, MaskVecVT, DAG, DL, true);
14843 return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1);
14845 return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2);
14855 static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14864 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
14869 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14874 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14879 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14885 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14894 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14899 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
14903 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14969 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14980 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
14997 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
15004 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
15009 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
15013 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
15042 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
15049 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
15062 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
15067 return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG);
15072 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
15082 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
15085 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
15103 SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
15105 WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);
15106 SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
15107 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),
15110 V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),
15114 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
15118 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
15125 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, Mask,
15147 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
15148 DAG.getConstant(0x00FF, DL, MVT::v8i16));
15163 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
15166 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
15168 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
15171 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
15172 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
15174 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
15181 static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
15188 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
15190 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
15192 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
15194 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
15196 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
15198 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
15210 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
15230 std::tie(LoV, HiV) = splitVector(peekThroughBitcasts(V), DAG, DL);
15272 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
15274 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
15279 DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
15289 DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
15297 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
15301 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
15312 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
15340 return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
15355 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
15359 return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, Subtarget,
15365 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,
15387 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
15388 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
15389 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS,
15390 DAG.getTargetConstant(SHUFPMask, DL, MVT::i8));
15402 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15452 SDValue LanePermute = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask);
15453 return DAG.getVectorShuffle(VT, DL, LanePermute, DAG.getUNDEF(VT), PermMask);
15464 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15477 lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG))
15489 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
15496 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
15518 DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1});
15520 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
15524 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
15545 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
15546 DAG.getIntPtrConstant(0, DL));
15547 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
15548 getZeroVector(VT, Subtarget, DAG, DL), LoV,
15549 DAG.getIntPtrConstant(0, DL));
15557 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
15573 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
15575 DAG.getIntPtrConstant(0, DL));
15576 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
15577 DAG.getIntPtrConstant(2, DL));
15586 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
15587 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15618 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
15619 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15630 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15756 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15773 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15788 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
15849 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15865 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V,
15866 DAG.getIntPtrConstant(HalfIdx, DL));
15872 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15878 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1);
15882 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
15883 DAG.getIntPtrConstant(Offset, DL));
15889 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15909 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15910 DAG.getIntPtrConstant(HalfNumElts, DL));
15911 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15912 DAG.getIntPtrConstant(0, DL));
15919 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15920 DAG.getIntPtrConstant(0, DL));
15921 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15922 DAG.getIntPtrConstant(HalfNumElts, DL));
15946 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15969 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15988 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
16016 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
16054 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
16061 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),
16165 SDValue RepeatedShuffle = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
16177 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),
16226 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
16242 V1 = getZeroVector(VT, Subtarget, DAG, DL);
16244 V2 = getZeroVector(VT, Subtarget, DAG, DL);
16246 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
16247 DAG.getTargetConstant(Immediate, DL, MVT::i8));
16253 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
16271 V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);
16272 V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);
16276 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
16280 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
16281 DAG.getConstant(0, DL, MVT::v32i8), Unpack,
16282 DAG.getIntPtrConstant(0, DL));
16290 static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16298 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
16304 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
16310 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
16317 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
16318 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
16323 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
16324 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
16329 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
16333 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
16338 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v4f64, V1, V2, Mask,
16343 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
16346 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
16351 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
16363 if (SDValue Op = lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2,
16370 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask,
16376 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
16386 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
16391 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
16398 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask,
16402 return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
16410 static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16419 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
16423 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
16428 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
16441 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
16443 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
16448 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
16449 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
16453 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
16459 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,
16463 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
16469 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
16474 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
16480 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask,
16486 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
16496 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
16500 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask,
16508 static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16516 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
16521 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
16534 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
16536 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
16539 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
16540 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16543 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
16548 return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
16554 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16561 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16562 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
16565 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16566 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
16569 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v8f32, V1, V2, Mask,
16576 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16581 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
16589 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16596 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, Mask,
16600 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16608 static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16620 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16629 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask,
16633 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16638 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16651 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
16652 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16655 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
16660 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
16666 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,
16670 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
16676 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16683 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16689 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
16694 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16695 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
16704 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16712 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16716 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, Mask,
16724 static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16737 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16741 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16745 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16750 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16754 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16759 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
16764 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16771 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16777 lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG))
16782 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
16789 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16792 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v16i16, V1, V2, Mask,
16802 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
16806 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16812 return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, DAG);
16817 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16822 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16826 return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
16834 static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16846 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16851 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16855 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16860 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
16864 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
16869 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
16874 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16881 lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG))
16887 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16895 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
16899 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16902 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
16906 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16912 return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, DAG);
16917 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16922 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16929 if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16934 return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
16943 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16954 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16959 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16973 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16976 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16978 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
16985 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
16990 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16992 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16994 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16996 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16998 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17000 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17008 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
17030 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
17031 DAG.getIntPtrConstant(0, DL));
17032 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17033 getZeroVector(VT, Subtarget, DAG, DL), LoV,
17034 DAG.getIntPtrConstant(0, DL));
17044 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
17045 DAG.getIntPtrConstant(0, DL));
17046 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
17047 DAG.getIntPtrConstant(4, DL));
17075 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
17076 DAG.getIntPtrConstant(0, DL));
17077 return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
17111 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
17112 DAG.getTargetConstant(PermMask, DL, MVT::i8));
17116 static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17127 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
17136 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
17137 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
17142 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
17143 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
17146 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
17150 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
17154 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
17158 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
17162 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
17166 return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG);
17170 static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17186 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
17188 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
17191 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
17192 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
17195 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
17198 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
17203 return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
17209 DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))
17216 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
17217 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
17221 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
17225 return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG);
17229 static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17247 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
17249 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
17254 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
17255 getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
17258 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
17263 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
17268 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,
17273 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
17277 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
17280 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
17284 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
17288 return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG);
17292 static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17304 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
17316 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
17317 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
17320 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
17325 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
17330 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,
17336 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
17345 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
17353 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
17357 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
17361 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
17365 return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
17369 static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17382 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
17386 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
17391 lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))
17395 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
17400 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
17407 lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG))
17415 return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v32i16, V1,
17420 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
17424 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
17428 return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG);
17432 static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17445 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
17449 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
17453 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
17458 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
17463 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
17470 lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))
17474 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
17478 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
17484 return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG);
17489 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17492 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
17500 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17504 return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
17512 static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17527 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
17532 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
17536 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
17543 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17546 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17549 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
17558 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17560 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17562 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17564 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17566 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17568 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17575 static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
17609 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
17611 DAG.getIntPtrConstant(0, DL));
17612 Res = DAG.getNode(X86ISD::KSHIFTR, DL, WideVT, Res,
17613 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17614 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17615 DAG.getIntPtrConstant(0, DL));
17655 static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17690 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
17692 DAG.getIntPtrConstant(0, DL));
17693 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17694 DAG.getConstant(0, DL, VT),
17695 Extract, DAG.getIntPtrConstant(0, DL));
17699 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
17712 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
17714 DAG.getIntPtrConstant(0, DL));
17719 Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
17720 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
17725 Res = DAG.getNode(Opcode, DL, WideVT, Res,
17726 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17727 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17728 DAG.getIntPtrConstant(0, DL));
17770 V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1);
17771 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
17773 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17778 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
17781 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
17863 SDLoc DL(Op);
17889 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
17907 return getZeroVector(VT, Subtarget, DAG, DL);
17922 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17948 V2 = getZeroVector(NewVT, Subtarget, DAG, DL);
17953 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
17964 if (SDValue V = lowerShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))
17969 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17972 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17975 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17978 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
18617 SDLoc DL(CP);
18618 Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result);
18622 DAG.getNode(ISD::ADD, DL, PtrVT,
18638 SDLoc DL(JT);
18639 Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result);
18644 DAG.getNode(ISD::ADD, DL, PtrVT,
18944 SDLoc DL(Op);
18945 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18948 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18952 Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
18960 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
18962 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
18963 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, DL, true),
18964 DAG.getIntPtrConstant(0, DL, true),
18965 Chain.getValue(1), DL);
18974 return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
19022 auto &DL = DAG.getDataLayout();
19024 DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);
19102 SDLoc DL(Op);
19118 return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, Op0,
19119 Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
19122 return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
19138 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
19139 SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
19140 Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
19141 Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
19142 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
19143 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
19144 Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
19146 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
19148 Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
19149 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
19151 return DAG.getZExtOrTrunc(Res, DL, VT);
19159 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
19160 DAG.getConstant(15, DL, Amt.getValueType()));
19162 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
19257 SDLoc DL(Cast);
19261 VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask);
19266 VecOp = extract128BitVector(VecOp, 0, DAG, DL);
19270 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
19271 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
19272 DAG.getIntPtrConstant(0, DL));
19317 SDLoc DL(CastToFP);
19318 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
19319 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19320 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19321 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19322 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx);
19327 SDLoc DL(Op);
19346 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19348 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
19349 DAG.getIntPtrConstant(0, DL));
19352 Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
19356 Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
19359 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19360 DAG.getIntPtrConstant(0, DL));
19363 return DAG.getMergeValues({Res, Chain}, DL);
19372 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19373 SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
19374 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19375 DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
19376 DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
19377 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19378 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19382 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19383 DAG.getIntPtrConstant(i, DL));
19386 DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},
19390 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt);
19393 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19397 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19398 Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},
19402 Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);
19405 IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);
19406 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19409 return DAG.getMergeValues({Cvt, Chain}, DL);
19503 EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
19515 DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo,
19533 DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO);
19535 DstVT, DL, Chain, StackSlot,
19690 const SDLoc &DL) {
19705 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19706 DAG.getConstant(0, DL, MVT::v2i32));
19707 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
19710 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,
19711 DAG.getIntPtrConstant(0, DL));
19712 return DAG.getMergeValues({Res, Chain}, DL);
19716 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19719 return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},
19721 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
19728 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
19730 DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), DL, MVT::v2f64);
19731 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
19736 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},
19738 return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);
19743 SDLoc DL(Op);
19766 IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
19767 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
19768 DAG.getIntPtrConstant(0, DL));
19771 Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
19775 Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
19778 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19779 DAG.getIntPtrConstant(0, DL));
19782 return DAG.getMergeValues({Res, Chain}, DL);
19788 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
19797 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
19801 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
19806 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},
19808 return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);
19838 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
19840 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
19843 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
19844 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
19854 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
19855 VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19862 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
19863 VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19865 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
19867 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
19868 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
19871 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
19876 APFloat(APFloat::IEEEsingle(), APInt(32, 0x53000080)), DL, VecFloatVT);
19888 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
19890 return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},
19895 DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub);
19896 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
20057 SDLoc DL(Op);
20132 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
20138 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT,
20142 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT);
20145 Adjust = DAG.getSelect(DL, MVT::i64, Cmp,
20146 DAG.getConstant(0, DL, MVT::i64),
20148 DL, MVT::i64));
20149 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp,
20150 DAG.getConstantFP(0.0, DL, TheVT),
20154 Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
20158 Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
20167 Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
20175 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
20183 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
20192 Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);
20280 SDLoc DL(Op);
20286 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20287 return DAG.getNode(ISD::SRL, DL, VT, Extend,
20288 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
20296 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);
20306 In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT),
20307 In, DAG.getIntPtrConstant(0, DL));
20312 SDValue One = DAG.getConstant(1, DL, WideVT);
20313 SDValue Zero = DAG.getConstant(0, DL, WideVT);
20315 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20320 SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal);
20325 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,
20326 DAG.getIntPtrConstant(0, DL));
20349 const SDLoc &DL, SelectionDAG &DAG,
20396 SDValue Res = DAG.getNode(Opcode, DL, OutVT, In, DAG.getUNDEF(InVT));
20397 Res = extractSubVector(Res, 0, DAG, DL, 64);
20403 std::tie(Lo, Hi) = splitVector(In, DAG, DL);
20413 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20422 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20430 Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
20438 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20444 Lo = truncateVectorWithPACK(Opcode, PackedVT, Lo, DL, DAG, Subtarget);
20445 Hi = truncateVectorWithPACK(Opcode, PackedVT, Hi, DL, DAG, Subtarget);
20448 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20449 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20455 SDLoc DL(Op);
20471 In = DAG.getNode(ISD::SHL, DL, ExtVT,
20473 DAG.getConstant(ShiftInx, DL, ExtVT));
20476 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
20496 Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In);
20498 InVT, DL, In, In,
20500 Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi);
20503 Lo = extract128BitVector(In, 0, DAG, DL);
20504 Hi = extract128BitVector(In, 8, DAG, DL);
20508 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);
20509 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);
20510 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20517 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
20524 In = DAG.getNode(ISD::SHL, DL, InVT, In,
20525 DAG.getConstant(ShiftInx, DL, InVT));
20529 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
20530 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);
20534 SDLoc DL(Op);
20554 std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
20559 Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo);
20560 Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi);
20561 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20596 truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget))
20603 truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget))
20614 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
20615 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
20616 DAG.getIntPtrConstant(0, DL));
20619 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20620 DAG.getIntPtrConstant(0, DL));
20621 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20622 DAG.getIntPtrConstant(2, DL));
20626 return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);
20639 In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
20643 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
20644 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20645 DAG.getIntPtrConstant(0, DL));
20649 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
20650 DAG.getIntPtrConstant(0, DL));
20652 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
20653 DAG.getIntPtrConstant(4, DL));
20662 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, OpLo, ShufMask1);
20663 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, OpHi, ShufMask1);
20670 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
20676 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(255, DL, InVT));
20678 SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
20679 DAG.getIntPtrConstant(0, DL));
20680 SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
20681 DAG.getIntPtrConstant(8, DL));
20682 return DAG.getNode(X86ISD::PACKUS, DL, VT, InLo, InHi);
20960 SDLoc DL(N);
20975 Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
20979 Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
20985 Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
20989 return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
20995 SDLoc DL(Op);
21008 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));
21010 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21012 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21161 SDLoc DL(Op);
21164 X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
21172 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
21173 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
21174 DAG.getIntPtrConstant(LExtIndex / 2, DL));
21440 static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC,
21455 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
21456 return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
21464 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
21466 DAG.getConstant(0, DL, IntVT));
21476 auto Split = DAG.SplitVector(V, DL);
21478 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second);
21485 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
21494 V = DAG.getNode(X86ISD::PCMPEQ, DL, MVT::v16i8, V,
21495 getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
21496 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
21497 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
21498 DAG.getConstant(0xFFFF, DL, MVT::i32));
21504 const SDLoc &DL,
21550 VecIns.push_back(DAG.getNode(ISD::OR, DL, VT, LHS, RHS));
21554 if (SDValue V = LowerVectorAllZero(DL, VecIns.back(), CC, Mask, Subtarget,
21556 X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8);
21567 LowerVectorAllZero(DL, Match, CC, Mask, Subtarget, DAG, CCode)) {
21568 X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8);
21919 SDLoc DL(N);
21921 SDValue Zero = DAG.getConstant(0, DL, VT);
21923 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
21926 SDValue Cmp = DAG.getSetCC(DL, MVT::i8, N0, Zero, ISD::SETLT);
21927 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
21928 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
21936 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, MVT::i8));
21944 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA);
22152 SDLoc DL(V);
22163 NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
22166 return DAG.getBuildVector(VT, DL, NewVecC);
22843 SDLoc DL(Op);
22850 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
22851 Carry, DAG.getAllOnesConstant(DL, CarryVT));
22854 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
22855 return getSETCC(CC, Cmp.getValue(1), DL, DAG);
22869 SDLoc DL(Op);
22901 Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
22913 SDLoc DL(Op);
22918 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
22920 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
22953 SDLoc DL(Op);
22970 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
22971 DAG.getTargetConstant(SSECC, DL, MVT::i8));
22973 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
22977 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
22978 DAG.getTargetConstant(SSECC, DL, MVT::i8));
22997 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
22998 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
22999 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
23004 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
23006 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
23007 VSel, DAG.getIntPtrConstant(0, DL));
23009 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
23010 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
23011 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
23017 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
23018 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
23070 SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
23071 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0);
23072 Zero = DAG.getConstant(0, DL, Op.getValueType());
23073 return DAG.getNode(X86ISD::SBB, DL, VTs, Zero, Zero, Neg.getValue(1));
23076 Cmp = DAG.getNode(X86ISD::SUB, DL, CmpVTs,
23077 CmpOp0, DAG.getConstant(1, DL, CmpOp0.getValueType()));
23079 SDValue Zero = DAG.getConstant(0, DL, Op.getValueType());
23081 DAG.getNode(X86ISD::SBB, DL, VTs, Zero, Zero, Cmp.getValue(1));
23084 Res = DAG.getNOT(DL, Res, Res.getValueType());
23086 return DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y);
23111 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0);
23113 Neg = DAG.getNode(ISD::AND, DL, VT,
23114 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)),
23115 DAG.getConstant(1, DL, VT));
23118 SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
23120 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
23121 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y
23157 CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
23170 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, BTCC)) {
23179 CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
23180 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
23194 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
23195 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
23197 return DAG.getNOT(DL, Res, Res.getValueType());
23211 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
23213 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
23226 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
23227 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
23229 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
23230 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
23236 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops);
23491 SDLoc DL(Store);
23493 std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL);
23496 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, HalfOffset, DL);
23498 DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(),
23501 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
23505 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);
23527 SDLoc DL(Store);
23531 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), Offset, DL);
23532 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
23533 DAG.getIntPtrConstant(i, DL));
23534 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
23540 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
23893 SDLoc DL(Op);
23900 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
23913 Op.getOperand(0), DL,
23914 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,
23919 FIN = DAG.getMemBasePlusOffset(FIN, 4, DL);
23921 Op.getOperand(0), DL,
23922 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,
23927 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
23930 DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8));
23934 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
23935 Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
23938 Op.getOperand(0), DL, RSFIN, FIN,
23941 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
24014 SDLoc DL(Op);
24016 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(24, DL),
25213 SDLoc DL(Op);
25216 DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs,
25219 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
25221 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
25223 return DAG.getMergeValues({Result0, Result1}, DL);
25233 SDLoc DL(Op);
25243 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
25245 DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
25280 ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
25281 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
25282 DAG.getConstant(NewIntrinsic, DL, MVT::i32),
25420 static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
25431 Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue);
25438 TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
25444 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
25445 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
25448 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
25449 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
25457 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
25458 DAG.getConstant(32, DL, MVT::i8));
25459 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
25466 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
25475 static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
25482 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
25491 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
25499 SDLoc DL(Op);
25500 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
25502 return DAG.getMergeValues(Results, DL);
25958 SDLoc DL(Op);
25969 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
25976 SDLoc DL(Op);
25977 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
25983 SDLoc DL(Op);
25984 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
26081 auto &DL = DAG.getDataLayout();
26083 InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32;
26166 SDLoc DL(Op);
26177 Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
26182 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
26187 DAG.getNode(ISD::SRL, DL, MVT::i16,
26188 DAG.getNode(ISD::AND, DL, MVT::i16,
26189 CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
26190 DAG.getConstant(9, DL, MVT::i8));
26191 Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
26193 SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
26195 DAG.getNode(ISD::AND, DL, MVT::i32,
26196 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
26197 DAG.getConstant(3, DL, MVT::i32));
26199 RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
26201 return DAG.getMergeValues({RetVal, Chain}, DL);
26240 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
26256 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
26257 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
26265 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
26267 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
26269 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
26273 HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ);
26274 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
26276 HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ);
26279 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
26280 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
26281 Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
26282 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
26294 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
26299 HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0),
26301 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
26303 HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0),
26312 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
26313 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
26314 R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1);
26315 Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1);
26322 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
26341 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
26448 SDLoc DL(Op);
26451 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, X, Y);
26452 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Add, ISD::SETUGT);
26453 return DAG.getSelect(DL, VT, Cmp, DAG.getAllOnesConstant(DL, VT), Add);
26457 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
26458 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
26459 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT));
26480 SDLoc DL(Op);
26482 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
26483 DAG.getConstant(0, DL, VT), N0);
26484 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_GE, DL, MVT::i8),
26486 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
26491 SDLoc DL(Op);
26494 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
26495 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src);
26521 SDLoc DL(Op);
26531 SDValue Sign = DAG.getConstant(APInt::getSignedMinValue(16), DL, VT);
26532 N0 = DAG.getNode(ISD::XOR, DL, VT, N0, Sign);
26533 N1 = DAG.getNode(ISD::XOR, DL, VT, N1, Sign);
26535 SDValue Result = DAG.getNode(Opcode, DL, VT, N0, N1);
26536 return DAG.getNode(ISD::XOR, DL, VT, Result, Sign);
26549 SDValue Cond = DAG.getSetCC(DL, VT, N0, N1, CC);
26550 return DAG.getSelect(DL, VT, Cond, N0, N1);
27764 SDLoc DL(Op);
27785 return DAG.getNode(RotOpc, DL, VT, R,
27786 DAG.getTargetConstant(RotAmt, DL, MVT::i8));
27806 return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
27807 DAG.getTargetConstant(RotAmt, DL, MVT::i8));
27846 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1));
27851 SDValue Z = DAG.getConstant(0, DL, SelVT);
27852 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
27853 return DAG.getSelect(DL, SelVT, C, V0, V1);
27860 Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT));
27866 ISD::OR, DL, VT,
27867 DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(4, DL, VT)),
27868 DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(4, DL, VT)));
27872 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
27876 ISD::OR, DL, VT,
27877 DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(2, DL, VT)),
27878 DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(6, DL, VT)));
27882 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
27886 ISD::OR, DL, VT,
27887 DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(1, DL, VT)),
27888 DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(7, DL, VT)));
27893 Amt = DAG.getNode(ISD::AND, DL, VT, Amt,
27894 DAG.getConstant(EltSizeInBits - 1, DL, VT));
27903 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
27904 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
27905 SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);
27906 SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);
27907 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
27911 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
27916 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
27917 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
27918 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
27926 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
27927 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
27929 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
27932 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
27938 return DAG.getNode(ISD::OR, DL, VT,
27939 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),
27940 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
28117 SDValue Chain, SDLoc DL) {
28147 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
28150 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
28152 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
28156 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
28161 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
28164 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
28166 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
28171 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
28202 SDLoc DL(Op);
28215 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
28220 DAG.getTargetConstant(size, DL, MVT::i8),
28224 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
28228 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
28229 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
28231 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
28233 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
28238 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
28244 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
28245 Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget);
28246 Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget);
28247 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
28248 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
28249 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
28250 DAG.getConstant(32, DL, MVT::i8));
28251 return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
28255 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
28256 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
28257 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
28258 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
28259 DAG.getConstant(16, DL, MVT::i8));
28260 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);
28263 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
28291 SDLoc DL(Op);
28292 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
28293 V = getPMOVMSKB(DL, V, DAG, Subtarget);
28294 return DAG.getZExtOrTrunc(V, DL, DstVT);
28339 SDLoc DL(V);
28352 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
28354 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
28364 SDValue Zeros = DAG.getConstant(0, DL, VT);
28366 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
28367 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
28370 Zeros = DAG.getConstant(0, DL, ByteVecVT);
28372 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
28374 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
28379 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
28393 SDValue ShifterV = DAG.getConstant(8, DL, VT);
28394 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
28395 V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
28397 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
28400 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
28426 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
28427 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
28428 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
28431 SDValue FourV = DAG.getConstant(4, DL, VT);
28432 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
28435 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
28440 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
28441 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
28442 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);
28452 SDLoc DL(Op.getNode());
28462 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
28463 Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op);
28464 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
28480 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
28488 return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
28501 SDLoc DL(Op);
28507 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
28508 Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res);
28509 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,
28510 DAG.getIntPtrConstant(0, DL));
28532 MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));
28536 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
28538 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
28553 SDLoc DL(Op);
28570 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
28571 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
28572 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
28587 LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));
28588 HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));
28591 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
28592 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
28593 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
28594 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
28595 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
28638 SDLoc DL(N);
28647 RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
28648 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS,
28672 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
28675 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
28679 SDValue NewChain = DAG.getNode(X86ISD::MEMBARRIER, DL, MVT::Other, Chain);
28682 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
28690 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
28780 SDLoc DL(N);
28785 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
28786 Carry, DAG.getAllOnesConstant(DL, CarryVT));
28789 SDValue Sum = DAG.getNode(Opc, DL, VTs, Op.getOperand(0),
28792 SDValue SetCC = getSETCC(X86::COND_B, Sum.getValue(1), DL, DAG);
28794 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
28796 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
30666 bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
31004 DebugLoc DL = MI.getDebugLoc();
31054 BuildMI(thisMBB, DL, TII->get(X86::XBEGIN_4)).addMBB(fallMBB);
31060 BuildMI(mainMBB, DL, TII->get(X86::MOV32ri), mainDstReg).addImm(-1);
31061 BuildMI(mainMBB, DL, TII->get(X86::JMP_1)).addMBB(sinkMBB);
31068 BuildMI(fallMBB, DL, TII->get(X86::XABORT_DEF));
31069 BuildMI(fallMBB, DL, TII->get(TargetOpcode::COPY), fallDstReg)
31075 BuildMI(*sinkMBB, sinkMBB->begin(), DL, TII->get(X86::PHI), DstReg)
31130 DebugLoc DL = MI.getDebugLoc();
31214 BuildMI(thisMBB, DL, TII->get(X86::MOV32rm), OffsetReg)
31223 BuildMI(thisMBB, DL, TII->get(X86::CMP32ri))
31229 BuildMI(thisMBB, DL, TII->get(X86::JCC_1))
31239 BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg)
31249 BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
31255 BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg)
31261 BuildMI(offsetMBB, DL, TII->get(X86::ADD32ri), NextOffsetReg)
31266 BuildMI(offsetMBB, DL, TII->get(X86::MOV32mr))
31276 BuildMI(offsetMBB, DL, TII->get(X86::JMP_1))
31286 BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg)
31301 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg)
31305 BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg)
31309 BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg)
31316 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg)
31321 BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr))
31332 BuildMI(*endMBB, endMBB->begin(), DL,
31377 DebugLoc DL = MI.getDebugLoc();
31385 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg);
31386 BuildMI(MBB, DL, TII->get(X86::JCC_1)).addMBB(EndMBB).addImm(X86::COND_E);
31404 BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc))
31482 DebugLoc DL = MIItBegin->getDebugLoc();
31515 MIB = BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(X86::PHI), DestReg)
31534 DebugLoc DL = FirstCMOV.getDebugLoc();
31651 BuildMI(ThisMBB, DL, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC);
31655 BuildMI(FirstInsertedMBB, DL, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(SecondCC);
31663 BuildMI(*SinkMBB, SinkMBB->begin(), DL, TII->get(X86::PHI), DestReg)
31673 BuildMI(*SinkMBB, std::next(MachineBasicBlock::iterator(MIB.getInstr())), DL,
31689 DebugLoc DL = MI.getDebugLoc();
31810 BuildMI(ThisMBB, DL, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC);
31844 DebugLoc DL = MI.getDebugLoc();
31868 BuildMI(*MBB, {MI}, DL, TII->get(TargetOpcode::COPY), TmpStackPtr)
31872 BuildMI(*MBB, {MI}, DL, TII->get(Opc), FinalStackPtr)
31879 BuildMI(testMBB, DL,
31884 BuildMI(testMBB, DL, TII->get(X86::JCC_1))
31904 addRegOffset(BuildMI(blockMBB, DL, TII->get(XORMIOpc)), physSPReg, false, 0)
31907 BuildMI(blockMBB, DL,
31913 BuildMI(blockMBB, DL, TII->get(X86::JMP_1)).addMBB(testMBB);
31917 BuildMI(tailMBB, DL, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg())
31937 DebugLoc DL = MI.getDebugLoc();
31992 BuildMI(BB, DL, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
31993 BuildMI(BB, DL, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
31995 BuildMI(BB, DL, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
31998 BuildMI(BB, DL, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
32002 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), physSPReg)
32004 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
32006 BuildMI(bumpMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
32012 BuildMI(mallocMBB, DL, TII->get(X86::MOV64rr), X86::RDI)
32014 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
32020 BuildMI(mallocMBB, DL, TII->get(X86::MOV32rr), X86::EDI)
32022 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
32028 BuildMI(mallocMBB, DL, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
32030 BuildMI(mallocMBB, DL, TII->get(X86::PUSH32r)).addReg(sizeVReg);
32031 BuildMI(mallocMBB, DL, TII->get(X86::CALLpcrel32))
32038 BuildMI(mallocMBB, DL, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
32041 BuildMI(mallocMBB, DL, TII->get(TargetOpcode::COPY), mallocPtrVReg)
32043 BuildMI(mallocMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
32052 BuildMI(*continueMBB, continueMBB->begin(), DL, TII->get(X86::PHI),
32072 DebugLoc DL = MI.getDebugLoc();
32097 BuildMI(*RestoreMBB, RestoreMBBI, DL, TII.get(X86::JMP_4)).addMBB(TargetMBB);
32110 DebugLoc DL = MI.getDebugLoc();
32116 BuildMI(MF, DL, TII.get(AdjStackDown)).addImm(0).addImm(0).addImm(0);
32124 BuildMI(MF, DL, TII.get(AdjStackUp)).addImm(0).addImm(0);
32139 DebugLoc DL = MI.getDebugLoc();
32153 BuildMI(*BB, MI, DL, TII->get(X86::MOV64rm), X86::RDI)
32160 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m));
32165 BuildMI(*BB, MI, DL, TII->get(X86::MOV32rm), X86::EAX)
32172 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
32177 BuildMI(*BB, MI, DL, TII->get(X86::MOV32rm), X86::EAX)
32184 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
32278 DebugLoc DL = MI.getDebugLoc();
32317 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), AvailableReg)
32340 DebugLoc DL = MI.getDebugLoc();
32355 BuildMI(*MBB, MI, DL, TII->get(XorRROpc))
32363 BuildMI(*MBB, MI, DL, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
32367 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrStoreOpc));
32383 DebugLoc DL = MI.getDebugLoc();
32459 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA64r), LabelReg)
32467 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA32r), LabelReg)
32477 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrStoreOpc));
32495 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::EH_SjLj_Setup))
32505 BuildMI(mainMBB, DL, TII->get(X86::MOV32r0), mainDstReg);
32509 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
32523 addRegOffset(BuildMI(restoreMBB, DL, TII->get(Opm), BasePtr),
32527 BuildMI(restoreMBB, DL, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
32528 BuildMI(restoreMBB, DL, TII->get(X86::JMP_1)).addMBB(sinkMBB);
32543 DebugLoc DL = MI.getDebugLoc();
32604 BuildMI(checkSspMBB, DL, TII->get(X86::MOV32r0), ZReg);
32608 BuildMI(checkSspMBB, DL, TII->get(X86::SUBREG_TO_REG), TmpZReg)
32618 BuildMI(checkSspMBB, DL, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
32623 BuildMI(checkSspMBB, DL, TII->get(TestRROpc))
32626 BuildMI(checkSspMBB, DL, TII->get(X86::JCC_1)).addMBB(sinkMBB).addImm(X86::COND_E);
32635 BuildMI(fallMBB, DL, TII->get(PtrLoadOpc), PrevSSPReg);
32651 BuildMI(fallMBB, DL, TII->get(SubRROpc), SspSubReg)
32656 BuildMI(fallMBB, DL, TII->get(X86::JCC_1)).addMBB(sinkMBB).addImm(X86::COND_BE);
32664 BuildMI(fixShadowMBB, DL, TII->get(ShrRIOpc), SspFirstShrReg)
32670 BuildMI(fixShadowMBB, DL, TII->get(IncsspOpc)).addReg(SspFirstShrReg);
32674 BuildMI(fixShadowMBB, DL, TII->get(ShrRIOpc), SspSecondShrReg)
32679 BuildMI(fixShadowMBB, DL, TII->get(X86::JCC_1)).addMBB(sinkMBB).addImm(X86::COND_E);
32686 BuildMI(fixShadowLoopPrepareMBB, DL, TII->get(ShlR1Opc), SspAfterShlReg)
32692 BuildMI(fixShadowLoopPrepareMBB, DL, TII->get(MovRIOpc), Value128InReg)
32700 BuildMI(fixShadowLoopMBB, DL, TII->get(X86::PHI), CounterReg)
32707 BuildMI(fixShadowLoopMBB, DL, TII->get(IncsspOpc)).addReg(Value128InReg);
32711 BuildMI(fixShadowLoopMBB, DL, TII->get(DecROpc), DecReg).addReg(CounterReg);
32714 BuildMI(fixShadowLoopMBB, DL, TII->get(X86::JCC_1)).addMBB(fixShadowLoopMBB).addImm(X86::COND_NE);
32724 DebugLoc DL = MI.getDebugLoc();
32761 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrLoadOpc), FP);
32773 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrLoadOpc), Tmp);
32787 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrLoadOpc), SP);
32798 BuildMI(*thisMBB, MI, DL, TII->get(IJmpOpc)).addReg(Tmp);
32808 DebugLoc DL = MI.getDebugLoc();
32831 BuildMI(*MBB, MI, DL, TII->get(X86::LEA64r), VR)
32838 BuildMI(*MBB, MI, DL, TII->get(X86::LEA32r), VR)
32846 MachineInstrBuilder MIB = BuildMI(*MBB, MI, DL, TII->get(Op));
32857 DebugLoc DL = MI.getDebugLoc();
32912 BuildMI(TrapBB, DL, TII->get(X86::TRAP));
32944 addRegOffset(BuildMI(DispatchBB, DL, TII->get(Op), BP), FP, true,
32948 BuildMI(DispatchBB, DL, TII->get(X86::NOOP))
32954 addFrameReference(BuildMI(DispatchBB, DL, TII->get(X86::MOV32rm), IReg), FI,
32956 BuildMI(DispatchBB, DL, TII->get(X86::CMP32ri))
32959 BuildMI(DispatchBB, DL, TII->get(X86::JCC_1)).addMBB(TrapBB).addImm(X86::COND_AE);
32966 BuildMI(DispContBB, DL, TII->get(X86::LEA64r), BReg)
32973 BuildMI(DispContBB, DL, TII->get(TargetOpcode::SUBREG_TO_REG), IReg64)
32981 BuildMI(DispContBB, DL, TII->get(X86::JMP64m))
32994 BuildMI(DispContBB, DL, TII->get(X86::MOV32rm), OReg)
33001 BuildMI(DispContBB, DL, TII->get(X86::MOVSX64rr32), OReg64).addReg(OReg);
33003 BuildMI(DispContBB, DL, TII->get(X86::ADD64rr), TReg)
33007 BuildMI(DispContBB, DL, TII->get(X86::JMP64r)).addReg(TReg);
33015 BuildMI(DispContBB, DL, TII->get(X86::JMP32m))
33087 DebugLoc DL = MI.getDebugLoc();
33146 MachineInstr *Push = BuildMI(*BB, MI, DL, TII->get(PushF));
33157 BuildMI(*BB, MI, DL, TII->get(Pop), MI.getOperand(0).getReg());
33169 BuildMI(*BB, MI, DL, TII->get(Push)).addReg(MI.getOperand(0).getReg());
33170 BuildMI(*BB, MI, DL, TII->get(PopF));
33189 addFrameReference(BuildMI(*BB, MI, DL,
33194 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOVZX32rm16), OldCW),
33199 BuildMI(*BB, MI, DL, TII->get(X86::OR32ri), NewCW)
33205 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), NewCW16)
33211 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)),
33216 addFrameReference(BuildMI(*BB, MI, DL,
33235 addFullAddress(BuildMI(*BB, MI, DL, TII->get(Opc)), AM)
33239 addFrameReference(BuildMI(*BB, MI, DL,
33329 BuildMI(*BB, *MBBI, DL, TII->get(X86::LEA32r), computedAddrVReg), AM);
33354 BuildMI(*BB, MI, DL, TII->get(X86::SUB32ri), X86::ESP)
33370 BuildMI(*BB, MI, DL, TII->get(X86::LEA32r), MI.getOperand(0).getReg()),
33380 const DebugLoc &DL = MI.getDebugLoc();
33390 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII->get(Opc));
33400 const DebugLoc &DL = MI.getDebugLoc();
33402 BuildMI(*BB, MI, DL, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
33409 const DebugLoc &DL = MI.getDebugLoc();
33417 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII->get(Opc));
33529 SDLoc DL(Op);
33530 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
33531 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
33952 SDValue &V1, const SDLoc &DL, SelectionDAG &DAG,
33991 V1 = extractSubVector(V1, 0, DAG, DL, SrcSize);
34216 SDValue &V1, SDValue &V2, const SDLoc &DL,
34268 if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG,
34283 const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
34340 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
34341 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
34347 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
34348 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
34375 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
34376 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
34401 return getZeroVector(MaskVT, Subtarget, DAG, DL);
34479 SDLoc DL(Root);
34526 return DAG.getBitcast(RootVT, DAG.getNode(X86ISD::SUBV_BROADCAST, DL,
34549 Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
34550 Res = widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
34560 auto MatchSHUF128 = [](MVT ShuffleVT, const SDLoc &DL, ArrayRef<int> Mask,
34583 return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
34586 DAG.getTargetConstant(PermMask, DL, MVT::i8));
34600 if (SDValue V = MatchSHUF128(ShuffleVT, DL, Mask, V1, V2, DAG))
34617 Res = extract128BitVector(Res, BaseMask[0] * 2, DAG, DL);
34619 DL, 256);
34637 Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res,
34639 DAG.getTargetConstant(PermMask, DL, MVT::i8));
34658 X86ISD::VPERM2X128, DL, ShuffleVT,
34661 DAG.getTargetConstant(PermMask, DL, MVT::i8));
34732 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
34739 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
34747 DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
34754 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res);
34766 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
34767 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
34783 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
34786 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
34794 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
34801 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2);
34808 AllowIntDomain, NewV1, NewV2, DL, DAG,
34815 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
34816 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
34832 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
34833 DAG.getTargetConstant(BitLen, DL, MVT::i8),
34834 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
34843 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
34844 DAG.getTargetConstant(BitLen, DL, MVT::i8),
34845 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
34862 Res = DAG.getNode(Opc, DL, ShuffleVT, V1);
34864 Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits);
34882 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2);
34883 Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res);
34911 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
34913 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
34935 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
34937 SDValue Zero = getZeroVector(MaskVT, Subtarget, DAG, DL);
34938 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, Res, VPermMask, Zero);
34961 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
34964 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2);
34989 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
34993 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask);
35006 M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);
35009 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
35011 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
35044 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
35045 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
35046 DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
35069 PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
35074 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
35078 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
35079 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
35100 VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
35104 VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));
35109 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
35110 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
35136 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
35139 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2);
35287 SDLoc DL(Root);
35327 return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL);
35340 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
35702 SDLoc DL(N);
35792 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
35793 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
35808 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
35814 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
35827 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
35834 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
35844 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
35845 DAG.getTargetConstant(Imm, DL, MVT::i8));
35852 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP,
35853 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
35862 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP,
35863 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
35865 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1,
35866 DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8));
35868 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP,
35869 DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8));
35882 SDLoc DL(N);
35927 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
35933 SDValue Horiz = DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
35940 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
35954 ? DAG.getNode(Opcode, DL, VT, Not)
35955 : DAG.getNode(Opcode, DL, VT, Not, N.getOperand(1));
35957 SDValue AllOnes = DAG.getConstant(-1, DL, IntVT);
35959 Not = DAG.getNode(ISD::XOR, DL, IntVT, Not, AllOnes);
35976 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
36004 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
36015 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
36020 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
36021 extract128BitVector(Src, 0, DAG, DL));
36025 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
36033 return extractSubVector(SDValue(User, 0), 0, DAG, DL,
36045 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
36054 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
36055 DAG.getIntPtrConstant(0, DL));
36077 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
36095 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
36118 SDValue Ptr = DAG.getMemBasePlusOffset(LN->getBasePtr(), Offset, DL);
36121 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
36140 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
36158 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
36195 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops,
36213 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
36215 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
36216 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
36240 return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment,
36263 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
36265 DAG.getTargetConstant(BlendMask, DL, MVT::i8)));
36280 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
36310 return DAG.getBitcast(VT, DAG.getNode(ISD::CONCAT_VECTORS, DL,
36337 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
36338 N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx);
36339 N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx);
36340 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
36341 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
36342 return DAG.getNode(Opcode, DL, VT, N0, SclVec);
36359 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
36360 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
36364 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
36365 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
36376 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
36377 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
36384 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
36385 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
36429 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
36430 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
36439 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
36442 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
36443 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
36445 DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
36480 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
36481 getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
36514 DL, VT, V, V);
36665 SDLoc DL(N);
36668 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),
36686 SDLoc DL(N);
36692 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
36704 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
36746 SDLoc DL(N);
36747 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
36749 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
37011 SDLoc DL(Op);
37015 BCVT, DL, TLO.DAG.getEntryNode(), LegalCV,
37323 SDLoc DL(Op);
37326 Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits);
37331 Src = TLO.DAG.getNode(X86ISD::SUBV_BROADCAST, DL, SrcVT, Src);
37334 TLO.DAG, DL, ExtSizeInBits));
37347 SDLoc DL(Op);
37349 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
37351 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1));
37354 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
37364 SDLoc DL(Op);
37365 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
37367 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
37395 SDLoc DL(Op);
37402 Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL,
37409 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
37412 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
37817 SDLoc DL(Op);
37819 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0,
37820 TLO.DAG.getConstant(MaskedVal1, DL, VT)));
37962 const SDLoc &DL) {
37983 SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
37984 SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
37986 return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0,
37996 SDValue Src, const SDLoc &DL) {
37999 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
38004 Src.getOpcode(), DL, SExtVT,
38005 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
38006 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
38018 const SDLoc &DL,
38027 if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
38028 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32,
38030 return DAG.getZExtOrTrunc(V, DL, VT);
38127 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
38128 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
38131 V = getPMOVMSKB(DL, V, DAG, Subtarget);
38134 V = DAG.getNode(X86ISD::PACKSS, DL, MVT::v16i8, V,
38136 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
38141 V = DAG.getZExtOrTrunc(V, DL, IntVT);
38221 SDLoc DL(BV);
38231 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);
38233 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
38237 V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);
38239 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
38256 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
38257 DAG.getConstant(Intrinsic::x86_mmx_punpcklbw, DL, MVT::i32), Splat,
38263 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
38264 DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL, MVT::i32),
38265 Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
38280 SDValue Intrin = DAG.getConstant(IntrinOp, DL, MVT::i32);
38282 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,
38294 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, SDLoc DL,
38315 combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget))
38316 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0,
38317 DAG.getIntPtrConstant(0, DL));
38328 combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget))
38329 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
38331 : DAG.getConstant(0, DL, VT),
38332 N0, DAG.getIntPtrConstant(0, DL));
38339 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
38340 if (SDValue N1 = combineBitcastToBoolVector(VT, Src1, DL, DAG, Subtarget))
38341 return DAG.getNode(Opc, DL, VT, N0, N1);
38348 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
38350 X86ISD::KSHIFTL, DL, VT, N0,
38351 DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8));
38475 SDLoc DL(N0);
38478 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
38479 DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));
38483 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));
38525 SDLoc DL(N0);
38526 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
38528 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
38660 const SDValue &Zext1, const SDLoc &DL,
38669 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
38672 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
38674 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
38677 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
38680 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
38683 return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 },
38711 SDLoc DL(Extract);
38717 std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
38719 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
38730 Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
38732 Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
38734 Mask = DAG.getConstant(APInt::getAllOnesValue(MaskEltsBits), DL, SrcVT);
38737 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
38745 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
38747 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
38752 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
38756 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
38758 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
38759 DAG.getIntPtrConstant(0, DL));
38790 SDLoc DL(Extract);
38819 DL, MatchVT, DAG.getBitcast(CmpVT, Match.getOperand(0)),
38827 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
38828 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
38832 Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget);
38836 Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
38857 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
38858 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
38871 Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget);
38880 SDValue Mask = DAG.getConstant(1, DL, CmpVT);
38881 SDValue Result = DAG.getNode(ISD::CTPOP, DL, CmpVT, Movmsk);
38882 Result = DAG.getNode(ISD::AND, DL, CmpVT, Result, Mask);
38883 return DAG.getZExtOrTrunc(Result, DL, ExtractVT);
38890 CmpC = DAG.getConstant(0, DL, CmpVT);
38895 DL, CmpVT);
38903 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
38904 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
38905 SDValue Zero = DAG.getConstant(0, DL, ExtractVT);
38906 return DAG.getNode(ISD::SUB, DL, ExtractVT, Zero, Zext);
38953 SDLoc DL(Extract);
38954 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
38969 DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask);
38970 SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle);
38979 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
39178 SDLoc DL(ExtElt);
39179 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
39181 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
39183 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2));
39200 SDLoc DL(ExtElt);
39201 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
39204 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
39206 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
39208 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
39242 SDLoc DL(ExtElt);
39245 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index));
39246 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps);
39279 SDLoc DL(ExtElt);
39287 Rdx = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
39288 DAG.getConstant(0, DL, MVT::v4i32), Rdx,
39289 DAG.getIntPtrConstant(0, DL));
39292 Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Rdx,
39293 DAG.getConstant(0, DL, VecVT));
39298 Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Rdx,
39301 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
39302 DAG.getConstant(0, DL, MVT::v16i8));
39304 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
39316 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
39318 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
39323 MVT::v16i8, DL, Rdx, Rdx,
39325 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
39326 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
39327 getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
39329 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
39345 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
39346 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
39347 Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
39357 Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
39359 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
39520 SDLoc DL(N);
39538 return DAG.getConstantFP(0.0, DL, VT);
39539 return DAG.getConstant(0, DL, VT);
39565 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
39587 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
39594 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
39604 AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
39607 AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
39639 auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL,
39641 return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops);
39651 SDLoc DL(N);
39689 Cond = DAG.getNOT(DL, Cond, MVT::i1);
39694 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
39698 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
39702 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
39822 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
39847 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
39863 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
39871 SDLoc DL(N);
39894 DL, DAG, Subtarget))
39901 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
40054 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
40071 AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);
40072 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);
40087 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
40088 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);
40115 LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL,
40117 RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL,
40119 Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT,
40121 DAG.getIntPtrConstant(0, DL));
40122 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
40123 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
40157 return DAG.getSelect(DL, VT, Cond, LHS, RHS);
40191 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
40206 OpRHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
40208 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
40222 OpRHS = DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT);
40223 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
40267 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
40279 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
40292 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
40294 return DAG.getSelect(DL, VT, CondNew, RHS, LHS);
40313 return DAG.getNode(N->getOpcode(), DL, VT,
40344 SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
40365 DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE);
40366 return DAG.getSelect(DL, VT, NotCond, RHS, LHS);
40390 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
40391 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
40393 DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT);
40394 return DAG.getSelect(DL, VT, NewCond, RHS, LHS);
40820 SDLoc DL(EFLAGS);
40824 Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
40825 Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res,
40826 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
40828 Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
40830 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res,
40831 DAG.getConstant(0, DL, MVT::i32));
40903 SDLoc DL(EFLAGS);
40905 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
40906 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC),
40907 DAG.getConstant(CmpMask, DL, MVT::i32));
40935 SDLoc DL(EFLAGS);
40937 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
40938 Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16);
40940 Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result,
40941 DAG.getConstant(0xAAAA, DL, MVT::i16));
40943 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
40944 DAG.getConstant(0, DL, MVT::i16));
40955 SDLoc DL(EFLAGS);
40957 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
40961 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,
40962 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
40964 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
40965 DAG.getConstant(CmpMask, DL, MVT::i32));
40984 SDLoc DL(EFLAGS);
40986 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
40988 DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
40989 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
41023 SDLoc DL(N);
41041 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
41043 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
41064 Cond = getSETCC(CC, Cond, DL, DAG);
41067 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
41070 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
41071 DAG.getConstant(ShAmt, DL, MVT::i8));
41078 Cond = getSETCC(CC, Cond, DL, DAG);
41081 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
41083 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
41112 Cond = getSETCC(CC, Cond, DL ,DAG);
41114 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
41118 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
41119 DAG.getConstant(Diff, DL, Cond.getValueType()));
41123 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
41163 DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
41164 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
41198 DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
41199 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
41200 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
41202 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
41232 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
41234 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
41235 DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
41236 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
41327 SDLoc DL(N);
41338 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
41339 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
41343 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
41347 DL, VT, MulLo);
41353 DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL,
41365 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
41373 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
41375 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
41379 EVT VT, const SDLoc &DL) {
41382 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41383 DAG.getConstant(Mult, DL, VT));
41384 Result = DAG.getNode(ISD::SHL, DL, VT, Result,
41385 DAG.getConstant(Shift, DL, MVT::i8));
41386 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
41392 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41393 DAG.getConstant(Mul1, DL, VT));
41394 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
41395 DAG.getConstant(Mul2, DL, VT));
41396 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
41415 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
41440 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
41453 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41454 DAG.getConstant(ShiftAmt, DL, MVT::i8));
41455 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41456 DAG.getConstant(ScaleShift, DL, MVT::i8));
41457 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
41508 auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
41511 return DAG.getNode(X86ISD::VPMADDWD, DL, OpVT, Ops);
41538 auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
41540 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
41549 auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
41551 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
41598 SDLoc DL(N);
41600 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41601 DAG.getConstant(AbsMulAmt, DL, VT));
41603 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
41638 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41639 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
41641 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41642 DAG.getConstant(MulAmt1, DL, VT));
41645 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
41646 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
41648 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
41649 DAG.getConstant(MulAmt2, DL, VT));
41653 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
41656 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL);
41666 ISD::ADD, DL, VT, N->getOperand(0),
41667 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41668 DAG.getConstant(Log2_64(AbsMulAmt - 1), DL,
41672 NewMul = DAG.getNode(ISD::SUB, DL, VT,
41673 DAG.getConstant(0, DL, VT), NewMul);
41676 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41678 DL, MVT::i8));
41681 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);
41683 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
41686 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41688 DL, MVT::i8));
41689 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0));
41690 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0));
41693 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41695 DL, MVT::i8));
41696 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
41697 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
41715 SDLoc DL(N);
41756 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
41759 return DAG.getNode(ExtOpc, DL, VT, Mulh);
41799 SDLoc DL(N);
41800 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));
41862 SDLoc DL(N);
41864 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
41869 return DAG.getNode(ISD::SHL, DL, VT, NN,
41870 DAG.getConstant(-SarConst, DL, CVT));
41872 return DAG.getNode(ISD::SRA, DL, VT, NN,
41873 DAG.getConstant(SarConst, DL, CVT));
41923 SDLoc DL(N);
41924 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
41925 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
41926 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);
41959 SDLoc DL(N);
41961 std::tie(Lo, Hi) = DAG.SplitVector(SVN->getOperand(0), DL);
41964 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
41966 Res = DAG.getVectorShuffle(MVT::v4i32, DL, Res, Res, ShuffleMask);
41992 SDLoc DL(N);
41993 SDValue Res = DAG.getNode(Opcode, DL, VT, Op00, Op01);
41995 Res = DAG.getVectorShuffle(MVT::v4i64, DL, Res, Res, ShuffleMask);
42282 SDLoc DL(N);
42328 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
42329 DAG.getTargetConstant(x86cc, DL, MVT::i8));
42332 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
42333 DAG.getConstant(0, DL, MVT::v16i1),
42334 FSetCC, DAG.getIntPtrConstant(0, DL));
42335 return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
42339 DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
42340 CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
42351 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
42354 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
42355 Vector32, DAG.getIntPtrConstant(0, DL));
42360 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
42361 DAG.getConstant(1, DL, IntVT));
42362 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
42436 SDLoc DL(N);
42468 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1);
42471 return DAG.getNode(N->getOpcode(), DL, VT, N0, N1);
42485 SDLoc DL(N);
42502 return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
42504 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
42528 SDLoc DL(N);
42545 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
42575 SDLoc DL(N);
42579 DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1));
42580 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
42612 SDLoc DL(N);
42614 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
42615 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
42764 SDLoc DL(N);
42770 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
42771 DAG.getNode(ISD::SRL, DL, VT, X,
42772 DAG.getConstant(32, DL, MVT::i8)));
42773 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
42774 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);
42777 SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32,
42778 DAG.getNode(ISD::CTPOP, DL, MVT::i32, X),
42779 DAG.getConstant(1, DL, MVT::i32));
42780 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Parity);
42785 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, VT, X,
42786 DAG.getConstant(16, DL, MVT::i8));
42787 X = DAG.getNode(ISD::XOR, DL, VT, X, Hi16);
42792 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
42793 DAG.getNode(ISD::SRL, DL, VT, X,
42794 DAG.getConstant(8, DL, MVT::i8)));
42795 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
42797 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
42800 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
42802 return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), Setnp);
43038 SDLoc DL(N);
43045 SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
43046 return DAG.getNode(X86ISD::VPTERNLOG, DL, VT, A, B, C, Imm);
43051 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
43053 return DAG.getNode(ISD::OR, DL, VT, X, Y);
43121 SDLoc DL(N);
43124 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
43141 Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X);
43387 SDLoc DL(N);
43393 SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
43394 DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT);
43396 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
43462 const SDLoc &DL) {
43493 return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1));
43542 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
43561 DL, DAG, Subtarget);
43563 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid);
43590 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
43593 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
43598 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
43602 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
43614 } else if (auto USatVal = detectUSatPattern(In, VT, DAG, DL)) {
43629 SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps);
43635 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
43636 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
43637 DAG.getIntPtrConstant(0, DL));
43649 const SDLoc &DL) {
43703 auto AVGBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
43705 return DAG.getNode(X86ISD::AVG, DL, Ops[0].getValueType(), Ops);
43715 SDValue VecOnes = DAG.getConstant(1, DL, InVT);
43716 Operands[1] = DAG.getNode(ISD::SUB, DL, InVT, Operands[1], VecOnes);
43717 Operands[1] = DAG.getNode(ISD::TRUNCATE, DL, VT, Operands[1]);
43718 return SplitOpsAndApply(DAG, Subtarget, DL, VT,
43767 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Operands[0], Operands[1]},
43930 SDLoc DL(ML);
43934 DAG.getLoad(EltVT, DL, ML->getChain(), Addr, ML->getPointerInfo(),
43938 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
43950 SDLoc DL(ML);
43961 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
43963 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
43983 VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
43986 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
44050 SDLoc DL(MS);
44053 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
44057 return DAG.getStore(MS->getChain(), DL, Extract, Addr, MS->getPointerInfo(),
44598 const SDLoc &DL) {
44627 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
44628 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
44629 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1);
44674 static SDValue combineVectorTruncationWithPACKUS(SDNode *N, const SDLoc &DL,
44683 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
44684 return truncateVectorWithPACK(X86ISD::PACKUS, OutVT, In, DL, DAG, Subtarget);
44688 static SDValue combineVectorTruncationWithPACKSS(SDNode *N, const SDLoc &DL,
44694 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, InVT, In,
44696 return truncateVectorWithPACK(X86ISD::PACKSS, OutVT, In, DL, DAG, Subtarget);
44736 SDLoc DL(N);
44741 return combineVectorTruncationWithPACKUS(N, DL, Subtarget, DAG);
44743 return combineVectorTruncationWithPACKSS(N, DL, Subtarget, DAG);
44751 static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL,
44798 return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget);
44811 return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget);
44823 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
44866 return DAG.getNode(Opc, DL, VT, LHS, RHS);
44879 const SDLoc &DL) {
44992 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45002 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
45004 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
45012 SDLoc DL(N);
45015 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
45019 if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))
45023 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
45027 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
45031 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
45039 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
45043 if (SDValue V = combineVectorSignBitsTruncation(N, DL, DAG, Subtarget))
45053 SDLoc DL(N);
45056 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
45057 if (auto USatVal = detectUSatPattern(In, VT, DAG, DL))
45058 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
45224 SDLoc DL(N);
45235 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
45236 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
45350 SDLoc DL(N);
45351 return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
45430 SDLoc DL(N);
45447 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
45451 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
45544 SDLoc DL(N);
45550 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
45555 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
45557 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags());
45586 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
45587 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
45591 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);
45785 SDLoc DL(N);
45790 CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0);
45791 CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1);
45794 CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1);
45795 CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1);
45801 CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0);
45802 CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1);
45805 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
45809 CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov);
45940 SDLoc DL(Extend);
45965 CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0);
45966 CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1);
45968 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
45973 Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res);
46013 SDLoc DL(N);
46029 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
46034 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
46045 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
46047 Vec = DAG.getVectorShuffle(BroadcastVT, DL, Vec, Vec, ShuffleMask);
46053 SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
46054 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
46056 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
46064 Bits.push_back(DAG.getConstant(Bit, DL, SVT));
46066 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
46067 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);
46071 Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ);
46072 Vec = DAG.getSExtOrTrunc(Vec, DL, VT);
46078 return DAG.getNode(ISD::SRL, DL, VT, Vec,
46079 DAG.getConstant(EltSizeInBits - 1, DL, VT));
46131 SDLoc DL(N);
46136 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
46165 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
46166 return DAG.getNode(ISD::SUB, DL, VT, Zext, DAG.getConstant(1, DL, VT));
46349 static SDValue emitOrXorXorTree(SDValue X, SDLoc &DL, SelectionDAG &DAG,
46354 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
46355 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
46357 return DAG.getNode(ISD::OR, DL, CmpVT, A, B);
46359 return DAG.getNode(ISD::OR, DL, VecVT, A, B);
46360 return DAG.getNode(ISD::AND, DL, CmpVT, A, B);
46365 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
46367 return DAG.getNode(ISD::XOR, DL, VecVT, A, B);
46368 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
46408 SDLoc DL(SetCC);
46468 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
46469 DAG.getConstant(0, DL, VecVT), X,
46470 DAG.getVectorIdxConstant(0, DL));
46479 Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector);
46484 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
46486 Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY);
46488 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ);
46495 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp),
46496 DAG.getConstant(0, DL, KRegVT), CC);
46501 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
46503 SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
46504 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0));
46511 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
46512 SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
46513 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);
46526 SDLoc DL(N);
46535 MatchVectorAllZeroTest(LHS, CC, DL, Subtarget, DAG, X86CC))
46536 return DAG.getNode(ISD::TRUNCATE, DL, VT,
46537 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, X86CC, V));
46563 return DAG.getConstant(0, DL, VT);
46565 return DAG.getConstant(1, DL, VT);
46567 return DAG.getNOT(DL, Op0.getOperand(0), VT);
46584 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
46585 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);
46627 SDLoc DL(N);
46630 return DAG.getNode(ISD::XOR, DL, VT,
46631 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc),
46632 DAG.getConstant(NotMask, DL, VT));
46664 SDLoc DL(GorS);
46670 Gather->getMemoryVT(), DL, Ops,
46678 Scatter->getMemoryVT(), DL,
46685 SDLoc DL(N);
46706 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
46721 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
46734 Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
46757 SDLoc DL(N);
46763 return getSETCC(CC, Flags, DL, DAG);
46771 SDLoc DL(N);
46779 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
46780 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
46820 SDLoc DL(N);
46826 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},
46829 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
46832 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
46836 return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
46872 SDLoc DL(N);
46873 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
46875 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt);
47166 SDLoc DL(N);
47174 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
47175 return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
47185 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
47227 SDLoc DL(N);
47229 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
47231 DAG.getNode(ISD::AND, DL, VT,
47232 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
47233 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
47235 DAG.getConstant(1, DL, VT));
47281 SDLoc DL(N);
47294 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
47295 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
47312 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
47313 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
47322 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
47324 DAG.getConstant(0, DL, VT), Y.getOperand(1));
47342 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
47344 DAG.getConstant(0, DL, VT), NewEFLAGS);
47351 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
47353 DAG.getConstant(-1, DL, VT), Y.getOperand(1));
47373 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
47375 DAG.getConstant(-1, DL, VT), NewEFLAGS);
47400 SDValue Zero = DAG.getConstant(0, DL, ZVT);
47402 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
47403 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
47404 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
47414 SDValue One = DAG.getConstant(1, DL, ZVT);
47416 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
47417 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
47418 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
47424 SDValue One = DAG.getConstant(1, DL, ZVT);
47426 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
47434 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
47435 DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1));
47439 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
47440 DAG.getConstant(0, DL, VT), Cmp1.getValue(1));
47444 const SDLoc &DL, EVT VT,
47534 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
47535 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
47537 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
47543 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
47545 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder);
47552 const SDLoc &DL, EVT VT,
47653 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
47663 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
47665 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
47682 auto HOpBuilder = [IsAdd](SelectionDAG &DAG, const SDLoc &DL,
47684 return DAG.getNode(IsAdd ? X86ISD::HADD : X86ISD::HSUB, DL,
47724 SDLoc DL(N);
47725 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
47726 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
47732 SDLoc DL(N);
47733 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
47734 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
47806 SDLoc DL(N);
47810 DL, TruncVT);
47811 SDValue UMin = DAG.getNode(ISD::UMIN, DL, TruncVT, OpToSaturate,
47813 SubusRHS = DAG.getNode(ISD::TRUNCATE, DL, VT, UMin);
47898 SDLoc DL(N);
47902 return DAG.getConstant(-1, DL, VT);
47904 return DAG.getConstant(0, DL, VT);
47913 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
47926 return getZeroVector(VT, Subtarget, DAG, DL);
47940 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false))
47951 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0));
47960 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(),
47963 Op0, extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits()));
47971 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
47972 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,
47974 DAG.getIntPtrConstant(0, DL)));
47981 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
48012 return DAG.getNode(Op0.getOpcode(), DL, VT,
48013 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS),
48014 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, RHS),
48027 return DAG.getNode(Op0.getOpcode(), DL, VT,
48028 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Src),
48039 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f32, Src);
48040 Res = DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, Res,
48057 return DAG.getNode(Op0.getOpcode(), DL, VT,
48058 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Src),
48072 return DAG.getNode(Op0.getOpcode(), DL, VT,
48073 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Src),
48089 return DAG.getNode(Op0.getOpcode(), DL, VT,
48090 DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, LHS),
48091 DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, RHS));
48106 return DAG.getNode(Op0.getOpcode(), DL, VT,
48107 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS),
48108 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, RHS),
48310 SDLoc DL(Ext);
48311 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
48312 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
48313 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
48314 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
48388 SDLoc DL(N);
48389 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
48390 getZeroVector(VT, Subtarget, DAG, DL),
48488 SDLoc DL(N);
48489 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
48490 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
48491 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
48492 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
48502 SDLoc DL(N);
48511 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1,
48520 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0),
48540 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
48541 DAG.getAnyExtOrTrunc(ExtSrc, DL, MVT::i32)));
48547 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0));
49544 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
49556 Flag = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Flag);
49559 Flag = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
49561 SDValue CC = getSETCC(Cond, Flag, DL, DAG);
49563 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
50127 int X86TargetLowering::getScalingFactorCost(const DataLayout &DL,
50148 if (isLegalAddressingMode(DL, AM, Ty, AS))