Lines Matching refs:SDValue
2040 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2045 return SDValue(Node, 0);
2382 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
2564 static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
2578 SDValue ValToCopy = DAG.getBitcast(TempValLoc, ValArg);
2596 const SDLoc &Dl, SelectionDAG &DAG, SDValue &Arg,
2597 SmallVectorImpl<std::pair<Register, SDValue>> &RegsToPass, CCValAssign &VA,
2609 SDValue Lo, Hi;
2620 SDValue
2621 X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2624 const SmallVectorImpl<SDValue> &OutVals,
2642 SmallVector<std::pair<Register, SDValue>, 4> RetVals;
2652 SDValue ValToCopy = OutVals[OutsIndex];
2730 SDValue Flag;
2731 SmallVector<SDValue, 6> RetOps;
2782 SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,
2824 bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2828 SDValue TCChain = Chain;
2886 /// our new SDValue to InFlag SDvalue.
2888 static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
2889 SDValue &Root, SelectionDAG &DAG,
2891 SDValue *InFlag = nullptr) {
2901 SDValue Lo, Hi;
2902 SDValue ArgValueLo, ArgValueHi;
2939 static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
2942 SDValue ValReturned = ValArg;
2975 SDValue X86TargetLowering::LowerCallResult(
2976 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2978 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
3031 SDValue Val;
3115 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
3116 SDValue Chain, ISD::ArgFlagsTy Flags,
3118 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
3170 SDValue
3171 X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
3228 SDValue PartAddr;
3255 SDValue Addr =
3276 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3277 SDValue Val = DAG.getLoad(
3360 void lowerVarArgsParameters(SDValue &Chain, unsigned StackSize);
3363 void createVarArgAreaAndStoreRegisters(SDValue &Chain, unsigned StackSize);
3365 void forwardMustTailParameters(SDValue &Chain);
3385 SDValue &Chain, unsigned StackSize) {
3432 SmallVector<SDValue, 6>
3433 LiveGPRs; // list of SDValue for GPR registers keeping live input value
3434 SmallVector<SDValue, 8> LiveXMMRegs; // list of SDValue for XMM registers
3436 SDValue ALVal; // if applicable keeps SDValue for %al register
3455 SmallVector<SDValue, 8> MemOps;
3456 SDValue RSFIN =
3460 for (SDValue Val : LiveGPRs) {
3461 SDValue FIN = DAG.getNode(ISD::ADD, DL,
3464 SDValue Store =
3475 SmallVector<SDValue, 12> SaveXMMOps;
3493 void VarArgsLoweringHelper::forwardMustTailParameters(SDValue &Chain) {
3527 SDValue RegVal = DAG.getCopyFromReg(Chain, DL, FR.VReg, FR.VT);
3534 void VarArgsLoweringHelper::lowerVarArgsParameters(SDValue &Chain,
3548 SDValue X86TargetLowering::LowerFormalArguments(
3549 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
3551 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3589 SDValue ArgValue;
3705 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[I]);
3772 SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
3773 SDValue Arg, const SDLoc &dl,
3779 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
3792 SDValue X86TargetLowering::EmitTailCallLoadRetAddr(
3793 SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
3801 return SDValue(OutRetAddr.getNode(), 1);
3806 static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
3807 SDValue Chain, SDValue RetAddrFrIdx,
3816 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
3825 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
3826 SDValue V2) {
3835 SDValue
3837 SmallVectorImpl<SDValue> &InVals) const {
3841 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
3843 SDValue Chain = CLI.Chain;
3844 SDValue Callee = CLI.Callee;
3981 SDValue RetAddrFrIdx;
3987 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
3988 SmallVector<SDValue, 8> MemOpChains;
3989 SDValue StackPtr;
4009 SDValue Arg = OutVals[OutIndex];
4045 SDValue StackSlot =
4054 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
4152 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
4167 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
4169 SmallVector<SDValue, 8> MemOpChains2;
4170 SDValue FIN;
4189 SDValue Arg = OutVals[OutsIndex];
4202 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);
4231 SDValue InFlag;
4259 SmallVector<SDValue, 8> Ops;
4341 SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
4438 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
4451 const SDValue &TruncInput = Arg.getOperand(0);
4491 SDValue Ptr = Ld->getBasePtr();
4532 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
4535 const SmallVectorImpl<SDValue> &OutVals,
4660 SDValue Arg = OutVals[i];
4734 static bool MayFoldLoad(SDValue Op) {
4738 static bool MayFoldIntoStore(SDValue Op) {
4742 static bool MayFoldIntoZeroExtend(SDValue Op) {
4809 static bool isTargetShuffleSplat(SDValue Op) {
4816 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
4924 bool isFP, SDValue &LHS, SDValue &RHS,
5104 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
5162 SDValue C) const {
5204 bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
5286 bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
5302 bool X86TargetLowering::hasAndNot(SDValue Y) const {
5319 bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
5325 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
5364 bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
5633 bool X86::isZeroNode(SDValue Elt) {
5640 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
5643 SmallVector<SDValue, 32> Ops;
5657 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
5664 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
5670 static SDValue getConstVector(ArrayRef<APInt> Bits, APInt &Undefs,
5674 SmallVector<SDValue, 32> Ops;
5707 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
5712 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
5721 SDValue Vec;
5737 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
5758 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
5768 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
5776 static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
5782 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5802 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
5812 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5820 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
5826 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
5834 static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
5849 static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops) {
5858 SDValue Src = N->getOperand(0);
5859 SDValue Sub = N->getOperand(1);
5887 static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
5895 SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
5896 SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
5901 static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG) {
5916 SDValue Lo, Hi;
5928 static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG) {
5939 SDValue LHS1, LHS2;
5943 SDValue RHS1, RHS2;
5959 // SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
5961 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
5962 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
5987 SmallVector<SDValue, 4> Subs;
5989 SmallVector<SDValue, 2> SubOps;
5990 for (SDValue Op : Ops) {
6002 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
6006 SDValue Vec = Op.getOperand(0);
6007 SDValue SubVec = Op.getOperand(1);
6008 SDValue Idx = Op.getOperand(2);
6020 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
6043 SDValue Undef = DAG.getUNDEF(WideOpVT);
6047 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
6099 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
6121 SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
6122 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
6142 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
6149 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
6162 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
6170 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
6177 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6183 SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts));
6203 static SDValue getExtendInVec(unsigned Opcode, const SDLoc &DL, EVT VT,
6204 SDValue In, SelectionDAG &DAG) {
6231 static SDValue IsNOT(SDValue V, SelectionDAG &DAG, bool OneUse = false) {
6238 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
6244 SmallVector<SDValue, 2> CatOps;
6246 for (SDValue &CatOp : CatOps) {
6247 SDValue NotCat = IsNOT(CatOp, DAG);
6248 if (!NotCat) return SDValue();
6253 return SDValue();
6286 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
6287 SDValue V1, SDValue V2) {
6294 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
6295 SDValue V1, SDValue V2) {
6305 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
6310 SDValue V1 = IsZero
6320 static const Constant *getTargetConstantFromBasePtr(SDValue Ptr) {
6338 static const Constant *getTargetConstantFromNode(SDValue Op) {
6350 static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
6468 const SDValue &Src = Op.getOperand(i);
6485 const SDValue &Src = Op.getOperand(i);
6524 SDValue Ptr = MemIntr->getBasePtr();
6660 bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs) {
6687 static bool getTargetShuffleMaskIndices(SDValue MaskNode,
6790 SmallVectorImpl<SDValue> &Ops,
6942 SDValue MaskNode = N->getOperand(1);
6955 SDValue MaskNode = N->getOperand(1);
7007 SDValue MaskNode = N->getOperand(2);
7008 SDValue CtrlNode = N->getOperand(3);
7024 SDValue MaskNode = N->getOperand(2);
7036 SDValue MaskNode = N->getOperand(0);
7051 SDValue MaskNode = N->getOperand(1);
7098 SDValue V1, SDValue V2,
7126 SDValue V = M < Size ? V1 : V2;
7137 SDValue Op = V.getOperand(M / Scale);
7163 SDValue Op = V.getOperand((M * Scale) + j);
7180 static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
7181 SmallVectorImpl<SDValue> &Ops,
7192 SDValue V1 = Ops[0];
7193 SDValue V2 = IsUnary ? V1 : Ops[1];
7227 SDValue V = M < Size ? V1 : V2;
7254 SDValue Vec = V.getOperand(0);
7314 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
7322 static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
7324 SmallVectorImpl<SDValue> &Ops,
7358 SDValue N0 = N.getOperand(0);
7359 SDValue N1 = N.getOperand(1);
7414 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
7415 SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
7419 SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
7427 if (llvm::any_of(SrcInputs0, [VT](SDValue Op) {
7431 if (llvm::any_of(SrcInputs1, [VT](SDValue Op) {
7457 SDValue Src = N.getOperand(0);
7458 SDValue Sub = N.getOperand(1);
7478 SmallVector<SDValue, 2> SubInputs;
7484 if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) {
7524 SDValue Scl = N.getOperand(Opcode == ISD::SCALAR_TO_VECTOR ? 0 : 1);
7558 SDValue SrcExtract;
7568 SDValue SrcVec = SrcExtract.getOperand(0);
7599 SDValue N0 = N.getOperand(0);
7600 SDValue N1 = N.getOperand(1);
7635 SDValue Src = N.getOperand(0);
7699 SDValue Src = N.getOperand(0);
7710 SDValue Src = N.getOperand(0);
7731 static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
7734 SmallVector<SDValue, 16> UsedInputs;
7775 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
7776 SmallVectorImpl<SDValue> &Inputs,
7798 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
7815 static SDValue getShuffleScalarElt(SDValue Op, unsigned Index,
7818 return SDValue(); // Limit search depth.
7831 SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1);
7841 SmallVector<SDValue, 16> ShuffleOps;
7846 return SDValue();
7856 SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
7862 SDValue Vec = Op.getOperand(0);
7863 SDValue Sub = Op.getOperand(1);
7883 SDValue Src = Op.getOperand(0);
7890 SDValue Src = Op.getOperand(0);
7894 return SDValue();
7915 return SDValue();
7919 static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros,
7930 SDValue V;
7961 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
7966 return SDValue();
7974 SDValue V;
7984 SDValue Elt;
7993 SDValue NextElt = Op.getOperand(i + 1);
8027 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
8032 return SDValue();
8040 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
8055 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
8057 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
8058 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
8065 SDValue Elt = Op.getOperand(i);
8074 SDValue FirstNonZero;
8079 SDValue Elt = Op.getOperand(i);
8082 return SDValue();
8086 return SDValue();
8094 SDValue V1 = FirstNonZero.getOperand(0);
8098 SDValue Elt;
8118 SDValue VZeroOrUndef = (Zeroable == Undefs)
8128 return SDValue();
8130 SDValue V2 = Elt.getOperand(0);
8132 V1 = SDValue();
8139 SDValue Current = Op->getOperand(i);
8140 SDValue SrcVector = Current->getOperand(0);
8147 return SDValue();
8161 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
8167 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
8175 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
8179 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
8186 SDValue Ptr = LD->getBasePtr();
8188 return SDValue();
8191 return SDValue();
8204 return SDValue();
8210 SDValue Chain = LD->getChain();
8219 return SDValue();
8228 return SDValue();
8230 return SDValue();
8242 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
8250 return SDValue();
8254 static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
8280 SDValue Src = Elt.getOperand(0);
8301 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
8306 return SDValue();
8321 SDValue Elt = peekThroughBitcasts(Elts[i]);
8323 return SDValue();
8337 return SDValue();
8340 return SDValue();
8343 return SDValue();
8363 SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
8376 return SDValue();
8413 SDValue NewLd =
8436 return SDValue();
8442 return SDValue();
8464 SDValue V = CreateLoad(VT, LDBase);
8465 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
8478 SDValue HalfLD =
8500 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
8501 SDValue ResNode = DAG.getMemIntrinsicNode(
8522 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
8526 SDValue Elt = peekThroughBitcasts(Elts[i]);
8550 if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
8554 SDValue Broadcast = DAG.getNode(Opcode, DL, BroadcastVT, RepeatLoad);
8561 return SDValue();
8567 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
8571 SmallVector<SDValue, 64> Elts;
8573 if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
8577 return SDValue();
8630 static SDValue isSplatZeroExtended(const BuildVectorSDNode *Op,
8632 SDValue ExtValue = Op->getOperand(0);
8642 return SDValue();
8645 return SDValue();
8650 return SDValue();
8653 return SDValue();
8668 /// or SDValue() otherwise.
8669 static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
8676 return SDValue();
8685 SDValue Ld = BVOp->getSplatValue(&UndefElements);
8696 SDValue BOperand;
8697 SDValue ZeroExtended = isSplatZeroExtended(BVOp, NumElts, EltType);
8708 SDValue Brdcst =
8729 return SDValue();
8742 SDValue CP = DAG.getConstantPool(C, PVT);
8748 SDValue Ops[] = {DAG.getEntryNode(), CP};
8751 SDValue Brdcst = DAG.getMemIntrinsicNode(
8761 SDValue VCP = DAG.getConstantPool(VecC, PVT);
8768 SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);
8780 return SDValue();
8783 return SDValue();
8794 return SDValue();
8831 SDValue CP =
8836 SDValue Ops[] = {DAG.getEntryNode(), CP};
8851 return SDValue();
8855 return SDValue();
8861 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
8862 SDValue BCast =
8865 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
8875 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
8876 SDValue BCast =
8879 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
8884 return SDValue();
8892 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
8893 SDValue ExtIdx) {
8909 SDValue ShuffleVec = SVOp->getOperand(0);
8922 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
8928 return SDValue();
8933 SDValue VecIn1;
8934 SDValue VecIn2;
8947 return SDValue();
8953 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
8954 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
8958 return SDValue();
8963 return SDValue();
8972 return SDValue();
8982 return SDValue();
8985 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
8995 static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
9013 SDValue In = Op.getOperand(idx);
9034 SDValue Cond = Op.getOperand(SplatIdx);
9042 SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
9049 SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
9060 SDValue DstVec;
9063 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
9064 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
9070 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
9111 SDValue &V0, SDValue &V1) {
9127 SDValue Op = N->getOperand(i + BaseIdx);
9143 SDValue Op0 = Op.getOperand(0);
9144 SDValue Op1 = Op.getOperand(1);
9175 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
9222 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
9231 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
9232 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
9233 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
9234 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
9237 SDValue LO = DAG.getUNDEF(NewVT);
9238 SDValue HI = DAG.getUNDEF(NewVT);
9265 SDValue &Opnd0, SDValue &Opnd1,
9274 SDValue InVec0 = DAG.getUNDEF(VT);
9275 SDValue InVec1 = DAG.getUNDEF(VT);
9285 SDValue Op = BV->getOperand(i);
9296 SDValue Op0 = Op.getOperand(0);
9297 SDValue Op1 = Op.getOperand(1);
9392 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
9418 static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
9421 SDValue Opnd0, Opnd1;
9426 return SDValue();
9432 SDValue Opnd2;
9440 return SDValue();
9448 return SDValue();
9454 unsigned &HOpcode, SDValue &V0, SDValue &V1) {
9472 SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
9492 SDValue Op0 = Op.getOperand(0);
9493 SDValue Op1 = Op.getOperand(1);
9511 SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
9540 static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
9542 SDValue V0, SDValue V1) {
9570 SDValue Half = DAG.getNode(HOpcode, SDLoc(BV), HalfVT, V0, V1);
9578 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
9583 count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
9585 return SDValue();
9596 SDValue V0, V1;
9603 return SDValue();
9619 SDValue InVec0, InVec1;
9621 SDValue InVec2, InVec3;
9645 return SDValue();
9650 SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
9651 SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
9675 return SDValue();
9680 return SDValue();
9690 return SDValue();
9693 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
9702 static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
9715 return SDValue();
9721 return SDValue();
9733 return SDValue();
9735 return SDValue();
9739 SmallVector<SDValue, 4> LHSElts, RHSElts;
9740 for (SDValue Elt : Op->ops()) {
9741 SDValue LHS = Elt.getOperand(0);
9742 SDValue RHS = Elt.getOperand(1);
9746 return SDValue();
9751 return SDValue();
9762 if (IsShift && any_of(RHSElts, [&](SDValue V) { return RHSElts[0] != V; }))
9763 return SDValue();
9765 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
9766 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
9767 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
9780 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
9799 return SDValue();
9805 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
9830 SDValue NewSrcVec =
9834 return SDValue();
9839 return SDValue();
9842 auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
9913 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
9914 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
9915 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
9916 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
9922 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
9923 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
9924 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
9925 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
9927 ArrayRef<SDValue> Ops) {
9931 SDValue Idx = Ops[2];
9938 SDValue Ops[] = {LoLo, HiHi, IndicesVec};
9961 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9963 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9971 SDValue Res = DAG.getSelectCC(
9988 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
9995 SDValue LoLo =
9997 SDValue HiHi =
10007 SDValue Res = DAG.getSelectCC(
10032 return SDValue();
10046 SDValue Res = Opcode == X86ISD::VPERMV
10063 static SDValue
10064 LowerBUILD_VECTORAsVariablePermute(SDValue V, SelectionDAG &DAG,
10066 SDValue SrcVec, IndicesVec;
10071 SDValue Op = V.getOperand(Idx);
10073 return SDValue();
10081 return SDValue();
10082 SDValue ExtractedIndex = Op->getOperand(1);
10088 return SDValue();
10096 return SDValue();
10100 return SDValue();
10108 SDValue
10109 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
10120 if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
10124 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG))
10126 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
10128 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
10130 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG))
10139 SmallSet<SDValue, 8> Values;
10142 SDValue Elt = Op.getOperand(i);
10178 SDValue VarElt;
10179 SDValue InsIndex;
10181 SDValue Elt = Op.getOperand(i);
10194 SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
10202 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
10205 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
10219 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
10226 SDValue Item = Op.getOperand(Idx);
10269 return SDValue();
10290 SDValue Item = Op.getOperand(Idx);
10294 return SDValue();
10300 return SDValue();
10302 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, DAG, Subtarget))
10307 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
10308 if (SDValue LD =
10317 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
10319 auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
10330 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
10345 SDValue Lower =
10347 SDValue Upper = DAG.getBuildVector(
10359 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
10363 return SDValue();
10368 if (SDValue V = LowerBuildVectorv16i8(Op, NonZeros, NumNonZero, NumZero,
10373 if (SDValue V = LowerBuildVectorv8i16(Op, NonZeros, NumNonZero, NumZero,
10379 if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget))
10384 SmallVector<SDValue, 8> Ops(NumElems);
10425 if (SDValue Sh = buildFromShuffleMostly(Op, DAG))
10430 SDValue Result;
10447 SmallVector<SDValue, 8> Ops(NumElems);
10477 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
10490 SDValue SubVec = Op.getOperand(i);
10506 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10508 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10514 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
10535 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
10548 SDValue SubVec = Op.getOperand(i);
10569 SDValue SubVec = Op.getOperand(Idx);
10582 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
10586 SDValue SubVec = Op.getOperand(Idx);
10595 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10597 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10607 SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT,
10614 static SDValue LowerCONCAT_VECTORS(SDValue Op,
10789 static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
10827 SDValue V1 = SDValue(),
10828 SDValue V2 = SDValue()) {
10865 SDValue Cond) {
10873 SDValue CondElt = Cond.getOperand(i);
10959 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
10997 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
10998 ArrayRef<int> Mask, SDValue V1,
10999 SDValue V2, const APInt &Zeroable,
11011 SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
11013 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
11015 SDValue V;
11028 SDValue SrcV = (M >= Size ? V2 : V1);
11030 return SDValue();
11036 return SDValue();
11050 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
11055 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
11057 ArrayRef<int> Mask, SDValue &V1,
11058 SDValue &V2, SelectionDAG &DAG,
11063 return SDValue();
11067 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
11071 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
11073 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
11074 SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
11078 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
11165 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
11166 ArrayRef<int> Mask, SDValue V1, SDValue V2,
11187 return SDValue();
11192 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
11193 ArrayRef<int> Mask, SDValue V1,
11194 SDValue V2, SelectionDAG &DAG) {
11205 return SDValue();
11294 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
11295 MVT VT, SDValue V1, SDValue V2,
11299 return SDValue();
11302 return SDValue();
11308 return SDValue();
11321 return SDValue();
11323 return SDValue();
11325 SDValue Src = V1.getOperand(0).getOperand(0);
11331 return SDValue();
11337 return SDValue();
11343 return SDValue();
11418 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
11428 auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {
11431 SDValue VV1 = DAG.getBitcast(PackVT, N1);
11432 SDValue VV2 = DAG.getBitcast(PackVT, N2);
11478 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
11479 SDValue V1, SDValue V2, SelectionDAG &DAG,
11488 return SDValue();
11495 return SDValue();
11505 SDValue Res;
11527 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
11528 SDValue V2, ArrayRef<int> Mask,
11534 SDValue Zero, AllOnes;
11554 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
11555 SDValue V;
11560 return SDValue(); // Not a blend.
11564 return SDValue(); // Can only let one input through the mask.
11569 return SDValue(); // No non-zeroable elements!
11571 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
11574 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
11583 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
11584 SDValue V2, ArrayRef<int> Mask,
11588 SDValue Zero = DAG.getConstant(0, DL, EltVT);
11589 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
11590 SmallVector<SDValue, 16> MaskOps;
11593 return SDValue(); // Shuffled input!
11597 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
11603 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
11604 SDValue PreservedSrc,
11608 static bool matchShuffleAsBlend(SDValue V1, SDValue V2,
11666 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
11667 SDValue V2, ArrayRef<int> Original,
11676 return SDValue();
11720 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11722 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11737 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11744 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11750 if (SDValue BitBlend =
11781 SmallVector<SDValue, 32> VSELECTMask;
11805 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11814 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11827 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
11828 SDValue V1, SDValue V2,
11846 return SDValue(); // Can't blend in the needed input!
11855 return SDValue();
11857 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11866 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
11867 SDValue V1, SDValue V2,
11876 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
11885 SDValue &Op = Ops[Elt & 1];
11891 return SDValue();
11899 return SDValue();
11914 return SDValue();
11923 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
11929 static SDValue lowerShuffleAsByteRotateAndPermute(
11930 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11935 return SDValue();
11939 return SDValue();
11978 return SDValue();
11981 return SDValue();
11984 auto RotateAndPermute = [&](SDValue Lo, SDValue Hi, int RotAmt, int Ofs) {
11986 SDValue Rotate = DAG.getBitcast(
12010 return SDValue();
12020 static SDValue lowerShuffleAsDecomposedShuffleBlend(
12021 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12044 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
12047 if (SDValue UnpackPerm = lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask,
12050 if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
12054 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
12113 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
12122 return SDValue();
12128 return SDValue();
12135 return SDValue();
12140 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
12142 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
12144 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
12148 SDValue Rot =
12157 static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
12169 SDValue Lo, Hi;
12195 SDValue MaskV = M < NumElts ? V1 : V2;
12200 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
12242 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
12264 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
12265 SDValue V2, ArrayRef<int> Mask,
12270 SDValue Lo = V1, Hi = V2;
12273 return SDValue();
12301 SDValue LoShift =
12304 SDValue HiShift =
12321 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
12322 SDValue V2, ArrayRef<int> Mask,
12332 SDValue Lo = V1, Hi = V2;
12335 return SDValue();
12342 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
12343 SDValue V2, ArrayRef<int> Mask,
12355 return SDValue();
12360 return SDValue();
12366 return SDValue();
12368 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
12401 return SDValue();
12491 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
12492 SDValue V2, ArrayRef<int> Mask,
12500 SDValue V = V1;
12515 return SDValue();
12527 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
12548 SDValue Src;
12554 SDValue &V = (M < Size ? V1 : V2);
12583 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
12595 SDValue Base;
12611 SDValue Insert;
12649 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
12650 SDValue V2, ArrayRef<int> Mask,
12664 return SDValue();
12676 static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
12677 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
12697 auto ShuffleOffset = [&](SDValue V) {
12715 return SDValue();
12757 SDValue Lo = DAG.getBitcast(
12766 SDValue Hi = DAG.getBitcast(
12779 SDValue PSHUFBMask[16];
12815 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
12838 static SDValue lowerShuffleAsZeroOrAnyExtend(
12839 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12852 auto Lower = [&](int Scale) -> SDValue {
12853 SDValue InputV;
12864 return SDValue();
12873 SDValue V = M < NumElements ? V1 : V2;
12879 return SDValue(); // Flip-flopping inputs.
12886 return SDValue();
12891 return SDValue();
12894 return SDValue(); // Non-consecutive strided elements.
12902 return SDValue();
12907 return SDValue();
12923 if (SDValue V = Lower(NumElements / NumExtElements))
12929 return SDValue();
12936 return SDValue();
12941 return SDValue();
12944 if (SDValue V = CanZExtLowHalf()) {
12951 return SDValue();
12957 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12967 return SDValue();
12973 SDValue S = V.getOperand(Idx);
12978 return SDValue();
12985 static bool isShuffleFoldableLoad(SDValue V) {
12994 static SDValue lowerShuffleAsElementInsertion(
12995 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13016 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
13025 return SDValue();
13036 return SDValue();
13045 return SDValue();
13049 return SDValue();
13051 return SDValue();
13062 return SDValue();
13092 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
13107 return SDValue();
13114 return SDValue();
13125 return SDValue();
13127 SDValue Scalar = V0.getOperand(V0BroadcastIdx);
13166 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
13167 SDValue N1, ArrayRef<int> Mask,
13179 return SDValue();
13181 SDValue WideVec = N0.getOperand(0);
13184 return SDValue();
13195 return SDValue();
13202 return SDValue();
13208 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
13220 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
13221 SDValue V2, ArrayRef<int> Mask,
13227 return SDValue();
13240 return SDValue();
13250 SDValue V = V1;
13274 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
13303 if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
13315 return SDValue();
13324 SDValue BaseAddr = Ld->getOperand(1);
13328 SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
13335 SDValue Ops[] = {Ld->getChain(), NewAddr};
13350 return SDValue();
13356 return SDValue();
13360 return SDValue();
13364 return SDValue();
13406 static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
13417 auto matchAsInsertPS = [&](SDValue VA, SDValue VB,
13494 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
13503 return SDValue();
13519 static SDValue lowerShuffleAsPermuteAndUnpack(
13520 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13553 return SDValue();
13566 return SDValue();
13587 if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
13594 return SDValue();
13624 return SDValue();
13634 static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13635 const APInt &Zeroable, SDValue V1, SDValue V2,
13644 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
13671 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13676 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13683 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13691 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
13699 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
13704 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
13718 static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13719 const APInt &Zeroable, SDValue V1, SDValue V2,
13728 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
13750 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13754 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
13760 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13766 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13774 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
13779 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
13786 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,
13790 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
13816 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
13817 ArrayRef<int> Mask, SDValue V1,
13818 SDValue V2, SelectionDAG &DAG) {
13819 SDValue LowV = V1, HighV = V2;
13902 static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13903 const APInt &Zeroable, SDValue V1, SDValue V2,
13914 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
13949 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13958 if (SDValue V = lowerShuffleAsElementInsertion(
13963 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
13968 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13972 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13987 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13998 static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13999 const APInt &Zeroable, SDValue V1, SDValue V2,
14009 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
14018 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
14040 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
14044 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
14050 if (SDValue V = lowerShuffleAsElementInsertion(
14058 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
14062 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
14067 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
14074 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,
14078 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
14094 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
14104 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
14105 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
14106 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
14126 static SDValue lowerV8I16GeneralSingleInputShuffle(
14127 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
14622 static SDValue lowerShuffleAsBlendOfPSHUFBs(
14623 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14632 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14633 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14663 SDValue V;
14685 static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14686 const APInt &Zeroable, SDValue V1, SDValue V2,
14695 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
14703 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,
14708 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
14713 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
14718 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14722 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14727 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
14742 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
14748 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
14754 if (SDValue V = lowerShuffleAsElementInsertion(
14762 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
14766 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
14771 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14775 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14780 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
14784 if (SDValue BitBlend =
14789 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
14799 SmallVector<SDValue, 8> DWordClearOps(4, DAG.getConstant(0, DL, MVT::i32));
14802 SDValue DWordClearMask = DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);
14808 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, V1, V2);
14817 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
14835 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
14836 ArrayRef<int> Mask, SDValue V1,
14837 SDValue V2, SelectionDAG &DAG) {
14841 SDValue MaskNode = getConstVector(Mask, MaskVecVT, DAG, DL, true);
14855 static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14856 const APInt &Zeroable, SDValue V1, SDValue V2,
14864 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
14869 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14874 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14879 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14885 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14894 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14899 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
14903 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14921 auto tryToWidenViaDuplication = [&]() -> SDValue {
14923 return SDValue();
14958 return SDValue();
15000 if (SDValue V = tryToWidenViaDuplication())
15004 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
15009 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
15013 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
15041 SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
15049 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
15061 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
15071 if (SDValue V = lowerShuffleAsByteRotateAndPermute(
15081 if (SDValue V = lowerShuffleAsElementInsertion(
15085 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
15103 SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
15106 SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
15114 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
15131 SDValue V = V1;
15139 SDValue VLoHalf, VHiHalf;
15163 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
15171 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
15172 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
15181 static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
15182 MVT VT, SDValue V1, SDValue V2,
15210 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
15211 SDValue V2, ArrayRef<int> Mask,
15228 auto SplitVector = [&](SDValue V) {
15229 SDValue LoV, HiV;
15235 SDValue LoV1, HiV1, LoV2, HiV2;
15276 SDValue V1Blend, V2Blend;
15299 SDValue Lo = HalfBlend(LoMask);
15300 SDValue Hi = HalfBlend(HiMask);
15312 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
15313 SDValue V2, ArrayRef<int> Mask,
15365 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,
15366 SDValue V1, SDValue V2,
15387 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
15388 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
15401 static SDValue lowerShuffleAsLanePermuteAndPermute(
15402 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15420 return SDValue();
15450 return SDValue();
15452 SDValue LanePermute = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask);
15463 static SDValue lowerShuffleAsLanePermuteAndShuffle(
15464 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15476 if (SDValue V =
15516 SDValue Flipped = DAG.getBitcast(PVT, V1);
15524 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
15525 SDValue V2, ArrayRef<int> Mask,
15531 return SDValue();
15537 return SDValue();
15545 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
15557 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
15573 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
15629 static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
15630 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15635 return SDValue();
15663 return SDValue();
15713 return SDValue();
15733 return SDValue();
15737 return SDValue();
15743 return SDValue();
15756 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15762 return SDValue();
15773 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15779 return SDValue();
15849 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15863 SDValue V = (HalfIdx < 2 ? V1 : V2);
15870 SDValue Half1 = getHalfVector(HalfIdx1);
15871 SDValue Half2 = getHalfVector(HalfIdx2);
15872 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15874 SDValue Op0 = V;
15875 SDValue Op1 = DAG.getUNDEF(HalfVT);
15889 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15890 SDValue V2, ArrayRef<int> Mask,
15898 return SDValue();
15909 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15919 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15928 return SDValue();
15957 return SDValue();
15963 return SDValue();
15967 return SDValue();
15975 return SDValue();
15983 return SDValue();
15986 return SDValue();
15993 return SDValue();
16015 static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
16016 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
16054 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
16068 return SDValue();
16073 return SDValue();
16101 return SDValue();
16147 return SDValue();
16165 SDValue RepeatedShuffle = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
16181 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
16226 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
16227 SDValue V2, ArrayRef<int> Mask,
16238 return SDValue();
16253 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
16254 SDValue V1, SDValue V2,
16262 return SDValue();
16266 return SDValue();
16276 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
16290 static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16291 const APInt &Zeroable, SDValue V1, SDValue V2,
16298 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
16304 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
16328 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16333 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
16343 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
16346 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
16351 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
16363 if (SDValue Op = lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2,
16375 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16385 if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
16391 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
16410 static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16411 const APInt &Zeroable, SDValue V1, SDValue V2,
16419 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
16423 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
16428 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
16453 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
16459 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,
16463 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
16469 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
16474 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
16485 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16495 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16508 static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16509 const APInt &Zeroable, SDValue V1, SDValue V2,
16516 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
16521 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
16543 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
16553 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16561 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16565 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16575 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16581 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
16589 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16608 static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16609 const APInt &Zeroable, SDValue V1, SDValue V2,
16620 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16629 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask,
16633 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16638 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16655 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
16660 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
16666 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,
16670 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
16676 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16682 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16689 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
16694 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16702 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16703 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16704 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16711 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16724 static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16725 const APInt &Zeroable, SDValue V1, SDValue V2,
16736 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16741 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16745 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16750 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16754 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16759 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
16764 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16770 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16776 if (SDValue Rotate =
16782 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
16788 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16806 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16816 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16821 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16834 static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16835 const APInt &Zeroable, SDValue V1, SDValue V2,
16846 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16851 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16855 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16860 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
16864 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
16869 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
16874 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16880 if (SDValue Rotate =
16886 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16895 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
16898 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16906 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16916 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16921 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16929 if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16943 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16944 SDValue V1, SDValue V2, const APInt &Zeroable,
16953 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16958 if (SDValue V =
16973 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16976 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17008 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
17009 const APInt &Zeroable, SDValue V1, SDValue V2,
17022 return SDValue();
17030 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
17043 SDValue SubVec =
17075 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
17091 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
17099 SDValue Op = Widened128Mask[i] >= 4 ? V2 : V1;
17104 return SDValue();
17116 static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17117 const APInt &Zeroable, SDValue V1, SDValue V2,
17146 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
17150 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
17154 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
17158 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
17162 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
17170 static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17171 const APInt &Zeroable, SDValue V1, SDValue V2,
17195 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
17198 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
17208 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17216 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
17221 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
17229 static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17230 const APInt &Zeroable, SDValue V1, SDValue V2,
17258 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
17263 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
17268 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,
17273 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
17277 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
17280 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
17284 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
17292 static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17293 const APInt &Zeroable, SDValue V1, SDValue V2,
17303 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17320 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
17325 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
17330 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,
17336 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
17343 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
17344 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
17345 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
17352 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17357 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
17361 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
17369 static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17370 const APInt &Zeroable, SDValue V1, SDValue V2,
17381 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17386 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
17390 if (SDValue V =
17395 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
17400 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
17406 if (SDValue Rotate =
17420 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
17424 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
17432 static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17433 const APInt &Zeroable, SDValue V1, SDValue V2,
17444 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17449 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
17453 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
17458 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
17463 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
17469 if (SDValue Rotate =
17474 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
17478 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
17488 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17492 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
17499 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
17512 static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17513 MVT VT, SDValue V1, SDValue V2,
17526 if (SDValue Insertion = lowerShuffleAsElementInsertion(
17531 if (SDValue V =
17536 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
17543 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17546 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17575 static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
17576 MVT VT, SDValue V1, SDValue V2,
17581 return SDValue();
17597 return SDValue();
17601 return SDValue();
17609 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
17655 static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17656 MVT VT, SDValue V1, SDValue V2,
17690 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
17699 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
17705 for (SDValue V : { V1, V2 }) {
17712 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
17765 return SDValue();
17773 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17855 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
17859 SDValue V1 = Op.getOperand(0);
17860 SDValue V2 = Op.getOperand(1);
17922 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17964 if (SDValue V = lowerShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))
17984 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17987 SDValue Cond = Op.getOperand(0);
17988 SDValue LHS = Op.getOperand(1);
17989 SDValue RHS = Op.getOperand(2);
17998 return SDValue();
18001 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
18002 SDValue Cond = Op.getOperand(0);
18003 SDValue LHS = Op.getOperand(1);
18004 SDValue RHS = Op.getOperand(2);
18011 return SDValue();
18015 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
18027 return SDValue();
18036 return SDValue();
18044 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
18055 return SDValue();
18076 return SDValue();
18085 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
18091 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
18093 SDValue Vec = Op.getOperand(0);
18094 SDValue Idx = Op.getOperand(1);
18099 return SDValue();
18110 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec, Idx);
18121 return SDValue();
18126 return SDValue();
18127 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18135 return SDValue();
18140 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
18142 SDValue Vec = Op.getOperand(0);
18145 SDValue Idx = Op.getOperand(1);
18160 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
18161 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
18187 SDValue
18188 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
18191 SDValue Vec = Op.getOperand(0);
18193 SDValue Idx = Op.getOperand(1);
18230 return SDValue();
18265 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec, Idx);
18270 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
18280 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18291 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
18328 return SDValue();
18333 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
18336 SDValue Vec = Op.getOperand(0);
18337 SDValue Elt = Op.getOperand(1);
18338 SDValue Idx = Op.getOperand(2);
18347 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
18354 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
18358 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
18368 SDValue N0 = Op.getOperand(0);
18369 SDValue N1 = Op.getOperand(1);
18370 SDValue N2 = Op.getOperand(2);
18374 return SDValue();
18388 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
18404 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18411 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
18501 return SDValue();
18504 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
18534 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
18542 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18549 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18555 SDValue Vec = Op.getOperand(0);
18606 SDValue
18607 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
18615 SDValue Result = DAG.getTargetConstantPool(
18629 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18637 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18650 SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
18655 SDValue
18656 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18664 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18678 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18705 SDValue Result;
18749 SDValue
18750 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
18754 static SDValue
18755 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
18756 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
18761 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18770 SDValue Ops[] = { Chain, TGA, *InFlag };
18773 SDValue Ops[] = { Chain, TGA };
18781 SDValue Flag = Chain.getValue(1);
18786 static SDValue
18789 SDValue InFlag;
18791 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18800 static SDValue
18807 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
18818 SDValue Base;
18823 SDValue InFlag;
18824 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18837 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18840 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18847 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18856 SDValue ThreadPointer =
18880 SDValue TGA =
18883 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18901 SDValue
18902 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
18945 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18948 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18958 SDValue Chain = DAG.getEntryNode();
18961 SDValue Args[] = { Chain, Offset };
18990 SDValue Chain = DAG.getEntryNode();
19001 SDValue TlsArray = Subtarget.is64Bit()
19007 SDValue ThreadPointer =
19010 SDValue res;
19015 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
19023 SDValue Scale =
19033 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
19036 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
19049 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
19055 SDValue ShOpLo = Op.getOperand(0);
19056 SDValue ShOpHi = Op.getOperand(1);
19057 SDValue ShAmt = Op.getOperand(2);
19061 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
19063 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
19067 SDValue Tmp2, Tmp3;
19079 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
19081 SDValue Cond = DAG.getSetCC(dl, MVT::i8, AndNode,
19084 SDValue Hi, Lo;
19096 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
19103 SDValue Op0 = Op.getOperand(0);
19104 SDValue Op1 = Op.getOperand(1);
19105 SDValue Amt = Op.getOperand(2);
19138 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
19139 SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
19143 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
19155 return SDValue();
19170 static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
19179 SDValue Src = Op.getOperand(OpNo);
19185 return SDValue();
19195 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
19197 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
19199 SDValue Chain = CvtVec.getValue(1);
19200 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19205 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
19236 static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
19240 SDValue Extract = Cast.getOperand(0);
19244 return SDValue();
19247 SDValue VecOp = Extract.getOperand(0);
19253 return SDValue();
19270 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
19278 static SDValue lowerFPToIntToFP(SDValue CastToFP, SelectionDAG &DAG,
19281 SDValue CastToInt = CastToFP.getOperand(0);
19284 return SDValue();
19287 SDValue X = CastToInt.getOperand(0);
19290 return SDValue();
19296 return SDValue();
19318 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
19319 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19320 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19321 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19325 static SDValue lowerINT_TO_FP_vXi64(SDValue Op, SelectionDAG &DAG,
19330 SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
19346 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19350 SDValue Res, Chain;
19370 return SDValue();
19372 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19373 SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
19374 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19377 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19378 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19379 SmallVector<SDValue, 4> SignCvts(4);
19380 SmallVector<SDValue, 4> Chains(4);
19382 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19393 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19395 SDValue Slow, Chain;
19406 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19414 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
19418 SDValue Src = Op.getOperand(OpNo);
19419 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19424 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19427 if (SDValue R = lowerFPToIntToFP(Op, DAG, Subtarget))
19446 return SDValue();
19461 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19466 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
19477 SDValue ValueToStore = Src;
19491 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19493 std::pair<SDValue, SDValue> Tmp =
19502 std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
19503 EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
19513 SDValue FILDOps[] = {Chain, Pointer};
19514 SDValue Result =
19525 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19527 SDValue FSTOps[] = {Chain, Result, StackSlot};
19554 static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG,
19578 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
19588 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
19591 SDValue XR1 =
19593 SDValue CLod0 =
19597 SDValue Unpck1 =
19600 SDValue CLod1 =
19604 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
19605 SDValue Sub;
19606 SDValue Chain;
19614 SDValue Result;
19621 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
19638 static SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG,
19643 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl,
19647 SDValue Load =
19654 SDValue Or = DAG.getNode(
19666 SDValue Chain = Op.getOperand(0);
19667 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
19674 std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
19682 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19688 static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,
19692 return SDValue();
19696 SDValue N0 = Op.getOperand(IsStrict ? 1 : 0);
19703 return SDValue();
19707 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
19709 SDValue Chain = Res.getValue(1);
19728 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
19729 SDValue VBias =
19731 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
19741 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
19745 SDValue V = Op->getOperand(IsStrict ? 1 : 0);
19765 SDValue Tmp =
19769 SDValue Res, Chain;
19788 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
19793 SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8));
19795 SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
19796 SDValue VBias = DAG.getMemIntrinsicNode(
19801 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
19828 return SDValue();
19838 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
19840 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
19843 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
19844 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
19846 SDValue Low, High;
19850 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
19851 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
19858 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
19859 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
19865 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
19867 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
19875 SDValue VecCstFSub = DAG.getConstantFP(
19882 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
19885 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
19888 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
19894 SDValue FHigh =
19899 static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
19902 SDValue N0 = Op.getOperand(OpNo);
19920 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
19924 SDValue Src = Op.getOperand(OpNo);
19929 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19937 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19956 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19964 return SDValue();
19967 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);
19972 SDValue OffsetSlot = DAG.getMemBasePlusOffset(StackSlot, 4, dl);
19973 SDValue Store1 =
19975 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
19977 std::pair<SDValue, SDValue> Tmp =
19986 SDValue ValueToStore = Src;
19993 SDValue Store =
20001 SDValue Ops[] = { Store, StackSlot };
20002 SDValue Fild =
20009 SDValue SignSet = DAG.getSetCC(
20015 SDValue FudgePtr = DAG.getConstantPool(
20020 SDValue Zero = DAG.getIntPtrConstant(0, dl);
20021 SDValue Four = DAG.getIntPtrConstant(4, dl);
20022 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero);
20026 SDValue Fudge = DAG.getExtLoad(
20034 SDValue Add = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::f80, MVT::Other},
20042 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
20049 // just return an SDValue().
20053 SDValue
20054 X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
20055 bool IsSigned, SDValue &Chain) const {
20060 SDValue Value = Op.getOperand(IsStrict ? 1 : 0);
20067 return SDValue();
20094 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
20098 SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
20132 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
20136 SDValue Cmp;
20149 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp,
20169 SDValue Ops[] = { Chain, StackSlot };
20182 SDValue Ops[] = { Chain, Value, StackSlot };
20183 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
20187 SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI);
20197 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
20200 SDValue In = Op.getOperand(0);
20242 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
20250 SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
20251 SDValue Undef = DAG.getUNDEF(InVT);
20253 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
20260 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
20263 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20265 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20269 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
20273 static SDValue LowerZERO_EXTEND_Mask(SDValue Op,
20277 SDValue In = Op->getOperand(0);
20286 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20312 SDValue One = DAG.getConstant(1, DL, WideVT);
20313 SDValue Zero = DAG.getConstant(0, DL, WideVT);
20315 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20331 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
20333 SDValue In = Op.getOperand(0);
20348 static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
20357 return SDValue();
20370 return SDValue();
20374 return SDValue();
20396 SDValue Res = DAG.getNode(Opcode, DL, OutVT, In, DAG.getUNDEF(InVT));
20402 SDValue Lo, Hi;
20413 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20422 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20448 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20452 static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
20457 SDValue In = Op.getOperand(0);
20494 SDValue Lo, Hi;
20533 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
20536 SDValue In = Op.getOperand(0);
20553 SDValue Lo, Hi;
20565 return SDValue();
20595 if (SDValue V =
20602 if (SDValue V =
20619 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20621 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20649 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
20652 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
20670 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
20678 SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
20680 SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
20688 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
20693 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
20716 SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
20721 SDValue Res, Chain;
20756 SDValue Tmp =
20761 SDValue Res, Chain;
20787 SDValue Tmp =
20792 SDValue Res, Chain;
20814 return SDValue();
20816 SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);
20817 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,
20821 SDValue Chain = Tmp.getValue(1);
20830 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
20841 return SDValue();
20855 return SDValue();
20863 SDValue Res, Chain;
20880 return SDValue();
20888 SDValue Res, Chain;
20914 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
20916 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, Src, CallOptions,
20926 SDValue Chain;
20927 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
20936 SDValue X86TargetLowering::LowerLRINT_LLRINT(SDValue Op,
20938 SDValue Src = Op.getOperand(0);
20948 SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
20951 SDValue Src = N->getOperand(0);
20957 return SDValue();
20961 SDValue Chain = DAG.getEntryNode();
20968 SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT);
20977 SDValue Ops[] = { Chain, StackPtr };
20984 SDValue StoreOps[] = { Chain, Src, StackPtr };
20992 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
20997 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21007 SDValue Res =
21015 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21019 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21033 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21035 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, In, CallOptions,
21044 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
21046 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21051 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
21055 SDValue Chain;
21073 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
21075 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21080 SDValue Res, Chain;
21107 static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
21110 SDValue LHS = Op.getOperand(0);
21111 SDValue RHS = Op.getOperand(1);
21151 SDValue X = LHS.getOperand(0);
21172 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
21179 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
21189 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
21190 SDValue N0 = Op.getOperand(0);
21201 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
21211 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
21251 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
21253 SDValue Op0 = Op.getOperand(0);
21258 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
21266 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21271 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
21272 SDValue Mag = Op.getOperand(0);
21273 SDValue Sign = Op.getOperand(1);
21307 SDValue SignMask = DAG.getConstantFP(
21309 SDValue MagMask = DAG.getConstantFP(
21315 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
21320 SDValue MagBits;
21333 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
21338 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
21339 SDValue N0 = Op.getOperand(0);
21349 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
21357 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
21367 static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
21368 SmallVectorImpl<SDValue> &SrcOps,
21370 SmallVector<SDValue, 8> Opnds;
21371 DenseMap<SDValue, APInt> SrcOpMap;
21382 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
21401 SDValue Src = I->getOperand(0);
21402 DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
21424 for (SDValue &SrcOp : SrcOps)
21428 for (DenseMap<SDValue, APInt>::const_iterator I = SrcOpMap.begin(),
21440 static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC,
21451 auto MaskBits = [&](SDValue Src) {
21455 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
21463 return SDValue();
21471 return SDValue();
21491 return SDValue();
21503 static SDValue MatchVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
21506 SelectionDAG &DAG, SDValue &X86CC) {
21510 return SDValue();
21517 SDValue Src = Op.getOperand(0);
21532 SmallVector<SDValue, 8> VecIns;
21536 [VT](SDValue V) { return VT == V.getValueType(); }) &&
21541 return SDValue();
21548 SDValue LHS = VecIns[Slot];
21549 SDValue RHS = VecIns[Slot + 1];
21554 if (SDValue V = LowerVectorAllZero(DL, VecIns.back(), CC, Mask, Subtarget,
21563 if (SDValue Match =
21566 if (SDValue V =
21574 return SDValue();
21578 static bool hasNonFlagsUse(SDValue Op) {
21599 static bool isProfitableToUseFlagOp(SDValue Op) {
21611 static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
21655 SDValue ArithOp = Op;
21692 return SDValue(Op.getNode(), 1);
21710 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
21712 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
21713 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
21714 return SDValue(New.getNode(), 1);
21719 static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
21745 SDValue In = Op0.getOperand(0);
21751 SDValue In = Op1.getOperand(0);
21781 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1);
21790 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1));
21796 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
21801 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
21815 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
21843 return SDValue();
21848 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
21868 return SDValue();
21877 return SDValue();
21890 SDValue
21896 return SDValue(N,0); // Lower SDIV as SDIV
21904 return SDValue();
21911 return SDValue();
21917 return SDValue();
21920 SDValue N0 = N->getOperand(0);
21921 SDValue Zero = DAG.getConstant(0, DL, VT);
21923 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
21926 SDValue Cmp = DAG.getSetCC(DL, MVT::i8, N0, Zero, ISD::SETLT);
21927 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
21928 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
21935 SDValue SRA =
21949 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
21951 SDValue &X86CC) {
21953 SDValue Op0 = And.getOperand(0);
21954 SDValue Op1 = And.getOperand(1);
21960 SDValue Src, BitNo;
21972 return SDValue();
21980 SDValue AndLHS = Op0;
22000 return SDValue();
22030 static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
22031 SDValue &Op1, bool &IsAlwaysSignaling) {
22091 static SDValue splitIntVSETCC(SDValue Op, SelectionDAG &DAG) {
22099 SDValue CC = Op.getOperand(2);
22102 SDValue LHS1, LHS2;
22106 SDValue RHS1, RHS2;
22117 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
22119 SDValue Op0 = Op.getOperand(0);
22120 SDValue Op1 = Op.getOperand(1);
22121 SDValue CC = Op.getOperand(2);
22143 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc) {
22146 return SDValue();
22151 SmallVector<SDValue, 8> NewVecC;
22156 return SDValue();
22161 return SDValue();
22173 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
22178 return SDValue();
22182 return SDValue();
22186 return SDValue();
22194 return SDValue();
22195 SDValue ULEOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/false);
22197 return SDValue();
22206 SDValue UGEOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/true);
22208 return SDValue();
22221 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
22226 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
22230 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
22231 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
22232 SDValue CC = Op.getOperand(IsStrict ? 3 : 2);
22245 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
22265 SDValue Cmp;
22277 return SDValue();
22281 SDValue SignalCmp = DAG.getNode(
22309 SDValue Cmp0, Cmp1;
22422 SDValue BC0 = peekThroughBitcasts(Op0);
22445 SDValue Result = Op0.getOperand(0);
22492 if (SDValue UGTOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/true)) {
22499 if (SDValue ULTOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/false)) {
22514 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
22525 if (SDValue V = LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
22553 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
22555 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
22564 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
22566 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
22574 SDValue SB;
22588 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
22589 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
22594 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
22595 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
22596 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
22598 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
22617 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
22621 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
22635 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
22641 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
22651 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
22654 SDValue &X86CC) {
22657 return SDValue();
22661 return SDValue();
22668 return SDValue();
22677 return SDValue();
22688 SDValue LHS = Op0.getOperand(0);
22689 SDValue RHS = Op0.getOperand(1);
22695 SDValue LHS = Op0;
22696 SDValue RHS = Op0;
22708 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
22711 SDValue &X86CC) const {
22718 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CC))
22725 if (SDValue CmpZ =
22730 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
22760 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
22762 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
22765 return SDValue(New.getNode(), 1);
22773 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget);
22778 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
22787 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
22788 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
22789 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
22811 SDValue X86CC;
22812 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
22813 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
22820 return SDValue();
22822 SDValue EFLAGS;
22833 SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
22834 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
22838 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
22839 SDValue LHS = Op.getOperand(0);
22840 SDValue RHS = Op.getOperand(1);
22841 SDValue Carry = Op.getOperand(2);
22842 SDValue Cond = Op.getOperand(3);
22854 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
22862 static std::pair<SDValue, SDValue>
22863 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
22865 SDValue Value, Overflow;
22866 SDValue LHS = Op.getOperand(0);
22867 SDValue RHS = Op.getOperand(1);
22908 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
22915 SDValue Value, Overflow;
22918 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
22924 static bool isX86LogicalCmp(SDValue Op) {
22938 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
22942 SDValue VOp0 = V.getOperand(0);
22948 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
22950 SDValue Cond = Op.getOperand(0);
22951 SDValue Op1 = Op.getOperand(1);
22952 SDValue Op2 = Op.getOperand(2);
22955 SDValue CC;
22962 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
22969 SDValue Cmp =
22977 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
22997 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
22998 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
22999 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
23004 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
23009 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
23010 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
23017 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
23022 if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
23041 SDValue Cmp = Cond.getOperand(1);
23042 SDValue CmpOp0 = Cmp.getOperand(0);
23050 auto MatchFFSMinus1 = [&](SDValue Op1, SDValue Op2) {
23060 SDValue Y = isAllOnesConstant(Op2) ? Op1 : Op2;
23070 SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
23071 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0);
23079 SDValue Zero = DAG.getConstant(0, DL, Op.getValueType());
23080 SDValue Res = // Res = 0 or -1.
23090 SDValue Src1, Src2;
23106 SDValue Neg;
23118 SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
23120 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
23139 SDValue Cmp = Cond.getOperand(1);
23153 SDValue Value;
23169 SDValue BTCC;
23170 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, BTCC)) {
23193 SDValue Res =
23207 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
23211 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
23228 SDValue Ops[] = { Op2, Op1, CC, Cond };
23229 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
23235 SDValue Ops[] = { Op2, Op1, CC, Cond };
23239 static SDValue LowerSIGN_EXTEND_Mask(SDValue Op,
23243 SDValue In = Op->getOperand(0);
23271 SDValue V;
23277 SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
23278 SDValue Zero = DAG.getConstant(0, dl, WideVT);
23296 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
23298 SDValue In = Op->getOperand(0);
23312 static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
23315 SDValue In = Op->getOperand(0);
23324 return SDValue();
23326 return SDValue();
23330 return SDValue();
23374 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
23375 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
23385 SDValue Curr = In;
23386 SDValue SignExt = Curr;
23415 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
23416 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
23424 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
23427 SDValue In = Op->getOperand(0);
23463 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
23470 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
23477 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
23478 SDValue StoredVal = Store->getValue();
23489 return SDValue();
23492 SDValue Value0, Value1;
23495 SDValue Ptr0 = Store->getBasePtr();
23496 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, HalfOffset, DL);
23497 SDValue Ch0 =
23501 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
23510 static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
23512 SDValue StoredVal = Store->getValue();
23521 return SDValue();
23528 SmallVector<SDValue, 4> Stores;
23531 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), Offset, DL);
23532 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
23534 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
23543 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
23547 SDValue StoredVal = St->getValue();
23570 return SDValue();
23580 SmallVector<SDValue, 4> CatOps;
23583 return SDValue();
23611 SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
23623 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
23640 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
23647 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
23654 return SDValue();
23659 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
23669 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
23670 SDValue Chain = Op.getOperand(0);
23671 SDValue Cond = Op.getOperand(1);
23672 SDValue Dest = Op.getOperand(2);
23677 SDValue LHS = Cond.getOperand(0);
23678 SDValue RHS = Cond.getOperand(1);
23687 SDValue Value, Overflow;
23694 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23700 SDValue CCVal;
23701 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
23718 SDValue FalseBB = User->getOperand(1);
23725 SDValue Cmp =
23727 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
23739 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
23740 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
23749 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
23750 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23757 SDValue Value, Overflow;
23761 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23777 SDValue LHS = Cond;
23778 SDValue RHS = DAG.getConstant(0, dl, CondVT);
23780 SDValue CCVal;
23781 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
23791 SDValue
23792 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
23803 SDValue Chain = Op.getOperand(0);
23804 SDValue Size = Op.getOperand(1);
23815 SDValue Result;
23833 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
23868 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
23881 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
23883 SDValue Ops[2] = {Result, Chain};
23887 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
23899 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
23909 SmallVector<SDValue, 8> MemOps;
23910 SDValue FIN = Op.getOperand(1);
23912 SDValue Store = DAG.getStore(
23928 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
23936 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
23944 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
23954 SDValue Chain = Op.getOperand(0);
23955 SDValue SrcPtr = Op.getOperand(1);
23986 SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32),
23990 SDValue VAARG = DAG.getMemIntrinsicNode(
23999 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
24009 SDValue Chain = Op.getOperand(0);
24010 SDValue DstPtr = Op.getOperand(1);
24011 SDValue SrcPtr = Op.getOperand(2);
24042 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
24043 SDValue SrcOp, uint64_t ShiftAmt,
24070 SmallVector<SDValue, 8> Elts;
24077 SDValue CurrentOp = SrcOp->getOperand(i);
24090 SDValue CurrentOp = SrcOp->getOperand(i);
24103 SDValue CurrentOp = SrcOp->getOperand(i);
24125 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
24126 SDValue SrcOp, SDValue ShAmt,
24165 SDValue ByteShift = DAG.getTargetConstant(
24179 SDValue ShOps[4] = {ShAmt, DAG.getConstant(0, dl, SVT), DAG.getUNDEF(SVT),
24195 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
24210 SDValue Lo, Hi;
24234 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
24235 SDValue PreservedSrc,
24246 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24260 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
24261 SDValue PreservedSrc,
24273 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
24309 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
24310 SDValue EntryEBP) {
24328 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
24329 SDValue ParentFrameOffset =
24342 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
24347 SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
24350 auto isRoundModeCurDirection = [](SDValue Rnd) {
24356 auto isRoundModeSAE = [](SDValue Rnd) {
24370 auto isRoundModeSAEToX = [](SDValue Rnd, unsigned &RC) {
24399 SDValue Rnd = Op.getOperand(2);
24406 return SDValue();
24412 SDValue Sae = Op.getOperand(2);
24420 return SDValue();
24425 SDValue Src2 = Op.getOperand(2);
24432 SDValue Rnd = Op.getOperand(3);
24439 return SDValue();
24446 SDValue Sae = Op.getOperand(3);
24454 return SDValue();
24461 SDValue Src1 = Op.getOperand(1);
24462 SDValue Src2 = Op.getOperand(2);
24463 SDValue Src3 = Op.getOperand(3);
24470 SDValue Rnd = Op.getOperand(4);
24477 return SDValue();
24487 SDValue Src = Op.getOperand(1);
24488 SDValue PassThru = Op.getOperand(2);
24489 SDValue Mask = Op.getOperand(3);
24495 SDValue Rnd = Op.getOperand(4);
24503 return SDValue();
24510 SDValue Src = Op.getOperand(1);
24511 SDValue PassThru = Op.getOperand(2);
24512 SDValue Mask = Op.getOperand(3);
24513 SDValue Rnd = Op.getOperand(4);
24521 return SDValue();
24527 SDValue Src1 = Op.getOperand(1);
24528 SDValue Src2 = Op.getOperand(2);
24529 SDValue passThru = Op.getOperand(3);
24530 SDValue Mask = Op.getOperand(4);
24538 SDValue Rnd = Op.getOperand(5);
24546 return SDValue();
24555 SDValue RoundingMode = Op.getOperand(5);
24558 SDValue Sae = Op.getOperand(6);
24562 return SDValue();
24569 SDValue Src1 = Op.getOperand(1);
24570 SDValue Src2 = Op.getOperand(2);
24571 SDValue passThru = Op.getOperand(3);
24572 SDValue Mask = Op.getOperand(4);
24573 SDValue Rnd = Op.getOperand(5);
24575 SDValue NewOp;
24583 return SDValue();
24588 SDValue Src1 = Op.getOperand(1);
24589 SDValue Src2 = Op.getOperand(2);
24590 SDValue passThru = Op.getOperand(3);
24591 SDValue Mask = Op.getOperand(4);
24592 SDValue Sae = Op.getOperand(5);
24599 return SDValue();
24605 SDValue Src1 = Op.getOperand(1);
24606 SDValue Src2 = Op.getOperand(2);
24607 SDValue PassThru = Op.getOperand(3);
24608 SDValue Mask = Op.getOperand(4);
24609 SDValue NewOp;
24611 SDValue Rnd = Op.getOperand(5);
24617 return SDValue();
24624 SDValue Src1 = Op.getOperand(1);
24625 SDValue Src2 = Op.getOperand(2);
24626 SDValue PassThru = Op.getOperand(3);
24627 SDValue Mask = Op.getOperand(4);
24631 SDValue Sae = Op.getOperand(5);
24635 return SDValue();
24642 SDValue Src1 = Op.getOperand(1);
24643 SDValue Src2 = Op.getOperand(2);
24644 SDValue Src3 = Op.getOperand(3);
24645 SDValue PassThru = Op.getOperand(4);
24646 SDValue Mask = Op.getOperand(5);
24647 SDValue Sae = Op.getOperand(6);
24654 return SDValue();
24660 SDValue Src1 = Op.getOperand(1);
24661 SDValue Src2 = Op.getOperand(2);
24662 SDValue Src3 = Op.getOperand(3);
24663 SDValue PassThru = Op.getOperand(4);
24664 SDValue Mask = Op.getOperand(5);
24668 SDValue Sae = Op.getOperand(6);
24672 return SDValue();
24678 SDValue Src1 = Op.getOperand(1);
24679 SDValue Src2 = Op.getOperand(2);
24680 SDValue Src3 = Op.getOperand(3);
24689 SDValue Src1 = Op.getOperand(1);
24690 SDValue Src2 = Op.getOperand(2);
24701 SDValue Src1 = Op.getOperand(1);
24702 SDValue Imm = Op.getOperand(2);
24703 SDValue Mask = Op.getOperand(3);
24704 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
24705 SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
24709 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
24717 SDValue CC = Op.getOperand(3);
24722 SDValue Sae = Op.getOperand(4);
24727 return SDValue();
24734 SDValue Src1 = Op.getOperand(1);
24735 SDValue Src2 = Op.getOperand(2);
24736 SDValue CC = Op.getOperand(3);
24737 SDValue Mask = Op.getOperand(4);
24739 SDValue Cmp;
24741 SDValue Sae = Op.getOperand(5);
24745 return SDValue();
24751 SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
24755 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
24762 SDValue LHS = Op.getOperand(1);
24763 SDValue RHS = Op.getOperand(2);
24768 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
24769 SDValue SetCC;
24773 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
24779 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
24798 SDValue LHS = Op.getOperand(1);
24799 SDValue RHS = Op.getOperand(2);
24801 SDValue Sae = Op.getOperand(4);
24803 SDValue FCmp;
24811 return SDValue();
24814 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
24825 SDValue Mask = Op.getOperand(3);
24826 SDValue DataToCompress = Op.getOperand(1);
24827 SDValue PassThru = Op.getOperand(2);
24840 SDValue Src1 = Op.getOperand(1);
24841 SDValue Src2 = Op.getOperand(2);
24842 SDValue Src3 = Op.getOperand(3);
24843 SDValue Imm = Op.getOperand(4);
24844 SDValue Mask = Op.getOperand(5);
24845 SDValue Passthru = (IntrData->Type == FIXUPIMM)
24851 SDValue Sae = Op.getOperand(6);
24855 return SDValue();
24858 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
24870 SDValue RoundingMode =
24880 SDValue RoundingMode =
24891 SDValue Control = DAG.getConstant(Imm, dl, Op.getValueType());
24900 SDValue Res;
24907 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
24912 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
24913 SDValue Results[] = { SetCC, Res };
24920 SDValue Src = Op.getOperand(1);
24921 SDValue PassThru = Op.getOperand(2);
24922 SDValue Mask = Op.getOperand(3);
24934 SDValue Src = Op.getOperand(1);
24935 SDValue Rnd = Op.getOperand(2);
24936 SDValue PassThru = Op.getOperand(3);
24937 SDValue Mask = Op.getOperand(4);
24950 SDValue Src = Op.getOperand(1);
24951 SDValue PassThru = Op.getOperand(2);
24952 SDValue Mask = Op.getOperand(3);
24970 default: return SDValue(); // Don't custom lower most intrinsics.
25055 SDValue LHS = Op.getOperand(1);
25056 SDValue RHS = Op.getOperand(2);
25057 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
25058 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
25117 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
25119 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
25120 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
25132 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
25145 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
25164 SDValue Op1 = Op.getOperand(1);
25171 SDValue Result = DAG.getMCSymbol(LSDASym, VT);
25176 SDValue FnOp = Op.getOperand(1);
25177 SDValue IncomingFPOp = Op.getOperand(2);
25215 SDValue Operation =
25219 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
25221 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
25234 SDValue ShAmt = Op.getOperand(2);
25289 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
25290 SDValue Src, SDValue Mask, SDValue Base,
25291 SDValue Index, SDValue ScaleOp, SDValue Chain,
25297 return SDValue();
25299 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
25314 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
25315 SDValue Res =
25321 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
25322 SDValue Src, SDValue Mask, SDValue Base,
25323 SDValue Index, SDValue ScaleOp, SDValue Chain,
25330 return SDValue();
25332 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
25352 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
25353 SDValue Res =
25359 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
25360 SDValue Src, SDValue Mask, SDValue Base,
25361 SDValue Index, SDValue ScaleOp, SDValue Chain,
25367 return SDValue();
25369 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
25383 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
25384 SDValue Res =
25390 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
25391 SDValue Mask, SDValue Base, SDValue Index,
25392 SDValue ScaleOp, SDValue Chain,
25398 return SDValue();
25400 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
25402 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
25403 SDValue Segment = DAG.getRegister(0, MVT::i32);
25406 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25407 SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
25409 return SDValue(Res, 0);
25420 static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
25425 SmallVectorImpl<SDValue> &Results) {
25426 SDValue Chain = N->getOperand(0);
25427 SDValue Glue;
25436 SDValue N1Ops[] = {Chain, Glue};
25438 TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
25439 Chain = SDValue(N1, 0);
25442 SDValue LO, HI;
25444 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
25448 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
25457 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
25465 SDValue Ops[] = { LO, HI };
25466 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
25478 SmallVectorImpl<SDValue> &Results) {
25482 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
25488 SDValue Chain = Results[1];
25491 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
25496 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
25498 SmallVector<SDValue, 3> Results;
25505 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
25507 SDValue Chain = Op.getOperand(0);
25508 SDValue RegNode = Op.getOperand(2);
25523 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
25525 SDValue Chain = Op.getOperand(0);
25526 SDValue EHGuard = Op.getOperand(2);
25542 static SDValue
25543 EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val,
25544 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
25547 SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
25548 SDValue Ops[] = { Chain, Val, Ptr, Undef };
25554 static SDValue
25555 EmitMaskedTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl,
25556 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
25559 SDValue Ops[] = { Chain, Val, Ptr, Mask };
25564 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
25607 SDValue Chain = Op->getOperand(0);
25625 SDValue Operation =
25628 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
25635 SDValue Chain = Op.getOperand(0);
25647 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
25649 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
25654 return SDValue();
25664 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
25668 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
25671 SDValue(Result.getNode(), 1)};
25672 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
25676 SDValue(Result.getNode(), 2));
25679 SDValue Chain = Op.getOperand(0);
25680 SDValue Src = Op.getOperand(2);
25681 SDValue Base = Op.getOperand(3);
25682 SDValue Index = Op.getOperand(4);
25683 SDValue Mask = Op.getOperand(5);
25684 SDValue Scale = Op.getOperand(6);
25690 SDValue Chain = Op.getOperand(0);
25691 SDValue Src = Op.getOperand(2);
25692 SDValue Base = Op.getOperand(3);
25693 SDValue Index = Op.getOperand(4);
25694 SDValue Mask = Op.getOperand(5);
25695 SDValue Scale = Op.getOperand(6);
25701 SDValue Chain = Op.getOperand(0);
25702 SDValue Base = Op.getOperand(2);
25703 SDValue Mask = Op.getOperand(3);
25704 SDValue Index = Op.getOperand(4);
25705 SDValue Src = Op.getOperand(5);
25706 SDValue Scale = Op.getOperand(6);
25715 SDValue Chain = Op.getOperand(0);
25716 SDValue Mask = Op.getOperand(2);
25717 SDValue Index = Op.getOperand(3);
25718 SDValue Base = Op.getOperand(4);
25719 SDValue Scale = Op.getOperand(5);
25725 SmallVector<SDValue, 2> Results;
25734 SmallVector<SDValue, 2> Results;
25746 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
25748 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
25749 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
25751 Ret, SDValue(InTrans.getNode(), 1));
25756 SDValue Mask = Op.getOperand(4);
25757 SDValue DataToTruncate = Op.getOperand(3);
25758 SDValue Addr = Op.getOperand(2);
25759 SDValue Chain = Op.getOperand(0);
25774 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25775 SDValue Offset = DAG.getUNDEF(VMask.getValueType());
25789 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25801 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
25807 return SDValue();
25814 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
25816 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
25823 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
25828 SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
25834 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
25865 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
25905 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
25930 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
25931 SDValue Chain = Op.getOperand(0);
25932 SDValue Offset = Op.getOperand(1);
25933 SDValue Handler = Op.getOperand(2);
25942 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
25945 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
25956 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
25974 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
25981 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
25988 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
25992 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
25994 SDValue Root = Op.getOperand(0);
25995 SDValue Trmp = Op.getOperand(1); // trampoline
25996 SDValue FPtr = Op.getOperand(2); // nested function
25997 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
26004 SDValue OutChains[6];
26017 SDValue Addr = Trmp;
26103 SDValue OutChains[4];
26104 SDValue Addr, Disp;
26140 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
26170 SDValue StackSlot =
26175 SDValue Chain = Op.getOperand(0);
26176 SDValue Ops[] = {Chain, StackSlot};
26182 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
26186 SDValue Shift =
26193 SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
26194 SDValue RetVal =
26210 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
26232 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
26233 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
26234 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
26240 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
26254 SmallVector<SDValue, 64> LUTVec;
26257 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
26264 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
26265 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
26267 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
26268 SDValue Lo = Op0;
26269 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
26270 SDValue HiZ;
26282 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
26294 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
26311 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
26312 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
26313 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
26322 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
26344 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
26368 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
26383 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
26387 SDValue N0 = Op.getOperand(0);
26398 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
26404 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
26423 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
26426 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
26451 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, X, Y);
26452 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Add, ISD::SETUGT);
26457 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
26458 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
26462 return SDValue();
26474 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
26481 SDValue N0 = Op.getOperand(0);
26482 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
26484 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_GE, DL, MVT::i8),
26485 SDValue(Neg.getNode(), 1)};
26492 SDValue Src = Op.getOperand(0);
26493 SDValue Sub =
26508 return SDValue();
26511 static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) {
26523 SDValue N0 = Op.getOperand(0);
26524 SDValue N1 = Op.getOperand(1);
26531 SDValue Sign = DAG.getConstant(APInt::getSignedMinValue(16), DL, VT);
26535 SDValue Result = DAG.getNode(Opcode, DL, VT, N0, N1);
26549 SDValue Cond = DAG.getSetCC(DL, VT, N0, N1, CC);
26553 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
26568 SDValue A = Op.getOperand(0);
26569 SDValue B = Op.getOperand(1);
26592 SDValue Undef = DAG.getUNDEF(VT);
26593 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
26594 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
26596 SDValue BLo, BHi;
26599 SmallVector<SDValue, 16> LoOps, HiOps;
26617 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
26618 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
26631 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
26632 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
26635 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
26639 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
26676 SDValue Zero = DAG.getConstant(0, dl, VT);
26679 SDValue AloBlo = Zero;
26683 SDValue AloBhi = Zero;
26685 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
26689 SDValue AhiBlo = Zero;
26691 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
26695 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
26701 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
26707 SDValue A = Op.getOperand(0);
26708 SDValue B = Op.getOperand(1);
26737 SDValue Odd0 = DAG.getVectorShuffle(VT, dl, A, A,
26740 SDValue Odd1 = DAG.getVectorShuffle(VT, dl, B, B,
26750 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
26755 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
26764 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
26769 SDValue Zero = DAG.getConstant(0, dl, VT);
26770 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
26772 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
26775 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
26797 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
26798 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
26799 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
26820 SDValue ALo, AHi;
26839 SDValue BLo, BHi;
26842 SmallVector<SDValue, 16> LoOps, HiOps;
26845 SDValue LoOp = B.getOperand(i + j);
26846 SDValue HiOp = B.getOperand(i + j + 8);
26883 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
26884 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
26892 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
26911 SDValue InChain = DAG.getEntryNode();
26919 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
26933 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
26947 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
26998 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
27002 SDValue R = Op.getOperand(0);
27003 SDValue Amt = Op.getOperand(1);
27009 SDValue Ex = DAG.getBitcast(ExVT, R);
27020 SDValue Upper =
27022 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
27031 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
27033 SDValue Lower =
27048 return SDValue();
27076 SDValue Zeros = DAG.getConstant(0, dl, VT);
27079 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
27087 return SDValue();
27091 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
27100 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
27109 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
27111 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
27119 return SDValue();
27122 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
27126 SDValue R = Op.getOperand(0);
27127 SDValue Amt = Op.getOperand(1);
27132 if (SDValue BaseShAmt = DAG.getSplatValue(Amt)) {
27158 SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
27168 SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
27177 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
27194 std::vector<SDValue> Vals(Ratio);
27200 return SDValue();
27206 return SDValue();
27210 static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
27217 return SDValue();
27220 SmallVector<SDValue, 8> Elts;
27227 SDValue Op = Amt->getOperand(i);
27257 SDValue Z = DAG.getConstant(0, dl, VT);
27258 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
27259 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
27270 return SDValue();
27273 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
27277 SDValue R = Op.getOperand(0);
27278 SDValue Amt = Op.getOperand(1);
27289 if (SDValue V = LowerScalarImmediateShift(Op, DAG, Subtarget))
27292 if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget))
27303 SDValue Zero = DAG.getConstant(0, dl, VT);
27316 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
27317 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
27318 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
27319 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
27328 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
27329 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
27348 SDValue Amt1, Amt2;
27352 SDValue A = Amt->getOperand(i);
27380 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
27382 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
27392 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
27399 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
27400 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
27401 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
27402 SDValue Zero = DAG.getConstant(0, dl, VT);
27403 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
27404 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
27418 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
27419 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
27420 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
27421 SDValue Amt0 =
27423 SDValue Amt1 =
27425 SDValue Sra1 =
27427 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
27439 SDValue Amt0, Amt1, Amt2, Amt3;
27451 SDValue Z = DAG.getConstant(0, dl, VT);
27457 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
27458 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
27472 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
27473 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
27474 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
27475 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
27480 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
27481 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
27484 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
27485 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
27516 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
27535 SmallVector<SDValue, 16> LoAmt, HiAmt;
27544 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
27545 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
27547 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
27548 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
27563 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
27587 SDValue Z = DAG.getConstant(0, dl, SelVT);
27588 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
27601 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
27624 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
27625 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
27626 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
27627 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
27634 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
27635 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
27669 SDValue Z = DAG.getConstant(0, dl, VT);
27670 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
27671 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
27672 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
27673 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
27678 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
27679 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
27691 auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
27705 SDValue C =
27723 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
27756 return SDValue();
27759 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
27765 SDValue R = Op.getOperand(0);
27766 SDValue Amt = Op.getOperand(1);
27825 return SDValue();
27833 return SDValue();
27838 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
27851 SDValue Z = DAG.getConstant(0, DL, SelVT);
27852 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
27864 SDValue M;
27903 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
27905 SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);
27906 SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);
27911 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
27916 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
27917 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
27926 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
27927 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
27929 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
27932 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
28115 static SDValue emitLockedStackOp(SelectionDAG &DAG,
28117 SDValue Chain, SDLoc DL) {
28147 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
28148 SDValue Ops[] = {
28158 return SDValue(Res, 1);
28161 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
28162 SDValue Ops[] = {
28173 return SDValue(Res, 1);
28176 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
28191 SDValue Chain = Op.getOperand(0);
28199 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
28215 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
28216 Op.getOperand(2), SDValue());
28217 SDValue Ops[] = { cpIn.getValue(0),
28224 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
28227 SDValue cpOut =
28229 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
28231 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
28238 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
28243 SDValue Lo, Hi;
28254 SDValue Lo, Hi;
28266 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
28268 SDValue Src = Op.getOperand(0);
28278 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,
28281 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,
28292 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
28304 return SDValue();
28336 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
28352 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
28364 SDValue Zeros = DAG.getConstant(0, DL, VT);
28365 SDValue V32 = DAG.getBitcast(VT, V);
28366 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
28367 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
28393 SDValue ShifterV = DAG.getConstant(8, DL, VT);
28394 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
28400 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
28424 SmallVector<SDValue, 64> LUTVec;
28427 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
28428 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
28431 SDValue FourV = DAG.getConstant(4, DL, VT);
28432 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
28435 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
28440 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
28441 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
28447 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
28453 SDValue Op0 = Op.getOperand(0);
28479 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
28480 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
28486 return SDValue();
28491 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
28498 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
28500 SDValue In = Op.getOperand(0);
28507 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
28527 SmallVector<SDValue, 16> MaskElts;
28536 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
28537 SDValue Res = DAG.getBitcast(MVT::v16i8, In);
28543 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
28552 SDValue In = Op.getOperand(0);
28570 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
28571 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
28572 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
28585 SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
28591 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
28592 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
28598 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
28630 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
28633 SDValue Chain = N->getOperand(0);
28634 SDValue LHS = N->getOperand(1);
28635 SDValue RHS = N->getOperand(2);
28672 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
28679 SDValue NewChain = DAG.getNode(X86ISD::MEMBARRIER, DL, MVT::Other, Chain);
28686 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
28694 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
28715 SDValue Chain;
28717 SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
28722 SDValue Ops[] = {Node->getChain(), SclToVec, Node->getBasePtr()};
28728 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
28736 SDValue LdOps[] = {Chain, StackPtr};
28737 SDValue Value =
28743 SDValue StoreOps[] = {Chain, Value, Node->getBasePtr()};
28763 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
28771 static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
28777 return SDValue();
28783 SDValue Carry = Op.getOperand(2);
28789 SDValue Sum = DAG.getNode(Opc, DL, VTs, Op.getOperand(0),
28792 SDValue SetCC = getSETCC(X86::COND_B, Sum.getValue(1), DL, DAG);
28799 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
28807 SDValue Arg = Op.getOperand(0);
28827 SDValue Callee =
28838 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
28845 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
28847 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
28855 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
28876 SDValue N1 = InOp.getOperand(1);
28886 SmallVector<SDValue, 16> Ops;
28892 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
28898 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) :
28904 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
28910 SDValue Src = N->getValue();
28915 SDValue Scale = N->getScale();
28916 SDValue Index = N->getIndex();
28917 SDValue Mask = N->getMask();
28918 SDValue Chain = N->getChain();
28919 SDValue BasePtr = N->getBasePtr();
28929 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
28933 return SDValue();
28941 return SDValue();
28962 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
28967 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
28973 SDValue Mask = N->getMask();
28975 SDValue PassThru = N->getPassThru();
28984 SDValue NewLoad = DAG.getMaskedLoad(
28990 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
29021 SDValue NewLoad = DAG.getMaskedLoad(
29026 SDValue Extract =
29029 SDValue RetOps[] = {Extract, NewLoad.getValue(1)};
29033 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
29036 SDValue DataToStore = N->getValue();
29039 SDValue Mask = N->getMask();
29075 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
29083 SDValue Index = N->getIndex();
29084 SDValue Mask = N->getMask();
29085 SDValue PassThru = N->getPassThru();
29092 return SDValue();
29114 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
29116 SDValue NewGather = DAG.getMemIntrinsicNode(
29119 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT,
29124 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
29126 SDValue Src = Op.getOperand(0);
29147 SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op,
29155 SmallVector<SDValue, 2> Ops;
29163 SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
29168 SDValue X86TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
29173 SmallVector<SDValue, 2> Ops(Op->op_begin() + Offset, Op->op_end());
29176 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
29178 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, Call, MVT::f128, Ops,
29188 static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) {
29191 SDValue Lo, Hi;
29195 SDValue RC = Op.getOperand(1);
29202 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
29346 SmallVectorImpl<SDValue> &Results,
29348 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
29373 SmallVectorImpl<SDValue>&Results,
29385 SDValue Lo, Hi;
29391 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
29397 SDValue Lo, Hi;
29405 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
29407 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
29419 SDValue Wide =
29439 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
29440 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
29441 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
29444 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
29470 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
29472 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
29474 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
29476 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
29484 SDValue Lo, Hi, Tmp;
29496 SDValue(Lo.getNode(), 1));
29509 SDValue UNDEF = DAG.getUNDEF(VT);
29510 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
29512 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
29531 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
29534 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
29535 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
29536 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
29546 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
29559 SDValue In = N->getOperand(0);
29570 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
29575 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, In,
29604 SDValue Lo, Hi;
29609 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
29627 SDValue In = N->getOperand(0);
29641 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
29642 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
29646 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
29649 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
29653 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
29679 SDValue Lo = getExtendInVec(N->getOpcode(), dl, LoVT, In, DAG);
29688 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
29691 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
29704 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
29715 SDValue Res;
29716 SDValue Chain;
29738 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
29774 SDValue Res;
29775 SDValue Chain;
29794 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},
29825 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
29826 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
29829 SDValue Chain;
29843 SDValue Chain;
29844 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
29853 if (SDValue V = LRINT_LLRINTHelper(N, DAG))
29868 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
29874 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
29886 SDValue Zero = DAG.getConstant(0, dl, SrcVT);
29887 SDValue One = DAG.getConstant(1, dl, SrcVT);
29888 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
29891 SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
29892 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
29893 SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
29895 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
29904 SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
29905 SDValue Slow, Chain;
29918 SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
29936 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
29944 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
29945 SDValue VBias =
29947 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
29951 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
29953 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
29960 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
29968 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
29971 SDValue V;
30021 SDValue cpInL, cpInH;
30028 cpInL, SDValue());
30032 SDValue swapInL, swapInH;
30047 SDValue Result;
30063 SDValue RBXSave = DAG.getCopyFromReg(swapInH.getValue(0), dl,
30066 SDValue Ops[] = {/*Chain*/ RBXSave.getValue(1), N->getOperand(1), swapInL,
30076 SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
30080 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
30083 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
30086 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
30088 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
30090 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
30110 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
30111 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
30114 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
30123 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,
30134 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
30135 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
30138 SDValue Chain = Result.getValue(1);
30144 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
30148 SDValue StoreOps[] = { Chain, Result, StackPtr };
30191 SDValue Lo, Hi;
30195 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
30206 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
30220 SDValue Index = Gather->getIndex();
30226 SDValue Mask = Gather->getMask();
30228 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
30238 SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
30240 SDValue Res = DAG.getMemIntrinsicNode(
30262 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
30265 SDValue Chain = Res.getValue(1);
30276 SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
30277 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
30284 SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
30293 Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
30831 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
30888 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
33444 X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
33456 auto NeedsSignExtension = [&](SDValue V, unsigned ActiveBits) {
33478 SDValue NewC =
33481 SDValue NewOp =
33530 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
33531 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
33535 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
33564 SDValue Src = Op.getOperand(0);
33664 SDValue Op0 = Op.getOperand(0);
33665 SDValue Op1 = Op.getOperand(1);
33740 SmallVector<SDValue, 2> Ops;
33788 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
33799 SDValue Src = Op.getOperand(0);
33829 SDValue Src = Op.getOperand(0);
33840 SDValue Src = Op.getOperand(0);
33879 SmallVector<SDValue, 2> Ops;
33926 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
33933 // specified VT and memory VT. Returns SDValue() on failure.
33934 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT,
33938 return SDValue();
33941 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
33952 SDValue &V1, const SDLoc &DL, SelectionDAG &DAG,
34216 SDValue &V1, SDValue &V2, const SDLoc &DL,
34282 bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
34412 return SDValue();
34416 SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
34417 SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
34442 static SDValue combineX86ShuffleChainWithExtract(
34443 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
34456 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
34468 SDValue V1 = peekThroughBitcasts(Inputs[0]);
34469 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
34480 SDValue Res;
34521 SDValue Src = Inputs[0];
34543 return SDValue(); // Nothing to do!
34561 SDValue V1, SDValue V2, SelectionDAG &DAG) {
34564 SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)};
34570 SDValue Op = Mask[i] >= 4 ? V2 : V1;
34575 return SDValue();
34600 if (SDValue V = MatchSHUF128(ShuffleVT, DL, Mask, V1, V2, DAG))
34614 return SDValue(); // Nothing to do!
34624 return SDValue(); // Nothing to do!
34644 return SDValue(); // Nothing to do!
34700 return SDValue();
34730 return SDValue(); // Nothing to do!
34737 return SDValue(); // Nothing to do!
34745 SDValue NewV1 = V1; // Save operand in case early exit happens.
34752 return SDValue(); // Nothing to do!
34764 return SDValue(); // Nothing to do!
34778 SDValue SrcV1 = V1, SrcV2 = V2;
34782 return SDValue(); // Nothing to do!
34791 SDValue NewV1 = V1; // Save operands in case early exit happens.
34792 SDValue NewV2 = V2;
34798 return SDValue(); // Nothing to do!
34812 return SDValue(); // Nothing to do!
34830 return SDValue(); // Nothing to do!
34840 return SDValue(); // Nothing to do!
34860 return SDValue(); // Nothing to do!
34875 return SDValue(); // Nothing to do!
34891 return SDValue();
34911 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
34935 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
34937 SDValue Zero = getZeroVector(MaskVT, Subtarget, DAG, DL);
34944 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
34961 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
34967 return SDValue();
34989 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
35003 SmallVector<SDValue, 16> VPermIdx;
35005 SDValue Idx =
35009 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
35044 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
35059 SmallVector<SDValue, 16> PSHUFBMask;
35078 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
35090 SmallVector<SDValue, 16> VPPERMMask;
35109 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
35116 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
35136 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
35144 return SDValue();
35155 static SDValue combineX86ShuffleChainWithExtract(
35156 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
35162 return SDValue();
35164 SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end());
35171 SDValue &Src = WideInputs[i];
35190 return SDValue();
35202 if (llvm::any_of(WideInputs, [&WideSVT, &DAG](SDValue Op) {
35206 return SDValue();
35208 for (SDValue &NewInput : WideInputs) {
35235 return SDValue();
35242 SDValue WideRoot = WideInputs[0];
35243 if (SDValue WideShuffle = combineX86ShuffleChain(
35250 return SDValue();
35256 static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops,
35257 ArrayRef<int> Mask, SDValue Root,
35273 SDValue SrcOp = Ops[i];
35277 return SDValue();
35284 return SDValue();
35338 return SDValue();
35340 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
35373 static SDValue combineX86ShufflesRecursively(
35374 ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
35386 return SDValue();
35389 SDValue Op = SrcOps[SrcOpIndex];
35394 return SDValue(); // Bail if we hit a non-vector.
35405 SmallVector<SDValue, 2> OpInputs;
35411 return SDValue();
35415 if (llvm::any_of(OpInputs, [RootSizeInBits](SDValue Op) {
35418 return SDValue();
35420 for (SDValue &Op : OpInputs)
35426 SmallVector<SDValue, 16> Ops;
35458 auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
35460 SDValue InputBC = peekThroughBitcasts(Input);
35475 for (SDValue OpInput : OpInputs)
35598 if (SDValue Res = combineX86ShufflesRecursively(
35606 if (SDValue Cst = combineX86ShufflesConstants(
35642 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
35653 static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
35656 SmallVector<SDValue, 2> Ops;
35697 static SDValue
35698 combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
35707 SmallVector<SDValue, 8> Chain;
35708 SDValue V = N.getOperand(0);
35712 return SDValue(); // Nothing combined!
35728 return SDValue();
35738 return SDValue();
35749 return SDValue();
35756 return SDValue();
35762 return SDValue(); // Nothing to combine.
35786 return SDValue();
35797 SDValue W = Chain.pop_back_val();
35827 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
35831 return SDValue();
35834 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
35836 return SDValue();
35837 SDValue N0 = V.getOperand(0);
35838 SDValue N1 = V.getOperand(1);
35842 return SDValue();
35850 if (SDValue NewSHUFP = commuteSHUFP(N, N.getOperand(0))) {
35857 SDValue N0 = N.getOperand(0);
35858 SDValue N1 = N.getOperand(1);
35861 if (SDValue NewSHUFP = commuteSHUFP(N, N0))
35864 } else if (SDValue NewSHUFP = commuteSHUFP(N, N0)) {
35867 } else if (SDValue NewSHUFP = commuteSHUFP(N, N1)) {
35875 return SDValue();
35879 static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
35889 SmallVector<SDValue, 2> TargetOps;
35908 SDValue BC0 = peekThroughBitcasts(TargetOps.front());
35909 SDValue BC1 = peekThroughBitcasts(TargetOps.back());
35920 SDValue Lo = isInRange(WidenedMask128[0], 0, 2) ? BC0 : BC1;
35921 SDValue Hi = isInRange(WidenedMask128[1], 0, 2) ? BC0 : BC1;
35926 SDValue Undef = DAG.getUNDEF(SrcVT);
35927 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
35933 SDValue Horiz = DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
35940 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
35949 SDValue Src = N.getOperand(0);
35951 if (SDValue Not = IsNOT(Src, DAG, /*OneUse*/ true)) {
35957 SDValue AllOnes = DAG.getConstant(-1, DL, IntVT);
35970 SDValue Src = N.getOperand(0);
35975 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {
35976 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
35978 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
35984 return SDValue();
35987 SDValue Src = N.getOperand(0);
35988 SDValue BC = peekThroughBitcasts(Src);
36001 if (SDValue Res = combineX86ShufflesRecursively(
36028 // Ensure the same SDValue from the SDNode use is being used.
36033 return extractSubVector(SDValue(User, 0), 0, DAG, DL,
36043 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
36044 SDValue BcastLd =
36051 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
36054 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
36066 SDValue TruncIn = Src.getOperand(0);
36075 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
36076 SDValue BcastLd = DAG.getMemIntrinsicNode(
36081 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
36093 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
36094 SDValue BcastLd =
36098 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
36118 SDValue Ptr = DAG.getMemBasePlusOffset(LN->getBasePtr(), Offset, DL);
36119 SDValue Ops[] = { LN->getChain(), Ptr };
36120 SDValue BcastLd = DAG.getMemIntrinsicNode(
36126 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
36138 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
36139 SDValue BcastLd =
36143 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
36156 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
36157 SDValue BcastLd = DAG.getMemIntrinsicNode(
36162 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
36168 return SDValue();
36171 SDValue N0 = N.getOperand(0);
36177 if (SDValue VZLoad =
36180 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
36193 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
36194 SDValue VZLoad =
36198 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
36210 SDValue In = N0.getOperand(0);
36213 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
36215 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
36216 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
36236 SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT);
36245 return SDValue();
36248 SDValue N0 = N.getOperand(0);
36249 SDValue N1 = N.getOperand(1);
36268 return SDValue();
36273 SDValue N0 = N.getOperand(0);
36274 SDValue N1 = N.getOperand(1);
36278 SDValue Src = N0.getOperand(0);
36280 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
36283 return SDValue();
36291 SDValue Ins0 = peekThroughBitcasts(N.getOperand(0));
36292 SDValue Ins1 = peekThroughBitcasts(N.getOperand(1));
36298 return SDValue();
36300 SDValue X = Ins0.getOperand(1);
36301 SDValue Y = Ins1.getOperand(1);
36308 return SDValue();
36321 SDValue N0 = N.getOperand(0);
36322 SDValue N1 = N.getOperand(1);
36330 SDValue N10 = N1.getOperand(0);
36331 SDValue N11 = N1.getOperand(1);
36337 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
36340 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
36341 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
36346 return SDValue();
36350 SDValue Op0 = N.getOperand(0);
36351 SDValue Op1 = N.getOperand(1);
36369 SmallVector<SDValue, 2> Ops1;
36390 SmallVector<SDValue, 2> Ops0;
36411 return SDValue();
36439 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
36442 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
36446 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
36451 return SDValue();
36454 return SDValue();
36462 SDValue V = N.getOperand(0);
36493 SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
36522 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG))
36528 return SDValue();
36572 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
36587 SDValue V1 = N->getOperand(0);
36588 SDValue V2 = N->getOperand(1);
36602 SDValue LHS, RHS;
36631 static SDValue combineShuffleToFMAddSub(SDNode *N,
36638 return SDValue();
36643 return SDValue();
36646 SDValue Op0 = N->getOperand(0);
36647 SDValue Op1 = N->getOperand(1);
36648 SDValue FMAdd = Op0, FMSub = Op1;
36656 return SDValue();
36662 return SDValue();
36674 static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N,
36677 if (SDValue V = combineShuffleToFMAddSub(N, Subtarget, DAG))
36680 SDValue Opnd0, Opnd1;
36683 return SDValue();
36689 SDValue Opnd2;
36696 return SDValue();
36702 return SDValue();
36710 static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
36713 return SDValue();
36719 return SDValue();
36725 return SDValue();
36727 SDValue N0 = N->getOperand(0);
36728 SDValue N1 = N->getOperand(1);
36735 return SDValue();
36747 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
36753 static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) {
36757 return SDValue();
36763 SDValue SrcOp = N->getOperand(0);
36771 SDValue HOp = N->getOperand(0);
36774 return SDValue();
36782 return SDValue();
36788 auto updateHOp = [](SDValue HorizOp, SelectionDAG &DAG) {
36789 SDValue X;
36815 return SDValue();
36836 return SDValue();
36842 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
36844 return SDValue();
36847 return SDValue();
36852 return SDValue();
36860 return SDValue();
36872 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
36876 if (SDValue V = narrowShuffle(Shuf, DAG))
36885 if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
36888 if (SDValue HAddSub = foldShuffleOfHorizOp(N, DAG))
36893 if (SDValue LD = combineToConsecutiveLoads(VT, SDValue(N, 0), dl, DAG,
36903 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, DAG, Subtarget))
36907 SDValue Op(N, 0);
36908 if (SDValue Shuffle = combineTargetShuffle(Op, DAG, DCI, Subtarget))
36916 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
36924 return SDValue(N, 0);
36933 SDValue V = peekThroughOneUseBitcasts(N->getOperand(0));
36937 SDValue In = V.getOperand(1);
36942 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, dl, SubVT, In);
36949 return SDValue();
36955 SDValue Op, const APInt &DemandedElts, unsigned MaskIndex,
36962 SDValue Mask = Op.getOperand(MaskIndex);
36974 SDValue BC = peekThroughOneUseBitcasts(Mask);
37012 SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT);
37013 SDValue LegalCV = LowerConstantPool(CV, TLO.DAG);
37014 SDValue NewMask = TLO.DAG.getLoad(
37022 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
37034 SDValue LHS = Op.getOperand(0);
37035 SDValue RHS = Op.getOperand(1);
37050 SDValue Amt = Op.getOperand(1);
37074 SDValue Src = Op.getOperand(0);
37083 SDValue Src = Op.getOperand(0);
37105 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
37122 SDValue Src = Op.getOperand(0);
37144 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
37162 SDValue Src = Op.getOperand(0);
37173 SDValue N0 = Op.getOperand(0);
37174 SDValue N1 = Op.getOperand(1);
37190 SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
37192 SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
37223 SDValue Src = Op.getOperand(0);
37257 SDValue Src = Op.getOperand(0);
37265 SDValue Src = Op.getOperand(0);
37283 if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
37324 SDValue Src = Op.getOperand(0);
37348 SDValue Ext0 =
37350 SDValue ExtOp =
37352 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
37353 SDValue Insert =
37365 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
37366 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
37367 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
37396 SmallVector<SDValue, 4> Ops;
37398 SDValue SrcOp = Op.getOperand(i);
37409 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
37410 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
37411 SDValue Insert =
37421 SmallVector<SDValue, 2> OpInputs;
37428 llvm::any_of(OpInputs, [VT](SDValue V) {
37488 SDValue NewShuffle = combineX86ShufflesRecursively(
37499 SDValue Op, const APInt &OriginalDemandedBits,
37508 SDValue Src = Op.getOperand(0);
37522 SDValue LHS = Op.getOperand(0);
37523 SDValue RHS = Op.getOperand(1);
37534 SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
37536 SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
37547 SDValue Op0 = Op.getOperand(0);
37567 SDValue NewShift = TLO.DAG.getNode(
37614 SDValue Op0 = Op.getOperand(0);
37615 SDValue Op1 = Op.getOperand(1);
37630 SDValue Op00 = Op0.getOperand(0);
37664 SDValue Vec = Op.getOperand(0);
37690 if (SDValue V = SimplifyMultipleUseDemandedBits(
37702 SDValue Vec = Op.getOperand(0);
37703 SDValue Scl = Op.getOperand(1);
37750 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
37752 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
37755 SDValue Op0 = DemandedOp0 ? DemandedOp0 : Op.getOperand(0);
37756 SDValue Op1 = DemandedOp1 ? DemandedOp1 : Op.getOperand(1);
37770 SDValue Src = Op.getOperand(0);
37802 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
37808 SDValue Op0 = Op.getOperand(0);
37809 SDValue Op1 = Op.getOperand(1);
37842 SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
37843 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
37853 SDValue Vec = Op.getOperand(0);
37864 SDValue Op0 = Op.getOperand(0);
37890 SmallVector<SDValue, 2> ShuffleOps;
37897 llvm::all_of(ShuffleOps, [VT](SDValue V) {
37936 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size) {
37961 static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src,
37965 return SDValue();
37972 SDValue Op0 = Src.getOperand(0);
37983 SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
37984 SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
37991 return SDValue();
37995 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
37996 SDValue Src, const SDLoc &DL) {
38017 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
38022 return SDValue();
38027 if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
38058 return SDValue();
38074 return SDValue();
38115 return SDValue();
38124 return SDValue();
38127 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
38146 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
38155 SDValue In = Op.getOperand(Idx);
38163 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
38169 return SDValue();
38173 return SDValue();
38176 SDValue Op = N->getOperand(0);
38180 return SDValue();
38186 return SDValue();
38193 return SDValue();
38195 SDValue LHS = Op.getOperand(0);
38196 SDValue RHS = Op.getOperand(1);
38216 return SDValue();
38219 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
38223 SDValue Splat = BV->getSplatValue();
38226 auto CreateMMXElement = [&](SDValue V) {
38243 SmallVector<SDValue, 8> Ops;
38280 SDValue Intrin = DAG.getConstant(IntrinOp, DL, MVT::i32);
38294 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, SDLoc DL,
38302 SDValue Src = V.getOperand(0);
38310 SDValue Src = V.getOperand(0);
38314 if (SDValue N0 =
38323 SDValue Src = V.getOperand(0);
38327 if (SDValue N0 =
38337 SDValue Src0 = V.getOperand(0);
38338 SDValue Src1 = V.getOperand(1);
38339 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
38340 if (SDValue N1 = combineBitcastToBoolVector(VT, Src1, DL, DAG, Subtarget))
38346 SDValue Src0 = V.getOperand(0);
38348 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
38355 return SDValue();
38358 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
38361 SDValue N0 = N->getOperand(0);
38373 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
38398 SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
38402 SmallVector<SDValue, 4> Ops(N0->op_begin(), N0->op_end());
38411 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
38423 if (SDValue V =
38458 SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
38459 SDValue ResNode =
38462 DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
38492 SDValue Op = N0.getOperand(i);
38497 SDValue N00 = N0.getOperand(0);
38517 SDValue N00 = N0.getOperand(0);
38526 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
38558 SDValue Src = N0;
38565 SDValue MovmskIn = Src.getOperand(0);
38577 SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
38585 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
38594 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
38607 default: return SDValue();
38612 return SDValue();
38614 SDValue LogicOp0 = N0.getOperand(0);
38615 SDValue LogicOp1 = N0.getOperand(1);
38622 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
38629 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
38633 return SDValue();
38639 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
38640 SDValue AbsOp1 = Abs->getOperand(0);
38659 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
38660 const SDValue &Zext1, const SDLoc &DL,
38669 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
38672 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
38674 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
38678 ArrayRef<SDValue> Ops) {
38689 static SDValue combineHorizontalMinMaxResult(SDNode *Extract, SelectionDAG &DAG,
38693 return SDValue();
38697 return SDValue();
38701 SDValue Src = DAG.matchBinOpReduction(
38704 return SDValue();
38709 return SDValue();
38712 SDValue MinPos = Src;
38716 SDValue Lo, Hi;
38727 SDValue Mask;
38744 SDValue Upper = DAG.getVectorShuffle(
38763 static SDValue combineHorizontalPredicateResult(SDNode *Extract,
38768 return SDValue();
38774 return SDValue();
38778 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
38782 return SDValue();
38787 return SDValue();
38789 SDValue Movmsk;
38799 return SDValue();
38812 SDValue Vec = Match.getOperand(0);
38826 SDValue Lo, Hi;
38835 return SDValue();
38842 return SDValue();
38849 return SDValue();
38853 return SDValue();
38856 SDValue Lo, Hi;
38870 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
38880 SDValue Mask = DAG.getConstant(1, DL, CmpVT);
38881 SDValue Result = DAG.getNode(ISD::CTPOP, DL, CmpVT, Movmsk);
38886 SDValue CmpC;
38903 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
38904 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
38905 SDValue Zero = DAG.getConstant(0, DL, ExtractVT);
38909 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
38913 return SDValue();
38919 return SDValue();
38923 return SDValue();
38927 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
38945 return SDValue();
38948 SDValue Zext0, Zext1;
38950 return SDValue();
38954 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
38968 SDValue Shuffle =
38985 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
38989 return SDValue();
38992 SDValue Src = N->getOperand(0);
38993 SDValue Idx = N->getOperand(1);
39003 return SDValue();
39007 return SDValue();
39009 SDValue SrcBC = peekThroughBitcasts(Src);
39013 SDValue SrcOp = SrcBC.getOperand(0);
39035 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
39040 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
39055 SDValue Scl = SrcBC.getOperand(0);
39078 SmallVector<SDValue, 2> Ops;
39080 return SDValue();
39083 if (llvm::any_of(Ops, [SrcVT](SDValue Op) {
39086 return SDValue();
39116 return SDValue();
39128 SDValue SrcOp = Ops[SrcIdx / Mask.size()];
39148 SDValue ExtOp = DAG.getNode(OpCode, dl, MVT::i32, SrcOp,
39153 return SDValue();
39158 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG) {
39160 SDValue Vec = ExtElt->getOperand(0);
39161 SDValue Index = ExtElt->getOperand(1);
39168 return SDValue();
39175 return SDValue();
39179 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
39181 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
39187 return SDValue();
39201 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
39204 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
39206 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
39243 SmallVector<SDValue, 4> ExtOps;
39244 for (SDValue Op : Vec->ops())
39249 return SDValue();
39256 static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
39262 return SDValue();
39265 SDValue Rdx =
39268 return SDValue();
39270 SDValue Index = ExtElt->getOperand(1);
39277 return SDValue();
39310 return SDValue();
39315 SDValue Lo, Hi;
39322 SDValue Hi = DAG.getVectorShuffle(
39334 return SDValue();
39345 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
39346 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
39352 return SDValue();
39366 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
39369 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
39372 SDValue InputVector = N->getOperand(0);
39373 SDValue EltIdx = N->getOperand(1);
39403 SDValue(N, 0), APInt::getAllOnesValue(VT.getSizeInBits()), DCI))
39404 return SDValue(N, 0);
39412 SDValue Scl = InputVector.getOperand(1);
39420 return SDValue();
39426 SDValue MMXSrc = InputVector.getOperand(0);
39436 SDValue MMXSrc = InputVector.getOperand(0);
39446 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
39450 if (SDValue Cmp = combineHorizontalPredicateResult(N, DAG, Subtarget))
39454 if (SDValue MinMax = combineHorizontalMinMaxResult(N, DAG, Subtarget))
39457 if (SDValue V = combineReductionToHorizontal(N, DAG, Subtarget))
39460 if (SDValue V = scalarizeExtEltFP(N, DAG))
39489 if (SDValue BC =
39495 SDValue Mask = DAG.getConstant(MaskBit, dl, BCVT);
39496 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
39500 return SDValue(N, 0);
39505 return SDValue();
39511 static SDValue
39515 SDValue Cond = N->getOperand(0);
39516 SDValue LHS = N->getOperand(1);
39517 SDValue RHS = N->getOperand(2);
39524 return SDValue();
39548 return SDValue();
39562 SDValue CC = Cond.getOperand(2);
39575 return SDValue();
39582 return SDValue();
39586 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
39587 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
39593 SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
39594 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
39600 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
39601 SDValue AndN;
39611 return SDValue();
39618 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG,
39622 return SDValue();
39627 return SDValue();
39630 SDValue Cond = N->getOperand(0);
39631 SDValue TVal = N->getOperand(1);
39632 SDValue FVal = N->getOperand(2);
39633 SmallVector<SDValue, 4> CatOpsT, CatOpsF;
39637 return SDValue();
39640 ArrayRef<SDValue> Ops) {
39647 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG) {
39648 SDValue Cond = N->getOperand(0);
39649 SDValue LHS = N->getOperand(1);
39650 SDValue RHS = N->getOperand(2);
39656 return SDValue();
39661 return SDValue();
39667 return SDValue();
39678 return SDValue();
39694 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
39702 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
39707 return SDValue();
39715 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
39718 SDValue Cond = N->getOperand(0);
39722 return SDValue();
39728 return SDValue();
39743 return SDValue();
39748 return SDValue();
39751 return SDValue();
39754 return SDValue();
39757 return SDValue();
39759 auto OnlyUsedAsSelectCond = [](SDValue Cond) {
39777 return SDValue();
39788 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
39790 DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
39794 return SDValue(N, 0);
39798 if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG))
39802 return SDValue();
39821 static SDValue combineLogicBlendIntoConditionalNegate(
39822 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
39830 return SDValue();
39832 return SDValue();
39834 auto IsNegV = [](SDNode *N, SDValue V) {
39839 SDValue V;
39845 return SDValue();
39847 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
39848 SDValue SubOp2 = Mask;
39863 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
39868 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
39872 SDValue Cond = N->getOperand(0);
39873 SDValue LHS = N->getOperand(1);
39874 SDValue RHS = N->getOperand(2);
39878 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
39893 if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS,
40065 SDValue AndNode = Cond.getOperand(0);
40097 auto SelectableOp = [&TLI](SDValue Op) {
40122 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
40127 if (SDValue V = combineSelectOfTwoConstants(N, DAG))
40173 SDValue Other;
40183 SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
40184 SDValue CondRHS = Cond->getOperand(1);
40240 SDValue CondLHS = Cond->getOperand(0);
40241 SDValue CondRHS = Cond->getOperand(1);
40245 SDValue Other;
40254 SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
40292 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
40299 return SDValue();
40301 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget))
40304 if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget))
40307 if (SDValue V = narrowVectorSelect(N, DAG, Subtarget))
40312 if (SDValue CondNot = IsNOT(Cond, DAG))
40344 SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
40360 SDValue And = Cond.getOperand(0);
40364 SDValue NotCond =
40382 SDValue Mask = And.getOperand(1);
40390 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
40391 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
40392 SDValue NewCond =
40398 return SDValue();
40407 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
40413 return SDValue();
40419 return SDValue();
40433 SDValue CmpLHS = Cmp.getOperand(0);
40434 SDValue CmpRHS = Cmp.getOperand(1);
40437 return SDValue();
40441 return SDValue();
40443 SDValue OpRHS = CmpLHS.getOperand(2);
40446 return SDValue();
40454 return SDValue();
40479 return SDValue();
40490 return SDValue();
40492 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
40514 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
40518 return SDValue();
40522 return SDValue();
40526 SDValue Op1 = Cmp.getOperand(0);
40527 SDValue Op2 = Cmp.getOperand(1);
40529 SDValue SetCC;
40539 return SDValue();
40546 return SDValue();
40590 return SDValue();
40593 SDValue Op = SetCC.getOperand(0);
40602 return SDValue();
40608 return SDValue();
40615 return SDValue();
40617 return SDValue();
40625 return SDValue();
40632 static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
40633 X86::CondCode &CC1, SDValue &Flags,
40644 SDValue SetCC0, SetCC1;
40673 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
40676 SDValue Carry = EFLAGS.getOperand(0);
40688 SDValue CarryOp1 = Carry.getOperand(1);
40702 SDValue SubCommute =
40705 return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
40718 return SDValue();
40723 static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
40729 return SDValue();
40736 SDValue Op0 = EFLAGS.getOperand(0);
40737 SDValue Op1 = EFLAGS.getOperand(1);
40741 if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
40779 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
40786 SDValue BC = peekThroughBitcasts(Op0);
40816 if (SDValue Res =
40845 return SDValue();
40849 static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
40855 return SDValue();
40857 return SDValue();
40860 return SDValue();
40863 return SDValue();
40866 SDValue CmpOp = EFLAGS.getOperand(0);
40876 return SDValue();
40878 SDValue Vec = CmpOp.getOperand(0);
40889 return SDValue();
40896 SDValue BC = peekThroughBitcasts(Vec);
40914 SDValue BC = peekThroughBitcasts(Vec);
40918 SDValue V = DAG.getBitcast(TestVT, BC.getOperand(0));
40929 SDValue VecOp0 = Vec.getOperand(0);
40930 SDValue VecOp1 = Vec.getOperand(1);
40936 SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);
40956 SDValue Result = DAG.getBitcast(MVT::v32i8, VecOp0.getOperand(0));
40971 SmallVector<SDValue, 2> ShuffleInputs;
40985 SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
40994 return SDValue();
41000 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
41004 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
41007 if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
41010 if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget))
41013 if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget))
41020 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
41025 SDValue FalseOp = N->getOperand(0);
41026 SDValue TrueOp = N->getOperand(1);
41028 SDValue Cond = N->getOperand(3);
41036 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
41041 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
41084 SDValue(FalseC, 0));
41124 SDValue(FalseC, 0));
41162 SDValue Ops[] = {FalseOp, Cond.getOperand(0),
41187 SDValue Flags;
41197 SDValue LOps[] = {FalseOp, TrueOp,
41199 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
41200 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
41202 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
41213 SDValue Add = TrueOp;
41214 SDValue Const = FalseOp;
41232 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
41233 SDValue CMov =
41240 return SDValue();
41255 SDValue Opd = N->getOperand(i);
41308 static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
41313 return SDValue();
41321 return SDValue();
41325 return SDValue();
41328 SDValue N0 = N->getOperand(0);
41329 SDValue N1 = N->getOperand(1);
41333 return SDValue();
41338 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
41339 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
41343 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
41352 SDValue MulHi =
41364 SDValue ResLo =
41372 SDValue ResHi =
41378 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
41382 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41392 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41453 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41455 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
41461 return SDValue();
41466 static SDValue combineMulToPMADDWD(SDNode *N, SelectionDAG &DAG,
41469 return SDValue();
41472 return SDValue();
41478 return SDValue();
41482 return SDValue();
41488 return SDValue();
41490 SDValue N0 = N->getOperand(0);
41491 SDValue N1 = N->getOperand(1);
41500 return SDValue();
41505 return SDValue();
41509 ArrayRef<SDValue> Ops) {
41518 static SDValue combineMulToPMULDQ(SDNode *N, SelectionDAG &DAG,
41521 return SDValue();
41529 return SDValue();
41531 SDValue N0 = N->getOperand(0);
41532 SDValue N1 = N->getOperand(1);
41539 ArrayRef<SDValue> Ops) {
41550 ArrayRef<SDValue> Ops) {
41557 return SDValue();
41562 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
41567 if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget))
41570 if (SDValue V = combineMulToPMULDQ(N, DAG, Subtarget))
41577 return SDValue();
41580 return SDValue();
41583 return SDValue();
41586 return SDValue();
41590 return SDValue();
41592 return SDValue();
41600 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
41622 SDValue NewMul;
41711 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
41720 return SDValue();
41723 SDValue ShiftOperand = N->getOperand(0);
41725 return SDValue();
41730 return SDValue();
41736 return SDValue();
41738 SDValue LHS = ShiftOperand.getOperand(0);
41739 SDValue RHS = ShiftOperand.getOperand(1);
41744 return SDValue();
41753 return SDValue();
41756 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
41762 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
41763 SDValue N0 = N->getOperand(0);
41764 SDValue N1 = N->getOperand(1);
41773 SDValue N00 = N0.getOperand(0);
41818 return SDValue();
41821 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
41823 SDValue N0 = N->getOperand(0);
41824 SDValue N1 = N->getOperand(1);
41828 if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget))
41845 return SDValue();
41847 SDValue N00 = N0.getOperand(0);
41848 SDValue N01 = N0.getOperand(1);
41854 return SDValue();
41863 SDValue NN =
41875 return SDValue();
41878 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
41881 SDValue N0 = N->getOperand(0);
41882 SDValue N1 = N->getOperand(1);
41885 if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget))
41891 return SDValue();
41898 return SDValue();
41903 return SDValue();
41914 return SDValue();
41924 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
41925 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
41928 return SDValue();
41931 static SDValue combineVectorPackWithShuffle(SDNode *N, SelectionDAG &DAG) {
41937 SDValue N0 = N->getOperand(0);
41938 SDValue N1 = N->getOperand(1);
41952 SDValue Vec = peekThroughBitcasts(N0.getOperand(0));
41960 SDValue Lo, Hi;
41964 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
41980 SDValue Op00 = SVN0->getOperand(0);
41981 SDValue Op01 = SVN0->getOperand(1);
41982 SDValue Op10 = SVN1->getOperand(0);
41983 SDValue Op11 = SVN1->getOperand(1);
41993 SDValue Res = DAG.getNode(Opcode, DL, VT, Op00, Op01);
42003 return SDValue();
42006 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
42014 SDValue N0 = N->getOperand(0);
42015 SDValue N1 = N->getOperand(1);
42080 if (SDValue V = combineVectorPackWithShuffle(N, DAG))
42096 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
42103 SDValue Op(N, 0);
42104 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42107 return SDValue();
42110 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
42117 SDValue N0 = N->getOperand(0);
42118 SDValue N1 = N->getOperand(1);
42136 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
42138 return SDValue(N, 0);
42140 return SDValue();
42143 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
42152 SDValue N0 = N->getOperand(0);
42201 SDValue Op(N, 0);
42202 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42233 if (TLI.SimplifyDemandedBits(SDValue(N, 0),
42235 return SDValue(N, 0);
42237 return SDValue();
42240 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
42252 if (TLI.SimplifyDemandedBits(SDValue(N, 0),
42254 return SDValue(N, 0);
42259 SDValue Op(N, 0);
42260 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42264 return SDValue();
42270 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
42277 if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
42278 SDValue N0 = N->getOperand(0);
42279 SDValue N1 = N->getOperand(1);
42280 SDValue CMP0 = N0.getOperand(1);
42281 SDValue CMP1 = N1.getOperand(1);
42286 return SDValue();
42288 SDValue CMP00 = CMP0->getOperand(0);
42289 SDValue CMP01 = CMP0->getOperand(1);
42327 SDValue FSetCC =
42332 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
42338 SDValue OnesOrZeroesF =
42351 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
42353 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
42359 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
42360 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
42362 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
42369 return SDValue();
42373 static SDValue combineANDXORWithAllOnesIntoANDNP(SDNode *N, SelectionDAG &DAG) {
42378 return SDValue();
42380 SDValue X, Y;
42381 SDValue N0 = N->getOperand(0);
42382 SDValue N1 = N->getOperand(1);
42384 auto GetNot = [&VT, &DAG](SDValue V) {
42386 if (SDValue Not = IsNOT(V, DAG))
42390 SDValue Src = V.getOperand(0);
42393 return SDValue();
42394 if (SDValue Not = IsNOT(Src, DAG))
42398 return SDValue();
42401 if (SDValue Not = GetNot(N0)) {
42404 } else if (SDValue Not = GetNot(N1)) {
42408 return SDValue();
42424 static SDValue PromoteMaskArithmetic(SDNode *N, EVT VT, SelectionDAG &DAG,
42428 return SDValue();
42432 return SDValue();
42434 SDValue N0 = N->getOperand(0);
42435 SDValue N1 = N->getOperand(1);
42440 return SDValue();
42442 if (SDValue NN0 = PromoteMaskArithmetic(N0.getNode(), VT, DAG, Depth + 1))
42447 return SDValue();
42451 return SDValue();
42456 if (SDValue NN1 = PromoteMaskArithmetic(N1.getNode(), VT, DAG, Depth + 1))
42463 return SDValue();
42480 static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
42490 SDValue Narrow = N->getOperand(0);
42494 SDValue Op = PromoteMaskArithmetic(Narrow.getNode(), VT, DAG, 0);
42496 return SDValue();
42523 static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
42526 SDValue N0 = N->getOperand(0);
42527 SDValue N1 = N->getOperand(1);
42531 return SDValue();
42533 SDValue N00 = N0.getOperand(0);
42534 SDValue N10 = N1.getOperand(0);
42542 return SDValue();
42545 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
42551 static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG) {
42556 SDValue N0 = N->getOperand(0);
42557 SDValue N1 = N->getOperand(1);
42562 return SDValue();
42564 SDValue Vec0 = N0.getOperand(0);
42565 SDValue Vec1 = N1.getOperand(0);
42573 return SDValue();
42578 SDValue Result =
42586 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
42588 SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
42589 SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
42594 return SDValue();
42599 return SDValue();
42603 return SDValue();
42606 return SDValue();
42610 return SDValue();
42614 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
42615 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
42621 static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
42623 return SDValue();
42625 SDValue Base = Ld->getBasePtr();
42628 return SDValue();
42630 SDValue ShiftedIndex = Base.getOperand(0);
42633 return SDValue();
42660 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
42667 return SDValue();
42671 SDValue N = Node->getOperand(i);
42676 return SDValue();
42681 return SDValue();
42715 SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0);
42716 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
42719 SDValue Index = getIndexFromUnindexedLoad(Ld);
42721 return SDValue();
42724 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
42727 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
42728 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
42735 return SDValue();
42740 static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
42747 return SDValue();
42751 return SDValue();
42753 SDValue N0 = N->getOperand(0);
42754 SDValue N1 = N->getOperand(1);
42758 return SDValue();
42762 return SDValue();
42765 SDValue X = N0.getOperand(0);
42770 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
42773 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
42777 SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32,
42785 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, VT, X,
42792 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
42795 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
42797 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
42800 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
42811 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
42820 return SDValue();
42828 return SDValue();
42831 SDValue Src = N->getOperand(0).getOperand(0);
42835 return SDValue();
42838 return SDValue();
42842 SDValue SubVec = Src.getOperand(0);
42849 return SDValue();
42854 return SDValue();
42857 return SDValue();
42862 SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
42865 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
42870 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
42890 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(0));
42891 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(1));
42898 if (SDValue V = combineParity(N, DAG, Subtarget))
42904 SmallVector<SDValue, 2> SrcOps;
42906 if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) &&
42912 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
42918 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
42925 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
42928 if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
42932 return SDValue();
42934 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
42937 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
42940 if (SDValue R = combineANDXORWithAllOnesIntoANDNP(N, DAG))
42943 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
42946 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
42951 SDValue Op(N, 0);
42952 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
42960 SDValue BitMask = N->getOperand(1);
42961 SDValue SrcVec = N->getOperand(0).getOperand(0);
42987 if (SDValue Shuffle = combineX86ShufflesRecursively(
42995 return SDValue();
42999 static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
43005 return SDValue();
43007 SDValue N0 = peekThroughBitcasts(N->getOperand(0));
43008 SDValue N1 = peekThroughBitcasts(N->getOperand(1));
43010 return SDValue();
43018 return SDValue();
43025 return SDValue();
43028 return SDValue();
43033 return SDValue();
43035 return SDValue();
43042 SDValue A = DAG.getBitcast(VT, N0.getOperand(1));
43043 SDValue B = DAG.getBitcast(VT, N0.getOperand(0));
43044 SDValue C = DAG.getBitcast(VT, N1.getOperand(0));
43045 SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
43049 SDValue X = N->getOperand(0);
43050 SDValue Y =
43057 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
43061 SDValue N0 = N->getOperand(0);
43062 SDValue N1 = N->getOperand(1);
43096 static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
43103 return SDValue();
43105 SDValue X, Y, Mask;
43107 return SDValue();
43119 return SDValue();
43124 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
43130 return SDValue();
43134 return SDValue();
43151 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, EVT ExtTy,
43153 SDValue Cmp = Op.getOperand(1);
43157 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
43160 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
43161 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
43173 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
43177 return SDValue();
43179 auto isORCandidate = [](SDValue N) {
43188 return SDValue();
43191 auto isSetCCCandidate = [](SDValue N) {
43200 SDValue LHS = OR->getOperand(0);
43201 SDValue RHS = OR->getOperand(1);
43215 !isORCandidate(SDValue(OR, 0)))
43216 return SDValue();
43224 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, VT, DAG);
43225 SDValue Ret, NewRHS;
43230 return SDValue();
43242 return SDValue();
43252 static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
43255 SDValue N0 = N->getOperand(0);
43256 SDValue N1 = N->getOperand(1);
43270 SmallVector<SDValue, 2> SrcOps;
43272 if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) &&
43278 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
43284 SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT);
43285 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
43292 if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
43296 return SDValue();
43298 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
43301 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
43304 if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
43307 if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
43340 SDValue Op(N, 0);
43341 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
43345 return SDValue();
43352 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
43356 return SDValue();
43358 SDValue N0 = N->getOperand(0);
43359 SDValue N1 = N->getOperand(1);
43363 return SDValue();
43367 return SDValue();
43370 SDValue Shift = N0.getOperand(0);
43372 return SDValue();
43377 return SDValue();
43382 return SDValue();
43388 SDValue ShiftOp = Shift.getOperand(0);
43393 SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
43407 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
43411 return SDValue();
43414 default: return SDValue();
43418 case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;
43422 case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;
43427 SDValue Shift = N->getOperand(0);
43428 SDValue Ones = N->getOperand(1);
43431 return SDValue();
43438 return SDValue();
43448 /// Return the source value x to be truncated or SDValue() if the pattern was
43459 /// So return the smax(x, C1) value to be truncated or SDValue() if the
43461 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
43470 auto MatchMinMax = [](SDValue V, unsigned Opcode, APInt &Limit) -> SDValue {
43474 return SDValue();
43478 if (SDValue UMin = MatchMinMax(In, ISD::UMIN, C2))
43484 if (SDValue SMin = MatchMinMax(In, ISD::SMIN, C2))
43489 if (SDValue SMax = MatchMinMax(In, ISD::SMAX, C1))
43490 if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, C2))
43496 return SDValue();
43506 /// Return the source value to be truncated or SDValue() if the pattern was not
43508 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
43513 auto MatchMinMax = [](SDValue V, unsigned Opcode,
43514 const APInt &Limit) -> SDValue {
43519 return SDValue();
43531 if (SDValue SMin = MatchMinMax(In, ISD::SMIN, SignedMax))
43532 if (SDValue SMax = MatchMinMax(SMin, ISD::SMAX, SignedMin))
43535 if (SDValue SMax = MatchMinMax(In, ISD::SMAX, SignedMin))
43536 if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, SignedMax))
43539 return SDValue();
43542 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
43546 return SDValue();
43560 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
43590 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
43593 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
43610 SDValue SatVal;
43625 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
43635 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
43641 return SDValue();
43647 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,
43651 return SDValue();
43658 return SDValue();
43664 return SDValue();
43667 return SDValue();
43680 return SDValue();
43682 // A lambda checking the given SDValue is a constant vector and each element
43684 auto IsConstVectorInRange = [](SDValue V, unsigned Min, unsigned Max) {
43694 return SDValue();
43696 return SDValue();
43699 SDValue Operands[3];
43704 ArrayRef<SDValue> Ops) {
43715 SDValue VecOnes = DAG.getConstant(1, DL, InVT);
43725 auto FindAddLike = [&](SDValue V, SDValue &Op0, SDValue &Op1) {
43742 SDValue Op0, Op1;
43746 return SDValue();
43762 return SDValue();
43771 return SDValue();
43774 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
43797 return SDValue();
43800 SDValue Ptr1 = Ld->getBasePtr();
43801 SDValue Ptr2 = DAG.getMemBasePlusOffset(Ptr1, HalfOffset, dl);
43804 SDValue Load1 =
43808 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
43812 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
43815 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
43826 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
43830 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
43841 SDValue Cast =
43849 return SDValue();
43855 static int getOneTrueElt(SDValue V) {
43870 const SDValue &Op = BV->getOperand(i);
43891 SelectionDAG &DAG, SDValue &Addr,
43892 SDValue &Index, unsigned &Alignment) {
43915 static SDValue
43923 SDValue Addr, VecIndex;
43926 return SDValue();
43933 SDValue Load =
43938 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
43943 static SDValue
43948 return SDValue();
43961 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
43963 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
43975 return SDValue();
43978 return SDValue();
43982 SDValue NewML = DAG.getMaskedLoad(
43986 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
43992 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
43999 return SDValue();
44002 if (SDValue ScalarLoad = reduceMaskedLoadToScalarLoad(Mld, DAG, DCI))
44007 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
44013 SDValue Mask = Mld->getMask();
44021 return SDValue(N, 0);
44023 if (SDValue NewMask =
44031 return SDValue();
44038 static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
44044 SDValue Addr, VecIndex;
44047 return SDValue();
44053 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
44061 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
44066 return SDValue();
44073 return SDValue();
44075 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG))
44080 SDValue Mask = Mst->getMask();
44086 return SDValue(N, 0);
44088 if (SDValue NewMask =
44096 SDValue Value = Mst->getValue();
44106 return SDValue();
44109 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
44115 SDValue StoredVal = St->getValue();
44146 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT));
44160 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
44163 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
44167 SDValue Ptr0 = St->getBasePtr();
44168 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, 4, dl);
44170 SDValue Ch0 =
44174 SDValue Ch1 =
44197 return SDValue();
44210 return SDValue();
44231 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32, St->getValue());
44255 if (SDValue Avg = detectAVGPattern(St->getValue(), St->getMemoryVT(), DAG,
44262 if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
44266 if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
44273 return SDValue();
44282 SDValue Cast =
44297 return SDValue();
44310 return SDValue();
44314 return SDValue();
44319 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
44336 SDValue OldExtract = St->getOperand(1);
44337 SDValue ExtOp0 = OldExtract.getOperand(0);
44340 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
44341 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
44348 return SDValue();
44351 static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
44356 SDValue StoredVal = N->getOperand(1);
44370 return SDValue(N, 0);
44373 return SDValue();
44389 static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
44411 auto GetShuffle = [&](SDValue Op, SDValue &N0, SDValue &N1,
44430 SmallVector<SDValue, 2> SrcOps;
44432 SDValue BC = peekThroughBitcasts(Op);
44438 N0 = SrcOps.size() > 0 ? SrcOps[0] : SDValue();
44439 N1 = SrcOps.size() > 1 ? SrcOps[1] : SDValue();
44456 // NOTE: A default initialized SDValue represents an UNDEF of type VT.
44457 SDValue A, B;
44463 SDValue C, D;
44565 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
44568 SDValue LHS = N->getOperand(0);
44569 SDValue RHS = N->getOperand(1);
44579 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
44588 return SDValue();
44596 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
44600 SDValue Src = N->getOperand(0);
44607 auto IsFreeTruncation = [VT](SDValue Op) {
44626 auto TruncateArithmetic = [&](SDValue N0, SDValue N1) {
44627 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
44628 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
44634 return SDValue();
44639 return SDValue();
44658 SDValue Op0 = Src.getOperand(0);
44659 SDValue Op1 = Src.getOperand(1);
44667 return SDValue();
44674 static SDValue combineVectorTruncationWithPACKUS(SDNode *N, const SDLoc &DL,
44677 SDValue In = N->getOperand(0);
44688 static SDValue combineVectorTruncationWithPACKSS(SDNode *N, const SDLoc &DL,
44691 SDValue In = N->getOperand(0);
44704 static SDValue combineVectorTruncation(SDNode *N, SelectionDAG &DAG,
44708 return SDValue();
44710 SDValue In = N->getOperand(0);
44712 return SDValue();
44721 return SDValue();
44728 return SDValue();
44734 return SDValue();
44745 return SDValue();
44751 static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL,
44756 return SDValue();
44759 return SDValue();
44761 SDValue In = N->getOperand(0);
44763 return SDValue();
44773 return SDValue();
44775 return SDValue();
44777 return SDValue();
44781 return SDValue();
44788 return SDValue();
44808 return SDValue();
44813 return SDValue();
44823 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
44828 return SDValue();
44831 return SDValue();
44836 return SDValue();
44841 return SDValue();
44847 return SDValue();
44849 SDValue LHS = Src.getOperand(0).getOperand(0);
44850 SDValue RHS = Src.getOperand(0).getOperand(1);
44855 return SDValue();
44863 return SDValue();
44877 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
44881 return SDValue();
44886 return SDValue();
44888 SDValue SSatVal = detectSSatPattern(In, VT);
44890 return SDValue();
44894 SDValue N0 = SSatVal.getOperand(0);
44895 SDValue N1 = SSatVal.getOperand(1);
44898 return SDValue();
44900 SDValue N00 = N0.getOperand(0);
44901 SDValue N01 = N0.getOperand(1);
44902 SDValue N10 = N1.getOperand(0);
44903 SDValue N11 = N1.getOperand(1);
44917 return SDValue();
44930 return SDValue();
44937 return SDValue();
44947 SDValue ZExtIn, SExtIn;
44949 SDValue N00Elt = N00.getOperand(i);
44950 SDValue N01Elt = N01.getOperand(i);
44951 SDValue N10Elt = N10.getOperand(i);
44952 SDValue N11Elt = N11.getOperand(i);
44958 return SDValue();
44964 return SDValue();
44977 return SDValue();
44978 SDValue N00In = N00Elt.getOperand(0);
44979 SDValue N01In = N01Elt.getOperand(0);
44980 SDValue N10In = N10Elt.getOperand(0);
44981 SDValue N11In = N11Elt.getOperand(0);
44989 return SDValue();
44993 ArrayRef<SDValue> Ops) {
45008 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
45011 SDValue Src = N->getOperand(0);
45015 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
45019 if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))
45023 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
45027 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
45031 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
45037 SDValue BCSrc = Src.getOperand(0);
45043 if (SDValue V = combineVectorSignBitsTruncation(N, DL, DAG, Subtarget))
45049 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
45052 SDValue In = N->getOperand(0);
45062 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
45063 return SDValue(N, 0);
45065 return SDValue();
45077 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
45083 return SDValue();
45087 SDValue Op = peekThroughBitcasts(SDValue(N, 0));
45092 return SDValue();
45100 return SDValue();
45101 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
45110 SDValue InsVector = Op.getOperand(0);
45111 SDValue InsVal = Op.getOperand(1);
45113 return SDValue();
45114 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
45123 SDValue Op1 = Op.getOperand(1);
45124 SDValue Op0 = Op.getOperand(0);
45142 return SDValue();
45149 return SDValue();
45213 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
45217 SDValue Arg = isFNEG(DAG, N);
45219 return SDValue();
45228 return SDValue();
45235 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
45236 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
45243 if (SDValue NegArg =
45247 return SDValue();
45250 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
45256 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
45280 SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
45300 if (SDValue NegOp0 =
45311 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
45316 return SDValue();
45324 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
45325 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
45334 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
45340 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
45342 return SDValue();
45344 SDValue LHS = N->getOperand(0);
45346 return SDValue();
45354 static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
45366 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
45369 if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
45373 return SDValue();
45375 if (SDValue SetCC = foldXor1SetCC(N, DAG))
45378 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
45381 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
45387 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
45398 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
45399 return SDValue(N, 0);
45401 return SDValue();
45404 static bool isNullFPScalarOrVectorConst(SDValue V) {
45414 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
45417 return SDValue();
45425 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
45427 SDValue N0 = N->getOperand(0);
45428 SDValue N1 = N->getOperand(1);
45436 return SDValue();
45438 auto isAllOnesConstantFP = [](SDValue V) {
45453 return SDValue();
45457 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
45460 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
45464 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
45467 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
45474 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
45481 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
45488 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
45501 if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget))
45508 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
45514 return SDValue();
45529 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
45532 return SDValue();
45540 return SDValue();
45542 SDValue Op0 = N->getOperand(0);
45543 SDValue Op1 = N->getOperand(1);
45562 return SDValue();
45586 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
45587 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
45594 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
45601 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
45603 return SDValue(N, 0);
45606 SDValue In = N->getOperand(0);
45615 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
45617 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
45620 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
45622 return SDValue(N, 0);
45626 return SDValue();
45629 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
45635 SDValue In = N->getOperand(IsStrict ? 1 : 0);
45644 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
45647 SDValue Convert =
45652 SDValue Convert =
45656 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
45658 return SDValue(N, 0);
45662 return SDValue();
45666 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
45680 if (SDValue Not = IsNOT(N->getOperand(0), DAG))
45686 SDValue Op(N, 0);
45687 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
45691 return SDValue();
45694 static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
45696 SDValue N1 = N->getOperand(1);
45704 return SDValue(N, 0);
45707 return SDValue();
45710 static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
45713 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
45723 return SDValue(N, 0);
45729 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {
45732 SDValue Convert = DAG.getNode(
45737 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
45742 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
45744 return SDValue(N, 0);
45749 return SDValue();
45753 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
45758 SDValue N0 = N->getOperand(0);
45759 SDValue N1 = N->getOperand(1);
45763 return SDValue();
45766 SDValue IntermediateBitwidthOp;
45775 return SDValue();
45777 SDValue CMovOp0 = N0.getOperand(0);
45778 SDValue CMovOp1 = N0.getOperand(1);
45783 return SDValue();
45805 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
45814 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
45818 if (SDValue V = combineSextInRegCmov(N, DAG))
45822 SDValue N0 = N->getOperand(0);
45823 SDValue N1 = N->getOperand(1);
45834 SDValue N00 = N0.getOperand(0);
45840 return SDValue();
45844 if (SDValue Promote = PromoteMaskArithmetic(N0.getNode(), DAG, Subtarget))
45848 SDValue Tmp =
45853 return SDValue();
45861 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
45865 return SDValue();
45870 return SDValue();
45872 SDValue Add = Ext->getOperand(0);
45874 return SDValue();
45883 return SDValue();
45890 return SDValue();
45905 return SDValue();
45909 SDValue AddOp0 = Add.getOperand(0);
45910 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
45911 SDValue NewConstant = DAG.getConstant(AddConstant, SDLoc(Add), VT);
45933 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
45934 SDValue CMovN = Extend->getOperand(0);
45936 return SDValue();
45943 SDValue CMovOp0 = CMovN.getOperand(0);
45944 SDValue CMovOp1 = CMovN.getOperand(1);
45948 return SDValue();
45952 return SDValue();
45957 return SDValue();
45968 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
45980 static SDValue
45987 return SDValue();
45989 return SDValue();
45991 return SDValue();
45993 SDValue N0 = N->getOperand(0);
46002 return SDValue();
46004 return SDValue();
46006 return SDValue();
46008 SDValue N00 = N0.getOperand(0);
46011 return SDValue();
46014 SDValue Vec;
46053 SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
46060 SmallVector<SDValue, 32> Bits;
46066 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
46084 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
46086 SDValue N0 = N->getOperand(0);
46092 return SDValue();
46098 return SDValue();
46103 return SDValue();
46109 return SDValue();
46115 return SDValue();
46117 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
46125 static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
46128 SDValue N0 = N->getOperand(0);
46136 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
46142 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
46147 return SDValue(N, 0);
46150 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
46154 return SDValue();
46156 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
46165 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
46169 if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
46173 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
46176 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
46179 return SDValue();
46182 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
46192 return SDValue();
46196 return SDValue();
46198 SDValue A = N->getOperand(IsStrict ? 1 : 0);
46199 SDValue B = N->getOperand(IsStrict ? 2 : 1);
46200 SDValue C = N->getOperand(IsStrict ? 3 : 2);
46202 auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
46205 if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations,
46214 SDValue Vec = V.getOperand(0);
46215 if (SDValue NegV = TLI.getCheaperNegatedExpression(
46233 return SDValue();
46251 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
46259 SDValue N2 = N->getOperand(2);
46261 SDValue NegN2 =
46264 return SDValue();
46274 static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
46278 SDValue N0 = N->getOperand(0);
46285 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
46291 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
46296 return SDValue(N, 0);
46299 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
46303 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
46306 if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
46310 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
46313 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
46316 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
46322 SDValue N00 = N0.getOperand(0);
46323 SDValue N01 = N0.getOperand(1);
46332 return SDValue();
46337 static bool isOrXorXorTree(SDValue X, bool Root = true) {
46349 static SDValue emitOrXorXorTree(SDValue X, SDLoc &DL, SelectionDAG &DAG,
46351 SDValue Op0 = X.getOperand(0);
46352 SDValue Op1 = X.getOperand(1);
46354 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
46355 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
46362 SDValue A = SToV(Op0);
46363 SDValue B = SToV(Op1);
46375 static SDValue combineVectorSizedSetCCEquality(SDNode *SetCC, SelectionDAG &DAG,
46381 SDValue X = SetCC->getOperand(0);
46382 SDValue Y = SetCC->getOperand(1);
46386 return SDValue();
46395 return SDValue();
46398 auto IsVectorBitCastCheap = [](SDValue X) {
46405 return SDValue();
46447 auto ScalarToVector = [&](SDValue X) -> SDValue {
46451 SDValue OrigX = X.getOperand(0);
46473 SDValue Cmp;
46481 SDValue VecX = ScalarToVector(X);
46482 SDValue VecY = ScalarToVector(Y);
46499 SDValue BCCmp = DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64,
46501 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
46503 SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
46511 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
46512 SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
46516 return SDValue();
46519 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
46522 const SDValue LHS = N->getOperand(0);
46523 const SDValue RHS = N->getOperand(1);
46529 if (SDValue V = combineVectorSizedSetCCEquality(N, DAG, Subtarget))
46533 SDValue X86CC;
46534 if (SDValue V =
46545 SDValue Op0 = LHS;
46546 SDValue Op1 = RHS;
46584 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
46592 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
46594 return SDValue();
46597 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
46600 SDValue Src = N->getOperand(0);
46626 if (SDValue NotSrc = IsNOT(Src, DAG)) {
46638 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
46639 return SDValue(N, 0);
46641 return SDValue();
46644 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
46647 SDValue Mask = cast<X86MaskedGatherScatterSDNode>(N)->getMask();
46654 return SDValue(N, 0);
46658 return SDValue();
46661 static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS,
46662 SDValue Index, SDValue Base, SDValue Scale,
46667 SDValue Ops[] = { Gather->getChain(), Gather->getPassThru(),
46675 SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(),
46683 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
46687 SDValue Index = GorS->getIndex();
46688 SDValue Base = GorS->getBasePtr();
46689 SDValue Scale = GorS->getScale();
46740 SDValue Mask = GorS->getMask();
46747 return SDValue(N, 0);
46751 return SDValue();
46755 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
46759 SDValue EFLAGS = N->getOperand(1);
46762 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
46765 return SDValue();
46769 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
46772 SDValue EFLAGS = N->getOperand(3);
46778 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
46779 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
46784 return SDValue();
46788 static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
46804 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
46808 return SDValue();
46817 return SDValue();
46824 SDValue SourceConst;
46827 {N->getOperand(0), SDValue(BV, 0)});
46829 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
46831 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
46832 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
46834 SDValue Res = DAG.getBitcast(VT, NewAnd);
46840 return SDValue();
46846 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
46850 SDValue Trunc = N->getOperand(0);
46852 return SDValue();
46854 SDValue ExtElt = Trunc.getOperand(0);
46857 return SDValue();
46864 return SDValue();
46871 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
46873 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
46878 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
46881 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
46892 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
46911 return SDValue();
46914 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
46920 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
46924 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
46935 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
46955 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
46964 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
46965 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
46982 return SDValue();
46987 return SDValue();
46991 std::pair<SDValue, SDValue> Tmp =
47001 return SDValue();
47003 if (SDValue V = combineToFPTruncExtElt(N, DAG))
47006 return SDValue();
47009 static bool needCarryOrOverflowFlag(SDValue Flags) {
47047 static bool onlyZeroFlagUsed(SDValue Flags) {
47073 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG) {
47076 return SDValue();
47083 SDValue Op = N->getOperand(0);
47091 onlyZeroFlagUsed(SDValue(N, 0))) {
47110 return SDValue();
47116 return SDValue();
47120 default: return SDValue();
47125 return SDValue();
47132 if (needCarryOrOverflowFlag(SDValue(N, 0)))
47133 return SDValue();
47138 if (needCarryOrOverflowFlag(SDValue(N, 0)))
47139 return SDValue();
47145 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
47146 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
47161 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
47167 SDValue LHS = N->getOperand(0);
47168 SDValue RHS = N->getOperand(1);
47174 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
47179 auto MatchGeneric = [&](SDValue N0, SDValue N1, bool Negate) {
47180 SDValue Ops[] = {N0, N1};
47183 SDValue Op(N, 0);
47192 return SDValue();
47195 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
47196 if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
47206 SDValue Op0 = N->getOperand(0);
47207 SDValue Op1 = N->getOperand(1);
47213 return SDValue();
47217 static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
47226 SDValue(N, 1).use_empty()) {
47229 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
47230 SDValue Res1 =
47239 if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
47247 return SDValue();
47253 static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
47255 SDValue X = N->getOperand(0);
47256 SDValue Y = N->getOperand(1);
47279 return SDValue();
47301 SDValue EFLAGS = Y->getOperand(1);
47308 SDValue NewSub = DAG.getNode(
47311 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
47328 SDValue EFLAGS = Y.getOperand(1);
47338 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
47341 SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
47359 SDValue EFLAGS = Y.getOperand(1);
47369 SDValue NewSub = DAG.getNode(
47372 SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
47380 return SDValue();
47382 SDValue Cmp = Y.getOperand(1);
47386 return SDValue();
47388 SDValue Z = Cmp.getOperand(0);
47400 SDValue Zero = DAG.getConstant(0, DL, ZVT);
47402 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
47405 SDValue(Neg.getNode(), 1));
47414 SDValue One = DAG.getConstant(1, DL, ZVT);
47416 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
47424 SDValue One = DAG.getConstant(1, DL, ZVT);
47426 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
47443 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
47458 return SDValue();
47462 return SDValue();
47467 return SDValue();
47480 SDValue Mul;
47482 SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
47489 return SDValue();
47495 return SDValue();
47510 return SDValue();
47518 return SDValue();
47523 return SDValue();
47530 return SDValue();
47534 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
47535 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
47538 ArrayRef<SDValue> Ops) {
47551 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
47555 return SDValue();
47558 return SDValue();
47563 return SDValue();
47565 SDValue N00 = N0.getOperand(0);
47566 SDValue N01 = N0.getOperand(1);
47567 SDValue N10 = N1.getOperand(0);
47568 SDValue N11 = N1.getOperand(1);
47576 return SDValue();
47588 return SDValue();
47595 return SDValue();
47603 SDValue In0, In1;
47605 SDValue N00Elt = N00.getOperand(i);
47606 SDValue N01Elt = N01.getOperand(i);
47607 SDValue N10Elt = N10.getOperand(i);
47608 SDValue N11Elt = N11.getOperand(i);
47614 return SDValue();
47620 return SDValue();
47633 return SDValue();
47634 SDValue N00In = N00Elt.getOperand(0);
47635 SDValue N01In = N01Elt.getOperand(0);
47636 SDValue N10In = N10Elt.getOperand(0);
47637 SDValue N11In = N11Elt.getOperand(0);
47650 return SDValue();
47654 ArrayRef<SDValue> Ops) {
47669 static SDValue combineAddOrSubToHADDorHSUB(SDNode *N, SelectionDAG &DAG,
47672 SDValue Op0 = N->getOperand(0);
47673 SDValue Op1 = N->getOperand(1);
47683 ArrayRef<SDValue> Ops) {
47687 SDValue HorizBinOp =
47695 return SDValue();
47698 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
47702 SDValue Op0 = N->getOperand(0);
47703 SDValue Op1 = N->getOperand(1);
47705 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))
47707 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))
47711 if (SDValue V = combineAddOrSubToHADDorHSUB(N, DAG, Subtarget))
47725 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
47733 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
47741 static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG,
47743 SDValue Op0 = N->getOperand(0);
47744 SDValue Op1 = N->getOperand(1);
47748 return SDValue();
47756 return SDValue();
47758 SDValue SubusLHS, SubusRHS;
47764 SDValue MaxLHS = Op0.getOperand(0);
47765 SDValue MaxRHS = Op0.getOperand(1);
47771 return SDValue();
47774 SDValue MinLHS = Op1.getOperand(0);
47775 SDValue MinRHS = Op1.getOperand(1);
47781 return SDValue();
47788 SDValue MinLHS = Op1.getOperand(0).getOperand(0);
47789 SDValue MinRHS = Op1.getOperand(0).getOperand(1);
47794 return SDValue();
47795 SDValue OpToSaturate;
47803 return SDValue();
47807 SDValue SaturationConst =
47811 SDValue UMin = DAG.getNode(ISD::UMIN, DL, TruncVT, OpToSaturate,
47815 return SDValue();
47832 return SDValue();
47843 SDValue SaturationConst =
47847 SDValue UMin = DAG.getNode(ISD::UMIN, SDLoc(SubusLHS), ExtType, SubusRHS,
47849 SDValue NewSubusLHS =
47851 SDValue NewSubusRHS = DAG.getZExtOrTrunc(UMin, SDLoc(SubusRHS), ShrinkedType);
47852 SDValue Psubus = DAG.getNode(ISD::USUBSAT, SDLoc(N), ShrinkedType,
47860 static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
47863 SDValue Op0 = N->getOperand(0);
47864 SDValue Op1 = N->getOperand(1);
47876 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
47885 if (SDValue V = combineAddOrSubToHADDorHSUB(N, DAG, Subtarget))
47889 if (SDValue V = combineSubToSubus(N, DAG, Subtarget))
47895 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
47907 return SDValue();
47913 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
47914 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
47920 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
47923 if (llvm::all_of(Ops, [](SDValue Op) {
47928 SDValue Op0 = Ops[0];
47929 bool IsSplat = llvm::all_of(Ops, [&Op0](SDValue Op) { return Op == Op0; });
47939 if (SDValue Ld =
47958 SDValue Ops[] = {MemIntr->getChain(), MemIntr->getBasePtr()};
47959 SDValue BcastLd = DAG.getMemIntrinsicNode(
47964 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
47996 if (llvm::all_of(Ops, [Op0](SDValue Op) {
48004 llvm::all_of(Ops, [Op0](SDValue Op) {
48007 SmallVector<SDValue, 2> LHS, RHS;
48024 SmallVector<SDValue, 2> Src;
48036 SmallVector<SDValue, 2> Src;
48039 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f32, Src);
48051 llvm::all_of(Ops, [Op0](SDValue Op) {
48054 SmallVector<SDValue, 2> Src;
48066 llvm::all_of(Ops, [Op0](SDValue Op) {
48069 SmallVector<SDValue, 2> Src;
48081 SmallVector<SDValue, 2> LHS, RHS;
48098 llvm::all_of(Ops, [Op0](SDValue Op) {
48101 SmallVector<SDValue, 2> LHS, RHS;
48115 return SDValue();
48118 static SDValue combineConcatVectors(SDNode *N, SelectionDAG &DAG,
48127 return SDValue();
48130 SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
48131 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
48136 return SDValue();
48139 static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
48143 return SDValue();
48150 SDValue Vec = N->getOperand(0);
48151 SDValue SubVec = N->getOperand(1);
48183 SDValue Ins = SubVec.getOperand(0);
48195 return SDValue();
48220 SmallVector<SDValue, 2> SubVectorOps;
48222 if (SDValue Fold =
48248 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
48249 SDValue BcastLd =
48253 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
48257 return SDValue();
48266 static SDValue narrowExtractedVectorSelect(SDNode *Ext, SelectionDAG &DAG) {
48267 SDValue Sel = peekThroughBitcasts(Ext->getOperand(0));
48268 SmallVector<SDValue, 4> CatOps;
48271 return SDValue();
48278 return SDValue();
48282 return SDValue();
48301 return SDValue();
48311 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
48312 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
48313 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
48314 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
48318 static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
48332 return SDValue();
48335 SDValue InVec = N->getOperand(0);
48337 SDValue InVecBC = peekThroughBitcasts(InVec);
48344 auto isConcatenatedNot = [] (SDValue V) {
48348 SDValue NotOp = V->getOperand(0);
48354 SDValue Concat = splitVectorIntBinary(InVecBC, DAG);
48361 return SDValue();
48363 if (SDValue V = narrowExtractedVectorSelect(N, DAG))
48405 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
48406 SDValue BcastLd =
48410 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
48432 SmallVector<SDValue, 2> ShuffleInputs;
48442 SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
48489 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
48490 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
48491 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
48496 return SDValue();
48499 static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
48501 SDValue Src = N->getOperand(0);
48526 auto IsAnyExt64 = [](SDValue Op) {
48528 return SDValue();
48536 return SDValue();
48538 if (SDValue ExtSrc = IsAnyExt64(peekThroughOneUseBitcasts(Src)))
48549 return SDValue();
48553 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
48556 SDValue LHS = N->getOperand(0);
48557 SDValue RHS = N->getOperand(1);
48571 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnesValue(64), DCI))
48572 return SDValue(N, 0);
48602 return SDValue();
48605 static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
48609 SDValue In = N->getOperand(0);
48624 SDValue Load =
48629 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
48638 SDValue Op(N, 0);
48640 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
48644 return SDValue();
48647 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
48657 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
48659 return SDValue(N, 0);
48661 return SDValue();
48667 static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
48670 return SDValue();
48673 return SDValue();
48677 return SDValue();
48680 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
48689 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
48692 return SDValue();
48696 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
48700 return SDValue();
48704 return SDValue();
48708 return SDValue();
48719 SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
48721 SmallVector<SDValue, 4> Ops(NumConcats, Fill);
48729 SDValue Cvt, Chain;
48761 static SDValue combineVBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
48765 return SDValue();
48769 SDValue Ptr = MemIntrin->getBasePtr();
48770 SDValue Chain = MemIntrin->getChain();
48784 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
48787 return DCI.CombineTo(N, Extract, SDValue(User, 1));
48790 return SDValue();
48793 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
48796 return SDValue();
48799 SDValue Src = N->getOperand(0);
48804 return SDValue();
48808 return SDValue();
48820 SDValue Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src,
48833 static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) {
48834 SDValue Src = N->getOperand(0);
48841 SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),
48846 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1));
48851 return SDValue();
48854 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
49022 return SDValue();
49071 SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc& dl,
49072 SDValue Value, SDValue Addr,
49087 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
49099 auto IsFoldableRMW = [](SDValue Load, SDValue Op) {
49110 auto IsFoldableAtomicRMW = [](SDValue Load, SDValue Op) {
49133 SDValue N0 = Op.getOperand(0);
49147 SDValue N0 = Op.getOperand(0);
49148 SDValue N1 = Op.getOperand(1);
49543 SDValue X86TargetLowering::LowerAsmOutputForConstraint(
49544 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
49548 return SDValue();
49561 SDValue CC = getSETCC(Cond, Flag, DL, DAG);
49563 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
49570 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
49572 std::vector<SDValue>&Ops,
49574 SDValue Result;