Lines Matching refs:SDValue
52 /// This corresponds to X86AddressMode, but uses SDValue's instead of register
61 SDValue Base_Reg;
65 SDValue IndexReg;
67 SDValue Segment;
102 void setBaseReg(SDValue Reg) {
195 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
208 bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
209 bool matchAddress(SDValue N, X86ISelAddressMode &AM);
210 bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
211 bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
212 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
214 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
215 bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
216 SDValue &Scale, SDValue &Index, SDValue &Disp,
217 SDValue &Segment);
218 bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp,
219 SDValue ScaleOp, SDValue &Base, SDValue &Scale,
220 SDValue &Index, SDValue &Disp, SDValue &Segment);
221 bool selectMOV64Imm32(SDValue N, SDValue &Imm);
222 bool selectLEAAddr(SDValue N, SDValue &Base,
223 SDValue &Scale, SDValue &Index, SDValue &Disp,
224 SDValue &Segment);
225 bool selectLEA64_32Addr(SDValue N, SDValue &Base,
226 SDValue &Scale, SDValue &Index, SDValue &Disp,
227 SDValue &Segment);
228 bool selectTLSADDRAddr(SDValue N, SDValue &Base,
229 SDValue &Scale, SDValue &Index, SDValue &Disp,
230 SDValue &Segment);
231 bool selectRelocImm(SDValue N, SDValue &Op);
233 bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
234 SDValue &Base, SDValue &Scale,
235 SDValue &Index, SDValue &Disp,
236 SDValue &Segment);
239 bool tryFoldLoad(SDNode *P, SDValue N,
240 SDValue &Base, SDValue &Scale,
241 SDValue &Index, SDValue &Disp,
242 SDValue &Segment) {
246 bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
247 SDValue &Base, SDValue &Scale,
248 SDValue &Index, SDValue &Disp,
249 SDValue &Segment);
254 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
256 std::vector<SDValue> &OutOps) override;
261 MVT VT, SDValue &Base, SDValue &Scale,
262 SDValue &Index, SDValue &Disp,
263 SDValue &Segment) {
277 SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32,
378 SDValue OtherOp = User->getOperand(0);
401 inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) {
406 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
411 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) {
415 SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth,
423 SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth,
502 bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
509 SDValue &InFlag);
513 bool onlyUsesZeroFlag(SDValue Flags) const;
514 bool hasNoSignFlagUses(SDValue Flags) const;
515 bool hasNoCarryFlagUses(SDValue Flags) const;
562 X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
591 SDValue Op1 = U->getOperand(1);
634 hasNoCarryFlagUses(SDValue(U, 1)))
649 SDValue Val = Op1.getOperand(0);
668 SDValue U0 = U->getOperand(0);
669 SDValue U1 = U->getOperand(1);
726 static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
727 SDValue Call, SDValue OrigChain) {
728 SmallVector<SDValue, 8> Ops;
729 SDValue Chain = OrigChain.getOperand(0);
740 SDValue NewChain =
751 Ops.push_back(SDValue(Load.getNode(), 1));
761 static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
807 SDValue Res = CurDAG->getNode(ISD::AND, SDLoc(N), N->getValueType(0),
810 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
833 SDValue AllOnes =
838 SDValue Res =
855 SDValue NarrowBCast =
857 SDValue Res =
881 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
882 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
885 SDValue Res =
893 SDValue To[] = {Res, NarrowBCast.getValue(1)};
908 SDValue Blendv =
938 SDValue Res;
968 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
971 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
994 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
997 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1030 SDValue Res;
1058 SDValue Op0 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT,
1060 SDValue Op1 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT,
1063 SDValue Res;
1084 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1119 SDValue Chain = N->getOperand(0);
1120 SDValue Load = N->getOperand(1);
1123 moveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain);
1171 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1179 SDValue Store = CurDAG->getTruncStore(
1181 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store,
1189 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1227 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1236 SDValue Store, Result;
1239 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp};
1256 SDValue Ops[] = {Store, MemTmp};
1299 SDValue N0 = N->getOperand(0);
1310 SDValue N00 = N0.getOperand(0);
1355 SDValue And = N->getOperand(0);
1378 SDValue Ops[] = { And.getOperand(1),
1404 onlyUsesZeroFlag(SDValue(N, 0))) {
1405 SDValue And = N->getOperand(0);
1438 SDValue Move = N->getOperand(1);
1463 SDValue In = Move.getOperand(0);
1499 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
1551 SDValue Address = N->getOperand(1);
1581 bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) {
1590 SDValue Val = N.getOperand(0);
1614 SDValue N0 = N.getOperand(0);
1653 bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) {
1687 bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM,
1727 static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) {
1745 static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N,
1747 SDValue Shift, SDValue X,
1761 SDValue Eight = DAG.getConstant(8, DL, MVT::i8);
1762 SDValue NewMask = DAG.getConstant(0xff, DL, VT);
1763 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, X, Eight);
1764 SDValue And = DAG.getNode(ISD::AND, DL, VT, Srl, NewMask);
1765 SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8);
1766 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And, ShlCount);
1789 static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
1791 SDValue Shift = N.getOperand(0);
1813 SDValue X = Shift.getOperand(0);
1829 SDValue NewX = DAG.getNode(ISD::ANY_EXTEND, DL, VT, X);
1834 SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT);
1835 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask);
1836 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1));
1881 static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
1883 SDValue Shift, SDValue X,
1938 SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X);
1943 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8);
1944 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt);
1945 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8);
1946 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewSRL, NewSHLAmt);
1968 static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N,
1970 SDValue Shift, SDValue X,
1998 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8);
1999 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt);
2000 SDValue NewMask = DAG.getConstant(Mask >> AMShiftAmt, DL, VT);
2001 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, NewSRL, NewMask);
2002 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8);
2003 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewAnd, NewSHLAmt);
2024 bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
2102 SDValue ShVal = N.getOperand(0);
2130 SDValue And = N.getOperand(0);
2132 SDValue X = And.getOperand(0);
2165 SDValue MulVal = N.getOperand(0);
2166 SDValue Reg;
2216 SDValue RHS = N.getOperand(1);
2287 SDValue Shift = N.getOperand(0);
2288 SDValue X = Shift.getOperand(0);
2321 SDValue Shl = N.getOperand(0);
2336 SDValue Zext = CurDAG->getNode(ISD::ZERO_EXTEND, DL, VT, Shl.getOperand(0));
2337 SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, Shl.getOperand(1));
2356 bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) {
2379 bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) {
2397 bool X86DAGToDAGISel::selectVectorAddr(MemSDNode *Parent, SDValue BasePtr,
2398 SDValue IndexOp, SDValue ScaleOp,
2399 SDValue &Base, SDValue &Scale,
2400 SDValue &Index, SDValue &Disp,
2401 SDValue &Segment) {
2432 bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
2433 SDValue &Scale, SDValue &Index,
2434 SDValue &Disp, SDValue &Segment) {
2468 bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
2493 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base,
2494 SDValue &Scale, SDValue &Index,
2495 SDValue &Disp, SDValue &Segment) {
2507 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
2519 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
2530 bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
2531 SDValue &Base, SDValue &Scale,
2532 SDValue &Index, SDValue &Disp,
2533 SDValue &Segment) {
2542 SDValue Copy = AM.Segment;
2543 SDValue T = CurDAG->getRegister(0, MVT::i32);
2581 auto isMathWithFlags = [](SDValue V) {
2596 return !SDValue(V.getNode(), 1).use_empty();
2620 bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base,
2621 SDValue &Scale, SDValue &Index,
2622 SDValue &Disp, SDValue &Segment) {
2641 bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) {
2678 bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
2679 SDValue &Base, SDValue &Scale,
2680 SDValue &Index, SDValue &Disp,
2681 SDValue &Segment) {
2692 bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
2693 SDValue &Base, SDValue &Scale,
2694 SDValue &Index, SDValue &Disp,
2695 SDValue &Segment) {
2752 bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const {
2788 bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
2844 bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
2901 SDValue StoredVal, SelectionDAG *CurDAG,
2904 SDValue &InputChain) {
2915 SDValue Load = StoredVal->getOperand(LoadOpNo);
2932 SmallVector<SDValue, 4> ChainOps;
2974 SDValue Chain = StoreNode->getChain();
2982 SDValue Op = Chain.getOperand(i);
2998 for (SDValue Op : StoredVal->ops())
3034 SDValue StoredVal = StoreNode->getOperand(1);
3066 SDValue InputChain;
3079 SDValue Base, Scale, Index, Disp, Segment;
3107 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
3124 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
3211 SDValue Operand = StoredVal->getOperand(1-LoadOpNo);
3242 SDValue CopyTo =
3244 StoredVal.getOperand(2), SDValue());
3246 const SDValue Ops[] = {Base, Scale, Index, Disp,
3251 const SDValue Ops[] = {Base, Scale, Index, Disp,
3267 ReplaceUses(SDValue(LoadNode, 1), SDValue(Result, 1));
3268 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
3269 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
3295 SDValue NBits;
3300 auto checkUses = [CanHaveExtraUses](SDValue Op, unsigned NUses) {
3304 auto checkOneUse = [checkUses](SDValue Op) { return checkUses(Op, 1); };
3305 auto checkTwoUse = [checkUses](SDValue Op) { return checkUses(Op, 2); };
3307 auto peekThroughOneUseTruncation = [checkOneUse](SDValue V) {
3319 &NBits](SDValue Mask) -> bool {
3327 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0));
3336 auto isAllOnes = [this, peekThroughOneUseTruncation, NVT](SDValue V) {
3345 &NBits](SDValue Mask) -> bool {
3353 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0));
3364 auto matchShiftAmt = [checkOneUse, &NBits](SDValue ShiftAmt,
3385 matchShiftAmt](SDValue Mask) -> bool {
3395 SDValue M1 = Mask.getOperand(1);
3402 SDValue X;
3409 SDValue N0 = Node->getOperand(0);
3413 SDValue N1 = Node->getOperand(1);
3414 SDValue N01 = N0->getOperand(1);
3426 matchPatternC](SDValue Mask) -> bool {
3432 SDValue Mask = Node->getOperand(1);
3448 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits);
3452 SDValue ImplDef = SDValue(
3454 insertDAGNode(*CurDAG, SDValue(Node, 0), ImplDef);
3456 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit, DL, MVT::i32);
3457 insertDAGNode(*CurDAG, SDValue(Node, 0), SRIdxVal);
3458 NBits = SDValue(
3461 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits);
3468 insertDAGNode(*CurDAG, SDValue(Node, 0), NBits);
3471 SDValue Extract = CurDAG->getNode(X86ISD::BZHI, DL, NVT, X, NBits);
3482 SDValue RealX = peekThroughOneUseTruncation(X);
3498 SDValue C8 = CurDAG->getConstant(8, DL, MVT::i8);
3499 insertDAGNode(*CurDAG, SDValue(Node, 0), C8);
3500 SDValue Control = CurDAG->getNode(ISD::SHL, DL, MVT::i32, NBits, C8);
3501 insertDAGNode(*CurDAG, SDValue(Node, 0), Control);
3506 SDValue ShiftAmt = X.getOperand(1);
3514 SDValue OrigShiftAmt = ShiftAmt;
3520 insertDAGNode(*CurDAG, SDValue(Node, 0), Control);
3526 insertDAGNode(*CurDAG, SDValue(Node, 0), Control);
3530 SDValue Extract = CurDAG->getNode(X86ISD::BEXTR, DL, XVT, X, Control);
3534 insertDAGNode(*CurDAG, SDValue(Node, 0), Extract);
3549 SDValue N0 = Node->getOperand(0);
3550 SDValue N1 = Node->getOperand(1);
3605 SDValue Control;
3617 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
3633 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
3638 SDValue Input = N0->getOperand(0);
3639 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3641 SDValue Ops[] = {
3646 ReplaceUses(Input.getValue(1), SDValue(NewNode, 2));
3655 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
3658 CurDAG->getMachineNode(NewOpc, dl, NVT, SDValue(NewNode, 0), ShAmt);
3668 SDValue N0 = Node->getOperand(0);
3669 SDValue N1 = Node->getOperand(1);
3670 SDValue Imm = Node->getOperand(2);
3675 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3677 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
3682 ReplaceUses(N1.getValue(1), SDValue(CNode, 2));
3688 SDValue Ops[] = { N0, N1, Imm };
3700 SDValue &InFlag) {
3701 SDValue N0 = Node->getOperand(0);
3702 SDValue N2 = Node->getOperand(2);
3703 SDValue Imm = Node->getOperand(4);
3708 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
3710 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
3714 InFlag = SDValue(CNode, 3);
3716 ReplaceUses(N2.getValue(1), SDValue(CNode, 2));
3722 SDValue Ops[] = { N0, N2, Imm, InFlag };
3725 InFlag = SDValue(CNode, 2);
3739 SDValue OrigShiftAmt = N->getOperand(1);
3740 SDValue ShiftAmt = OrigShiftAmt;
3750 SDValue NewShiftAmt;
3752 SDValue Add0 = ShiftAmt->getOperand(0);
3753 SDValue Add1 = ShiftAmt->getOperand(1);
3769 SDValue Zero = CurDAG->getConstant(0, DL, SubVT);
3770 SDValue Neg = CurDAG->getNode(ISD::SUB, DL, SubVT, Zero, Add1);
3823 SDValue Shift = N->getOperand(0);
3824 SDValue N1 = N->getOperand(1);
3911 SDValue X = Shift.getOperand(0);
3913 SDValue NewX = CurDAG->getNode(ISD::ANY_EXTEND, dl, NVT, X);
3914 insertDAGNode(*CurDAG, SDValue(N, 0), NewX);
3918 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
3919 insertDAGNode(*CurDAG, SDValue(N, 0), NewCst);
3920 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst);
3921 insertDAGNode(*CurDAG, SDValue(N, 0), NewBinOp);
3922 SDValue NewSHL = CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp,
3945 SDValue N0 = N->getOperand(0);
3946 SDValue N1 = N->getOperand(1);
3953 SDValue A, B, C;
4001 SDValue New = CurDAG->getNode(X86ISD::VPTERNLOG, DL, NVT, A, B, C,
4042 SDValue And0 = And->getOperand(0);
4071 SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT);
4072 SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask);
4128 bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
4129 SDValue InMask) {
4139 SDValue SetccOp0 = Setcc.getOperand(0);
4140 SDValue SetccOp1 = Setcc.getOperand(1);
4150 SDValue N0 = SetccOp0;
4156 SDValue Src0 = N0;
4157 SDValue Src1 = N0;
4161 SDValue N0Temp = N0;
4180 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Load;
4195 auto findBroadcastedOp = [](SDValue Src, MVT CmpSVT, SDNode *&Parent) {
4208 return SDValue();
4259 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
4270 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4271 InMask = SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
4285 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4289 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
4295 ReplaceUses(Load.getValue(1), SDValue(CNode, 1));
4308 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
4310 dl, ResVT, SDValue(CNode, 0), RC);
4313 ReplaceUses(SDValue(Root, 0), SDValue(CNode, 0));
4333 SDValue N0 = N->getOperand(0);
4334 SDValue N1 = N->getOperand(1);
4346 SDValue A = N1.getOperand(0);
4347 SDValue C = N1.getOperand(1);
4351 SDValue B;
4360 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
4361 SDValue Ternlog = CurDAG->getNode(X86ISD::VPTERNLOG, dl, NVT, A, B, C, Imm);
4411 SDValue Chain = CurDAG->getCopyToReg(Node->getOperand(0), dl, PtrReg,
4412 Node->getOperand(2), SDValue());
4413 SDValue InFlag = Chain.getValue(1);
4448 SDValue TReg = getI8Imm(TIndex, dl);
4449 SDValue Base = Node->getOperand(3);
4450 SDValue Scale = getI8Imm(1, dl);
4451 SDValue Index = Node->getOperand(4);
4452 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
4453 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
4454 SDValue Chain = Node->getOperand(0);
4457 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, TReg, Chain };
4460 SDValue Ops[] = { TReg, Base, Scale, Index, Disp, Segment, Chain };
4478 SDValue Target = Node->getOperand(1);
4480 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, MVT::i64);
4481 SDValue Brind = CurDAG->getNode(ISD::BRIND, dl, MVT::Other,
4498 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
4517 SDValue N0 = Node->getOperand(0);
4518 SDValue N1 = Node->getOperand(1);
4528 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0));
4563 SDValue N0 = Node->getOperand(0);
4564 SDValue N1 = Node->getOperand(1);
4635 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4637 SDValue Ops[] = { N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
4641 ReplaceUses(N0.getValue(1), SDValue(CNode, 2));
4644 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0));
4660 SDValue N0 = Node->getOperand(0);
4661 SDValue N1 = Node->getOperand(1);
4688 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4697 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
4698 N0, SDValue()).getValue(1);
4710 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
4715 ReplaceUses(N1.getValue(1), SDValue(CNode, NVT == MVT::i8 ? 2 : 3));
4730 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0));
4731 ReplaceUses(SDValue(Node, 1), SDValue(CNode, NVT == MVT::i8 ? 1 : 2));
4738 SDValue N0 = Node->getOperand(0);
4739 SDValue N1 = Node->getOperand(1);
4745 bool UseMULXHi = UseMULX && SDValue(Node, 0).use_empty();
4770 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4779 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
4780 N0, SDValue()).getValue(1);
4781 SDValue ResHi, ResLo;
4783 SDValue Chain;
4785 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
4790 ResHi = SDValue(CNode, 0);
4791 Chain = SDValue(CNode, 1);
4795 ResHi = SDValue(CNode, 0);
4796 ResLo = SDValue(CNode, 1);
4797 Chain = SDValue(CNode, 2);
4801 Chain = SDValue(CNode, 0);
4802 InFlag = SDValue(CNode, 1);
4810 SDValue Ops[] = { N1, InFlag };
4814 ResHi = SDValue(CNode, 0);
4818 ResHi = SDValue(CNode, 0);
4819 ResLo = SDValue(CNode, 1);
4823 InFlag = SDValue(CNode, 0);
4828 if (!SDValue(Node, 0).use_empty()) {
4835 ReplaceUses(SDValue(Node, 0), ResLo);
4840 if (!SDValue(Node, 1).use_empty()) {
4847 ReplaceUses(SDValue(Node, 1), ResHi);
4858 SDValue N0 = Node->getOperand(0);
4859 SDValue N1 = Node->getOperand(1);
4904 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4908 SDValue InFlag;
4912 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
4915 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
4919 Chain = SDValue(Move, 1);
4929 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, SDValue(Move, 0),
4930 SDValue());
4935 LoReg, N0, SDValue()).getValue(1);
4939 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0);
4943 SDValue ClrNode =
4944 SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, None), 0);
4948 SDValue(CurDAG->getMachineNode(
4958 SDValue(CurDAG->getMachineNode(
4975 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
4979 InFlag = SDValue(CNode, 1);
4981 ReplaceUses(N1.getValue(1), SDValue(CNode, 0));
4986 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InFlag), 0);
4996 if (HiReg == X86::AH && !SDValue(Node, 1).use_empty()) {
4997 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5003 SDValue Result(RNode, 0);
5004 InFlag = SDValue(RNode, 1);
5009 ReplaceUses(SDValue(Node, 1), Result);
5014 if (!SDValue(Node, 0).use_empty()) {
5015 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5018 ReplaceUses(SDValue(Node, 0), Result);
5023 if (!SDValue(Node, 1).use_empty()) {
5024 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
5027 ReplaceUses(SDValue(Node, 1), Result);
5040 SDValue N0 = Node->getOperand(IsStrictCmp ? 1 : 0);
5041 SDValue N1 = Node->getOperand(IsStrictCmp ? 2 : 1);
5066 SDValue Cmp;
5067 SDValue Chain =
5071 Cmp = SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
5074 Cmp = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i16, N0, N1), 0);
5078 SDValue FPSW = CurDAG->getCopyToReg(Chain, dl, X86::FPSW, Cmp, SDValue());
5080 SDValue FNSTSW =
5081 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, FPSW,
5086 SDValue Extract =
5093 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract, SDValue());
5095 SDValue SAHF = SDValue(
5099 ReplaceUses(SDValue(Node, 1), Chain);
5101 ReplaceUses(SDValue(Node, 0), SAHF);
5107 SDValue N0 = Node->getOperand(0);
5108 SDValue N1 = Node->getOperand(1);
5125 SDValue BEXTR = SDValue(NewNode, 0);
5127 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0));
5151 onlyUsesZeroFlag(SDValue(Node, 0))) {
5154 SDValue Imm = CurDAG->getTargetConstant(TrailingZeros, dl, MVT::i64);
5155 SDValue Shift =
5156 SDValue(CurDAG->getMachineNode(X86::SHR64ri, dl, MVT::i64, MVT::i32,
5165 SDValue Imm = CurDAG->getTargetConstant(LeadingZeros, dl, MVT::i64);
5166 SDValue Shift =
5167 SDValue(CurDAG->getMachineNode(X86::SHL64ri, dl, MVT::i64, MVT::i32,
5187 hasNoSignFlagUses(SDValue(Node, 0)))) {
5195 hasNoSignFlagUses(SDValue(Node, 0)))) {
5210 hasNoSignFlagUses(SDValue(Node, 0)))) {
5225 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
5226 SDValue Reg = N0.getOperand(0);
5230 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5243 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
5247 ReplaceUses(Reg.getValue(1), SDValue(NewNode, 1));
5268 bool NeedIndex = !SDValue(Node, 0).use_empty();
5269 bool NeedMask = !SDValue(Node, 1).use_empty();
5278 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 0));
5284 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0));
5288 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 1));
5297 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
5299 SDValue()).getValue(1);
5303 bool NeedIndex = !SDValue(Node, 0).use_empty();
5304 bool NeedMask = !SDValue(Node, 1).use_empty();
5314 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 0));
5320 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0));
5323 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 1));
5329 if (NVT.isVector() && tryVPTESTM(Node, SDValue(Node, 0), SDValue()))
5346 SDValue EFLAGS =
5348 Node->getOperand(1), SDValue());
5354 SDValue Result = SDValue(
5363 ReplaceUses(SDValue(Node, 0), Result);
5374 SDValue Zero =
5375 SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, None), 0);
5377 Zero = SDValue(
5386 SDValue EFLAGS =
5388 Node->getOperand(2), SDValue());
5395 SDValue Result =
5396 SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {Zero, Zero, EFLAGS,
5401 ReplaceUses(SDValue(Node, 1), Result.getValue(1));
5404 if (!SDValue(Node, 0).use_empty()) {
5410 ReplaceUses(SDValue(Node, 0), Result);
5420 SDValue IndexOp = Mgt->getIndex();
5421 SDValue Mask = Mgt->getMask();
5489 SDValue Base, Scale, Index, Disp, Segment;
5494 SDValue PassThru = Mgt->getPassThru();
5495 SDValue Chain = Mgt->getChain();
5501 SDValue Ops[] = {PassThru, Mask, Base, Scale,
5505 SDValue Ops[] = {PassThru, Base, Scale, Index,
5510 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0));
5511 ReplaceUses(SDValue(Node, 1), SDValue(NewNode, 2));
5517 SDValue Value = Sc->getValue();
5518 SDValue IndexOp = Sc->getIndex();
5562 SDValue Base, Scale, Index, Disp, Segment;
5567 SDValue Mask = Sc->getMask();
5568 SDValue Chain = Sc->getChain();
5571 SDValue Ops[] = {Base, Scale, Index, Disp, Segment, Mask, Value, Chain};
5575 ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 1));
5583 SDValue Chain = Node->getOperand(0);
5584 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
5587 ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); // Chain
5595 SDValue Chain = Node->getOperand(0);
5596 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
5597 SDValue ArgIndex = Node->getOperand(2);
5598 SDValue Ops[3];
5607 ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); // Arg pointer
5608 ReplaceUses(SDValue(Node, 1), SDValue(New, 1)); // Chain
5618 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
5619 std::vector<SDValue> &OutOps) {
5620 SDValue Op0, Op1, Op2, Op3, Op4;