1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86BaseInfo.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
22#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24#include "llvm/CodeGen/GlobalISel/RegisterBank.h"
25#include "llvm/CodeGen/GlobalISel/Utils.h"
26#include "llvm/CodeGen/MachineBasicBlock.h"
27#include "llvm/CodeGen/MachineConstantPool.h"
28#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
31#include "llvm/CodeGen/MachineMemOperand.h"
32#include "llvm/CodeGen/MachineOperand.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/TargetOpcodes.h"
35#include "llvm/CodeGen/TargetRegisterInfo.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/InstrTypes.h"
38#include "llvm/IR/IntrinsicsX86.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/CodeGen.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/LowLevelTypeImpl.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include <cassert>
47#include <cstdint>
48#include <tuple>
49
50#define DEBUG_TYPE "X86-isel"
51
52using namespace llvm;
53
54namespace {
55
56#define GET_GLOBALISEL_PREDICATE_BITSET
57#include "X86GenGlobalISel.inc"
58#undef GET_GLOBALISEL_PREDICATE_BITSET
59
60class X86InstructionSelector : public InstructionSelector {
61public:
62  X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
63                         const X86RegisterBankInfo &RBI);
64
65  bool select(MachineInstr &I) override;
66  static const char *getName() { return DEBUG_TYPE; }
67
68private:
69  /// tblgen-erated 'select' implementation, used as the initial selector for
70  /// the patterns that don't require complex C++.
71  bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
72
73  // TODO: remove after supported by Tablegen-erated instruction selection.
74  unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
75                          Align Alignment) const;
76
77  bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
78                         MachineFunction &MF) const;
79  bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
80                             MachineFunction &MF) const;
81  bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
82                         MachineFunction &MF) const;
83  bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
84                      MachineFunction &MF) const;
85  bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
86                             MachineFunction &MF) const;
87  bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
88                  MachineFunction &MF) const;
89  bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
90                    MachineFunction &MF) const;
91  bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
92                 MachineFunction &MF) const;
93  bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
94                  MachineFunction &MF) const;
95  bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
96                   MachineFunction &MF) const;
97  bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
98  bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
99                           MachineFunction &MF);
100  bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
101                         MachineFunction &MF);
102  bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
103                    MachineFunction &MF) const;
104  bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
105                     MachineFunction &MF) const;
106  bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
107                        MachineFunction &MF) const;
108  bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
109                          const unsigned DstReg,
110                          const TargetRegisterClass *DstRC,
111                          const unsigned SrcReg,
112                          const TargetRegisterClass *SrcRC) const;
113  bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
114                     MachineFunction &MF) const;
115  bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
116  bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
117                    MachineFunction &MF) const;
118  bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
119                                   MachineFunction &MF) const;
120
121  // emit insert subreg instruction and insert it before MachineInstr &I
122  bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
123                        MachineRegisterInfo &MRI, MachineFunction &MF) const;
124  // emit extract subreg instruction and insert it before MachineInstr &I
125  bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
126                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
127
128  const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
129  const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
130                                         MachineRegisterInfo &MRI) const;
131
132  const X86TargetMachine &TM;
133  const X86Subtarget &STI;
134  const X86InstrInfo &TII;
135  const X86RegisterInfo &TRI;
136  const X86RegisterBankInfo &RBI;
137
138#define GET_GLOBALISEL_PREDICATES_DECL
139#include "X86GenGlobalISel.inc"
140#undef GET_GLOBALISEL_PREDICATES_DECL
141
142#define GET_GLOBALISEL_TEMPORARIES_DECL
143#include "X86GenGlobalISel.inc"
144#undef GET_GLOBALISEL_TEMPORARIES_DECL
145};
146
147} // end anonymous namespace
148
149#define GET_GLOBALISEL_IMPL
150#include "X86GenGlobalISel.inc"
151#undef GET_GLOBALISEL_IMPL
152
153X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
154                                               const X86Subtarget &STI,
155                                               const X86RegisterBankInfo &RBI)
156    : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
157      TRI(*STI.getRegisterInfo()), RBI(RBI),
158#define GET_GLOBALISEL_PREDICATES_INIT
159#include "X86GenGlobalISel.inc"
160#undef GET_GLOBALISEL_PREDICATES_INIT
161#define GET_GLOBALISEL_TEMPORARIES_INIT
162#include "X86GenGlobalISel.inc"
163#undef GET_GLOBALISEL_TEMPORARIES_INIT
164{
165}
166
167// FIXME: This should be target-independent, inferred from the types declared
168// for each class in the bank.
169const TargetRegisterClass *
170X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
171  if (RB.getID() == X86::GPRRegBankID) {
172    if (Ty.getSizeInBits() <= 8)
173      return &X86::GR8RegClass;
174    if (Ty.getSizeInBits() == 16)
175      return &X86::GR16RegClass;
176    if (Ty.getSizeInBits() == 32)
177      return &X86::GR32RegClass;
178    if (Ty.getSizeInBits() == 64)
179      return &X86::GR64RegClass;
180  }
181  if (RB.getID() == X86::VECRRegBankID) {
182    if (Ty.getSizeInBits() == 32)
183      return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
184    if (Ty.getSizeInBits() == 64)
185      return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
186    if (Ty.getSizeInBits() == 128)
187      return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
188    if (Ty.getSizeInBits() == 256)
189      return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
190    if (Ty.getSizeInBits() == 512)
191      return &X86::VR512RegClass;
192  }
193
194  llvm_unreachable("Unknown RegBank!");
195}
196
197const TargetRegisterClass *
198X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
199                                    MachineRegisterInfo &MRI) const {
200  const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
201  return getRegClass(Ty, RegBank);
202}
203
204static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
205  unsigned SubIdx = X86::NoSubRegister;
206  if (RC == &X86::GR32RegClass) {
207    SubIdx = X86::sub_32bit;
208  } else if (RC == &X86::GR16RegClass) {
209    SubIdx = X86::sub_16bit;
210  } else if (RC == &X86::GR8RegClass) {
211    SubIdx = X86::sub_8bit;
212  }
213
214  return SubIdx;
215}
216
217static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
218  assert(Register::isPhysicalRegister(Reg));
219  if (X86::GR64RegClass.contains(Reg))
220    return &X86::GR64RegClass;
221  if (X86::GR32RegClass.contains(Reg))
222    return &X86::GR32RegClass;
223  if (X86::GR16RegClass.contains(Reg))
224    return &X86::GR16RegClass;
225  if (X86::GR8RegClass.contains(Reg))
226    return &X86::GR8RegClass;
227
228  llvm_unreachable("Unknown RegClass for PhysReg!");
229}
230
231// Set X86 Opcode and constrain DestReg.
232bool X86InstructionSelector::selectCopy(MachineInstr &I,
233                                        MachineRegisterInfo &MRI) const {
234  Register DstReg = I.getOperand(0).getReg();
235  const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
236  const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
237
238  Register SrcReg = I.getOperand(1).getReg();
239  const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
240  const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
241
242  if (Register::isPhysicalRegister(DstReg)) {
243    assert(I.isCopy() && "Generic operators do not allow physical registers");
244
245    if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
246        DstRegBank.getID() == X86::GPRRegBankID) {
247
248      const TargetRegisterClass *SrcRC =
249          getRegClass(MRI.getType(SrcReg), SrcRegBank);
250      const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
251
252      if (SrcRC != DstRC) {
253        // This case can be generated by ABI lowering, performe anyext
254        Register ExtSrc = MRI.createVirtualRegister(DstRC);
255        BuildMI(*I.getParent(), I, I.getDebugLoc(),
256                TII.get(TargetOpcode::SUBREG_TO_REG))
257            .addDef(ExtSrc)
258            .addImm(0)
259            .addReg(SrcReg)
260            .addImm(getSubRegIndex(SrcRC));
261
262        I.getOperand(1).setReg(ExtSrc);
263      }
264    }
265
266    return true;
267  }
268
269  assert((!Register::isPhysicalRegister(SrcReg) || I.isCopy()) &&
270         "No phys reg on generic operators");
271  assert((DstSize == SrcSize ||
272          // Copies are a mean to setup initial types, the number of
273          // bits may not exactly match.
274          (Register::isPhysicalRegister(SrcReg) &&
275           DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
276         "Copy with different width?!");
277
278  const TargetRegisterClass *DstRC =
279      getRegClass(MRI.getType(DstReg), DstRegBank);
280
281  if (SrcRegBank.getID() == X86::GPRRegBankID &&
282      DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
283      Register::isPhysicalRegister(SrcReg)) {
284    // Change the physical register to performe truncate.
285
286    const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
287
288    if (DstRC != SrcRC) {
289      I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
290      I.getOperand(1).substPhysReg(SrcReg, TRI);
291    }
292  }
293
294  // No need to constrain SrcReg. It will get constrained when
295  // we hit another of its use or its defs.
296  // Copies do not have constraints.
297  const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
298  if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
299    if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
300      LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
301                        << " operand\n");
302      return false;
303    }
304  }
305  I.setDesc(TII.get(X86::COPY));
306  return true;
307}
308
309bool X86InstructionSelector::select(MachineInstr &I) {
310  assert(I.getParent() && "Instruction should be in a basic block!");
311  assert(I.getParent()->getParent() && "Instruction should be in a function!");
312
313  MachineBasicBlock &MBB = *I.getParent();
314  MachineFunction &MF = *MBB.getParent();
315  MachineRegisterInfo &MRI = MF.getRegInfo();
316
317  unsigned Opcode = I.getOpcode();
318  if (!isPreISelGenericOpcode(Opcode)) {
319    // Certain non-generic instructions also need some special handling.
320
321    if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
322      return false;
323
324    if (I.isCopy())
325      return selectCopy(I, MRI);
326
327    return true;
328  }
329
330  assert(I.getNumOperands() == I.getNumExplicitOperands() &&
331         "Generic instruction has unexpected implicit operands\n");
332
333  if (selectImpl(I, *CoverageInfo))
334    return true;
335
336  LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
337
338  // TODO: This should be implemented by tblgen.
339  switch (I.getOpcode()) {
340  default:
341    return false;
342  case TargetOpcode::G_STORE:
343  case TargetOpcode::G_LOAD:
344    return selectLoadStoreOp(I, MRI, MF);
345  case TargetOpcode::G_PTR_ADD:
346  case TargetOpcode::G_FRAME_INDEX:
347    return selectFrameIndexOrGep(I, MRI, MF);
348  case TargetOpcode::G_GLOBAL_VALUE:
349    return selectGlobalValue(I, MRI, MF);
350  case TargetOpcode::G_CONSTANT:
351    return selectConstant(I, MRI, MF);
352  case TargetOpcode::G_FCONSTANT:
353    return materializeFP(I, MRI, MF);
354  case TargetOpcode::G_PTRTOINT:
355  case TargetOpcode::G_TRUNC:
356    return selectTruncOrPtrToInt(I, MRI, MF);
357  case TargetOpcode::G_INTTOPTR:
358    return selectCopy(I, MRI);
359  case TargetOpcode::G_ZEXT:
360    return selectZext(I, MRI, MF);
361  case TargetOpcode::G_ANYEXT:
362    return selectAnyext(I, MRI, MF);
363  case TargetOpcode::G_ICMP:
364    return selectCmp(I, MRI, MF);
365  case TargetOpcode::G_FCMP:
366    return selectFCmp(I, MRI, MF);
367  case TargetOpcode::G_UADDE:
368    return selectUadde(I, MRI, MF);
369  case TargetOpcode::G_UNMERGE_VALUES:
370    return selectUnmergeValues(I, MRI, MF);
371  case TargetOpcode::G_MERGE_VALUES:
372  case TargetOpcode::G_CONCAT_VECTORS:
373    return selectMergeValues(I, MRI, MF);
374  case TargetOpcode::G_EXTRACT:
375    return selectExtract(I, MRI, MF);
376  case TargetOpcode::G_INSERT:
377    return selectInsert(I, MRI, MF);
378  case TargetOpcode::G_BRCOND:
379    return selectCondBranch(I, MRI, MF);
380  case TargetOpcode::G_IMPLICIT_DEF:
381  case TargetOpcode::G_PHI:
382    return selectImplicitDefOrPHI(I, MRI);
383  case TargetOpcode::G_SDIV:
384  case TargetOpcode::G_UDIV:
385  case TargetOpcode::G_SREM:
386  case TargetOpcode::G_UREM:
387    return selectDivRem(I, MRI, MF);
388  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
389    return selectIntrinsicWSideEffects(I, MRI, MF);
390  }
391
392  return false;
393}
394
395unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
396                                                const RegisterBank &RB,
397                                                unsigned Opc,
398                                                Align Alignment) const {
399  bool Isload = (Opc == TargetOpcode::G_LOAD);
400  bool HasAVX = STI.hasAVX();
401  bool HasAVX512 = STI.hasAVX512();
402  bool HasVLX = STI.hasVLX();
403
404  if (Ty == LLT::scalar(8)) {
405    if (X86::GPRRegBankID == RB.getID())
406      return Isload ? X86::MOV8rm : X86::MOV8mr;
407  } else if (Ty == LLT::scalar(16)) {
408    if (X86::GPRRegBankID == RB.getID())
409      return Isload ? X86::MOV16rm : X86::MOV16mr;
410  } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
411    if (X86::GPRRegBankID == RB.getID())
412      return Isload ? X86::MOV32rm : X86::MOV32mr;
413    if (X86::VECRRegBankID == RB.getID())
414      return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
415                       HasAVX    ? X86::VMOVSSrm_alt :
416                                   X86::MOVSSrm_alt)
417                    : (HasAVX512 ? X86::VMOVSSZmr :
418                       HasAVX    ? X86::VMOVSSmr :
419                                   X86::MOVSSmr);
420  } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
421    if (X86::GPRRegBankID == RB.getID())
422      return Isload ? X86::MOV64rm : X86::MOV64mr;
423    if (X86::VECRRegBankID == RB.getID())
424      return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
425                       HasAVX    ? X86::VMOVSDrm_alt :
426                                   X86::MOVSDrm_alt)
427                    : (HasAVX512 ? X86::VMOVSDZmr :
428                       HasAVX    ? X86::VMOVSDmr :
429                                   X86::MOVSDmr);
430  } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
431    if (Alignment >= Align(16))
432      return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
433                              : HasAVX512
434                                    ? X86::VMOVAPSZ128rm_NOVLX
435                                    : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
436                    : (HasVLX ? X86::VMOVAPSZ128mr
437                              : HasAVX512
438                                    ? X86::VMOVAPSZ128mr_NOVLX
439                                    : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
440    else
441      return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
442                              : HasAVX512
443                                    ? X86::VMOVUPSZ128rm_NOVLX
444                                    : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
445                    : (HasVLX ? X86::VMOVUPSZ128mr
446                              : HasAVX512
447                                    ? X86::VMOVUPSZ128mr_NOVLX
448                                    : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
449  } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
450    if (Alignment >= Align(32))
451      return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
452                              : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
453                                          : X86::VMOVAPSYrm)
454                    : (HasVLX ? X86::VMOVAPSZ256mr
455                              : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
456                                          : X86::VMOVAPSYmr);
457    else
458      return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
459                              : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
460                                          : X86::VMOVUPSYrm)
461                    : (HasVLX ? X86::VMOVUPSZ256mr
462                              : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
463                                          : X86::VMOVUPSYmr);
464  } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
465    if (Alignment >= Align(64))
466      return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
467    else
468      return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
469  }
470  return Opc;
471}
472
473// Fill in an address from the given instruction.
474static void X86SelectAddress(const MachineInstr &I,
475                             const MachineRegisterInfo &MRI,
476                             X86AddressMode &AM) {
477  assert(I.getOperand(0).isReg() && "unsupported opperand.");
478  assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
479         "unsupported type.");
480
481  if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
482    if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
483      int64_t Imm = *COff;
484      if (isInt<32>(Imm)) { // Check for displacement overflow.
485        AM.Disp = static_cast<int32_t>(Imm);
486        AM.Base.Reg = I.getOperand(1).getReg();
487        return;
488      }
489    }
490  } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
491    AM.Base.FrameIndex = I.getOperand(1).getIndex();
492    AM.BaseType = X86AddressMode::FrameIndexBase;
493    return;
494  }
495
496  // Default behavior.
497  AM.Base.Reg = I.getOperand(0).getReg();
498}
499
500bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
501                                               MachineRegisterInfo &MRI,
502                                               MachineFunction &MF) const {
503  unsigned Opc = I.getOpcode();
504
505  assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
506         "unexpected instruction");
507
508  const Register DefReg = I.getOperand(0).getReg();
509  LLT Ty = MRI.getType(DefReg);
510  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
511
512  assert(I.hasOneMemOperand());
513  auto &MemOp = **I.memoperands_begin();
514  if (MemOp.isAtomic()) {
515    // Note: for unordered operations, we rely on the fact the appropriate MMO
516    // is already on the instruction we're mutating, and thus we don't need to
517    // make any changes.  So long as we select an opcode which is capable of
518    // loading or storing the appropriate size atomically, the rest of the
519    // backend is required to respect the MMO state.
520    if (!MemOp.isUnordered()) {
521      LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
522      return false;
523    }
524    if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
525      LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
526      return false;
527    }
528  }
529
530  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
531  if (NewOpc == Opc)
532    return false;
533
534  X86AddressMode AM;
535  X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
536
537  I.setDesc(TII.get(NewOpc));
538  MachineInstrBuilder MIB(MF, I);
539  if (Opc == TargetOpcode::G_LOAD) {
540    I.RemoveOperand(1);
541    addFullAddress(MIB, AM);
542  } else {
543    // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
544    I.RemoveOperand(1);
545    I.RemoveOperand(0);
546    addFullAddress(MIB, AM).addUse(DefReg);
547  }
548  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
549}
550
551static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
552  if (Ty == LLT::pointer(0, 64))
553    return X86::LEA64r;
554  else if (Ty == LLT::pointer(0, 32))
555    return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
556  else
557    llvm_unreachable("Can't get LEA opcode. Unsupported type.");
558}
559
560bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
561                                                   MachineRegisterInfo &MRI,
562                                                   MachineFunction &MF) const {
563  unsigned Opc = I.getOpcode();
564
565  assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
566         "unexpected instruction");
567
568  const Register DefReg = I.getOperand(0).getReg();
569  LLT Ty = MRI.getType(DefReg);
570
571  // Use LEA to calculate frame index and GEP
572  unsigned NewOpc = getLeaOP(Ty, STI);
573  I.setDesc(TII.get(NewOpc));
574  MachineInstrBuilder MIB(MF, I);
575
576  if (Opc == TargetOpcode::G_FRAME_INDEX) {
577    addOffset(MIB, 0);
578  } else {
579    MachineOperand &InxOp = I.getOperand(2);
580    I.addOperand(InxOp);        // set IndexReg
581    InxOp.ChangeToImmediate(1); // set Scale
582    MIB.addImm(0).addReg(0);
583  }
584
585  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
586}
587
588bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
589                                               MachineRegisterInfo &MRI,
590                                               MachineFunction &MF) const {
591  assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
592         "unexpected instruction");
593
594  auto GV = I.getOperand(1).getGlobal();
595  if (GV->isThreadLocal()) {
596    return false; // TODO: we don't support TLS yet.
597  }
598
599  // Can't handle alternate code models yet.
600  if (TM.getCodeModel() != CodeModel::Small)
601    return false;
602
603  X86AddressMode AM;
604  AM.GV = GV;
605  AM.GVOpFlags = STI.classifyGlobalReference(GV);
606
607  // TODO: The ABI requires an extra load. not supported yet.
608  if (isGlobalStubReference(AM.GVOpFlags))
609    return false;
610
611  // TODO: This reference is relative to the pic base. not supported yet.
612  if (isGlobalRelativeToPICBase(AM.GVOpFlags))
613    return false;
614
615  if (STI.isPICStyleRIPRel()) {
616    // Use rip-relative addressing.
617    assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
618    AM.Base.Reg = X86::RIP;
619  }
620
621  const Register DefReg = I.getOperand(0).getReg();
622  LLT Ty = MRI.getType(DefReg);
623  unsigned NewOpc = getLeaOP(Ty, STI);
624
625  I.setDesc(TII.get(NewOpc));
626  MachineInstrBuilder MIB(MF, I);
627
628  I.RemoveOperand(1);
629  addFullAddress(MIB, AM);
630
631  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
632}
633
634bool X86InstructionSelector::selectConstant(MachineInstr &I,
635                                            MachineRegisterInfo &MRI,
636                                            MachineFunction &MF) const {
637  assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
638         "unexpected instruction");
639
640  const Register DefReg = I.getOperand(0).getReg();
641  LLT Ty = MRI.getType(DefReg);
642
643  if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
644    return false;
645
646  uint64_t Val = 0;
647  if (I.getOperand(1).isCImm()) {
648    Val = I.getOperand(1).getCImm()->getZExtValue();
649    I.getOperand(1).ChangeToImmediate(Val);
650  } else if (I.getOperand(1).isImm()) {
651    Val = I.getOperand(1).getImm();
652  } else
653    llvm_unreachable("Unsupported operand type.");
654
655  unsigned NewOpc;
656  switch (Ty.getSizeInBits()) {
657  case 8:
658    NewOpc = X86::MOV8ri;
659    break;
660  case 16:
661    NewOpc = X86::MOV16ri;
662    break;
663  case 32:
664    NewOpc = X86::MOV32ri;
665    break;
666  case 64:
667    // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
668    if (isInt<32>(Val))
669      NewOpc = X86::MOV64ri32;
670    else
671      NewOpc = X86::MOV64ri;
672    break;
673  default:
674    llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
675  }
676
677  I.setDesc(TII.get(NewOpc));
678  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
679}
680
681// Helper function for selectTruncOrPtrToInt and selectAnyext.
682// Returns true if DstRC lives on a floating register class and
683// SrcRC lives on a 128-bit vector class.
684static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
685                            const TargetRegisterClass *SrcRC) {
686  return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
687          DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
688         (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
689}
690
691bool X86InstructionSelector::selectTurnIntoCOPY(
692    MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
693    const TargetRegisterClass *DstRC, const unsigned SrcReg,
694    const TargetRegisterClass *SrcRC) const {
695
696  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
697      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
698    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
699                      << " operand\n");
700    return false;
701  }
702  I.setDesc(TII.get(X86::COPY));
703  return true;
704}
705
706bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
707                                                   MachineRegisterInfo &MRI,
708                                                   MachineFunction &MF) const {
709  assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
710          I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
711         "unexpected instruction");
712
713  const Register DstReg = I.getOperand(0).getReg();
714  const Register SrcReg = I.getOperand(1).getReg();
715
716  const LLT DstTy = MRI.getType(DstReg);
717  const LLT SrcTy = MRI.getType(SrcReg);
718
719  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
720  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
721
722  if (DstRB.getID() != SrcRB.getID()) {
723    LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
724                      << " input/output on different banks\n");
725    return false;
726  }
727
728  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
729  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
730
731  if (!DstRC || !SrcRC)
732    return false;
733
734  // If that's truncation of the value that lives on the vector class and goes
735  // into the floating class, just replace it with copy, as we are able to
736  // select it as a regular move.
737  if (canTurnIntoCOPY(DstRC, SrcRC))
738    return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
739
740  if (DstRB.getID() != X86::GPRRegBankID)
741    return false;
742
743  unsigned SubIdx;
744  if (DstRC == SrcRC) {
745    // Nothing to be done
746    SubIdx = X86::NoSubRegister;
747  } else if (DstRC == &X86::GR32RegClass) {
748    SubIdx = X86::sub_32bit;
749  } else if (DstRC == &X86::GR16RegClass) {
750    SubIdx = X86::sub_16bit;
751  } else if (DstRC == &X86::GR8RegClass) {
752    SubIdx = X86::sub_8bit;
753  } else {
754    return false;
755  }
756
757  SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
758
759  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
760      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
761    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
762                      << "\n");
763    return false;
764  }
765
766  I.getOperand(1).setSubReg(SubIdx);
767
768  I.setDesc(TII.get(X86::COPY));
769  return true;
770}
771
772bool X86InstructionSelector::selectZext(MachineInstr &I,
773                                        MachineRegisterInfo &MRI,
774                                        MachineFunction &MF) const {
775  assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
776
777  const Register DstReg = I.getOperand(0).getReg();
778  const Register SrcReg = I.getOperand(1).getReg();
779
780  const LLT DstTy = MRI.getType(DstReg);
781  const LLT SrcTy = MRI.getType(SrcReg);
782
783  assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
784         "8=>32 Zext is handled by tablegen");
785  assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
786         "16=>32 Zext is handled by tablegen");
787
788  const static struct ZextEntry {
789    LLT SrcTy;
790    LLT DstTy;
791    unsigned MovOp;
792    bool NeedSubregToReg;
793  } OpTable[] = {
794      {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false},  // i8  => i16
795      {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true},   // i8  => i64
796      {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64
797      {LLT::scalar(32), LLT::scalar(64), 0, true}                 // i32 => i64
798  };
799
800  auto ZextEntryIt =
801      std::find_if(std::begin(OpTable), std::end(OpTable),
802                   [SrcTy, DstTy](const ZextEntry &El) {
803                     return El.DstTy == DstTy && El.SrcTy == SrcTy;
804                   });
805
806  // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction.
807  if (ZextEntryIt != std::end(OpTable)) {
808    const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
809    const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
810    const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
811    const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
812
813    if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
814        !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
815      LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
816                        << " operand\n");
817      return false;
818    }
819
820    unsigned TransitRegTo = DstReg;
821    unsigned TransitRegFrom = SrcReg;
822    if (ZextEntryIt->MovOp) {
823      // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have
824      // a transit register in between: create it here.
825      if (ZextEntryIt->NeedSubregToReg) {
826        TransitRegFrom = MRI.createVirtualRegister(
827            getRegClass(LLT::scalar(32), DstReg, MRI));
828        TransitRegTo = TransitRegFrom;
829      }
830
831      BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp))
832          .addDef(TransitRegTo)
833          .addReg(SrcReg);
834    }
835    if (ZextEntryIt->NeedSubregToReg) {
836      BuildMI(*I.getParent(), I, I.getDebugLoc(),
837              TII.get(TargetOpcode::SUBREG_TO_REG))
838          .addDef(DstReg)
839          .addImm(0)
840          .addReg(TransitRegFrom)
841          .addImm(X86::sub_32bit);
842    }
843    I.eraseFromParent();
844    return true;
845  }
846
847  if (SrcTy != LLT::scalar(1))
848    return false;
849
850  unsigned AndOpc;
851  if (DstTy == LLT::scalar(8))
852    AndOpc = X86::AND8ri;
853  else if (DstTy == LLT::scalar(16))
854    AndOpc = X86::AND16ri8;
855  else if (DstTy == LLT::scalar(32))
856    AndOpc = X86::AND32ri8;
857  else if (DstTy == LLT::scalar(64))
858    AndOpc = X86::AND64ri8;
859  else
860    return false;
861
862  unsigned DefReg = SrcReg;
863  if (DstTy != LLT::scalar(8)) {
864    DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
865    BuildMI(*I.getParent(), I, I.getDebugLoc(),
866            TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
867        .addImm(0)
868        .addReg(SrcReg)
869        .addImm(X86::sub_8bit);
870  }
871
872  MachineInstr &AndInst =
873      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
874           .addReg(DefReg)
875           .addImm(1);
876
877  constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
878
879  I.eraseFromParent();
880  return true;
881}
882
883bool X86InstructionSelector::selectAnyext(MachineInstr &I,
884                                          MachineRegisterInfo &MRI,
885                                          MachineFunction &MF) const {
886  assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
887
888  const Register DstReg = I.getOperand(0).getReg();
889  const Register SrcReg = I.getOperand(1).getReg();
890
891  const LLT DstTy = MRI.getType(DstReg);
892  const LLT SrcTy = MRI.getType(SrcReg);
893
894  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
895  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
896
897  assert(DstRB.getID() == SrcRB.getID() &&
898         "G_ANYEXT input/output on different banks\n");
899
900  assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
901         "G_ANYEXT incorrect operand size");
902
903  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
904  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
905
906  // If that's ANY_EXT of the value that lives on the floating class and goes
907  // into the vector class, just replace it with copy, as we are able to select
908  // it as a regular move.
909  if (canTurnIntoCOPY(SrcRC, DstRC))
910    return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
911
912  if (DstRB.getID() != X86::GPRRegBankID)
913    return false;
914
915  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
916      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
917    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
918                      << " operand\n");
919    return false;
920  }
921
922  if (SrcRC == DstRC) {
923    I.setDesc(TII.get(X86::COPY));
924    return true;
925  }
926
927  BuildMI(*I.getParent(), I, I.getDebugLoc(),
928          TII.get(TargetOpcode::SUBREG_TO_REG))
929      .addDef(DstReg)
930      .addImm(0)
931      .addReg(SrcReg)
932      .addImm(getSubRegIndex(SrcRC));
933
934  I.eraseFromParent();
935  return true;
936}
937
938bool X86InstructionSelector::selectCmp(MachineInstr &I,
939                                       MachineRegisterInfo &MRI,
940                                       MachineFunction &MF) const {
941  assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
942
943  X86::CondCode CC;
944  bool SwapArgs;
945  std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
946      (CmpInst::Predicate)I.getOperand(1).getPredicate());
947
948  Register LHS = I.getOperand(2).getReg();
949  Register RHS = I.getOperand(3).getReg();
950
951  if (SwapArgs)
952    std::swap(LHS, RHS);
953
954  unsigned OpCmp;
955  LLT Ty = MRI.getType(LHS);
956
957  switch (Ty.getSizeInBits()) {
958  default:
959    return false;
960  case 8:
961    OpCmp = X86::CMP8rr;
962    break;
963  case 16:
964    OpCmp = X86::CMP16rr;
965    break;
966  case 32:
967    OpCmp = X86::CMP32rr;
968    break;
969  case 64:
970    OpCmp = X86::CMP64rr;
971    break;
972  }
973
974  MachineInstr &CmpInst =
975      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
976           .addReg(LHS)
977           .addReg(RHS);
978
979  MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
980                                   TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
981
982  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
983  constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
984
985  I.eraseFromParent();
986  return true;
987}
988
989bool X86InstructionSelector::selectFCmp(MachineInstr &I,
990                                        MachineRegisterInfo &MRI,
991                                        MachineFunction &MF) const {
992  assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
993
994  Register LhsReg = I.getOperand(2).getReg();
995  Register RhsReg = I.getOperand(3).getReg();
996  CmpInst::Predicate Predicate =
997      (CmpInst::Predicate)I.getOperand(1).getPredicate();
998
999  // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1000  static const uint16_t SETFOpcTable[2][3] = {
1001      {X86::COND_E, X86::COND_NP, X86::AND8rr},
1002      {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1003  const uint16_t *SETFOpc = nullptr;
1004  switch (Predicate) {
1005  default:
1006    break;
1007  case CmpInst::FCMP_OEQ:
1008    SETFOpc = &SETFOpcTable[0][0];
1009    break;
1010  case CmpInst::FCMP_UNE:
1011    SETFOpc = &SETFOpcTable[1][0];
1012    break;
1013  }
1014
1015  // Compute the opcode for the CMP instruction.
1016  unsigned OpCmp;
1017  LLT Ty = MRI.getType(LhsReg);
1018  switch (Ty.getSizeInBits()) {
1019  default:
1020    return false;
1021  case 32:
1022    OpCmp = X86::UCOMISSrr;
1023    break;
1024  case 64:
1025    OpCmp = X86::UCOMISDrr;
1026    break;
1027  }
1028
1029  Register ResultReg = I.getOperand(0).getReg();
1030  RBI.constrainGenericRegister(
1031      ResultReg,
1032      *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1033  if (SETFOpc) {
1034    MachineInstr &CmpInst =
1035        *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1036             .addReg(LhsReg)
1037             .addReg(RhsReg);
1038
1039    Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1040    Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1041    MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1042                                  TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1043    MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1044                                  TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1045    MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1046                                  TII.get(SETFOpc[2]), ResultReg)
1047                              .addReg(FlagReg1)
1048                              .addReg(FlagReg2);
1049    constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1050    constrainSelectedInstRegOperands(Set1, TII, TRI, RBI);
1051    constrainSelectedInstRegOperands(Set2, TII, TRI, RBI);
1052    constrainSelectedInstRegOperands(Set3, TII, TRI, RBI);
1053
1054    I.eraseFromParent();
1055    return true;
1056  }
1057
1058  X86::CondCode CC;
1059  bool SwapArgs;
1060  std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1061  assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1062
1063  if (SwapArgs)
1064    std::swap(LhsReg, RhsReg);
1065
1066  // Emit a compare of LHS/RHS.
1067  MachineInstr &CmpInst =
1068      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1069           .addReg(LhsReg)
1070           .addReg(RhsReg);
1071
1072  MachineInstr &Set =
1073      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1074  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1075  constrainSelectedInstRegOperands(Set, TII, TRI, RBI);
1076  I.eraseFromParent();
1077  return true;
1078}
1079
1080bool X86InstructionSelector::selectUadde(MachineInstr &I,
1081                                         MachineRegisterInfo &MRI,
1082                                         MachineFunction &MF) const {
1083  assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
1084
1085  const Register DstReg = I.getOperand(0).getReg();
1086  const Register CarryOutReg = I.getOperand(1).getReg();
1087  const Register Op0Reg = I.getOperand(2).getReg();
1088  const Register Op1Reg = I.getOperand(3).getReg();
1089  Register CarryInReg = I.getOperand(4).getReg();
1090
1091  const LLT DstTy = MRI.getType(DstReg);
1092
1093  if (DstTy != LLT::scalar(32))
1094    return false;
1095
1096  // find CarryIn def instruction.
1097  MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1098  while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1099    CarryInReg = Def->getOperand(1).getReg();
1100    Def = MRI.getVRegDef(CarryInReg);
1101  }
1102
1103  unsigned Opcode;
1104  if (Def->getOpcode() == TargetOpcode::G_UADDE) {
1105    // carry set by prev ADD.
1106
1107    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
1108        .addReg(CarryInReg);
1109
1110    if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
1111      return false;
1112
1113    Opcode = X86::ADC32rr;
1114  } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
1115    // carry is constant, support only 0.
1116    if (*val != 0)
1117      return false;
1118
1119    Opcode = X86::ADD32rr;
1120  } else
1121    return false;
1122
1123  MachineInstr &AddInst =
1124      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1125           .addReg(Op0Reg)
1126           .addReg(Op1Reg);
1127
1128  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1129      .addReg(X86::EFLAGS);
1130
1131  if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
1132      !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
1133    return false;
1134
1135  I.eraseFromParent();
1136  return true;
1137}
1138
1139bool X86InstructionSelector::selectExtract(MachineInstr &I,
1140                                           MachineRegisterInfo &MRI,
1141                                           MachineFunction &MF) const {
1142  assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1143         "unexpected instruction");
1144
1145  const Register DstReg = I.getOperand(0).getReg();
1146  const Register SrcReg = I.getOperand(1).getReg();
1147  int64_t Index = I.getOperand(2).getImm();
1148
1149  const LLT DstTy = MRI.getType(DstReg);
1150  const LLT SrcTy = MRI.getType(SrcReg);
1151
1152  // Meanwile handle vector type only.
1153  if (!DstTy.isVector())
1154    return false;
1155
1156  if (Index % DstTy.getSizeInBits() != 0)
1157    return false; // Not extract subvector.
1158
1159  if (Index == 0) {
1160    // Replace by extract subreg copy.
1161    if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1162      return false;
1163
1164    I.eraseFromParent();
1165    return true;
1166  }
1167
1168  bool HasAVX = STI.hasAVX();
1169  bool HasAVX512 = STI.hasAVX512();
1170  bool HasVLX = STI.hasVLX();
1171
1172  if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1173    if (HasVLX)
1174      I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1175    else if (HasAVX)
1176      I.setDesc(TII.get(X86::VEXTRACTF128rr));
1177    else
1178      return false;
1179  } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1180    if (DstTy.getSizeInBits() == 128)
1181      I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1182    else if (DstTy.getSizeInBits() == 256)
1183      I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1184    else
1185      return false;
1186  } else
1187    return false;
1188
1189  // Convert to X86 VEXTRACT immediate.
1190  Index = Index / DstTy.getSizeInBits();
1191  I.getOperand(2).setImm(Index);
1192
1193  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1194}
1195
1196bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1197                                               MachineInstr &I,
1198                                               MachineRegisterInfo &MRI,
1199                                               MachineFunction &MF) const {
1200  const LLT DstTy = MRI.getType(DstReg);
1201  const LLT SrcTy = MRI.getType(SrcReg);
1202  unsigned SubIdx = X86::NoSubRegister;
1203
1204  if (!DstTy.isVector() || !SrcTy.isVector())
1205    return false;
1206
1207  assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1208         "Incorrect Src/Dst register size");
1209
1210  if (DstTy.getSizeInBits() == 128)
1211    SubIdx = X86::sub_xmm;
1212  else if (DstTy.getSizeInBits() == 256)
1213    SubIdx = X86::sub_ymm;
1214  else
1215    return false;
1216
1217  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1218  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1219
1220  SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1221
1222  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1223      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1224    LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1225    return false;
1226  }
1227
1228  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1229      .addReg(SrcReg, 0, SubIdx);
1230
1231  return true;
1232}
1233
1234bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1235                                              MachineInstr &I,
1236                                              MachineRegisterInfo &MRI,
1237                                              MachineFunction &MF) const {
1238  const LLT DstTy = MRI.getType(DstReg);
1239  const LLT SrcTy = MRI.getType(SrcReg);
1240  unsigned SubIdx = X86::NoSubRegister;
1241
1242  // TODO: support scalar types
1243  if (!DstTy.isVector() || !SrcTy.isVector())
1244    return false;
1245
1246  assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1247         "Incorrect Src/Dst register size");
1248
1249  if (SrcTy.getSizeInBits() == 128)
1250    SubIdx = X86::sub_xmm;
1251  else if (SrcTy.getSizeInBits() == 256)
1252    SubIdx = X86::sub_ymm;
1253  else
1254    return false;
1255
1256  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1257  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1258
1259  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1260      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1261    LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1262    return false;
1263  }
1264
1265  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1266      .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1267      .addReg(SrcReg);
1268
1269  return true;
1270}
1271
1272bool X86InstructionSelector::selectInsert(MachineInstr &I,
1273                                          MachineRegisterInfo &MRI,
1274                                          MachineFunction &MF) const {
1275  assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1276
1277  const Register DstReg = I.getOperand(0).getReg();
1278  const Register SrcReg = I.getOperand(1).getReg();
1279  const Register InsertReg = I.getOperand(2).getReg();
1280  int64_t Index = I.getOperand(3).getImm();
1281
1282  const LLT DstTy = MRI.getType(DstReg);
1283  const LLT InsertRegTy = MRI.getType(InsertReg);
1284
1285  // Meanwile handle vector type only.
1286  if (!DstTy.isVector())
1287    return false;
1288
1289  if (Index % InsertRegTy.getSizeInBits() != 0)
1290    return false; // Not insert subvector.
1291
1292  if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1293    // Replace by subreg copy.
1294    if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1295      return false;
1296
1297    I.eraseFromParent();
1298    return true;
1299  }
1300
1301  bool HasAVX = STI.hasAVX();
1302  bool HasAVX512 = STI.hasAVX512();
1303  bool HasVLX = STI.hasVLX();
1304
1305  if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1306    if (HasVLX)
1307      I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1308    else if (HasAVX)
1309      I.setDesc(TII.get(X86::VINSERTF128rr));
1310    else
1311      return false;
1312  } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1313    if (InsertRegTy.getSizeInBits() == 128)
1314      I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1315    else if (InsertRegTy.getSizeInBits() == 256)
1316      I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1317    else
1318      return false;
1319  } else
1320    return false;
1321
1322  // Convert to X86 VINSERT immediate.
1323  Index = Index / InsertRegTy.getSizeInBits();
1324
1325  I.getOperand(3).setImm(Index);
1326
1327  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1328}
1329
1330bool X86InstructionSelector::selectUnmergeValues(
1331    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1332  assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1333         "unexpected instruction");
1334
1335  // Split to extracts.
1336  unsigned NumDefs = I.getNumOperands() - 1;
1337  Register SrcReg = I.getOperand(NumDefs).getReg();
1338  unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1339
1340  for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1341    MachineInstr &ExtrInst =
1342        *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1343                 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1344             .addReg(SrcReg)
1345             .addImm(Idx * DefSize);
1346
1347    if (!select(ExtrInst))
1348      return false;
1349  }
1350
1351  I.eraseFromParent();
1352  return true;
1353}
1354
1355bool X86InstructionSelector::selectMergeValues(
1356    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1357  assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1358          I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1359         "unexpected instruction");
1360
1361  // Split to inserts.
1362  Register DstReg = I.getOperand(0).getReg();
1363  Register SrcReg0 = I.getOperand(1).getReg();
1364
1365  const LLT DstTy = MRI.getType(DstReg);
1366  const LLT SrcTy = MRI.getType(SrcReg0);
1367  unsigned SrcSize = SrcTy.getSizeInBits();
1368
1369  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1370
1371  // For the first src use insertSubReg.
1372  Register DefReg = MRI.createGenericVirtualRegister(DstTy);
1373  MRI.setRegBank(DefReg, RegBank);
1374  if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1375    return false;
1376
1377  for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1378    Register Tmp = MRI.createGenericVirtualRegister(DstTy);
1379    MRI.setRegBank(Tmp, RegBank);
1380
1381    MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1382                                        TII.get(TargetOpcode::G_INSERT), Tmp)
1383                                    .addReg(DefReg)
1384                                    .addReg(I.getOperand(Idx).getReg())
1385                                    .addImm((Idx - 1) * SrcSize);
1386
1387    DefReg = Tmp;
1388
1389    if (!select(InsertInst))
1390      return false;
1391  }
1392
1393  MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1394                                    TII.get(TargetOpcode::COPY), DstReg)
1395                                .addReg(DefReg);
1396
1397  if (!select(CopyInst))
1398    return false;
1399
1400  I.eraseFromParent();
1401  return true;
1402}
1403
1404bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1405                                              MachineRegisterInfo &MRI,
1406                                              MachineFunction &MF) const {
1407  assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1408
1409  const Register CondReg = I.getOperand(0).getReg();
1410  MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1411
1412  MachineInstr &TestInst =
1413      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1414           .addReg(CondReg)
1415           .addImm(1);
1416  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1417      .addMBB(DestMBB).addImm(X86::COND_NE);
1418
1419  constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1420
1421  I.eraseFromParent();
1422  return true;
1423}
1424
1425bool X86InstructionSelector::materializeFP(MachineInstr &I,
1426                                           MachineRegisterInfo &MRI,
1427                                           MachineFunction &MF) const {
1428  assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1429         "unexpected instruction");
1430
1431  // Can't handle alternate code models yet.
1432  CodeModel::Model CM = TM.getCodeModel();
1433  if (CM != CodeModel::Small && CM != CodeModel::Large)
1434    return false;
1435
1436  const Register DstReg = I.getOperand(0).getReg();
1437  const LLT DstTy = MRI.getType(DstReg);
1438  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1439  Align Alignment = Align(DstTy.getSizeInBytes());
1440  const DebugLoc &DbgLoc = I.getDebugLoc();
1441
1442  unsigned Opc =
1443      getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment);
1444
1445  // Create the load from the constant pool.
1446  const ConstantFP *CFP = I.getOperand(1).getFPImm();
1447  unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment);
1448  MachineInstr *LoadInst = nullptr;
1449  unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1450
1451  if (CM == CodeModel::Large && STI.is64Bit()) {
1452    // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1453    // they cannot be folded into immediate fields.
1454
1455    Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1456    BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1457        .addConstantPoolIndex(CPI, 0, OpFlag);
1458
1459    MachineMemOperand *MMO = MF.getMachineMemOperand(
1460        MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1461        MF.getDataLayout().getPointerSize(), Alignment);
1462
1463    LoadInst =
1464        addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1465                     AddrReg)
1466            .addMemOperand(MMO);
1467
1468  } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1469    // Handle the case when globals fit in our immediate field.
1470    // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1471
1472    // x86-32 PIC requires a PIC base register for constant pools.
1473    unsigned PICBase = 0;
1474    if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1475      // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1476      // In DAGISEL the code that initialize it generated by the CGBR pass.
1477      return false; // TODO support the mode.
1478    } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1479      PICBase = X86::RIP;
1480
1481    LoadInst = addConstantPoolReference(
1482        BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1483        OpFlag);
1484  } else
1485    return false;
1486
1487  constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1488  I.eraseFromParent();
1489  return true;
1490}
1491
1492bool X86InstructionSelector::selectImplicitDefOrPHI(
1493    MachineInstr &I, MachineRegisterInfo &MRI) const {
1494  assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1495          I.getOpcode() == TargetOpcode::G_PHI) &&
1496         "unexpected instruction");
1497
1498  Register DstReg = I.getOperand(0).getReg();
1499
1500  if (!MRI.getRegClassOrNull(DstReg)) {
1501    const LLT DstTy = MRI.getType(DstReg);
1502    const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1503
1504    if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1505      LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1506                        << " operand\n");
1507      return false;
1508    }
1509  }
1510
1511  if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1512    I.setDesc(TII.get(X86::IMPLICIT_DEF));
1513  else
1514    I.setDesc(TII.get(X86::PHI));
1515
1516  return true;
1517}
1518
1519bool X86InstructionSelector::selectDivRem(MachineInstr &I,
1520                                          MachineRegisterInfo &MRI,
1521                                          MachineFunction &MF) const {
1522  // The implementation of this function is taken from X86FastISel.
1523  assert((I.getOpcode() == TargetOpcode::G_SDIV ||
1524          I.getOpcode() == TargetOpcode::G_SREM ||
1525          I.getOpcode() == TargetOpcode::G_UDIV ||
1526          I.getOpcode() == TargetOpcode::G_UREM) &&
1527         "unexpected instruction");
1528
1529  const Register DstReg = I.getOperand(0).getReg();
1530  const Register Op1Reg = I.getOperand(1).getReg();
1531  const Register Op2Reg = I.getOperand(2).getReg();
1532
1533  const LLT RegTy = MRI.getType(DstReg);
1534  assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1535         "Arguments and return value types must match");
1536
1537  const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1538  if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1539    return false;
1540
1541  const static unsigned NumTypes = 4; // i8, i16, i32, i64
1542  const static unsigned NumOps = 4;   // SDiv, SRem, UDiv, URem
1543  const static bool S = true;         // IsSigned
1544  const static bool U = false;        // !IsSigned
1545  const static unsigned Copy = TargetOpcode::COPY;
1546  // For the X86 IDIV instruction, in most cases the dividend
1547  // (numerator) must be in a specific register pair highreg:lowreg,
1548  // producing the quotient in lowreg and the remainder in highreg.
1549  // For most data types, to set up the instruction, the dividend is
1550  // copied into lowreg, and lowreg is sign-extended into highreg.  The
1551  // exception is i8, where the dividend is defined as a single register rather
1552  // than a register pair, and we therefore directly sign-extend the dividend
1553  // into lowreg, instead of copying, and ignore the highreg.
1554  const static struct DivRemEntry {
1555    // The following portion depends only on the data type.
1556    unsigned SizeInBits;
1557    unsigned LowInReg;  // low part of the register pair
1558    unsigned HighInReg; // high part of the register pair
1559    // The following portion depends on both the data type and the operation.
1560    struct DivRemResult {
1561      unsigned OpDivRem;        // The specific DIV/IDIV opcode to use.
1562      unsigned OpSignExtend;    // Opcode for sign-extending lowreg into
1563                                // highreg, or copying a zero into highreg.
1564      unsigned OpCopy;          // Opcode for copying dividend into lowreg, or
1565                                // zero/sign-extending into lowreg for i8.
1566      unsigned DivRemResultReg; // Register containing the desired result.
1567      bool IsOpSigned;          // Whether to use signed or unsigned form.
1568    } ResultTable[NumOps];
1569  } OpTable[NumTypes] = {
1570      {8,
1571       X86::AX,
1572       0,
1573       {
1574           {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1575           {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1576           {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U},  // UDiv
1577           {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U},  // URem
1578       }},                                                // i8
1579      {16,
1580       X86::AX,
1581       X86::DX,
1582       {
1583           {X86::IDIV16r, X86::CWD, Copy, X86::AX, S},    // SDiv
1584           {X86::IDIV16r, X86::CWD, Copy, X86::DX, S},    // SRem
1585           {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1586           {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1587       }},                                                // i16
1588      {32,
1589       X86::EAX,
1590       X86::EDX,
1591       {
1592           {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S},    // SDiv
1593           {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S},    // SRem
1594           {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1595           {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1596       }},                                                 // i32
1597      {64,
1598       X86::RAX,
1599       X86::RDX,
1600       {
1601           {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S},    // SDiv
1602           {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S},    // SRem
1603           {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1604           {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1605       }},                                                 // i64
1606  };
1607
1608  auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable),
1609                                [RegTy](const DivRemEntry &El) {
1610                                  return El.SizeInBits == RegTy.getSizeInBits();
1611                                });
1612  if (OpEntryIt == std::end(OpTable))
1613    return false;
1614
1615  unsigned OpIndex;
1616  switch (I.getOpcode()) {
1617  default:
1618    llvm_unreachable("Unexpected div/rem opcode");
1619  case TargetOpcode::G_SDIV:
1620    OpIndex = 0;
1621    break;
1622  case TargetOpcode::G_SREM:
1623    OpIndex = 1;
1624    break;
1625  case TargetOpcode::G_UDIV:
1626    OpIndex = 2;
1627    break;
1628  case TargetOpcode::G_UREM:
1629    OpIndex = 3;
1630    break;
1631  }
1632
1633  const DivRemEntry &TypeEntry = *OpEntryIt;
1634  const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1635
1636  const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1637  if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1638      !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1639      !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1640    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1641                      << " operand\n");
1642    return false;
1643  }
1644
1645  // Move op1 into low-order input register.
1646  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1647          TypeEntry.LowInReg)
1648      .addReg(Op1Reg);
1649  // Zero-extend or sign-extend into high-order input register.
1650  if (OpEntry.OpSignExtend) {
1651    if (OpEntry.IsOpSigned)
1652      BuildMI(*I.getParent(), I, I.getDebugLoc(),
1653              TII.get(OpEntry.OpSignExtend));
1654    else {
1655      Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1656      BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1657              Zero32);
1658
1659      // Copy the zero into the appropriate sub/super/identical physical
1660      // register. Unfortunately the operations needed are not uniform enough
1661      // to fit neatly into the table above.
1662      if (RegTy.getSizeInBits() == 16) {
1663        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1664                TypeEntry.HighInReg)
1665            .addReg(Zero32, 0, X86::sub_16bit);
1666      } else if (RegTy.getSizeInBits() == 32) {
1667        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1668                TypeEntry.HighInReg)
1669            .addReg(Zero32);
1670      } else if (RegTy.getSizeInBits() == 64) {
1671        BuildMI(*I.getParent(), I, I.getDebugLoc(),
1672                TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1673            .addImm(0)
1674            .addReg(Zero32)
1675            .addImm(X86::sub_32bit);
1676      }
1677    }
1678  }
1679  // Generate the DIV/IDIV instruction.
1680  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem))
1681      .addReg(Op2Reg);
1682  // For i8 remainder, we can't reference ah directly, as we'll end
1683  // up with bogus copies like %r9b = COPY %ah. Reference ax
1684  // instead to prevent ah references in a rex instruction.
1685  //
1686  // The current assumption of the fast register allocator is that isel
1687  // won't generate explicit references to the GR8_NOREX registers. If
1688  // the allocator and/or the backend get enhanced to be more robust in
1689  // that regard, this can be, and should be, removed.
1690  if ((I.getOpcode() == Instruction::SRem ||
1691       I.getOpcode() == Instruction::URem) &&
1692      OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) {
1693    Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1694    Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1695    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1696        .addReg(X86::AX);
1697
1698    // Shift AX right by 8 bits instead of using AH.
1699    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1700            ResultSuperReg)
1701        .addReg(SourceSuperReg)
1702        .addImm(8);
1703
1704    // Now reference the 8-bit subreg of the result.
1705    BuildMI(*I.getParent(), I, I.getDebugLoc(),
1706            TII.get(TargetOpcode::SUBREG_TO_REG))
1707        .addDef(DstReg)
1708        .addImm(0)
1709        .addReg(ResultSuperReg)
1710        .addImm(X86::sub_8bit);
1711  } else {
1712    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1713            DstReg)
1714        .addReg(OpEntry.DivRemResultReg);
1715  }
1716  I.eraseFromParent();
1717  return true;
1718}
1719
1720bool X86InstructionSelector::selectIntrinsicWSideEffects(
1721    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
1722
1723  assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1724         "unexpected instruction");
1725
1726  if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1727    return false;
1728
1729  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
1730
1731  I.eraseFromParent();
1732  return true;
1733}
1734
1735InstructionSelector *
1736llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1737                                   X86Subtarget &Subtarget,
1738                                   X86RegisterBankInfo &RBI) {
1739  return new X86InstructionSelector(TM, Subtarget, RBI);
1740}
1741